support pool version archive

This commit is contained in:
Albert Yi 2016-12-14 18:09:45 -08:00
parent 3c038f76f9
commit ee4ebce4d7
22 changed files with 300 additions and 64 deletions

1
.gitignore vendored
View File

@ -20,3 +20,4 @@ danbooru.db
iqdb.db
danbooru.sublime-project
danbooru.sublime-workspace
script/scratch.rb

View File

@ -42,4 +42,8 @@ debug your Nginx configuration file.
IQDB integration is now delegated to the [IQDBS service](https://github.com/r888888888/iqdbs).
You will need to install your own copy and enable the appropriate
configuration settings.
configuration settings.
### Archive Service
In order to access versioned data for pools (and eventually posts) you will need to install and configure the [Archive service](https://github.com/r888888888/archives).

View File

@ -1,16 +1,35 @@
class PoolVersionsController < ApplicationController
respond_to :html, :xml, :json
before_filter :check_availabililty
def index
if params[:search] && params[:search][:pool_id].present?
@pool = Pool.find(params[:search][:pool_id])
end
@pool_versions = PoolVersion.search(params[:search]).order("updated_at desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search])
@pool_versions = PoolArchive.search(params[:search]).order("updated_at desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search])
respond_with(@pool_versions) do |format|
format.xml do
render :xml => @pool_versions.to_xml(:root => "pool-versions")
end
end
end
private
def check_availabililty
if !PoolArchive.enabled?
respond_to do |format|
format.html do
flash[:notice] = "Archive service is not configured. Pool versions are not saved."
redirect_to :back
end
format.json do
render json: {success: false, reason: "Archive service is not configured"}.to_json, status: 501
end
end
return false
end
end
end

View File

@ -2,13 +2,13 @@ module PoolVersionsHelper
def pool_version_diff(pool_version)
html = ""
html << pool_version.changes[:added_posts].map do |post_id|
html << pool_version.added_post_ids.map do |post_id|
'<ins><a href="/posts/' + post_id.to_s + '">' + post_id.to_s + '</a></ins>'
end.join(" ")
html << " "
html << pool_version.changes[:removed_posts].map do |post_id|
html << pool_version.removed_post_ids.map do |post_id|
'<del><a href="/posts/' + post_id.to_s + '">' + post_id.to_s + '</a></del>'
end.join(" ")

View File

@ -11,7 +11,6 @@ class Pool < ActiveRecord::Base
validate :updater_can_remove_posts
belongs_to :creator, :class_name => "User"
belongs_to :updater, :class_name => "User"
has_many :versions, lambda {order("pool_versions.id ASC")}, :class_name => "PoolVersion", :dependent => :destroy
before_validation :normalize_post_ids
before_validation :normalize_name
before_validation :initialize_is_active, :on => :create
@ -158,6 +157,14 @@ class Pool < ActiveRecord::Base
end
end
def versions
if PoolArchive.enabled?
PoolArchive.where("pool_id = ?", id).order("id asc")
else
raise "Archive service not configured"
end
end
def is_series?
category == "series"
end
@ -200,8 +207,10 @@ class Pool < ActiveRecord::Base
raise RevertError.new("You cannot revert to a previous version of another pool.")
end
self.post_ids = version.post_ids
self.post_ids = version.post_ids.join(" ")
self.name = version.name
self.description = version.description
synchronize!
end
@ -294,7 +303,7 @@ class Pool < ActiveRecord::Base
def synchronize!
synchronize
save
save if post_ids_changed?
end
def post_id_array
@ -336,17 +345,10 @@ class Pool < ActiveRecord::Base
end
def create_version(force = false)
if post_ids_changed? || name_changed? || description_changed? || is_active_changed? || is_deleted_changed? || category_changed? || force
last_version = versions.last
if last_version && last_version.updater_ip_addr == CurrentUser.ip_addr && CurrentUser.user.id == last_version.updater_id && last_version.created_at > 1.hour.ago
# merge
last_version.update_column(:post_ids, post_ids)
last_version.update_column(:name, name)
else
# create
versions.create(:post_ids => post_ids, :name => name)
end
if PoolArchive.enabled?
PoolArchive.queue(self)
else
Rails.logger.warn("Archive service is not configured. Pool versions will not be saved.")
end
end

View File

@ -0,0 +1,77 @@
class PoolArchive < ActiveRecord::Base
establish_connection "archive_#{Rails.env}".to_sym
self.table_name = "pool_versions"
module SearchMethods
def for_user(user_id)
where("updater_id = ?", user_id)
end
def search(params)
q = where("true")
return q if params.blank?
if params[:updater_id].present?
q = q.for_user(params[:updater_id].to_i)
end
if params[:updater_name].present?
q = q.where("updater_id = ?", User.name_to_id(params[:updater_name]))
end
if params[:pool_id].present?
q = q.where("pool_id = ?", params[:pool_id].to_i)
end
q
end
end
extend SearchMethods
def self.enabled?
Danbooru.config.aws_sqs_archives_url.present?
end
def self.sqs_service
SqsService.new(Danbooru.config.aws_sqs_archives_url)
end
def self.queue(pool)
# queue updates to sqs so that if archives goes down for whatever reason it won't
# block pool updates
raise "Archive service is not configured" if !enabled?
json = {
pool_id: pool.id,
post_ids: pool.post_ids.scan(/\d+/).map(&:to_i),
updater_id: CurrentUser.id,
updater_ip_addr: CurrentUser.ip_addr.to_s,
created_at: pool.created_at.try(:iso8601),
updated_at: pool.updated_at.try(:iso8601),
description: pool.description,
name: pool.name,
is_active: pool.is_active?,
is_deleted: pool.is_deleted?,
category: pool.category
}
msg = "add pool version\n#{json.to_json}"
sqs_service.send_message(msg)
end
def pool
Pool.find(pool_id)
end
def updater
User.find(updater_id)
end
def updater_name
User.id_to_name(updater_id)
end
def pretty_name
name.tr("_", " ")
end
end

View File

@ -34,6 +34,46 @@ class PoolVersion < ActiveRecord::Base
extend SearchMethods
def self.export_to_archives(starting_version_id = 0)
raise "SQS URL not setup" if Danbooru.config.aws_sqs_archives_url.nil?
credentials = Aws::Credentials.new(
Danbooru.config.aws_access_key_id,
Danbooru.config.aws_secret_access_key
)
sqs = Aws::SQS::Client.new(
credentials: credentials,
region: Danbooru.config.aws_sqs_region
)
last_version_id = 0
where("id > ?", starting_version_id).find_each do |version|
last_version_id = version.id
json = {
id: version.id,
pool_id: version.pool_id,
post_ids: version.post_ids.scan(/\d+/).map(&:to_i),
updater_id: version.updater_id,
updater_ip_addr: version.updater_ip_addr.to_s,
created_at: version.created_at.try(:iso8601),
updated_at: version.updated_at.try(:iso8601),
description: version.pool.description,
name: version.name,
is_active: version.pool.is_active?,
is_deleted: version.pool.is_deleted?,
category: version.pool.category
}
msg = "add pool version\n#{json.to_json}"
sqs.send_message(
message_body: msg,
queue_url: Danbooru.config.aws_sqs_archives_url
)
end
puts "last version id: #{last_version_id}"
end
def updater_name
User.id_to_name(updater_id)
end
@ -81,7 +121,7 @@ class PoolVersion < ActiveRecord::Base
intersect << id
other_array.delete_at(index)
end
intersect
intersect
end
end

View File

@ -178,7 +178,11 @@ class UserPresenter
end
def pool_version_count(template)
template.link_to(user.pool_version_count, template.pool_versions_path(:search => {:updater_id => user.id}))
if PoolArchive.enabled?
template.link_to(user.pool_version_count, template.pool_versions_path(:search => {:updater_id => user.id}))
else
"N/A"
end
end
def inviter(template)

View File

@ -22,9 +22,9 @@
<% @pool_versions.each do |pool_version| %>
<tr>
<td><%= link_to pool_version.pretty_name, pool_path(pool_version.pool_id), :class => "pool-category-#{pool_version.pool.category}" %></td>
<td><%= link_to pool_version.post_id_array.size, pool_versions_path(:search => {:pool_id => pool_version.pool_id}) %></td>
<td><%= link_to pool_version.post_ids.size, pool_versions_path(:search => {:pool_id => pool_version.pool_id}) %></td>
<td><%= pool_version_diff(pool_version) %></td>
<td><%= link_to_user pool_version.updater%></td>
<td><%= link_to_user pool_version.updater %></td>
<% if CurrentUser.is_moderator? %>
<td>
<%= pool_version.updater_ip_addr %>

View File

@ -19,7 +19,9 @@
<li><%= link_to "Delete", pool_path(@pool), :method => :delete, :data => {:confirm => "Are you sure you want to delete this pool?"}, :remote => true %></li>
<% end %>
<% end %>
<li><%= link_to "History", pool_versions_path(:search => {:pool_id => @pool.id}) %></li>
<% if PoolArchive.enabled? %>
<li><%= link_to "History", pool_versions_path(:search => {:pool_id => @pool.id}) %></li>
<% end %>
<% if @pool.post_count <= 100 && CurrentUser.is_member? %>
<li><%= link_to "Order", edit_pool_order_path(@pool) %></li>
<% end %>

View File

@ -53,7 +53,9 @@
<li><h1>Pools</h1></li>
<li><%= link_to("Help", wiki_pages_path(:title => "help:pools")) %></li>
<li><%= link_to("Listing", pools_path) %></li>
<li><%= link_to("Changes", pool_versions_path) %></li>
<% if PoolArchive.enabled? %>
<li><%= link_to("Changes", pool_versions_path) %></li>
<% end %>
</ul>
<ul>
<li><h1>Reports</h1></li>

View File

@ -84,10 +84,12 @@
<td><%= presenter.artist_commentary_version_count(self) %></td>
</tr>
<tr>
<th>Pool Changes</th>
<td><%= presenter.pool_version_count(self) %></td>
</tr>
<% if PoolArchive.enabled? %>
<tr>
<th>Pool Changes</th>
<td><%= presenter.pool_version_count(self) %></td>
</tr>
<% end %>
<tr>
<th>Forum Posts</th>

View File

@ -494,5 +494,8 @@ module Danbooru
def aws_sqs_iqdb_url
end
def aws_sqs_archives_url
end
end
end

View File

@ -1,8 +1,13 @@
require 'test_helper'
require 'helpers/pool_archive_test_helper'
class PoolElementsControllerTest < ActionController::TestCase
include PoolArchiveTestHelper
context "The pools posts controller" do
setup do
mock_pool_archive_service!
start_pool_archive_transaction
@user = Timecop.travel(1.month.ago) {FactoryGirl.create(:user)}
@mod = FactoryGirl.create(:moderator_user)
CurrentUser.user = @user
@ -12,6 +17,7 @@ class PoolElementsControllerTest < ActionController::TestCase
end
teardown do
rollback_pool_archive_transaction
CurrentUser.user = nil
end

View File

@ -1,14 +1,20 @@
require 'test_helper'
require 'helpers/pool_archive_test_helper'
class PoolVersionsControllerTest < ActionController::TestCase
include PoolArchiveTestHelper
context "The pool versions controller" do
setup do
mock_pool_archive_service!
start_pool_archive_transaction
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
end
teardown do
rollback_pool_archive_transaction
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end

View File

@ -1,6 +1,9 @@
require 'test_helper'
require 'helpers/pool_archive_test_helper'
class PoolsControllerTest < ActionController::TestCase
include PoolArchiveTestHelper
context "The pools controller" do
setup do
Timecop.travel(1.month.ago) do
@ -10,9 +13,12 @@ class PoolsControllerTest < ActionController::TestCase
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
@post = FactoryGirl.create(:post)
mock_pool_archive_service!
start_pool_archive_transaction
end
teardown do
rollback_pool_archive_transaction
CurrentUser.user = nil
end
@ -99,10 +105,10 @@ class PoolsControllerTest < ActionController::TestCase
end
should "revert to a previous version" do
assert_equal(2, PoolVersion.count)
assert_equal(2, PoolArchive.count)
@pool.reload
version = @pool.versions.first
assert_equal("#{@post.id}", version.post_ids)
assert_equal([@post.id], version.post_ids)
post :revert, {:id => @pool.id, :version_id => version.id}, {:user_id => @mod.id}
@pool.reload
assert_equal([@post.id], @pool.post_id_array)
@ -111,7 +117,7 @@ class PoolsControllerTest < ActionController::TestCase
should "not allow reverting to a previous version of another pool" do
@pool2 = FactoryGirl.create(:pool)
post :revert, { :id => @pool.id, :version_id => @pool2.versions(true).first.id }, {:user_id => @user.id}
post :revert, { :id => @pool.id, :version_id => @pool2.versions.first.id }, {:user_id => @user.id}
@pool.reload
assert_not_equal(@pool.name, @pool2.name)

View File

@ -1,4 +0,0 @@
require 'test_helper'
class BulkUpdateRequestsHelperTest < ActionView::TestCase
end

View File

@ -0,0 +1,26 @@
module PoolArchiveTestHelper
def mock_pool_archive_service!
mock_sqs_service = Class.new do
def send_message(msg)
_, json = msg.split(/\n/)
json = JSON.parse(json)
prev = PoolArchive.where(pool_id: json["pool_id"]).order("id desc").first
if prev && prev.updater_ip_addr.to_s == json["updater_ip_addr"]
prev.update_columns(json)
else
PoolArchive.create(json)
end
end
end
PoolArchive.stubs(:sqs_service).returns(mock_sqs_service.new)
end
def start_pool_archive_transaction
PoolArchive.connection.begin_transaction joinable: false
end
def rollback_pool_archive_transaction
PoolArchive.connection.rollback_transaction
end
end

View File

@ -1,4 +0,0 @@
require 'test_helper'
class SavedSearchesHelperTest < ActionView::TestCase
end

View File

@ -1,18 +1,26 @@
# encoding: utf-8
require 'test_helper'
require 'helpers/pool_archive_test_helper'
class PoolTest < ActiveSupport::TestCase
include PoolArchiveTestHelper
setup do
Timecop.travel(1.month.ago) do
user = FactoryGirl.create(:user)
CurrentUser.user = user
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
end
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
mock_pool_archive_service!
start_pool_archive_transaction
end
teardown do
rollback_pool_archive_transaction
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
@ -53,33 +61,41 @@ class PoolTest < ActiveSupport::TestCase
context "Reverting a pool" do
setup do
PoolArchive.stubs(:enabled?).returns(true)
@pool = FactoryGirl.create(:pool)
@p1 = FactoryGirl.create(:post)
@p2 = FactoryGirl.create(:post)
@p3 = FactoryGirl.create(:post)
CurrentUser.ip_addr = "1.2.3.4"
@pool.add!(@p1)
CurrentUser.ip_addr = "1.2.3.5"
@pool.add!(@p2)
CurrentUser.ip_addr = "1.2.3.6"
@pool.add!(@p3)
CurrentUser.ip_addr = "1.2.3.7"
@pool.remove!(@p1)
CurrentUser.ip_addr = "1.2.3.8"
@pool.revert_to!(@pool.versions.all[1])
CurrentUser.scoped(@user, "1.2.3.4") do
@pool.add!(@p1)
end
CurrentUser.scoped(@user, "1.2.3.5") do
@pool.add!(@p2)
end
CurrentUser.scoped(@user, "1.2.3.6") do
@pool.add!(@p3)
end
CurrentUser.scoped(@user, "1.2.3.7") do
@pool.remove!(@p1)
end
CurrentUser.scoped(@user, "1.2.3.8") do
version = @pool.versions[1]
@pool.revert_to!(version)
end
end
should "have the correct versions" do
assert_equal(6, @pool.versions.size)
assert_equal("", @pool.versions.all[0].post_ids)
assert_equal("#{@p1.id}", @pool.versions.all[1].post_ids)
assert_equal("#{@p1.id} #{@p2.id}", @pool.versions.all[2].post_ids)
assert_equal("#{@p1.id} #{@p2.id} #{@p3.id}", @pool.versions.all[3].post_ids)
assert_equal("#{@p2.id} #{@p3.id}", @pool.versions.all[4].post_ids)
assert_equal([], @pool.versions.all[0].post_ids)
assert_equal([@p1.id], @pool.versions.all[1].post_ids)
assert_equal([@p1.id, @p2.id], @pool.versions.all[2].post_ids)
assert_equal([@p1.id, @p2.id, @p3.id], @pool.versions.all[3].post_ids)
assert_equal([@p2.id, @p3.id], @pool.versions.all[4].post_ids)
end
should "update its post_ids" do
assert_equal("#{@p1.id}", @pool.post_ids)
assert_equal([@p1.id], @pool.post_id_array)
end
should "update any old posts that were removed" do

View File

@ -1,7 +1,10 @@
require 'test_helper'
require 'helpers/pool_archive_test_helper'
module PostSets
class PoolTest < ActiveSupport::TestCase
include PoolArchiveTestHelper
context "In all cases" do
setup do
@user = FactoryGirl.create(:user)
@ -9,6 +12,9 @@ module PostSets
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
mock_pool_archive_service!
start_pool_archive_transaction
@post_1 = FactoryGirl.create(:post)
@post_2 = FactoryGirl.create(:post)
@post_3 = FactoryGirl.create(:post)
@ -19,6 +25,7 @@ module PostSets
end
teardown do
rollback_pool_archive_transaction
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end

View File

@ -1,6 +1,9 @@
require 'test_helper'
require 'helpers/pool_archive_test_helper'
class PostTest < ActiveSupport::TestCase
include PoolArchiveTestHelper
setup do
Timecop.travel(2.weeks.ago) do
@user = FactoryGirl.create(:user)
@ -35,6 +38,7 @@ class PostTest < ActiveSupport::TestCase
context "that belongs to a pool" do
setup do
SqsService.any_instance.stubs(:send_message)
@pool = FactoryGirl.create(:pool)
@pool.add!(@post)
@post.reload
@ -590,6 +594,15 @@ class PostTest < ActiveSupport::TestCase
end
context "for a pool" do
setup do
mock_pool_archive_service!
start_pool_archive_transaction
end
teardown do
rollback_pool_archive_transaction
end
context "on creation" do
setup do
@pool = FactoryGirl.create(:pool)
@ -1143,11 +1156,11 @@ class PostTest < ActiveSupport::TestCase
should "clear the pixiv id" do
@post.pixiv_id = 1234
@post.update(source: "http://fc06.deviantart.net/fs71/f/2013/295/d/7/you_are_already_dead__by_mar11co-d6rgm0e.jpg")
assert_equal(nil, @post.pixiv_id)
assert_nil(@post.pixiv_id)
@post.pixiv_id = 1234
@post.update(source: "http://pictures.hentai-foundry.com//a/AnimeFlux/219123.jpg")
assert_equal(nil, @post.pixiv_id)
assert_nil(@post.pixiv_id)
end
end
@ -1232,19 +1245,19 @@ class PostTest < ActiveSupport::TestCase
should "not save the pixiv id" do
@post.pixiv_id = 1234
@post.update(source: "http://i1.pixiv.net/novel-cover-original/img/2016/11/03/20/10/58/7436075_f75af69f3eacd1656d3733c72aa959cf.jpg")
assert_equal(nil, @post.pixiv_id)
assert_nil(@post.pixiv_id)
@post.pixiv_id = 1234
@post.update(source: "http://i2.pixiv.net/background/img/2016/10/30/12/27/30/7059005_da9946b806c10d391a81ed1117cd33d6.jpg")
assert_equal(nil, @post.pixiv_id)
assert_nil(@post.pixiv_id)
@post.pixiv_id = 1234
@post.update(source: "http://i1.pixiv.net/img15/img/omega777/novel/2612734.jpg")
assert_equal(nil, @post.pixiv_id)
assert_nil(@post.pixiv_id)
@post.pixiv_id = 1234
@post.update(source: "http://img08.pixiv.net/profile/nice/1408837.jpg")
assert_equal(nil, @post.pixiv_id)
assert_nil(@post.pixiv_id)
end
end
end
@ -1440,6 +1453,10 @@ class PostTest < ActiveSupport::TestCase
end
context "Pools:" do
setup do
SqsService.any_instance.stubs(:send_message)
end
context "Removing a post from a pool" do
should "update the post's pool string" do
post = FactoryGirl.create(:post)
@ -1615,6 +1632,8 @@ class PostTest < ActiveSupport::TestCase
end
should "return posts for the <pool> metatag" do
SqsService.any_instance.stubs(:send_message)
post1 = FactoryGirl.create(:post)
post2 = FactoryGirl.create(:post)
post3 = FactoryGirl.create(:post)
@ -1626,6 +1645,8 @@ class PostTest < ActiveSupport::TestCase
end
should "return posts for the <pool> metatag with a wildcard" do
SqsService.any_instance.stubs(:send_message)
post1 = FactoryGirl.create(:post)
post2 = FactoryGirl.create(:post)
post3 = FactoryGirl.create(:post)
@ -2020,7 +2041,7 @@ class PostTest < ActiveSupport::TestCase
should "correctly revert all fields" do
assert_equal("aaa bbb ccc ddd", @post.tag_string)
assert_equal(nil, @post.source)
assert_nil(@post.source)
assert_equal("q", @post.rating)
end
end