diff --git a/.gitignore b/.gitignore index e36181c54..b4ad7db95 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ danbooru.db iqdb.db danbooru.sublime-project danbooru.sublime-workspace +script/scratch.rb diff --git a/README.md b/README.md index 13251c12b..616b863b3 100644 --- a/README.md +++ b/README.md @@ -42,4 +42,8 @@ debug your Nginx configuration file. IQDB integration is now delegated to the [IQDBS service](https://github.com/r888888888/iqdbs). You will need to install your own copy and enable the appropriate -configuration settings. \ No newline at end of file +configuration settings. + +### Archive Service + +In order to access versioned data for pools (and eventually posts) you will need to install and configure the [Archive service](https://github.com/r888888888/archives). diff --git a/app/controllers/pool_versions_controller.rb b/app/controllers/pool_versions_controller.rb index e89296fdc..5ab18aa7a 100644 --- a/app/controllers/pool_versions_controller.rb +++ b/app/controllers/pool_versions_controller.rb @@ -1,16 +1,35 @@ class PoolVersionsController < ApplicationController respond_to :html, :xml, :json + before_filter :check_availabililty def index if params[:search] && params[:search][:pool_id].present? @pool = Pool.find(params[:search][:pool_id]) end - @pool_versions = PoolVersion.search(params[:search]).order("updated_at desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search]) + @pool_versions = PoolArchive.search(params[:search]).order("updated_at desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search]) respond_with(@pool_versions) do |format| format.xml do render :xml => @pool_versions.to_xml(:root => "pool-versions") end end end + +private + + def check_availabililty + if !PoolArchive.enabled? + respond_to do |format| + format.html do + flash[:notice] = "Archive service is not configured. Pool versions are not saved." + redirect_to :back + end + format.json do + render json: {success: false, reason: "Archive service is not configured"}.to_json, status: 501 + end + end + + return false + end + end end diff --git a/app/helpers/pool_versions_helper.rb b/app/helpers/pool_versions_helper.rb index e80871825..e7939f784 100644 --- a/app/helpers/pool_versions_helper.rb +++ b/app/helpers/pool_versions_helper.rb @@ -2,13 +2,13 @@ module PoolVersionsHelper def pool_version_diff(pool_version) html = "" - html << pool_version.changes[:added_posts].map do |post_id| + html << pool_version.added_post_ids.map do |post_id| '' + post_id.to_s + '' end.join(" ") html << " " - html << pool_version.changes[:removed_posts].map do |post_id| + html << pool_version.removed_post_ids.map do |post_id| '' + post_id.to_s + '' end.join(" ") diff --git a/app/models/pool.rb b/app/models/pool.rb index 6c29c9d23..11696c2fb 100644 --- a/app/models/pool.rb +++ b/app/models/pool.rb @@ -11,7 +11,6 @@ class Pool < ActiveRecord::Base validate :updater_can_remove_posts belongs_to :creator, :class_name => "User" belongs_to :updater, :class_name => "User" - has_many :versions, lambda {order("pool_versions.id ASC")}, :class_name => "PoolVersion", :dependent => :destroy before_validation :normalize_post_ids before_validation :normalize_name before_validation :initialize_is_active, :on => :create @@ -158,6 +157,14 @@ class Pool < ActiveRecord::Base end end + def versions + if PoolArchive.enabled? + PoolArchive.where("pool_id = ?", id).order("id asc") + else + raise "Archive service not configured" + end + end + def is_series? category == "series" end @@ -200,8 +207,10 @@ class Pool < ActiveRecord::Base raise RevertError.new("You cannot revert to a previous version of another pool.") end - self.post_ids = version.post_ids + self.post_ids = version.post_ids.join(" ") self.name = version.name + self.description = version.description + synchronize! end @@ -294,7 +303,7 @@ class Pool < ActiveRecord::Base def synchronize! synchronize - save + save if post_ids_changed? end def post_id_array @@ -336,17 +345,10 @@ class Pool < ActiveRecord::Base end def create_version(force = false) - if post_ids_changed? || name_changed? || description_changed? || is_active_changed? || is_deleted_changed? || category_changed? || force - last_version = versions.last - - if last_version && last_version.updater_ip_addr == CurrentUser.ip_addr && CurrentUser.user.id == last_version.updater_id && last_version.created_at > 1.hour.ago - # merge - last_version.update_column(:post_ids, post_ids) - last_version.update_column(:name, name) - else - # create - versions.create(:post_ids => post_ids, :name => name) - end + if PoolArchive.enabled? + PoolArchive.queue(self) + else + Rails.logger.warn("Archive service is not configured. Pool versions will not be saved.") end end diff --git a/app/models/pool_archive.rb b/app/models/pool_archive.rb new file mode 100644 index 000000000..3fd15737f --- /dev/null +++ b/app/models/pool_archive.rb @@ -0,0 +1,77 @@ +class PoolArchive < ActiveRecord::Base + establish_connection "archive_#{Rails.env}".to_sym + self.table_name = "pool_versions" + + module SearchMethods + def for_user(user_id) + where("updater_id = ?", user_id) + end + + def search(params) + q = where("true") + return q if params.blank? + + if params[:updater_id].present? + q = q.for_user(params[:updater_id].to_i) + end + + if params[:updater_name].present? + q = q.where("updater_id = ?", User.name_to_id(params[:updater_name])) + end + + if params[:pool_id].present? + q = q.where("pool_id = ?", params[:pool_id].to_i) + end + + q + end + end + + extend SearchMethods + + def self.enabled? + Danbooru.config.aws_sqs_archives_url.present? + end + + def self.sqs_service + SqsService.new(Danbooru.config.aws_sqs_archives_url) + end + + def self.queue(pool) + # queue updates to sqs so that if archives goes down for whatever reason it won't + # block pool updates + raise "Archive service is not configured" if !enabled? + + json = { + pool_id: pool.id, + post_ids: pool.post_ids.scan(/\d+/).map(&:to_i), + updater_id: CurrentUser.id, + updater_ip_addr: CurrentUser.ip_addr.to_s, + created_at: pool.created_at.try(:iso8601), + updated_at: pool.updated_at.try(:iso8601), + description: pool.description, + name: pool.name, + is_active: pool.is_active?, + is_deleted: pool.is_deleted?, + category: pool.category + } + msg = "add pool version\n#{json.to_json}" + sqs_service.send_message(msg) + end + + def pool + Pool.find(pool_id) + end + + def updater + User.find(updater_id) + end + + def updater_name + User.id_to_name(updater_id) + end + + def pretty_name + name.tr("_", " ") + end +end diff --git a/app/models/pool_version.rb b/app/models/pool_version.rb index 3d6b7ce9e..9f5105c05 100644 --- a/app/models/pool_version.rb +++ b/app/models/pool_version.rb @@ -34,6 +34,46 @@ class PoolVersion < ActiveRecord::Base extend SearchMethods + def self.export_to_archives(starting_version_id = 0) + raise "SQS URL not setup" if Danbooru.config.aws_sqs_archives_url.nil? + + credentials = Aws::Credentials.new( + Danbooru.config.aws_access_key_id, + Danbooru.config.aws_secret_access_key + ) + sqs = Aws::SQS::Client.new( + credentials: credentials, + region: Danbooru.config.aws_sqs_region + ) + last_version_id = 0 + + where("id > ?", starting_version_id).find_each do |version| + last_version_id = version.id + + json = { + id: version.id, + pool_id: version.pool_id, + post_ids: version.post_ids.scan(/\d+/).map(&:to_i), + updater_id: version.updater_id, + updater_ip_addr: version.updater_ip_addr.to_s, + created_at: version.created_at.try(:iso8601), + updated_at: version.updated_at.try(:iso8601), + description: version.pool.description, + name: version.name, + is_active: version.pool.is_active?, + is_deleted: version.pool.is_deleted?, + category: version.pool.category + } + msg = "add pool version\n#{json.to_json}" + sqs.send_message( + message_body: msg, + queue_url: Danbooru.config.aws_sqs_archives_url + ) + end + + puts "last version id: #{last_version_id}" + end + def updater_name User.id_to_name(updater_id) end @@ -81,7 +121,7 @@ class PoolVersion < ActiveRecord::Base intersect << id other_array.delete_at(index) end - intersect + intersect end end diff --git a/app/presenters/user_presenter.rb b/app/presenters/user_presenter.rb index da8587a2c..dc821a6d6 100644 --- a/app/presenters/user_presenter.rb +++ b/app/presenters/user_presenter.rb @@ -178,7 +178,11 @@ class UserPresenter end def pool_version_count(template) - template.link_to(user.pool_version_count, template.pool_versions_path(:search => {:updater_id => user.id})) + if PoolArchive.enabled? + template.link_to(user.pool_version_count, template.pool_versions_path(:search => {:updater_id => user.id})) + else + "N/A" + end end def inviter(template) diff --git a/app/views/pool_versions/index.html.erb b/app/views/pool_versions/index.html.erb index 012313445..bae6fa1dd 100644 --- a/app/views/pool_versions/index.html.erb +++ b/app/views/pool_versions/index.html.erb @@ -22,9 +22,9 @@ <% @pool_versions.each do |pool_version| %> <%= link_to pool_version.pretty_name, pool_path(pool_version.pool_id), :class => "pool-category-#{pool_version.pool.category}" %> - <%= link_to pool_version.post_id_array.size, pool_versions_path(:search => {:pool_id => pool_version.pool_id}) %> + <%= link_to pool_version.post_ids.size, pool_versions_path(:search => {:pool_id => pool_version.pool_id}) %> <%= pool_version_diff(pool_version) %> - <%= link_to_user pool_version.updater%> + <%= link_to_user pool_version.updater %> <% if CurrentUser.is_moderator? %> <%= pool_version.updater_ip_addr %> diff --git a/app/views/pools/_secondary_links.html.erb b/app/views/pools/_secondary_links.html.erb index 2f18d126b..8f8eae873 100644 --- a/app/views/pools/_secondary_links.html.erb +++ b/app/views/pools/_secondary_links.html.erb @@ -19,7 +19,9 @@
  • <%= link_to "Delete", pool_path(@pool), :method => :delete, :data => {:confirm => "Are you sure you want to delete this pool?"}, :remote => true %>
    • <% end %> <% end %> -
  • <%= link_to "History", pool_versions_path(:search => {:pool_id => @pool.id}) %>
    • + <% if PoolArchive.enabled? %> +
  • <%= link_to "History", pool_versions_path(:search => {:pool_id => @pool.id}) %>
    • + <% end %> <% if @pool.post_count <= 100 && CurrentUser.is_member? %>
  • <%= link_to "Order", edit_pool_order_path(@pool) %>
    • <% end %> diff --git a/app/views/static/site_map.html.erb b/app/views/static/site_map.html.erb index 7f5b6c61f..1d5a7fbc5 100644 --- a/app/views/static/site_map.html.erb +++ b/app/views/static/site_map.html.erb @@ -53,7 +53,9 @@

  • Pools

  • <%= link_to("Help", wiki_pages_path(:title => "help:pools")) %>
  • <%= link_to("Listing", pools_path) %>
    • -
  • <%= link_to("Changes", pool_versions_path) %>
    • + <% if PoolArchive.enabled? %> +
  • <%= link_to("Changes", pool_versions_path) %>
    • + <% end %>