diff --git a/Gemfile b/Gemfile index 11cbcb3b7..a8a28cd37 100644 --- a/Gemfile +++ b/Gemfile @@ -55,6 +55,8 @@ gem 'sidekiq' # gem 'sidekiq-unique-jobs' gem 'redis' +gem 'elasticsearch-model' + # needed for looser jpeg header compat gem 'ruby-imagespec', :require => "image_spec", :git => "https://github.com/r888888888/ruby-imagespec.git", :branch => "exif-fixes" diff --git a/Gemfile.lock b/Gemfile.lock index 740b45fde..a6b31df81 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,6 +126,18 @@ GEM dotenv-rails (2.4.0) dotenv (= 2.4.0) railties (>= 3.2, < 6.0) + elasticsearch (6.1.0) + elasticsearch-api (= 6.1.0) + elasticsearch-transport (= 6.1.0) + elasticsearch-api (6.1.0) + multi_json + elasticsearch-model (6.0.0) + activesupport (> 3) + elasticsearch (> 1) + hashie + elasticsearch-transport (6.1.0) + faraday + multi_json equalizer (0.0.11) erubi (1.7.1) execjs (2.7.0) @@ -161,6 +173,7 @@ GEM os (~> 0.9) signet (~> 0.7) hashdiff (0.3.7) + hashie (3.6.0) highline (1.7.10) http (3.3.0) addressable (~> 2.3) @@ -465,6 +478,7 @@ DEPENDENCIES diff-lcs dotenv-rails dtext_rb! + elasticsearch-model factory_bot ffaker foreman diff --git a/app/indexes/indexable.rb b/app/indexes/indexable.rb new file mode 100644 index 000000000..000ebf966 --- /dev/null +++ b/app/indexes/indexable.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'elasticsearch/model' + +# Base Elasticsearch indexing definitions +# +# Make sure to include your custom index file +# in your model alongside Indexable. +module Indexable + def self.included(base) + base.include Elasticsearch::Model + + base.after_commit on: [:create] do + __elasticsearch__.index_document + end + + base.after_commit on: [:update] do + update_index # XXX + end + + base.after_commit on: [:destroy] do + __elasticsearch__.delete_document + end + end + + def update_index(defer: true, priority: :high) + # if defer + # if priority == :high + # IndexUpdateJob.perform_later(self.class.to_s, id) + # elsif priority == :rebuild + # IndexRebuildJob.perform_later(self.class.to_s, id) + # else + # raise ArgumentError, 'No such priority known' + # end + # else + __elasticsearch__.index_document + # end + end +end diff --git a/app/indexes/post_index.rb b/app/indexes/post_index.rb new file mode 100644 index 000000000..b7ca9967e --- /dev/null +++ b/app/indexes/post_index.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module PostIndex + def self.included(base) + base.settings index: { number_of_shards: 5, number_of_replicas: 1 } do + mappings dynamic: false, _all: { enabled: false } do + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + indexes :id, type: 'integer' + indexes :up_score, type: 'integer' + indexes :down_score, type: 'integer' + indexes :score, type: 'integer' + indexes :fav_count, type: 'integer' + indexes :tag_count, type: 'integer' + + indexes :tag_count_general, type: 'integer' + indexes :tag_count_artist, type: 'integer' + indexes :tag_count_character, type: 'integer' + indexes :tag_count_copyright, type: 'integer' + indexes :tag_count_meta, type: 'integer' + indexes :tag_count_species, type: 'integer' + indexes :comment_count, type: 'integer' + + indexes :file_size, type: 'integer' + indexes :pixiv_id, type: 'integer' + indexes :parent_id, type: 'integer' + indexes :child_ids, type: 'integer' + indexes :pool_ids, type: 'integer' + indexes :set_ids, type: 'integer' + indexes :width, type: 'integer' + indexes :height, type: 'integer' + indexes :aspect_ratio, type: 'float' + + indexes :tags, type: 'keyword' + indexes :pools, type: 'keyword' + indexes :sets, type: 'keyword' + indexes :md5, type: 'keyword' + indexes :rating, type: 'keyword' + indexes :file_ext, type: 'keyword' + indexes :source, type: 'keyword' + indexes :faves, type: 'keyword' + indexes :upvotes, type: 'keyword' + indexes :downvotes, type: 'keyword' + indexes :approver, type: 'keyword' + indexes :deleter, type: 'keyword' + indexes :uploader, type: 'keyword' + + indexes :rating_locked, type: 'boolean' + indexes :note_locked, type: 'boolean' + indexes :hide_anon, type: 'boolean' + indexes :hide_google, type: 'boolean' + indexes :flagged, type: 'boolean' + indexes :pending, type: 'boolean' + indexes :deleted, type: 'boolean' + indexes :has_description, type: 'boolean' + + indexes :description, type: 'text', analyzer: 'snowball' + end + end + end + + def as_indexed_json(options = {}) + { + created_at: created_at, + updated_at: updated_at, + id: id, + up_score: up_score, + down_score: down_score, + score: score, + fav_count: fav_count, + tag_count: tag_count, + + tag_count_general: tag_count_general, + tag_count_artist: tag_count_artist, + tag_count_character: tag_count_character, + tag_count_copyright: tag_count_copyright, + tag_count_meta: tag_count_meta, + # tag_count_species: tag_count_species, + # comment_count: comment_count, + + file_size: file_size, + pixiv_id: pixiv_id, + parent_id: parent_id, + # child_ids: child_ids, + # pool_ids: pool_ids, + # set_ids: set_ids, + width: image_width, + height: image_height, + aspect_ratio: image_width.to_f / [image_height, 1].max, + + tags: tag_string.split(' '), + pools: pool_string.split(' '), + # sets: set_string.split(' '), + md5: md5, + rating: rating, + file_ext: file_ext, + source: source.presence, + faves: fav_string.split(' '), + # upvotes: upvotes, + # downvotes: downvotes, + approver: approver&.name, + # deleter: deleter&.name, + uploader: uploader&.name, + + rating_locked: is_rating_locked, + note_locked: is_note_locked, + # hide_anon: hide_anon, + # hide_google: hide_google, + flagged: is_flagged, + pending: is_pending, + deleted: is_deleted, + # has_description: description.present?, + + # description: description.presence, + } + end +end diff --git a/app/models/post.rb b/app/models/post.rb index 6127aef2e..6af6f371f 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -1914,6 +1914,8 @@ class Post < ApplicationRecord include IqdbMethods include ValidationMethods include Danbooru::HasBitFlags + include Indexable + include PostIndex BOOLEAN_ATTRIBUTES = %w( has_embedded_notes diff --git a/config/application.rb b/config/application.rb index 4dd350695..09d984a5e 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,7 +15,7 @@ module Danbooru config.filter_parameters += [:password] config.assets.enabled = true config.assets.version = '1.0' - config.autoload_paths += %W(#{config.root}/app/presenters #{config.root}/app/logical #{config.root}/app/mailers) + config.autoload_paths += %W(#{config.root}/app/presenters #{config.root}/app/logical #{config.root}/app/mailers #{config.root}/app/indexes) config.plugins = [:all] config.time_zone = 'UTC' config.action_mailer.delivery_method = :smtp