diff --git a/Gemfile b/Gemfile index a5791940b..a87bad6e0 100644 --- a/Gemfile +++ b/Gemfile @@ -65,6 +65,10 @@ group :production do gem 'capistrano-deploytags', '~> 1.0.0', require: false end +group :development do + gem 'sinatra' +end + group :development, :test do gem 'awesome_print' gem 'pry-byebug' diff --git a/Gemfile.lock b/Gemfile.lock index 9d60a85e4..034fa2351 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -237,6 +237,7 @@ GEM multi_json (1.13.1) multi_xml (0.6.0) multipart-post (2.0.0) + mustermann (1.0.2) naught (1.1.0) net-http-digest_auth (1.4.1) net-http-persistent (2.9.4) @@ -274,6 +275,8 @@ GEM win32-file (>= 0.7.0) public_suffix (3.0.2) rack (2.0.5) + rack-protection (2.0.3) + rack rack-test (1.0.0) rack (>= 1.0, < 3) radix62 (1.0.1) @@ -354,6 +357,11 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) + sinatra (2.0.3) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.3) + tilt (~> 2.0) sprockets (3.7.1) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -485,6 +493,7 @@ DEPENDENCIES shoulda-matchers simple_form simplecov + sinatra sprockets-rails statistics2 streamio-ffmpeg diff --git a/Procfile b/Procfile index c8f3a7913..54767a5b0 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,6 @@ -unicorn: bundle exec rails server -jobs: bundle exec rake jobs:work +unicorn: bin/rails server -p 3000 +jobs: bin/rake jobs:work +recommender: bundle exec ruby script/mock_services/recommender.rb +iqdbs: bundle exec ruby script/mock_services/iqdbs.rb +reportbooru: bundle exec ruby script/mock_services/reportbooru.rb +listbooru: bundle exec ruby script/mock_services/listbooru.rb \ No newline at end of file diff --git a/app/assets/javascripts/posts.js.erb b/app/assets/javascripts/posts.js.erb index 0941d4581..aa7a0f2ab 100644 --- a/app/assets/javascripts/posts.js.erb +++ b/app/assets/javascripts/posts.js.erb @@ -452,6 +452,9 @@ $("#edit").hide(); $("#share").hide(); $("#recommended").show(); + $.get("/recommended_posts", {context: "post", post_id: Danbooru.meta("post-id")}, function(data) { + $("#recommended").html(data); + }); } else { $("#edit").hide(); $("#comments").hide(); diff --git a/app/controllers/recommended_posts_controller.rb b/app/controllers/recommended_posts_controller.rb new file mode 100644 index 000000000..8695ab6aa --- /dev/null +++ b/app/controllers/recommended_posts_controller.rb @@ -0,0 +1,23 @@ +class RecommendedPostsController < ApplicationController + before_action :member_only + respond_to :html + + def show + @posts = load_posts() + + if request.xhr? + render partial: "show", layout: false + end + end + +private + + def load_posts + if params[:context] == "post" + @posts = RecommenderService.recommend(post_id: params[:post_id]) + + elsif params[:context] == "user" + @posts = RecommenderService.recommend(user_id: CurrentUser.id) + end + end +end diff --git a/app/logical/post_sets/recommended.rb b/app/logical/post_sets/recommended.rb index 01ffe92c1..87b17803c 100644 --- a/app/logical/post_sets/recommended.rb +++ b/app/logical/post_sets/recommended.rb @@ -1,16 +1,10 @@ module PostSets class Recommended < PostSets::Post - def initialize(post) + attr_reader :posts + + def initialize(posts) super("") - @post = post - end - - def posts - @posts ||= begin - response = RecommenderService.similar(@post) - post_ids = response.reject {|x| x[0] == @post.id}.slice(0, 6).map {|x| x[0]} - ::Post.find(post_ids) - end + @posts = posts end def presenter diff --git a/app/models/recommender_service.rb b/app/models/recommender_service.rb index c0cbd6d51..97fd28bc0 100644 --- a/app/models/recommender_service.rb +++ b/app/models/recommender_service.rb @@ -1,25 +1,26 @@ module RecommenderService extend self - SCORE_THRESHOLD = 10 + SCORE_THRESHOLD = 5 def enabled? Danbooru.config.recommender_server.present? end - def available?(post) + def available_for_post?(post) return true if Rails.env.development? + enabled? && CurrentUser.enable_recommended_posts? && post.created_at > Date.civil(2018, 1, 1) && post.score >= SCORE_THRESHOLD end - def similar(post) - if Danbooru.config.recommender_server == "development" - return Post.order("random()").limit(6).map {|x| [x.id, "1.000"]} - end + def available_for_user? + enabled? && CurrentUser.is_gold? + end - Cache.get("rss:#{post.id}", 1.day) do + def recommend_for_user(user_id) + ids = Cache.get("rsu:#{user_id}", 1.day) do resp = HTTParty.get( - "#{Danbooru.config.recommender_server}/similar/#{post.id}", + "#{Danbooru.config.recommender_server}/recommend/#{user_id}", Danbooru.config.httparty_options.merge( basic_auth: { username: "danbooru", @@ -29,5 +30,30 @@ module RecommenderService ) JSON.parse(resp.body) end + Post.find(ids.map(&:first)) + end + + def recommend_for_post(post_id) + ids = Cache.get("rss:#{post_id}", 1.day) do + resp = HTTParty.get( + "#{Danbooru.config.recommender_server}/similar/#{post_id}", + Danbooru.config.httparty_options.merge( + basic_auth: { + username: "danbooru", + password: Danbooru.config.recommender_key + } + ) + ) + JSON.parse(resp.body) + end + Post.find(ids.reject {|x| x[0] == post_id}.map(&:first)) + end + + def recommend(post_id: nil, user_id: nil) + if post_id + recommend_for_post(post_id) + elsif user_id + recommend_for_user(user_id) + end end end diff --git a/app/views/posts/partials/common/_secondary_links.html.erb b/app/views/posts/partials/common/_secondary_links.html.erb index 754dac440..e8fb0cac8 100644 --- a/app/views/posts/partials/common/_secondary_links.html.erb +++ b/app/views/posts/partials/common/_secondary_links.html.erb @@ -3,6 +3,9 @@
  • <%= link_to "Listing", posts_path %>
  • + <% if RecommenderService.available_for_user? %> +
  • <%= link_to "Recommended", recommended_posts_path(context: "user") %>
  • + <% end %> <% unless CurrentUser.is_anonymous? %> diff --git a/app/views/posts/show.html.erb b/app/views/posts/show.html.erb index 83120e0e4..069f7a343 100644 --- a/app/views/posts/show.html.erb +++ b/app/views/posts/show.html.erb @@ -89,7 +89,7 @@
  • Comments
  • - <% if RecommenderService.enabled? %> + <% if RecommenderService.available_for_post?(@post) %>
  • Recommended
  • <% end %> @@ -100,13 +100,11 @@
  • Share
  • - + <% if RecommenderService.available_for_post?(@post) %> + + <% end %>
    <% if !CurrentUser.user.is_builder? %> diff --git a/app/views/recommended_posts/_show.html.erb b/app/views/recommended_posts/_show.html.erb new file mode 100644 index 000000000..6349d4fc5 --- /dev/null +++ b/app/views/recommended_posts/_show.html.erb @@ -0,0 +1,3 @@ + diff --git a/app/views/recommended_posts/show.html.erb b/app/views/recommended_posts/show.html.erb new file mode 100644 index 000000000..f9733b86f --- /dev/null +++ b/app/views/recommended_posts/show.html.erb @@ -0,0 +1,15 @@ +
    +
    +

    Recommended Posts

    + +

    Based on your voting history, you may enjoy these posts. Vote more to get more accurate results. These recommendations update every hour.

    + + <%= render partial: "show" %> +
    +
    + +<%= render "posts/partials/common/secondary_links" %> + +<% content_for(:page_title) do %> + Recommended Posts - <%= Danbooru.config.app_name %> +<% end %> diff --git a/config/routes.rb b/config/routes.rb index 073ad1883..1c924d6b2 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -251,6 +251,7 @@ Rails.application.routes.draw do post "reports/post_versions_create" => "reports#post_versions_create" get "reports/down_voting_post" => "reports#down_voting_post" post "reports/down_voting_post_create" => "reports#down_voting_post_create" + resource :recommended_posts, only: [:show] resources :saved_searches, :except => [:show] do collection do get :labels diff --git a/script/mock_services/README.md b/script/mock_services/README.md new file mode 100644 index 000000000..6f2e10259 --- /dev/null +++ b/script/mock_services/README.md @@ -0,0 +1,7 @@ +These are mocked services to be used for development purposes. + +- danbooru: port 3000 +- recommender: port 3001 +- iqdbs: port 3002 +- reportbooru: port 3003 +- listbooru: port 3004 diff --git a/script/mock_services/iqdbs.rb b/script/mock_services/iqdbs.rb new file mode 100644 index 000000000..03e8ea3f6 --- /dev/null +++ b/script/mock_services/iqdbs.rb @@ -0,0 +1,14 @@ +require 'sinatra' +require 'json' +require_relative './mock_service_helper' + +set :port, 3002 + +configure do + POST_IDS = MockServiceHelper.fetch_post_ids() +end + +get '/similar' do + content_type :json + POST_IDS[0..10].map {|x| {post_id: x}}.to_json +end diff --git a/script/mock_services/listbooru.rb b/script/mock_services/listbooru.rb new file mode 100644 index 000000000..b25cffc7f --- /dev/null +++ b/script/mock_services/listbooru.rb @@ -0,0 +1,8 @@ +require 'sinatra' +require 'json' + +set :port, 3004 + +post '/v2/search' do + # todo +end diff --git a/script/mock_services/mock_service_helper.rb b/script/mock_services/mock_service_helper.rb new file mode 100644 index 000000000..40b04da22 --- /dev/null +++ b/script/mock_services/mock_service_helper.rb @@ -0,0 +1,22 @@ +require 'socket' +require 'timeout' +require 'httparty' + +module MockServiceHelper + extend self + + DANBOORU_PORT = 3000 + + def fetch_post_ids() + begin + s = TCPSocket.new("localhost", DANBOORU_PORT) + s.close + rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH + sleep 1 + retry + end + + json = HTTParty.get("http://localhost:#{DANBOORU_PORT}/posts.json?random=true&limit=10").body + return JSON.parse(json).map {|x| x["id"]} + end +end diff --git a/script/mock_services/recommender.rb b/script/mock_services/recommender.rb new file mode 100644 index 000000000..77b30ed9d --- /dev/null +++ b/script/mock_services/recommender.rb @@ -0,0 +1,19 @@ +require 'sinatra' +require 'json' +require_relative './mock_service_helper' + +set :port, 3001 + +configure do + POST_IDS = MockServiceHelper.fetch_post_ids() +end + +get '/recommend/:user_id' do + content_type :json + POST_IDS[0..10].map {|x| [x, "1.000"]}.to_json +end + +get '/similar/:post_id' do + content_type :json + POST_IDS[0..6].map {|x| [x, "1.000"]}.to_json +end diff --git a/script/mock_services/reportbooru.rb b/script/mock_services/reportbooru.rb new file mode 100644 index 000000000..def0462f2 --- /dev/null +++ b/script/mock_services/reportbooru.rb @@ -0,0 +1,26 @@ +require 'sinatra' +require 'json' + +set :port, 3003 + +get '/missed_searches' do + content_type :text + return "abcdefg 10.0\nblahblahblah 20.0\n" +end + +get '/post_searches/rank' do + content_type :json + return [["abc", 100], ["def", 200]].to_json +end + +get '/reports/user_similarity' do + # todo +end + +get '/reports/uploads' do + # todo +end + +post '/post_views' do + # todo +end diff --git a/test/functional/posts_controller_test.rb b/test/functional/posts_controller_test.rb index 19e76597e..dad1b9d83 100644 --- a/test/functional/posts_controller_test.rb +++ b/test/functional/posts_controller_test.rb @@ -120,15 +120,12 @@ class PostsControllerTest < ActionDispatch::IntegrationTest setup do @post2 = create(:post) RecommenderService.stubs(:enabled?).returns(true) - RecommenderService.stubs(:available?).returns(true) - RecommenderService.stubs(:similar).returns([[@post.id, "1.0"], [@post2.id, "0.01"]]) + RecommenderService.stubs(:available_for_post?).returns(true) end - should "render a section for similar posts" do + should "not error out" do get_auth post_path(@post), @user assert_response :success - assert_select ".similar-posts" - assert_select ".similar-posts #post_#{@post2.id}" end end end diff --git a/test/functional/recommended_posts_controller_test.rb b/test/functional/recommended_posts_controller_test.rb new file mode 100644 index 000000000..a6d328f63 --- /dev/null +++ b/test/functional/recommended_posts_controller_test.rb @@ -0,0 +1,41 @@ +require "test_helper" + +class RecommendedPostsControllerTest < ActionDispatch::IntegrationTest + context "The recommended posts controller" do + setup do + @user = travel_to(1.month.ago) {create(:user)} + as_user do + @post = create(:post, :tag_string => "aaaa") + end + RecommenderService.stubs(:enabled?).returns(true) + end + + context "post context" do + setup do + RecommenderService.stubs(:available_for_post?).returns(true) + RecommenderService.stubs(:recommend_for_post).returns([@post]) + end + + should "render" do + get_auth recommended_posts_path, @user, xhr: true, params: {context: "post", post_id: @post.id} + assert_response :success + assert_select ".recommended-posts" + assert_select ".recommended-posts #post_#{@post.id}" + end + end + + context "user context" do + setup do + RecommenderService.stubs(:available_for_user?).returns(true) + RecommenderService.stubs(:recommend_for_user).returns([@post]) + end + + should "render" do + get_auth recommended_posts_path, @user, params: {context: "user"} + assert_response :success + assert_select ".recommended-posts" + assert_select ".recommended-posts #post_#{@post.id}" + end + end + end +end