Merge pull request #353 from zwagoth/docker-iqdb

[Docker] Add iqdb/s to composer

Procfile

@@ -1,2 +1,2 @@
 unicorn: bin/rails server -p 9000 -b 0.0.0.0
-jobs: bundle exec sidekiq -c 1 -q low_prio -q tags -q default -q high_prio -q video -q iqdb
+jobs: bundle exec sidekiq -c 1 -q low_prio -q tags -q default -q high_prio -q video

app/models/post.rb

@@ -1969,7 +1969,7 @@ class Post < ApplicationRecord
 
     module ClassMethods
       def iqdb_enabled?
-        Danbooru.config.iqdb_enabled?
+        Danbooru.config.iqdbs_server.present?
       end
 
       def remove_iqdb(post_id)

app/views/posts/partials/show/_options.html.erb

@@ -48,7 +48,7 @@
         <% end %>
 
         <% if CurrentUser.is_admin? %>
-          <% if Danbooru.config.iqdb_enabled? %>
+          <% if Danbooru.config.iqdbs_server.present? %>
             <li><%= link_to "Update IQDB", update_iqdb_post_path(@post) %></li>
           <% end %>
           <li><%= tag.a "Destroy", href: '#', id: 'destroy-post-link', 'data-pid': post.id %></li>

config/danbooru_default_config.rb

@@ -469,7 +469,7 @@ fart'
       # base_url - where to serve files from (default: http://#{hostname}/data)
       # hierarchical: false - store files in a single directory
       # hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash
-      StorageManager::Local.new(base_url: "#{CurrentUser.root_url}/", base_dir: "#{Rails.root}/public/data", hierarchical: false)
+      StorageManager::Local.new(base_url: "#{CurrentUser.root_url}/", base_dir: "#{Rails.root}/public/data", hierarchical: true)
 
       # Store files on one or more remote host(s). Configure SSH settings in
       # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
@@ -958,10 +958,6 @@ fart'
     def reportbooru_server
     end
 
-    def iqdb_enabled?
-      false
-    end
-
     def iqdbs_server
     end
 

docker-compose.yml

@@ -16,6 +16,7 @@ services:
       DANBOORU_REDIS_URL: redis://redis
       DANBOORU_ELASTICSEARCH_HOST: elastic
       DANBOORU_MEMCACHED_SERVERS: memcached
+      DANBOORU_IQDBS_SERVER: http://iqdb:4567
       # These are just development secrets, do not use them in production
       SECRET_TOKEN: 1c58518a891eff4520cadc59afa9e378a9325f1247544ff258096e497f095f45
       SESSION_SECRET_KEY: 44b4f44e9f253c406cbe727d403d500c1cecff943e4d2aea8f5447f28846fffe
@@ -24,6 +25,7 @@ services:
       - redis
       - memcached
       - elastic
+      - iqdb
 
   nginx:
     image: nginx:stable-alpine
@@ -69,9 +71,28 @@ services:
       - ES_JAVA_OPTS=-Xms1g -Xmx1g
     volumes:
       - elastic_data:/usr/share/elasticsearch/data
+  
+  iqdb:
+    build: ./docker/iqdb
+    environment:
+      - IQDB_HOSTNAME=localhost
+      - IQDB_PORT=62125
+      # Hardcoded in iqdbs Procfile
+      - IQDB_DATABASE_FILE=/home/vagrant/iqdbs/e621.db
+      - SINATRA_PORT=4567
+      - IMAGES_FOLDER=/data
+      - REDIS_URL=redis://redis
+      # Sinatra only binds to localhost in development, but to 0.0.0.0 for anything else
+      - APP_ENV=docker
+    depends_on:
+      - redis
+    volumes:
+      - post_data:/data
+      - iqdb_data:/home/vagrant/iqdbs
 
 volumes:
   post_data:
+  iqdb_data:
   elastic_data:
   db_data:
   redis_data:

docker/iqdb/Dockerfile

@@ -0,0 +1,32 @@
+FROM ruby:2.7.3-alpine
+
+ADD https://api.github.com/repos/zwagoth/iqdbs/git/refs/heads/master /tmp/iqdbs_version.json
+COPY iqdb.patch /tmp/iqdb.patch
+ENV IQDB_VERSION=20161008
+ENV IQDB_CHECKSUM=daa4812b35f84bc7e2f07661fd9abf73a06738c76beb92252666a86ebaea3c64
+
+RUN apk --no-cache add build-base git libjpeg-turbo-dev gd-dev bash \
+  # iqdb
+  && cd /tmp \
+  && wget -q  https://iqdb.org/code/iqdb-$IQDB_VERSION.tar.bz2 \
+  && echo "$IQDB_CHECKSUM  iqdb-$IQDB_VERSION.tar.bz2" | sha256sum -c - \
+  && tar xjf iqdb-$IQDB_VERSION.tar.bz2 \
+  && cd iqdb \
+  && patch -N -i /tmp/iqdb.patch \
+  && make EXTRADEFS="-include stdint.h" -j$(nproc) \
+  && cp iqdb /usr/bin \
+  # iqdbs
+  && git clone https://github.com/zwagoth/iqdbs /iqdbs \
+  && bundler install --gemfile /iqdbs/Gemfile \
+  # cleanup
+  && rm  /tmp/iqdb-$IQDB_VERSION.tar.bz2 \
+  && rm -rf /tmp/iqdb \
+  && apk del build-base git
+
+# shoreman
+RUN wget -O /usr/bin/shoreman https://github.com/chrismytton/shoreman/raw/master/shoreman.sh \
+  && chmod +x /usr/bin/shoreman
+
+WORKDIR /iqdbs
+
+CMD [ "shoreman" ]

docker/iqdb/iqdb.patch

@@ -0,0 +1,13 @@
+diff --git a/imglib.h b/imglib.h
+index 2a03f93..e5468c4 100644
+--- a/imglib.h
++++ b/imglib.h
+@@ -98,7 +98,7 @@ union image_id_index {
+ 	bool operator==(const image_id_index& other) const { return id == other.id; }
+ 	bool operator!=(const image_id_index& other) const { return id != other.id; }
+ 
+-	operator size_t&() { return index; }
++	//operator size_t&() { return index; }
+ 	operator imageId&() { return id; }
+ };
+ 

test/test_helpers/iqdb_test_helper.rb

@@ -16,7 +16,6 @@ module IqdbTestHelper
 
     service = mock_sqs_service.new
     Post.stubs(:iqdb_sqs_service).returns(service)
-    Post.stubs(:iqdb_enabled?).returns(true)
 
     Danbooru.config.stubs(:iqdbs_server).returns("http://localhost:3004")
   end