forked from e621ng/e621ng
[Prod] Replace unicorn with pitchfork
pitchfork is a unicorn fork with some things removed and some things added. We don't need any of these things so it should be an easy replace. There is no worker-killer for pitchfork, it is however trivial to implement ourselves. unicorn is pretty much dead. The author implies as much in the various readme updates he made. It also doesn't work with rack 3 and I doubt a new release will even be made. If it will, it highly likely won't be on rubygems.
This commit is contained in:
Earlopain
parent
d7ece18a1c
commit
2c60ec69c8
@ -1,6 +1,5 @@
|
||||
# The settings here, if defined, override the settings in config/database.yml,
|
||||
# config/unicorn/unicorn.rb, config/danbooru_local_config.rb, and
|
||||
# ~/.danbooru/{secret_token,session_secret_key}.
|
||||
# config/danbooru_local_config.rb, and ~/.danbooru/{secret_token,session_secret_key}.
|
||||
#
|
||||
# `.env.$RAILS_ENV` takes precedence over .env, and .env.local takes
|
||||
# precedence over .env and `.env.$RAILS_ENV`.
|
||||
|
||||
6
Gemfile
6
Gemfile
@ -30,9 +30,6 @@ gem 'request_store'
|
||||
gem "diffy"
|
||||
gem "rugged"
|
||||
|
||||
# Blocked by unicorn which lacks a release with Rack 3 support
|
||||
gem "rack", "~> 2.0"
|
||||
|
||||
gem "datadog", require: "datadog/auto_instrument"
|
||||
|
||||
gem 'opensearch-ruby'
|
||||
@ -44,8 +41,7 @@ gem "faraday-follow_redirects"
|
||||
gem "faraday-retry"
|
||||
|
||||
group :production do
|
||||
gem 'unicorn'
|
||||
gem 'unicorn-worker-killer'
|
||||
gem "pitchfork"
|
||||
end
|
||||
|
||||
group :development, :test do
|
||||
|
||||
28
Gemfile.lock
28
Gemfile.lock
@ -152,8 +152,6 @@ GEM
|
||||
faraday-retry (2.2.1)
|
||||
faraday (~> 2.0)
|
||||
ffi (1.16.3)
|
||||
get_process_mem (0.2.7)
|
||||
ffi (~> 1.0)
|
||||
globalid (1.2.1)
|
||||
activesupport (>= 6.1)
|
||||
hashdiff (1.1.0)
|
||||
@ -168,7 +166,6 @@ GEM
|
||||
reline (>= 0.4.2)
|
||||
json (2.7.2)
|
||||
jsonapi-renderer (0.2.2)
|
||||
kgio (2.11.4)
|
||||
language_server-protocol (3.17.0.3)
|
||||
libdatadog (7.0.0.1.0)
|
||||
libddwaf (1.14.0.0.0)
|
||||
@ -222,6 +219,9 @@ GEM
|
||||
ast (~> 2.4.1)
|
||||
racc
|
||||
pg (1.5.6)
|
||||
pitchfork (0.13.0)
|
||||
rack (>= 2.0)
|
||||
raindrops (~> 0.7)
|
||||
prism (0.27.0)
|
||||
psych (5.1.2)
|
||||
stringio
|
||||
@ -229,16 +229,16 @@ GEM
|
||||
puma (6.4.2)
|
||||
nio4r (~> 2.0)
|
||||
racc (1.7.3)
|
||||
rack (2.2.9)
|
||||
rack (3.0.10)
|
||||
rack-proxy (0.7.7)
|
||||
rack
|
||||
rack-session (1.0.2)
|
||||
rack (< 3)
|
||||
rack-session (2.0.0)
|
||||
rack (>= 3.0.0)
|
||||
rack-test (2.1.0)
|
||||
rack (>= 1.3)
|
||||
rackup (1.0.0)
|
||||
rack (< 3)
|
||||
webrick
|
||||
rackup (2.1.0)
|
||||
rack (>= 3)
|
||||
webrick (~> 1.8)
|
||||
rails (7.1.3.2)
|
||||
actioncable (= 7.1.3.2)
|
||||
actionmailbox (= 7.1.3.2)
|
||||
@ -354,12 +354,6 @@ GEM
|
||||
tzinfo (2.0.6)
|
||||
concurrent-ruby (~> 1.0)
|
||||
unicode-display_width (2.5.0)
|
||||
unicorn (6.1.0)
|
||||
kgio (~> 2.6)
|
||||
raindrops (~> 0.7)
|
||||
unicorn-worker-killer (0.4.5)
|
||||
get_process_mem (~> 0)
|
||||
unicorn (>= 4, < 7)
|
||||
uri (0.13.0)
|
||||
webmock (3.23.0)
|
||||
addressable (>= 2.8.0)
|
||||
@ -401,8 +395,8 @@ DEPENDENCIES
|
||||
mocha
|
||||
opensearch-ruby
|
||||
pg
|
||||
pitchfork
|
||||
puma
|
||||
rack (~> 2.0)
|
||||
rails (~> 7.1.0)
|
||||
recaptcha
|
||||
redis
|
||||
@ -421,8 +415,6 @@ DEPENDENCIES
|
||||
sidekiq-unique-jobs
|
||||
simple_form
|
||||
streamio-ffmpeg
|
||||
unicorn
|
||||
unicorn-worker-killer
|
||||
webmock
|
||||
webpacker (>= 4.0.x)
|
||||
|
||||
|
||||
2
Procfile
2
Procfile
@ -1,4 +1,4 @@
|
||||
server: bin/rails server -p 9000 -b 0.0.0.0 --pid=/tmp/rails-server.pid
|
||||
# server: bundle exec unicorn -c config/unicorn/development.rb
|
||||
# server: bundle exec pitchfork -c config/pitchfork/development.rb
|
||||
jobs: SIDEKIQ_QUEUES="low_prio:1;video:1;iqdb:1;tags:2;default:3;high_prio:5" bundle exec sidekiq
|
||||
cron: run-parts /etc/periodic/daily && crond -f
|
||||
|
||||
@ -28,7 +28,7 @@ class Cache
|
||||
|
||||
def self.redis
|
||||
# Using a shared variable like this here is OK
|
||||
# since unicorn spawns a new process for each worker
|
||||
# since pitchfork spawns a new process for each worker
|
||||
@redis ||= Redis.new(url: Danbooru.config.redis_url)
|
||||
end
|
||||
end
|
||||
|
||||
14
config.ru
14
config.ru
@ -2,17 +2,7 @@
|
||||
|
||||
# This file is used by Rack-based servers to start the application.
|
||||
|
||||
if defined?(Unicorn) && ENV["RAILS_ENV"] == "production"
|
||||
# Unicorn self-process killer
|
||||
require 'unicorn/worker_killer'
|
||||
|
||||
# Max requests per worker
|
||||
use Unicorn::WorkerKiller::MaxRequests, 5_000, 10_000
|
||||
|
||||
# Max memory size (RSS) per worker
|
||||
use Unicorn::WorkerKiller::Oom, (386*(1024**2)), (768*(1024**2))
|
||||
end
|
||||
|
||||
require ::File.expand_path('../config/environment', __FILE__)
|
||||
require_relative "config/environment"
|
||||
|
||||
run Rails.application
|
||||
Rails.application.load_server
|
||||
|
||||
4
config/pitchfork/development.rb
Normal file
4
config/pitchfork/development.rb
Normal file
@ -0,0 +1,4 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
worker_processes 2
|
||||
listen "0.0.0.0:9000"
|
||||
36
config/pitchfork/production.rb
Normal file
36
config/pitchfork/production.rb
Normal file
@ -0,0 +1,36 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "dotenv"
|
||||
|
||||
# Should be "production" by default, otherwise use other env
|
||||
rails_env = ENV.fetch("RAILS_ENV", "production")
|
||||
|
||||
Dotenv.load(".env.#{rails_env}")
|
||||
|
||||
timeout 180
|
||||
listen ENV.fetch("PITCHFORK_LISTEN_ADDRESS"), tcp_nopush: true, backlog: 2048
|
||||
worker_processes ENV.fetch("PITCHFORK_WORKER_COUNT").to_i
|
||||
|
||||
after_worker_ready do |server, worker|
|
||||
max_requests = Random.rand(5_000..10_000)
|
||||
worker.instance_variable_set(:@_max_requests, max_requests)
|
||||
max_mem = Random.rand((386 * (1024**2))..(768 * (1024**2)))
|
||||
worker.instance_variable_set(:@_max_mem, max_mem)
|
||||
|
||||
server.logger.info("worker=#{worker.nr} gen=#{worker.generation} ready, serving #{max_requests} requests, #{max_mem} bytes")
|
||||
end
|
||||
|
||||
after_request_complete do |server, worker, _env|
|
||||
if worker.requests_count > worker.instance_variable_get(:@_max_requests)
|
||||
server.logger.info("worker=#{worker.nr} gen=#{worker.generation}) exit: request limit (#{worker.instance_variable_get(:@_max_requests)})")
|
||||
exit # rubocop:disable Rails/Exit
|
||||
end
|
||||
|
||||
if worker.requests_count % 16 == 0
|
||||
mem_info = Pitchfork::MemInfo.new(worker.pid)
|
||||
if mem_info.pss > worker.instance_variable_get(:@_max_mem)
|
||||
server.logger.info("worker=#{worker.nr} gen=#{worker.generation}) exit: memory limit (#{mem_info.pss} bytes > #{worker.instance_variable_get(:@_max_mem)} bytes)")
|
||||
exit # rubocop:disable Rails/Exit
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,21 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Set your full path to application.
|
||||
app_path = "/app"
|
||||
|
||||
# Set unicorn options
|
||||
worker_processes 2
|
||||
|
||||
preload_app false
|
||||
timeout 180
|
||||
listen "0.0.0.0:9000"
|
||||
|
||||
# Fill path to your app
|
||||
working_directory app_path
|
||||
|
||||
# Log everything to one file
|
||||
stderr_path "log/unicorn.log"
|
||||
stdout_path "log/unicorn.log"
|
||||
|
||||
# Set master PID location
|
||||
pid "#{app_path}/tmp/pids/unicorn.pid"
|
||||
@ -1,88 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "dotenv"
|
||||
|
||||
# Set your full path to application.
|
||||
app_path = "/home/e621/e621ng"
|
||||
|
||||
# Should be "production" by default, otherwise use other env
|
||||
rails_env = ENV.fetch("RAILS_ENV", "production")
|
||||
|
||||
Dotenv.load("#{app_path}/.env.#{rails_env}")
|
||||
|
||||
# Set unicorn options
|
||||
worker_processes ENV.fetch("UNICORN_WORKER_COUNT").to_i
|
||||
|
||||
timeout 180
|
||||
listen ENV.fetch("UNICORN_LISTEN_ADDRESS"), tcp_nopush: true, backlog: 2048
|
||||
|
||||
# Spawn unicorn master worker for user apps (group: apps)
|
||||
user "e621", "e621"
|
||||
|
||||
# Fill path to your app
|
||||
working_directory app_path
|
||||
|
||||
# Log everything to one file
|
||||
stderr_path "/dev/null"
|
||||
stdout_path "/dev/null"
|
||||
|
||||
# Set master PID location
|
||||
pid "#{app_path}/tmp/pids/unicorn.pid"
|
||||
|
||||
# combine Ruby 2.0.0+ with "preload_app true" for memory savings
|
||||
preload_app true
|
||||
|
||||
# Enable this flag to have unicorn test client connections by writing the
|
||||
# beginning of the HTTP headers before calling the application. This
|
||||
# prevents calling the application for connections that have disconnected
|
||||
# while queued. This is only guaranteed to detect clients on the same
|
||||
# host unicorn runs on, and unlikely to detect disconnects even on a
|
||||
# fast LAN.
|
||||
check_client_connection false
|
||||
|
||||
# local variable to guard against running a hook multiple times
|
||||
run_once = true
|
||||
|
||||
before_fork do |server, worker|
|
||||
# the following is highly recomended for Rails + "preload_app true"
|
||||
# as there's no need for the master process to hold a connection
|
||||
defined?(ActiveRecord::Base) and ActiveRecord::Base.connection.disconnect!
|
||||
|
||||
# Occasionally, it may be necessary to run non-idempotent code in the
|
||||
# master before forking. Keep in mind the above disconnect! example
|
||||
# is idempotent and does not need a guard.
|
||||
if run_once
|
||||
# do_something_once_here ...
|
||||
run_once = false # prevent from firing again
|
||||
end
|
||||
|
||||
# The following is only recommended for memory/DB-constrained
|
||||
# installations. It is not needed if your system can house
|
||||
# twice as many worker_processes as you have configured.
|
||||
#
|
||||
# # This allows a new master process to incrementally
|
||||
# # phase out the old master process with SIGTTOU to avoid a
|
||||
# # thundering herd (especially in the "preload_app false" case)
|
||||
# # when doing a transparent upgrade. The last worker spawned
|
||||
# # will then kill off the old master process with a SIGQUIT.
|
||||
# old_pid = "#{server.config[:pid]}.oldbin"
|
||||
# if old_pid != server.pid
|
||||
# begin
|
||||
# sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
|
||||
# Process.kill(sig, File.read(old_pid).to_i)
|
||||
# rescue Errno::ENOENT, Errno::ESRCH
|
||||
# end
|
||||
# end
|
||||
#
|
||||
# Throttle the master from forking too quickly by sleeping. Due
|
||||
# to the implementation of standard Unix signal handlers, this
|
||||
# helps (but does not completely) prevent identical, repeated signals
|
||||
# from being lost when the receiving process is busy.
|
||||
sleep 1
|
||||
end
|
||||
|
||||
after_fork do |server, worker|
|
||||
if defined?(ActiveRecord::Base)
|
||||
ActiveRecord::Base.establish_connection
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue
Block a user