From 897d3d3a995881598524afa5003e6ee62ccdcb82 Mon Sep 17 00:00:00 2001 From: Albert Yi Date: Fri, 21 Sep 2018 12:47:03 -0700 Subject: [PATCH] add production2 env for testing new servers --- INSTALL.debian | 48 ++++++++++++-- config/deploy.rb | 2 +- config/docker/danbooru-base | 6 +- config/environments/production2.rb | 96 +++++++++++++++++++++++++++ config/unicorn/production.rb | 2 +- lib/capistrano/tasks/delayed_job.rake | 10 +-- lib/capistrano/tasks/symlink.rake | 4 +- script/install/vrack-cfg.yaml | 10 +++ 8 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 config/environments/production2.rb create mode 100644 script/install/vrack-cfg.yaml diff --git a/INSTALL.debian b/INSTALL.debian index f7b96df15..7cf337d5e 100644 --- a/INSTALL.debian +++ b/INSTALL.debian @@ -2,6 +2,10 @@ # Run: curl -L -s https://raw.githubusercontent.com/r888888888/danbooru/master/INSTALL.debian -o install.sh ; chmod +x install.sh ; ./install.sh +export RUBY_VERSION=2.5.1 +export GITHUB_INSTALL_SCRIPTS=https://raw.githubusercontent.com/r888888888/danbooru/master/script/install +export VIPS_VERSION=8.7.0 + if [[ "$(whoami)" != "root" ]] ; then echo "You must run this script as root" exit 1 @@ -28,20 +32,54 @@ if [[ -z "$HOSTNAME" ]] ; then exit 1 fi -export RUBY_VERSION=2.5.1 -export GITHUB_INSTALL_SCRIPTS=https://raw.githubusercontent.com/r888888888/danbooru/master/script/install +echo -n "* Enter the VLAN IP address for this server (ex: 172.16.0.1, enter nothing to skip): " +read VLAN_IP_ADDR # Install packages echo "* Installing packages..." -LIBSSL_DEV_PKG=$( verlt `lsb_release -sr` 9.0 && echo libssl-dev || echo libssl1.0-dev ) + +if [ -n "$(uname -a | grep Ubuntu)" ] ; then + LIBSSL_DEV_PKG=libssl-dev +else + LIBSSL_DEV_PKG=$( verlt `lsb_release -sr` 9.0 && echo libssl-dev || echo libssl1.0-dev ) +fi apt-get update -apt-get -y install build-essential automake $LIBSSL_DEV_PKG libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel memcached libmemcached-dev git curl libcurl4-openssl-dev libvips-dev libvips-tools sendmail-bin sendmail postgresql postgresql-contrib libpq-dev postgresql-server-dev-all nginx ssh coreutils ffmpeg mkvtoolnix libvips42 libvips-tools libvips-dev +apt-get -y install $LIBSSL_DEV_PKG build-essential automake libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel memcached libmemcached-dev git curl libcurl4-openssl-dev sendmail-bin sendmail nginx ssh coreutils ffmpeg mkvtoolnix +apt-get -y install libpq-dev postgresql-client +apt-get -y install libjpeg-turbo8-dev libexpat1-dev libgif-dev libpng-dev libexif-dev + +# vrack specific stuff +if [ -n "$VLAN_IP_ADDR" ] ; then + apt-get -y install vlan + modprobe 8021q + echo "8021q" >> /etc/modules + vconfig add eno2 99 + ip addr add $VLAN_IP_ADDR/24 dev eno2.99 + ip link set up eno2.99 + curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml +fi + +curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash - +apt-get update +apt-get -y install nodejs yarn +apt-get remove cmdtest if [ $? -ne 0 ]; then echo "* Error installing packages; aborting" exit 1 fi +# compile and install libvips (the version in apt is too old) +cd /tmp +wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz +tar xzf vips-$VIPS_VERSION.tar.gz +cd vips-$VIPS_VERSION +./configure --prefix=/usr +make install +ldconfig + # Create user account useradd -m danbooru chsh -s /bin/bash danbooru @@ -58,7 +96,7 @@ fi # Install rbenv echo "* Installing rbenv..." -cd / +cd /tmp sudo -u danbooru git clone git://github.com/sstephenson/rbenv.git ~danbooru/.rbenv sudo -u danbooru touch ~danbooru/.bash_profile echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~danbooru/.bash_profile diff --git a/config/deploy.rb b/config/deploy.rb index 029771b47..48c07feb4 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,4 +1,4 @@ -set :stages, %w(production development staging) +set :stages, %w(production production2 development staging) set :default_stage, "staging" set :application, "danbooru" set :repo_url, "git://github.com/r888888888/danbooru.git" diff --git a/config/docker/danbooru-base b/config/docker/danbooru-base index 23383deff..e883a695c 100644 --- a/config/docker/danbooru-base +++ b/config/docker/danbooru-base @@ -1,14 +1,14 @@ FROM ruby:2.5.1-slim-stretch ENV DOCKERIZE_VERSION v0.6.1 -ENV VIPS_VERSION 8.6.4 +ENV VIPS_VERSION 8.7.0 RUN apt-get update && \ apt-get -y install wget && \ wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ - apt-get -y install -qq curl gnupg apt-transport-https apt-utils build-essential automake libssl-dev libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel memcached libmemcached-dev git libcurl4-openssl-dev imagemagick libmagickcore-dev libmagickwand-dev sendmail-bin sendmail postgresql-client libpq-dev nginx ssh coreutils ffmpeg mkvtoolnix emacs24-nox telnet libgif-dev && \ + apt-get -y install -qq curl gnupg apt-transport-https apt-utils build-essential automake libssl-dev libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel memcached libmemcached-dev git libcurl4-openssl-dev sendmail-bin sendmail postgresql-client libpq-dev nginx ssh coreutils ffmpeg mkvtoolnix emacs24-nox telnet libgif-dev && \ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash - && \ @@ -17,7 +17,7 @@ RUN apt-get update && \ apt-get remove cmdtest && \ apt-get clean && \ cd /tmp && \ - wget -q https://github.com/jcupitt/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz && \ + wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz && \ tar xzf vips-$VIPS_VERSION.tar.gz && \ cd vips-$VIPS_VERSION && \ ./configure --prefix=/usr && \ diff --git a/config/environments/production2.rb b/config/environments/production2.rb new file mode 100644 index 000000000..d35ae37d2 --- /dev/null +++ b/config/environments/production2.rb @@ -0,0 +1,96 @@ +Rails.application.configure do + # Verifies that versions and hashed value of the package contents in the project's package.json + config.webpacker.check_yarn_integrity = false + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.cache_classes = true + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + config.action_controller.perform_caching = true + + # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] + # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true + + # Disable serving static files from the `/public` folder by default since + # Apache or NGINX already handles this. + config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? + + # Compress JavaScripts and CSS. + config.assets.js_compressor = :uglifier + # config.assets.css_compressor = :sass + + # Do not fallback to assets pipeline if a precompiled asset is missed. + config.assets.compile = false + + # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb + + # Enable serving of images, stylesheets, and JavaScripts from an asset server. + # config.action_controller.asset_host = 'http://assets.example.com' + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache + # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + + # Store uploaded files on the local file system (see config/storage.yml for options) + config.active_storage.service = :local + + # Mount Action Cable outside main process or domain + # config.action_cable.mount_path = nil + # config.action_cable.url = 'wss://example.com/cable' + # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + # config.force_ssl = true + + # Use the lowest log level to ensure availability of diagnostic information + # when problems arise. + config.log_level = :error + + # Prepend all log lines with the following tags. + config.log_tags = [ :request_id ] + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + + # Use a real queuing backend for Active Job (and separate queues per environment) + # config.active_job.queue_adapter = :resque + # config.active_job.queue_name_prefix = "danbooru_#{Rails.env}" + + config.action_mailer.perform_caching = false + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + # config.action_mailer.raise_delivery_errors = false + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = true + + # Send deprecation notices to registered listeners. + config.active_support.deprecation = :notify + + # Use default logging formatter so that PID and timestamp are not suppressed. + config.log_formatter = ::Logger::Formatter.new + + # Use a different logger for distributed setups. + # require 'syslog/logger' + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + + # Do not dump schema after migrations. + config.active_record.dump_schema_after_migration = false +end diff --git a/config/unicorn/production.rb b/config/unicorn/production.rb index 1ecc2c75b..87860c1ff 100644 --- a/config/unicorn/production.rb +++ b/config/unicorn/production.rb @@ -2,7 +2,7 @@ app_path = "/var/www/danbooru2/current" # Set unicorn options -worker_processes 22 +worker_processes 10 timeout 180 #listen "127.0.0.1:9000", :tcp_nopush => true diff --git a/lib/capistrano/tasks/delayed_job.rake b/lib/capistrano/tasks/delayed_job.rake index 091510410..4109a9a8d 100644 --- a/lib/capistrano/tasks/delayed_job.rake +++ b/lib/capistrano/tasks/delayed_job.rake @@ -1,12 +1,12 @@ namespace :delayed_job do desc "Start delayed_job process" task :start do - on roles(:app) do + on roles(:worker) do if test("[ -d #{current_path} ]") within current_path do with rails_env: fetch(:rails_env) do hostname = capture("hostname").strip - execute :bundle, "exec", "script/delayed_job", "--queues=default,#{hostname}", "-n 2", "start" + execute :bundle, "exec", "script/delayed_job", "--queues=default,#{hostname}", "-n 8", "start" end end end @@ -15,7 +15,7 @@ namespace :delayed_job do desc "Stop delayed_job process" task :stop do - on roles(:app) do + on roles(:worker) do if test("[ -d #{current_path} ]") within current_path do with rails_env: fetch(:rails_env) do @@ -28,7 +28,7 @@ namespace :delayed_job do desc "Restart delayed_job process" task :restart do - on roles(:app) do + on roles(:worker) do find_and_execute_task("delayed_job:stop") find_and_execute_task("delayed_job:start") end @@ -36,7 +36,7 @@ namespace :delayed_job do desc "Kill delayed_job process" task :kill do - on roles(:app) do + on roles(:worker) do procs = capture("ps -A -o pid,command").split(/\r\n|\r|\n/).grep(/delayed_job/).map(&:to_i) if procs.any? diff --git a/lib/capistrano/tasks/symlink.rake b/lib/capistrano/tasks/symlink.rake index 537577976..656f41257 100644 --- a/lib/capistrano/tasks/symlink.rake +++ b/lib/capistrano/tasks/symlink.rake @@ -1,7 +1,7 @@ namespace :symlink do desc "Link the local config files" task :local_files do - on roles(:app) do + on roles(:app, :worker) do execute :ln, "-s", "#{deploy_to}/shared/config/danbooru_local_config.rb", "#{release_path}/config/danbooru_local_config.rb" execute :ln, "-s", "#{deploy_to}/shared/config/database.yml", "#{release_path}/config/database.yml" if test("[ -f #{deploy_to}/shared/config/newrelic.yml ]") @@ -12,7 +12,7 @@ namespace :symlink do desc "Link the local directories" task :directories do - on roles(:app) do + on roles(:app, :worker) do execute :rm, "-f", "#{release_path}/public/data" execute :ln, "-s", "#{deploy_to}/shared/data", "#{release_path}/public/data" execute :mkdir, "-p", "#{release_path}/public/cache" diff --git a/script/install/vrack-cfg.yaml b/script/install/vrack-cfg.yaml new file mode 100644 index 000000000..cbd7434a3 --- /dev/null +++ b/script/install/vrack-cfg.yaml @@ -0,0 +1,10 @@ +network: + version: 2 + renderer: networkd + ethernets: + eno2: {} + vlans: + eno2.99: + id: 99 + link: eno2 + addresses: [172.16.0.1]