[Seeding] Rewrite the seeding script (#676)

.rubocop.yml

@@ -85,6 +85,7 @@ Rails/InverseOf:
 
 Rails/Output:
   Exclude:
+    - db/populate.rb
     - db/seeds.rb
     - db/fixes/*.rb
   Include:

Gemfile

@@ -56,6 +56,7 @@ group :development do
   gem "rubocop-rails", require: false
   gem "ruby-lsp"
   gem "ruby-lsp-rails"
+  gem 'faker', require: false
 end
 
 group :test do

Gemfile.lock

@@ -143,6 +143,8 @@ GEM
     factory_bot_rails (6.4.3)
       factory_bot (~> 6.4)
       railties (>= 5.0.0)
+    faker (3.4.2)
+      i18n (>= 1.8.11, < 2)
     faraday (2.9.0)
       faraday-net_http (>= 2.0, < 3.2)
     faraday-follow_redirects (0.3.0)
@@ -386,6 +388,7 @@ DEPENDENCIES
   draper
   dtext_rb!
   factory_bot_rails
+  faker
   faraday
   faraday-follow_redirects
   faraday-retry

README.md

@@ -17,11 +17,17 @@
 1. Copy the sample environment file with `cp .env.sample .env`.
 1. Run the following commands:
     ```
-    docker compose run --rm -e SEED_POST_COUNT=100 e621 /app/bin/setup
+    docker compose run --rm e621 /app/bin/setup
     docker compose up
     ```
     After running the commands once only `docker compose up` is needed to bring up the containers.
-1. To confirm the installation worked, open the web browser of your choice and enter `http://localhost:3000` into the address bar and see if the website loads correctly. An admin account has been created automatically, the username and password are `admin` and `e621test` respectively.
+1. To confirm the installation worked, open the web browser of your choice and enter `http://localhost:3000` into the address bar and see if the website loads correctly. An admin account has been created automatically, the username and password are `admin` and `qwerty` respectively.
+1. By default, the site will lack any content. For testing purposes, you can generate some using the following command:
+    ```
+    docker exec -it e621ng-e621-1 /app/bin/populate
+    ```
+    The command can be run multiple times to generate more content.  
+    Environmental variables are available to customize what kind of content is generated.
 
 Note: When gems or js packages are updated you need to execute `docker compose build` to reflect them in the container.
 

bin/populate

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+system('RAILS_ENV=development DANBOORU_DISABLE_THROTTLES=true bin/rails runner db/populate.rb', exception: true)

db/populate.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+# This script populates the database with random data for testing or development purposes.
+# Usage: docker exec -it e621ng-e621-1 /app/bin/populate
+
+require "faker"
+
+# Environmental variables that govern how much content to generate
+presets = {
+  users: ENV.fetch("USERS", 0).to_i,
+  posts: ENV.fetch("POSTS", 0).to_i,
+  comments: ENV.fetch("COMMENTS", 0).to_i,
+  favorites: ENV.fetch("FAVORITES", 0).to_i,
+  forums: ENV.fetch("FORUMS", 0).to_i,
+}
+if presets.values.sum == 0
+  puts "DEFAULTS"
+  presets = {
+    users: 10,
+    posts: 100,
+    comments: 100,
+    favorites: 100,
+    forums: 100,
+  }
+end
+
+USERS     = presets[:users]
+POSTS     = presets[:posts]
+COMMENTS  = presets[:comments]
+FAVORITES = presets[:favorites]
+FORUMS    = presets[:forums]
+
+DISTRIBUTION = ENV.fetch("DISTRIBUTION", 10).to_i
+DEFAULT_PASSWORD = ENV.fetch("PASSWORD", "qwerty")
+
+CurrentUser.user = User.system
+
+def api_request(path)
+  response = Faraday.get("https://e621.net#{path}", nil, user_agent: "e621ng/seeding")
+  JSON.parse(response.body)
+end
+
+def populate_users(number, password: DEFAULT_PASSWORD)
+  return [] unless number > 0
+  puts "* Creating #{number} users\n  This may take some time."
+
+  output = []
+
+  number.times do
+    user_name = generate_username
+    puts "  - #{user_name}"
+    user_obj = User.create do |user|
+      user.name = user_name
+      user.password = password
+      user.password_confirmation = password
+      user.email = "#{user_name}@e621.local"
+      user.level = User::Levels::MEMBER
+      user.created_at = Faker::Date.between(from: "2007-02-10", to: 2.weeks.ago)
+
+      user.profile_about = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false) if Faker::Boolean.boolean(true_ratio: 0.2)
+      user.profile_artinfo = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false) if Faker::Boolean.boolean(true_ratio: 0.2)
+    end
+
+    if user_obj.errors.empty?
+      output << user_obj
+      puts "    user ##{user_obj.id}"
+    else
+      puts "    error: #{user_obj.errors.full_messages.join('; ')}"
+    end
+  end
+
+  output
+end
+
+def generate_username
+  loop do
+    @username = [
+      Faker::Adjective.positive.split.each(&:capitalize!),
+      Faker::Creature::Animal.name.split.each(&:capitalize!),
+    ].concat.join("_")
+
+    next unless @username.length >= 3 && @username.length <= 20
+    next unless User.find_by(name: @username).nil?
+    break
+  end
+
+  @username
+end
+
+def populate_posts(number, users: [], batch_size: 320)
+  return [] unless number > 0
+  puts "* Creating #{number} posts"
+
+  admin = User.find(1)
+  users = User.where("users.created_at < ?", 7.days.ago).limit(DISTRIBUTION).order("random()") if users.empty?
+  output = []
+
+  # Generate posts in batches of 200 (by default)
+  number.times.each_slice(batch_size).map(&:size).each do |count|
+    posts = api_request("/posts.json?tags=rating:s+order:random+score:>250+-grandfathered_content&limit=#{count}")["posts"]
+
+    posts.each do |post|
+      post["tags"].each do |category, tags|
+        Tag.find_or_create_by_name_list(tags.map { |tag| "#{category}:#{tag}" })
+      end
+
+      CurrentUser.user = users.sample # Stupid, but I can't be bothered
+      CurrentUser.user = users.sample unless CurrentUser.user.can_upload_with_reason
+      puts "  - #{CurrentUser.user.name} : #{post['file']['url']}"
+
+      Post.transaction do
+        service = UploadService.new(generate_upload(post))
+        @upload = service.start!
+      end
+
+      if @upload.invalid? || @upload.post.nil?
+        puts "    #{@upload.errors.full_messages.join('; ')}"
+      else
+        puts "    post: ##{@upload.post.id}"
+        CurrentUser.scoped(admin) do
+          @upload.post.approve!
+        end
+        output << @upload.post
+      end
+    end
+  end
+
+  output
+end
+
+def generate_upload(post)
+  {
+    uploader: CurrentUser.user,
+    uploader_ip_addr: "127.0.0.1",
+    direct_url: post["file"]["url"],
+    tag_string: post["tags"].values.flatten.join(" "),
+    source: post["sources"].join("\n"),
+    description: post["description"],
+    rating: post["rating"],
+  }
+end
+
+def fill_avatars(users = [], posts = [])
+  return if users.empty?
+  puts "* Filling in #{users.size} avatars"
+
+  posts = Post.limit(users.size).order("random()") if posts.empty?
+  puts posts
+
+  users.each do |user|
+    post = posts.sample
+    puts "post: #{post}"
+    puts "  - #{user.name} : ##{post.id}"
+    user.update({ avatar_id: post.id })
+  end
+end
+
+def populate_comments(number, users: [])
+  return unless number > 0
+  puts "* Creating #{number} comments"
+
+  users = User.where("users.created_at < ?", 14.days.ago).limit(DISTRIBUTION).order("random()") if users.empty?
+  posts = Post.limit(DISTRIBUTION).order("random()")
+
+  number.times do |index|
+    post = posts[index % DISTRIBUTION]
+    CurrentUser.user = users[index % DISTRIBUTION]
+
+    comment_obj = Comment.create do |comment|
+      comment.creator = CurrentUser.user
+      comment.updater = CurrentUser.user
+      comment.post = post
+      comment.body = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false)
+      comment.creator_ip_addr = "127.0.0.1"
+    end
+
+    puts "  - ##{comment_obj.id} by #{CurrentUser.user.name}"
+  end
+end
+
+def populate_favorites(number, users: [])
+  return unless number > 0
+  puts "* Creating #{number} favorites"
+
+  users = User.limit(DISTRIBUTION).order("random()") if users.empty?
+
+  number.times do |index|
+    CurrentUser.user = users[index % DISTRIBUTION]
+    post = Post.order("random()").first
+    puts "  - ##{post.id} faved by #{CurrentUser.user.name}"
+
+    begin
+      Favorite.create do |fav|
+        fav.user = CurrentUser.user
+        fav.post = post
+      end
+    rescue StandardError
+      puts "    Favorite already exists"
+    end
+  end
+end
+
+def populate_forums(number, users: [])
+  return unless number > 0
+  number -= 1 # Accounts for the first post in the thread
+  puts "* Creating a topic with #{number} replies"
+
+  users = User.where("users.created_at < ?", 14.days.ago).limit(DISTRIBUTION).order("random()") if users.empty?
+
+  category = ForumCategory.find_or_create_by!(name: "General") do |cat|
+    cat.can_view = 0
+  end
+
+  CurrentUser.user = users.sample
+  CurrentUser.ip_addr = "127.0.0.1"
+  forum_topic = ForumTopic.create do |topic|
+    topic.creator = CurrentUser.user
+    topic.creator_ip_addr = "127.0.0.1"
+    topic.title = Faker::Lorem.sentence(word_count: 3, supplemental: true, random_words_to_add: 4)
+    topic.category = category
+    topic.original_post_attributes = {
+      creator: CurrentUser.user,
+      body: Faker::Lorem.paragraphs.join("\n\n"),
+    }
+  end
+
+  puts "  topic ##{forum_topic.id} by #{CurrentUser.user.name}"
+
+  unless forum_topic.valid?
+    puts "  #{forum_topic.errors.full_messages.join('; ')}"
+  end
+
+  number.times do
+    CurrentUser.user = users.sample
+
+    forum_post = ForumPost.create do |post|
+      post.creator = CurrentUser.user
+      post.topic_id = forum_topic.id
+      post.body = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false)
+    end
+
+    puts "  - #{CurrentUser.user.name} | forum post ##{forum_post.id}"
+
+    unless forum_post.valid?
+      puts "    #{forum_post.errors.full_messages.join('; ')}"
+    end
+  end
+end
+
+puts "Populating the Database"
+CurrentUser.user = User.find(1)
+CurrentUser.ip_addr = "127.0.0.1"
+
+users = populate_users(USERS)
+posts = populate_posts(POSTS, users: users)
+fill_avatars(users, posts)
+
+populate_comments(COMMENTS, users: users)
+populate_favorites(FAVORITES, users: users)
+populate_forums(FORUMS, users: users)

db/seeds.rb

@@ -9,10 +9,10 @@ require "tempfile"
 
 admin = User.find_or_create_by!(name: "admin") do |user|
   user.created_at = 2.weeks.ago
-  user.password = "e621test"
-  user.password_confirmation = "e621test"
+  user.password = "qwerty"
+  user.password_confirmation = "qwerty"
   user.password_hash = ""
-  user.email = "admin@e621.net"
+  user.email = "admin@e621.local"
   user.can_upload_free = true
   user.can_approve_posts = true
   user.level = User::Levels::ADMIN
@@ -22,7 +22,7 @@ User.find_or_create_by!(name: Danbooru.config.system_user) do |user|
   user.password = "ae3n4oie2n3oi4en23oie4noienaorshtaioresnt"
   user.password_confirmation = "ae3n4oie2n3oi4en23oie4noienaorshtaioresnt"
   user.password_hash = ""
-  user.email = "system@e621.net"
+  user.email = "system@e621.local"
   user.can_upload_free = true
   user.can_approve_posts = true
   user.level = User::Levels::JANITOR
@@ -37,33 +37,8 @@ def api_request(path)
   JSON.parse(response.body)
 end
 
-def import_posts
-  resources = YAML.load_file Rails.root.join("db/seeds.yml")
-  json = api_request("/posts.json?limit=#{ENV.fetch('SEED_POST_COUNT', 100)}&tags=id:#{resources['post_ids'].join(',')}")
-
-  json["posts"].each do |post|
-    puts post["file"]["url"]
-
-    post["tags"].each do |category, tags|
-      Tag.find_or_create_by_name_list(tags.map { |tag| "#{category}:#{tag}" })
-    end
-
-    service = UploadService.new({
-      uploader: CurrentUser.user,
-      uploader_ip_addr: CurrentUser.ip_addr,
-      direct_url: post["file"]["url"],
-      tag_string: post["tags"].values.flatten.join(" "),
-      source: post["sources"].join("\n"),
-      description: post["description"],
-      rating: post["rating"],
-    })
-
-    service.start!
-  end
-end
-
 def import_mascots
-  api_request("/mascots.json").each do |mascot|
+  api_request("/mascots.json?limit=1").each do |mascot|
     puts mascot["url_path"]
     Mascot.create!(
       creator: CurrentUser.user,
@@ -78,12 +53,18 @@ def import_mascots
   end
 end
 
+def setup_upload_whitelist
+  UploadWhitelist.create do |entry|
+    entry.pattern = "https://static1.e621.net/*"
+  end
+end
+
 unless Rails.env.test?
   CurrentUser.user = admin
   CurrentUser.ip_addr = "127.0.0.1"
   begin
-    import_posts
     import_mascots
+    setup_upload_whitelist
   rescue StandardError => e
     puts "--------"
     puts "#{e.class}: #{e.message}"

db/seeds.yml

@@ -1,81 +0,0 @@
-post_ids:
-  - 864982
-  - 831434
-  - 604569
-  - 1384351
-  - 713232
-  - 2058923
-  - 348854
-  - 677272
-  - 721020
-  - 783617
-  - 524011
-  - 738084
-  - 1324650
-  - 447317
-  - 902353
-  - 608312
-  - 1196171
-  - 539888
-  - 426556
-  - 1158932
-  - 1787376
-  - 844336
-  - 133831
-  - 543720
-  - 675477
-  - 468397
-  - 480698
-  - 1201721
-  - 2104542
-  - 1910123
-  - 1162821
-  - 1559520
-  - 1559522
-  - 1337331
-  - 1289163
-  - 1124466
-  - 922038
-  - 1051272
-  - 1014268
-  - 972377
-  - 891419
-  - 902353
-  - 896454
-  - 882158
-  - 864033
-  - 854142
-  - 853653
-  - 818700
-  - 830459
-  - 818006
-  - 818294
-  - 802009
-  - 810790
-  - 799347
-  - 1401572
-  - 766674
-  - 735523
-  - 722531
-  - 721020
-  - 2038195
-  - 701921
-  - 692610
-  - 687969
-  - 677272
-  - 672692
-  - 668603
-  - 664515
-  - 604569
-  - 605113
-  - 589740
-  - 586160
-  - 584859
-  - 577089
-  - 559155
-  - 494915
-  - 492433
-  - 464910
-  - 444992
-  - 429880
-  - 437103