From b852254ad0a593848913d77f1b98cef46bf95692 Mon Sep 17 00:00:00 2001 From: Cinder Date: Sat, 20 Jul 2024 08:30:21 -0700 Subject: [PATCH] [Seeding] Rewrite the seeding script (#676) --- .rubocop.yml | 1 + Gemfile | 1 + Gemfile.lock | 3 + README.md | 10 +- bin/populate | 3 + db/populate.rb | 260 +++++++++++++++++++++++++++++++++++++++++++++++++ db/seeds.rb | 43 +++----- db/seeds.yml | 81 --------------- 8 files changed, 288 insertions(+), 114 deletions(-) create mode 100644 bin/populate create mode 100644 db/populate.rb delete mode 100644 db/seeds.yml diff --git a/.rubocop.yml b/.rubocop.yml index b9b805cf0..6a4974108 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -85,6 +85,7 @@ Rails/InverseOf: Rails/Output: Exclude: + - db/populate.rb - db/seeds.rb - db/fixes/*.rb Include: diff --git a/Gemfile b/Gemfile index 29f489bf0..217afb164 100644 --- a/Gemfile +++ b/Gemfile @@ -56,6 +56,7 @@ group :development do gem "rubocop-rails", require: false gem "ruby-lsp" gem "ruby-lsp-rails" + gem 'faker', require: false end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 02fde0e7f..c792d552c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -143,6 +143,8 @@ GEM factory_bot_rails (6.4.3) factory_bot (~> 6.4) railties (>= 5.0.0) + faker (3.4.2) + i18n (>= 1.8.11, < 2) faraday (2.9.0) faraday-net_http (>= 2.0, < 3.2) faraday-follow_redirects (0.3.0) @@ -386,6 +388,7 @@ DEPENDENCIES draper dtext_rb! factory_bot_rails + faker faraday faraday-follow_redirects faraday-retry diff --git a/README.md b/README.md index fd0bc9d68..3840c60b4 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,17 @@ 1. Copy the sample environment file with `cp .env.sample .env`. 1. Run the following commands: ``` - docker compose run --rm -e SEED_POST_COUNT=100 e621 /app/bin/setup + docker compose run --rm e621 /app/bin/setup docker compose up ``` After running the commands once only `docker compose up` is needed to bring up the containers. -1. To confirm the installation worked, open the web browser of your choice and enter `http://localhost:3000` into the address bar and see if the website loads correctly. An admin account has been created automatically, the username and password are `admin` and `e621test` respectively. +1. To confirm the installation worked, open the web browser of your choice and enter `http://localhost:3000` into the address bar and see if the website loads correctly. An admin account has been created automatically, the username and password are `admin` and `qwerty` respectively. +1. By default, the site will lack any content. For testing purposes, you can generate some using the following command: + ``` + docker exec -it e621ng-e621-1 /app/bin/populate + ``` + The command can be run multiple times to generate more content. + Environmental variables are available to customize what kind of content is generated. Note: When gems or js packages are updated you need to execute `docker compose build` to reflect them in the container. diff --git a/bin/populate b/bin/populate new file mode 100644 index 000000000..f10a5c583 --- /dev/null +++ b/bin/populate @@ -0,0 +1,3 @@ +#!/usr/bin/env ruby + +system('RAILS_ENV=development DANBOORU_DISABLE_THROTTLES=true bin/rails runner db/populate.rb', exception: true) diff --git a/db/populate.rb b/db/populate.rb new file mode 100644 index 000000000..b314e44d2 --- /dev/null +++ b/db/populate.rb @@ -0,0 +1,260 @@ +# frozen_string_literal: true + +# This script populates the database with random data for testing or development purposes. +# Usage: docker exec -it e621ng-e621-1 /app/bin/populate + +require "faker" + +# Environmental variables that govern how much content to generate +presets = { + users: ENV.fetch("USERS", 0).to_i, + posts: ENV.fetch("POSTS", 0).to_i, + comments: ENV.fetch("COMMENTS", 0).to_i, + favorites: ENV.fetch("FAVORITES", 0).to_i, + forums: ENV.fetch("FORUMS", 0).to_i, +} +if presets.values.sum == 0 + puts "DEFAULTS" + presets = { + users: 10, + posts: 100, + comments: 100, + favorites: 100, + forums: 100, + } +end + +USERS = presets[:users] +POSTS = presets[:posts] +COMMENTS = presets[:comments] +FAVORITES = presets[:favorites] +FORUMS = presets[:forums] + +DISTRIBUTION = ENV.fetch("DISTRIBUTION", 10).to_i +DEFAULT_PASSWORD = ENV.fetch("PASSWORD", "qwerty") + +CurrentUser.user = User.system + +def api_request(path) + response = Faraday.get("https://e621.net#{path}", nil, user_agent: "e621ng/seeding") + JSON.parse(response.body) +end + +def populate_users(number, password: DEFAULT_PASSWORD) + return [] unless number > 0 + puts "* Creating #{number} users\n This may take some time." + + output = [] + + number.times do + user_name = generate_username + puts " - #{user_name}" + user_obj = User.create do |user| + user.name = user_name + user.password = password + user.password_confirmation = password + user.email = "#{user_name}@e621.local" + user.level = User::Levels::MEMBER + user.created_at = Faker::Date.between(from: "2007-02-10", to: 2.weeks.ago) + + user.profile_about = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false) if Faker::Boolean.boolean(true_ratio: 0.2) + user.profile_artinfo = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false) if Faker::Boolean.boolean(true_ratio: 0.2) + end + + if user_obj.errors.empty? + output << user_obj + puts " user ##{user_obj.id}" + else + puts " error: #{user_obj.errors.full_messages.join('; ')}" + end + end + + output +end + +def generate_username + loop do + @username = [ + Faker::Adjective.positive.split.each(&:capitalize!), + Faker::Creature::Animal.name.split.each(&:capitalize!), + ].concat.join("_") + + next unless @username.length >= 3 && @username.length <= 20 + next unless User.find_by(name: @username).nil? + break + end + + @username +end + +def populate_posts(number, users: [], batch_size: 320) + return [] unless number > 0 + puts "* Creating #{number} posts" + + admin = User.find(1) + users = User.where("users.created_at < ?", 7.days.ago).limit(DISTRIBUTION).order("random()") if users.empty? + output = [] + + # Generate posts in batches of 200 (by default) + number.times.each_slice(batch_size).map(&:size).each do |count| + posts = api_request("/posts.json?tags=rating:s+order:random+score:>250+-grandfathered_content&limit=#{count}")["posts"] + + posts.each do |post| + post["tags"].each do |category, tags| + Tag.find_or_create_by_name_list(tags.map { |tag| "#{category}:#{tag}" }) + end + + CurrentUser.user = users.sample # Stupid, but I can't be bothered + CurrentUser.user = users.sample unless CurrentUser.user.can_upload_with_reason + puts " - #{CurrentUser.user.name} : #{post['file']['url']}" + + Post.transaction do + service = UploadService.new(generate_upload(post)) + @upload = service.start! + end + + if @upload.invalid? || @upload.post.nil? + puts " #{@upload.errors.full_messages.join('; ')}" + else + puts " post: ##{@upload.post.id}" + CurrentUser.scoped(admin) do + @upload.post.approve! + end + output << @upload.post + end + end + end + + output +end + +def generate_upload(post) + { + uploader: CurrentUser.user, + uploader_ip_addr: "127.0.0.1", + direct_url: post["file"]["url"], + tag_string: post["tags"].values.flatten.join(" "), + source: post["sources"].join("\n"), + description: post["description"], + rating: post["rating"], + } +end + +def fill_avatars(users = [], posts = []) + return if users.empty? + puts "* Filling in #{users.size} avatars" + + posts = Post.limit(users.size).order("random()") if posts.empty? + puts posts + + users.each do |user| + post = posts.sample + puts "post: #{post}" + puts " - #{user.name} : ##{post.id}" + user.update({ avatar_id: post.id }) + end +end + +def populate_comments(number, users: []) + return unless number > 0 + puts "* Creating #{number} comments" + + users = User.where("users.created_at < ?", 14.days.ago).limit(DISTRIBUTION).order("random()") if users.empty? + posts = Post.limit(DISTRIBUTION).order("random()") + + number.times do |index| + post = posts[index % DISTRIBUTION] + CurrentUser.user = users[index % DISTRIBUTION] + + comment_obj = Comment.create do |comment| + comment.creator = CurrentUser.user + comment.updater = CurrentUser.user + comment.post = post + comment.body = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false) + comment.creator_ip_addr = "127.0.0.1" + end + + puts " - ##{comment_obj.id} by #{CurrentUser.user.name}" + end +end + +def populate_favorites(number, users: []) + return unless number > 0 + puts "* Creating #{number} favorites" + + users = User.limit(DISTRIBUTION).order("random()") if users.empty? + + number.times do |index| + CurrentUser.user = users[index % DISTRIBUTION] + post = Post.order("random()").first + puts " - ##{post.id} faved by #{CurrentUser.user.name}" + + begin + Favorite.create do |fav| + fav.user = CurrentUser.user + fav.post = post + end + rescue StandardError + puts " Favorite already exists" + end + end +end + +def populate_forums(number, users: []) + return unless number > 0 + number -= 1 # Accounts for the first post in the thread + puts "* Creating a topic with #{number} replies" + + users = User.where("users.created_at < ?", 14.days.ago).limit(DISTRIBUTION).order("random()") if users.empty? + + category = ForumCategory.find_or_create_by!(name: "General") do |cat| + cat.can_view = 0 + end + + CurrentUser.user = users.sample + CurrentUser.ip_addr = "127.0.0.1" + forum_topic = ForumTopic.create do |topic| + topic.creator = CurrentUser.user + topic.creator_ip_addr = "127.0.0.1" + topic.title = Faker::Lorem.sentence(word_count: 3, supplemental: true, random_words_to_add: 4) + topic.category = category + topic.original_post_attributes = { + creator: CurrentUser.user, + body: Faker::Lorem.paragraphs.join("\n\n"), + } + end + + puts " topic ##{forum_topic.id} by #{CurrentUser.user.name}" + + unless forum_topic.valid? + puts " #{forum_topic.errors.full_messages.join('; ')}" + end + + number.times do + CurrentUser.user = users.sample + + forum_post = ForumPost.create do |post| + post.creator = CurrentUser.user + post.topic_id = forum_topic.id + post.body = Faker::Hipster.paragraph_by_chars(characters: rand(100..2_000), supplemental: false) + end + + puts " - #{CurrentUser.user.name} | forum post ##{forum_post.id}" + + unless forum_post.valid? + puts " #{forum_post.errors.full_messages.join('; ')}" + end + end +end + +puts "Populating the Database" +CurrentUser.user = User.find(1) +CurrentUser.ip_addr = "127.0.0.1" + +users = populate_users(USERS) +posts = populate_posts(POSTS, users: users) +fill_avatars(users, posts) + +populate_comments(COMMENTS, users: users) +populate_favorites(FAVORITES, users: users) +populate_forums(FORUMS, users: users) diff --git a/db/seeds.rb b/db/seeds.rb index 295938428..fa4c38a26 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -9,10 +9,10 @@ require "tempfile" admin = User.find_or_create_by!(name: "admin") do |user| user.created_at = 2.weeks.ago - user.password = "e621test" - user.password_confirmation = "e621test" + user.password = "qwerty" + user.password_confirmation = "qwerty" user.password_hash = "" - user.email = "admin@e621.net" + user.email = "admin@e621.local" user.can_upload_free = true user.can_approve_posts = true user.level = User::Levels::ADMIN @@ -22,7 +22,7 @@ User.find_or_create_by!(name: Danbooru.config.system_user) do |user| user.password = "ae3n4oie2n3oi4en23oie4noienaorshtaioresnt" user.password_confirmation = "ae3n4oie2n3oi4en23oie4noienaorshtaioresnt" user.password_hash = "" - user.email = "system@e621.net" + user.email = "system@e621.local" user.can_upload_free = true user.can_approve_posts = true user.level = User::Levels::JANITOR @@ -37,33 +37,8 @@ def api_request(path) JSON.parse(response.body) end -def import_posts - resources = YAML.load_file Rails.root.join("db/seeds.yml") - json = api_request("/posts.json?limit=#{ENV.fetch('SEED_POST_COUNT', 100)}&tags=id:#{resources['post_ids'].join(',')}") - - json["posts"].each do |post| - puts post["file"]["url"] - - post["tags"].each do |category, tags| - Tag.find_or_create_by_name_list(tags.map { |tag| "#{category}:#{tag}" }) - end - - service = UploadService.new({ - uploader: CurrentUser.user, - uploader_ip_addr: CurrentUser.ip_addr, - direct_url: post["file"]["url"], - tag_string: post["tags"].values.flatten.join(" "), - source: post["sources"].join("\n"), - description: post["description"], - rating: post["rating"], - }) - - service.start! - end -end - def import_mascots - api_request("/mascots.json").each do |mascot| + api_request("/mascots.json?limit=1").each do |mascot| puts mascot["url_path"] Mascot.create!( creator: CurrentUser.user, @@ -78,12 +53,18 @@ def import_mascots end end +def setup_upload_whitelist + UploadWhitelist.create do |entry| + entry.pattern = "https://static1.e621.net/*" + end +end + unless Rails.env.test? CurrentUser.user = admin CurrentUser.ip_addr = "127.0.0.1" begin - import_posts import_mascots + setup_upload_whitelist rescue StandardError => e puts "--------" puts "#{e.class}: #{e.message}" diff --git a/db/seeds.yml b/db/seeds.yml deleted file mode 100644 index d26540e3b..000000000 --- a/db/seeds.yml +++ /dev/null @@ -1,81 +0,0 @@ -post_ids: - - 864982 - - 831434 - - 604569 - - 1384351 - - 713232 - - 2058923 - - 348854 - - 677272 - - 721020 - - 783617 - - 524011 - - 738084 - - 1324650 - - 447317 - - 902353 - - 608312 - - 1196171 - - 539888 - - 426556 - - 1158932 - - 1787376 - - 844336 - - 133831 - - 543720 - - 675477 - - 468397 - - 480698 - - 1201721 - - 2104542 - - 1910123 - - 1162821 - - 1559520 - - 1559522 - - 1337331 - - 1289163 - - 1124466 - - 922038 - - 1051272 - - 1014268 - - 972377 - - 891419 - - 902353 - - 896454 - - 882158 - - 864033 - - 854142 - - 853653 - - 818700 - - 830459 - - 818006 - - 818294 - - 802009 - - 810790 - - 799347 - - 1401572 - - 766674 - - 735523 - - 722531 - - 721020 - - 2038195 - - 701921 - - 692610 - - 687969 - - 677272 - - 672692 - - 668603 - - 664515 - - 604569 - - 605113 - - 589740 - - 586160 - - 584859 - - 577089 - - 559155 - - 494915 - - 492433 - - 464910 - - 444992 - - 429880 - - 437103