edshot99/eBooru
1
0
Fork 0
forked from e621ng/e621ng
Code Issues 11 Releases 1 Activity

[Cleanup] Remove remaining upload preprocess code

Browse Source
This commit is contained in:
Earlopain 2022-02-19 17:30:49 +01:00
parent de0311a268
commit caa51c4f2c
No known key found for this signature in database
GPG Key ID: 6CFB948E15246897
5 changed files with 15 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/logical/upload_service/utils.rb
View File

@ -33,12 +33,6 @@ class UploadService
return
end
if upload_id.present? && Upload.where(id: upload_id).exists?
CurrentUser.as_system do
Upload.find(upload_id).update(status: "preprocessed + deleted")
end
end
Danbooru.config.storage_manager.delete_post_files(md5, file_ext)
end

8
app/models/upload.rb
View File

@ -151,14 +151,6 @@ class Upload < ApplicationRecord
status == "completed"
end
def is_preprocessed?
status == "preprocessed"
end
def is_preprocessing?
status == "preprocessing"
end
def is_duplicate?
status.match?(/duplicate: \d+/)
end

4
app/views/uploads/show.html.erb
View File

@ -23,7 +23,7 @@
<p>This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.</p>
<% elsif @upload.is_pending? %>
<p>This upload is waiting to be processed. Please wait a few seconds.</p>
<% elsif @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
<% elsif @upload.is_processing? %>
<p>This upload is being processed. Please wait a few seconds.</p>
<% elsif @upload.is_duplicate? %>
<p>This upload is a duplicate: <%= link_to "post ##{@upload.duplicate_post_id}", post_path(@upload.duplicate_post_id) %></p>
@ -49,7 +49,7 @@
Upload
<% end %>
<% if @upload.is_pending? || @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
<% if @upload.is_pending? || @upload.is_processing? %>
<% content_for(:html_header) do %>
<meta http-equiv="refresh" content="2">
<% end %>

77
test/functional/uploads_controller_test.rb
View File

@ -16,15 +16,6 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
mock_iqdb_service!
end
context "preprocess action" do
should "prefer the file over the source when preprocessing" do
file = Rack::Test::UploadedFile.new("#{Rails.root}/test/files/test.jpg", "image/jpeg")
post_auth preprocess_uploads_path, @user, params: {:upload => {:source => "https://raikou1.donmai.us/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg", :file => file}}
assert_response :success
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", Upload.last.md5)
end
end
context "new action" do
should "render" do
get_auth new_upload_path, @user
@ -50,19 +41,6 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
end
end
context "for a direct link twitter post" do
setup do
@ref = "https://twitter.com/onsen_musume_jp/status/865534101918330881"
@source = "https://pbs.twimg.com/media/DAL-ntWV0AEbhes.jpg:orig"
end
should "trigger the preprocessor" do
assert_difference(-> { Upload.preprocessed.count }, 1) do
get_auth new_upload_path, @user, params: {:url => @source, :ref => @ref}
end
end
end
context "for a twitter post" do
setup do
@source = "https://twitter.com/frappuccino/status/566030116182949888"
@ -81,19 +59,6 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
end
end
context "for a pixiv post" do
setup do
@ref = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482"
@source = "https://i.pximg.net/img-original/img/2015/03/14/17/53/32/49270482_p0.jpg"
end
should "trigger the preprocessor" do
assert_difference(-> { Upload.preprocessed.count }, 1) do
get_auth new_upload_path, @user, params: {:url => @source, :ref => @ref}
end
end
end
context "for a post that has already been uploaded" do
setup do
as_user do
@ -154,48 +119,6 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
end
context "create action" do
context "when a preprocessed upload already exists" do
context "for twitter" do
setup do
as_user do
@ref = "https://twitter.com/onsen_musume_jp/status/865534101918330881"
@source = "https://pbs.twimg.com/media/DAL-ntWV0AEbhes.jpg:orig"
@upload = create(:upload, status: "preprocessed", source: @source, referer_url: @ref, image_width: 0, image_height: 0, file_size: 0, md5: "something", file_ext: "jpg")
end
end
should "update the predecessor" do
assert_difference(->{ Post.count }, 1) do
assert_difference(->{ Upload.count }, 0) do
post_auth uploads_path, @user, params: {:upload => {:tag_string => "aaa", :rating => "q", :source => @source, :referer_url => @ref}}
end
end
post = Post.last
assert_match(/aaa/, post.tag_string)
end
end
context "for pixiv" do
setup do
@ref = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482"
@source = "https://i.pximg.net/img-original/img/2015/03/14/17/53/32/49270482_p0.jpg"
as_user do
@upload = create(:upload, status: "preprocessed", source: @source, referer_url: @ref, image_width: 0, image_height: 0, file_size: 0, md5: "something", file_ext: "jpg")
end
end
should "update the predecessor" do
assert_difference(->{ Post.count }, 1) do
assert_difference(->{ Upload.count }, 0) do
post_auth uploads_path, @user, params: {:upload => {:tag_string => "aaa", :rating => "q", :source => @source, :referer_url => @ref}}
end
end
post = Post.last
assert_match(/aaa/, post.tag_string)
end
end
end
should "create a new upload" do
assert_difference("Upload.count", 1) do
file = Rack::Test::UploadedFile.new("#{Rails.root}/test/files/test.jpg", "image/jpeg")

229
test/models/upload_service_test.rb
View File

@ -369,163 +369,6 @@ class UploadServiceTest < ActiveSupport::TestCase
end
end
context "::Preprocessor" do
subject { UploadService::Preprocessor }
context "#start!" do
setup do
CurrentUser.user = travel_to(1.month.ago) do
FactoryBot.create(:user)
end
CurrentUser.ip_addr = "127.0.0.1"
end
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end
context "for twitter" do
setup do
@source = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large"
@ref = "https://twitter.com/nounproject/status/540944400767922176"
end
should "download the file" do
@service = subject.new(source: @source, referer_url: @ref)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(9800, @upload.file_size)
assert_equal("png", @upload.file_ext)
assert_equal("f5fe24f3a3a13885285f6627e04feec9", @upload.md5)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "png", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "png", :preview)))
end
end
context "for pixiv" do
setup do
@source = "https://i.pximg.net/img-original/img/2014/10/29/09/27/19/46785915_p0.jpg"
@ref = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=46785915"
end
should "download the file" do
begin
@service = subject.new(source: @source, referer_url: @ref)
@upload = @service.start!
rescue Net::OpenTimeout
skip "network failure"
end
assert_equal("preprocessed", @upload.status)
assert_equal(294591, @upload.file_size)
assert_equal("jpg", @upload.file_ext)
assert_equal("3cb1ef624714c15dbb2d6e7b1d57faef", @upload.md5)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :preview)))
end
end
context "for pixiv ugoira" do
setup do
@source = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364"
end
should "download the file" do
@service = subject.new(source: @source)
begin
@upload = @service.start!
rescue Net::OpenTimeout
skip "network problems"
end
assert_equal("preprocessed", @upload.status)
assert_equal(2804, @upload.file_size)
assert_equal("zip", @upload.file_ext)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @upload.md5)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "zip", :large)))
end
end
context "for null" do
setup do
@source = "https://raikou1.donmai.us/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
end
should "download the file" do
@service = subject.new(source: @source)
begin
@upload = @service.start!
rescue Net::OpenTimeout
skip "network problems"
end
assert_equal("preprocessed", @upload.status)
assert_equal(181309, @upload.file_size)
assert_equal("jpg", @upload.file_ext)
assert_equal("93f4dd66ef1eb11a89e56d31f9adc8d0", @upload.md5)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :large)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "jpg", :preview)))
end
end
context "for a video" do
setup do
@source = "https://raikou2.donmai.us/b7/cb/b7cb80092be273771510952812380fa2.mp4"
end
should "work for a video" do
@service = subject.new(source: @source)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_not_nil(@upload.md5)
assert_equal("mp4", @upload.file_ext)
assert_operator(@upload.file_size, :>, 0)
assert_not_nil(@upload.source)
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :original)))
assert(File.exists?(Danbooru.config.storage_manager.file_path(@upload.md5, "mp4", :preview)))
end
end
context "on timeout errors" do
setup do
@source = "https://raikou1.donmai.us/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
HTTParty.stubs(:get).raises(Net::ReadTimeout)
end
should "leave the upload in an error state" do
@service = subject.new(source: @source)
@upload = @service.start!
assert_match(/error:/, @upload.status)
end
end
context "for an invalid content type" do
should "fail" do
upload = subject.new(source: "http://www.example.com").start!
assert_match(/\Aerror:.*File ext is invalid/, upload.status)
end
end
end
context "#finish!" do
setup do
CurrentUser.user = travel_to(1.month.ago) do
FactoryBot.create(:user)
end
CurrentUser.ip_addr = "127.0.0.1"
@source = "https://twitter.com/nounproject/status/540944400767922176"
end
should "overwrite the attributes" do
@service = subject.new(source: @source, rating: 'e')
@upload = @service.start!
@service.finish!
@upload.reload
assert_equal('e', @upload.rating)
end
end
end
context "::Replacer" do
context "for a file replacement" do
setup do
@ -989,70 +832,24 @@ class UploadServiceTest < ActiveSupport::TestCase
end
end
context "with a preprocessing predecessor" do
setup do
@predecessor = FactoryBot.create(:source_upload, status: "preprocessing", source: @source, image_height: 0, image_width: 0, file_ext: "jpg")
end
should "create an upload" do
service = subject.new(source: @source)
# should "schedule a job later" do
# service = subject.new(source: @source)
#
# assert_difference(-> { Delayed::Job.count }) do
# predecessor = service.start!
# assert_equal(@predecessor, predecessor)
# end
# end
assert_difference(-> { Upload.count }) do
service.start!
end
end
context "with a preprocessed predecessor" do
setup do
@predecessor = FactoryBot.create(:source_upload, status: "preprocessed", source: @source, image_height: 0, image_width: 0, file_size: 1, md5: 'd34e4cf0a437a5d65f8e82b7bcd02606', file_ext: "jpg")
@tags = 'hello world'
end
should "assign the rating from tags" do
service = subject.new(source: @source, tag_string: "rating:safe blah")
upload = service.start!
should "update the predecessor" do
service = subject.new(source: @source, tag_string: @tags)
assert_equal(true, upload.valid?)
assert_equal("s", upload.rating)
assert_equal("rating:safe blah ", upload.tag_string)
predecessor = service.start!
assert_equal(@predecessor, predecessor)
assert_equal(@tags, predecessor.tag_string.strip)
end
context "when the file has already been uploaded" do
setup do
@post = create(:post, md5: "d34e4cf0a437a5d65f8e82b7bcd02606")
@service = subject.new(source: @source)
end
should "point to the dup post in the upload" do
@upload = subject.new(source: @source, tag_string: @tags).start!
@predecessor.reload
assert_equal("error: ActiveRecord::RecordInvalid - Validation failed: Md5 duplicate: #{@post.id}", @predecessor.status)
end
end
end
context "with no predecessor" do
should "create an upload" do
service = subject.new(source: @source)
assert_difference(-> { Upload.count }) do
service.start!
end
end
should "assign the rating from tags" do
service = subject.new(source: @source, tag_string: "rating:safe blah")
upload = service.start!
assert_equal(true, upload.valid?)
assert_equal("s", upload.rating)
assert_equal("rating:safe blah ", upload.tag_string)
assert_equal("s", upload.post.rating)
assert_equal("blah", upload.post.tag_string)
end
assert_equal("s", upload.post.rating)
assert_equal("blah", upload.post.tag_string)
end
context "with a source containing unicode characters" do