edshot99/eBooru
1
0
Fork 0
forked from e621ng/e621ng
Code Issues 11 Releases 1 Activity

[Cleanup] Remove upload referer_url

Browse Source
This commit is contained in:
Earlopain 2022-03-16 17:39:58 +01:00
parent 3aa731c408
commit 92a0a36589
No known key found for this signature in database
GPG Key ID: 6CFB948E15246897
10 changed files with 19 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/uploads_controller.rb
View File

@ -78,7 +78,7 @@ class UploadsController < ApplicationController
def upload_params
permitted_params = %i[
file direct_url source tag_string rating parent_id description description referer_url md5_confirmation as_pending
file direct_url source tag_string rating parent_id description description md5_confirmation as_pending
]
permitted_params << :locked_tags if CurrentUser.is_admin?

7
app/logical/downloads/file.rb
View File

@ -6,13 +6,12 @@ module Downloads
RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError]
delegate :data, to: :strategy
attr_reader :url, :referer
attr_reader :url
validate :validate_url
def initialize(url, referer=nil)
def initialize(url)
@url = Addressable::URI.parse(url) rescue nil
@referer = referer
validate!
end
@ -72,7 +71,7 @@ module Downloads
end
def strategy
@strategy ||= Sources::Strategies.find(url.to_s, referer)
@strategy ||= Sources::Strategies.find(url.to_s)
end
def httparty_options

12
app/logical/sources/strategies.rb
View File

@ -1,18 +1,14 @@
module Sources
module Strategies
def self.all
return [
[
Strategies::PixivSlim
]
end
def self.find(url, referer=nil, default: Strategies::Null)
strategy = all.map { |strategy| strategy.new(url, referer) }.detect(&:match?)
strategy || default&.new(url, referer)
end
def self.canonical(url, referer)
find(url, referer).canonical_url
def self.find(url, default: Strategies::Null)
strategy = all.map { |strategy_class| strategy_class.new(url) }.detect(&:match?)
strategy || default&.new(url)
end
end
end

16
app/logical/sources/strategies/base.rb
View File

@ -6,7 +6,7 @@
# Design Principles
#
# In general you should minimize state. You can safely assume that <tt>url</tt>
# and <tt>referer_url</tt> will not change over the lifetime of an instance,
# will not change over the lifetime of an instance,
# so you can safely memoize methods and their results. A common pattern is
# conditionally making an external API call and parsing its response. You should
# make this call on demand and memoize the response.
@ -14,7 +14,7 @@
module Sources
module Strategies
class Base
attr_reader :url, :referer_url, :urls, :parsed_url, :parsed_referer, :parsed_urls
attr_reader :url, :urls, :parsed_url, :parsed_urls
extend Memoist
@ -30,18 +30,12 @@ module Sources
# where this class is being used to normalize artist urls.
# Implementations should be smart enough to detect this and
# behave accordingly.
# * <tt>referer_url</tt> - Sometimes the HTML page cannot be
# determined from <tt>url</tt>. You should generally pass in a
# <tt>referrer_url</tt> so the strategy can discover the HTML
# page and other information.
def initialize(url, referer_url = nil)
def initialize(url)
@url = url
@referer_url = referer_url
@urls = [url, referer_url].select(&:present?)
@urls = [url].select(&:present?)
@parsed_url = Addressable::URI.heuristic_parse(url) rescue nil
@parsed_referer = Addressable::URI.heuristic_parse(referer_url) rescue nil
@parsed_urls = [parsed_url, parsed_referer].select(&:present?)
@parsed_urls = [parsed_url].select(&:present?)
end
# Should return true if this strategy should be used. By default, checks

2
app/logical/upload_service/utils.rb
View File

@ -145,7 +145,7 @@ class UploadService
return file if file.present?
raise RuntimeError, "No file or source URL provided" if upload.direct_url_parsed.blank?
download = Downloads::File.new(upload.direct_url_parsed, upload.referer_url)
download = Downloads::File.new(upload.direct_url_parsed)
download.download!
end
end

2
app/models/post_replacement.rb
View File

@ -102,7 +102,7 @@ class PostReplacement < ApplicationRecord
throw :abort
end
download = Downloads::File.new(replacement_url_parsed, "")
download = Downloads::File.new(replacement_url_parsed)
file = download.download!
self.replacement_file = file

2
app/models/upload.rb
View File

@ -318,7 +318,7 @@ class Upload < ApplicationRecord
def fixup_source
if direct_url_parsed.present?
canonical = Sources::Strategies.find(direct_url_parsed, referer_url).canonical_url
canonical = Sources::Strategies.find(direct_url_parsed).canonical_url
self.source += "\n#{canonical}" if canonical
end
end

8
app/views/uploads/index.html.erb
View File

@ -47,14 +47,6 @@
</span>
<br>
<% if upload.referer_url.present? %>
<span class="info">
<strong>Referer</strong>
<%= URI.parse(upload.referer_url).host rescue nil %>
</span>
<br>
<% end %>
<span class="info">
<strong>Tags</strong>
<%= upload.presenter.inline_tag_list_html %>

19
test/models/upload_service_test.rb
View File

@ -35,7 +35,6 @@ class UploadServiceTest < ActiveSupport::TestCase
@source = "https://raikou1.donmai.us/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
@mock_upload = mock("upload")
@mock_upload.stubs(:direct_url_parsed).returns(@source)
@mock_upload.stubs(:referer_url).returns(nil)
@bad_file = File.open("#{Rails.root}/test/files/test-corrupt.jpg", "rb")
Downloads::File.any_instance.stubs(:download!).returns(@bad_file)
end
@ -762,8 +761,7 @@ class UploadServiceTest < ActiveSupport::TestCase
context "for a pixiv" do
setup do
@source = "https://i.pximg.net/img-original/img/2017/11/21/05/12/37/65981735_p0.jpg"
@ref = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=65981735"
@upload = FactoryBot.create(:jpg_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, source: @source, referer_url: @ref)
@upload = FactoryBot.create(:jpg_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, source: @source)
end
should "record the canonical source" do
@ -776,24 +774,11 @@ class UploadServiceTest < ActiveSupport::TestCase
end
end
context "for a twitter" do
setup do
@source = "https://pbs.twimg.com/media/C1kt72yVEAEGpOv.jpg:large"
@ref = "https://twitter.com/aranobu/status/817736083567820800"
@upload = FactoryBot.create(:jpg_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, source: @source, referer_url: @ref)
end
should "record the canonical source" do
post = subject.new({}).create_post_from_upload(@upload)
assert_equal(@ref, post.source)
end
end
context "for nijie" do
should "record the canonical source" do
page_url = "https://nijie.info/view.php?id=728995"
image_url = "https://pic03.nijie.info/nijie_picture/728995_20170505014820_0.jpg"
upload = FactoryBot.create(:jpg_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, source: image_url, referer_url: page_url)
upload = FactoryBot.create(:jpg_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, source: image_url)
post = UploadService.new({}).create_post_from_upload(upload)
assert_equal(page_url, post.source)

2
test/unit/sources/null_test.rb
View File

@ -4,7 +4,7 @@ module Sources
class NullTest < ActiveSupport::TestCase
context "A source from an unknown site" do
setup do
@site = Sources::Strategies.find("http://oremuhax.x0.com/yoro1603.jpg", "http://oremuhax.x0.com/yo125.htm")
@site = Sources::Strategies.find("http://oremuhax.x0.com/yoro1603.jpg")
end
should "be handled by the null strategy" do