edshot99/eBooru
1
0
Fork 0
forked from e621ng/e621ng
Code Issues 11 Releases 1 Activity

Teach uploaders and editor pages about multiple sources

Browse Source 
This somewhat gimps the current source information fetching tool,
however it was already a bit suspect, and needs to be reworked before
being released as a public tool because it relies on API calls.
This commit is contained in:
Kira 2019-04-26 07:27:06 -07:00
parent 8274d3b3c5
commit bf100446b9
10 changed files with 54 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/controllers/uploads_controller.rb
View File

@ -6,9 +6,7 @@ class UploadsController < ApplicationController
def new
@source = Sources::Strategies.find(params[:url], params[:ref]) if params[:url].present?
@upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
@upload, @remote_size = UploadService::ControllerHelper.prepare(
url: params[:url], ref: params[:ref]
)
@upload = Upload.new
respond_with(@upload)
end
@ -61,7 +59,7 @@ class UploadsController < ApplicationController
def upload_params
permitted_params = %i[
file source tag_string rating status parent_id artist_commentary_title
file direct_url source tag_string rating status parent_id artist_commentary_title
artist_commentary_desc include_artist_commentary referer_url
md5_confirmation as_pending
]

2
app/indexes/post_index.rb
View File

@ -197,7 +197,7 @@ module PostIndex
md5: md5,
rating: rating,
file_ext: file_ext,
source: source.downcase.presence,
source: source_array,
rating_locked: is_rating_locked,
note_locked: is_note_locked,

13
app/javascript/src/javascripts/uploads.js
View File

@ -5,8 +5,7 @@ let Upload = {};
Upload.initialize_all = function() {
if ($("#c-uploads,#c-posts").length) {
this.initialize_enter_on_tags();
$("#upload_source").on("change.danbooru", Upload.fetch_data_manual);
// TODO: This fires off when the editor is first displayed on the post page?
$("#upload_direct_url").on("change.danbooru", Upload.fetch_data_manual);
$(document).on("click.danbooru", "#fetch-data-manual", Upload.fetch_data_manual);
}
@ -41,7 +40,7 @@ Upload.initialize_submit = function() {
Upload.validate_upload = function (e) {
var error_messages = [];
if (($("#upload_file").val() === "") && !/^https?:\/\//i.test($("#upload_source").val()) && $("#upload_md5_confirmation").val() === "") {
if (($("#upload_file").val() === "") && !/^https?:\/\//i.test($("#upload_direct_url").val()) && $("#upload_md5_confirmation").val() === "") {
error_messages.push("Must choose file or specify source");
}
if (!$("#upload_rating_s").prop("checked") && !$("#upload_rating_q").prop("checked") && !$("#upload_rating_e").prop("checked") &&
@ -60,8 +59,8 @@ Upload.validate_upload = function (e) {
}
Upload.initialize_iqdb_source = function() {
if (/^https?:\/\//.test($("#upload_source").val())) {
$.get("/iqdb_queries", {"url": $("#upload_source").val()}).done(function(html) {$("#iqdb-similar").html(html)});
if (/^https?:\/\//.test($("#upload_direct_url").val())) {
$.get("/iqdb_queries", {"url": $("#upload_direct_url").val()}).done(function(html) {$("#iqdb-similar").html(html)});
}
}
@ -77,7 +76,7 @@ Upload.initialize_enter_on_tags = function() {
Upload.initialize_similar = function() {
$("#similar-button").on("click.danbooru", function(e) {
$.get("/iqdb_queries", {"url": $("#upload_source").val()}).done(function(html) {$("#iqdb-similar").html(html).show()});
$.get("/iqdb_queries", {"url": $("#upload_direct_url").val()}).done(function(html) {$("#iqdb-similar").html(html).show()});
e.preventDefault();
});
}
@ -104,7 +103,7 @@ Upload.showWhitelistWarning = function(allowed, reason, domain) {
}
Upload.fetch_data_manual = function(e) {
var url = $("#upload_source,#post_source").val();
var url = $("#upload_direct_url").val();
var ref = $("#upload_referer_url").val();
if(!url.length)

4
app/logical/upload_service.rb
View File

@ -74,9 +74,7 @@ class UploadService
p.image_width = upload.image_width
p.image_height = upload.image_height
p.rating = upload.rating
if upload.source.present?
p.source = Sources::Strategies.find(upload.source, upload.referer_url).canonical_url || upload.source
end
p.source = upload.source
p.file_size = upload.file_size
p.uploader_id = upload.uploader_id
p.uploader_ip_addr = upload.uploader_ip_addr

4
app/logical/upload_service/utils.rb
View File

@ -208,12 +208,12 @@ class UploadService
def get_file_for_upload(upload, file: nil)
return file if file.present?
raise RuntimeError, "No file or source URL provided" if upload.source_url.blank?
raise RuntimeError, "No file or source URL provided" if upload.direct_url_parsed.blank?
attempts = 0
begin
download = Downloads::File.new(upload.source_url, upload.referer_url)
download = Downloads::File.new(upload.direct_url_parsed, upload.referer_url)
file, strategy = download.download!
if !DanbooruImageResizer.validate_shell(file)

3
app/models/post.rb
View File

@ -323,7 +323,8 @@ class Post < ApplicationRecord
module SourceMethods
def source_array
self.source.split("\n")
return [] if source.blank?
source.split("\n")
end
def strip_source

30
app/models/upload.rb
View File

@ -55,14 +55,15 @@ class Upload < ApplicationRecord
end
attr_accessor :as_pending, :replaced_post, :file
attr_accessor :as_pending, :replaced_post, :file, :direct_url
belongs_to :uploader, :class_name => "User"
belongs_to :post, optional: true
before_validation :initialize_attributes, on: :create
before_validation :assign_rating_from_tags
before_validation :fixup_source, on: :create
validate :uploader_is_not_limited, on: :create
validate :source_is_whitelisted, on: :create
validate :direct_url_is_whitelisted, on: :create
# validates :source, format: { with: /\Ahttps?/ }, if: ->(record) {record.file.blank?}, on: :create
validates :rating, inclusion: { in: %w(q e s) }, allow_nil: false
validates :md5, confirmation: true, if: -> (rec) { rec.md5_confirmation.present? }
@ -135,8 +136,8 @@ class Upload < ApplicationRecord
end
end
module SourceMethods
def source=(source)
module DirectURLMethods
def direct_url=(source)
source = source.unicode_normalize(:nfc)
# percent encode unicode characters in urls
@ -147,9 +148,9 @@ class Upload < ApplicationRecord
super(source)
end
def source_url
return nil unless source =~ %r!\Ahttps?://!i
Addressable::URI.heuristic_parse(source) rescue nil
def direct_url_parsed
return nil unless direct_url =~ %r!\Ahttps?://!i
Addressable::URI.heuristic_parse(direct_url) rescue nil
end
end
@ -251,7 +252,7 @@ class Upload < ApplicationRecord
include VideoMethods
extend SearchMethods
include ApiMethods
include SourceMethods
include DirectURLMethods
def uploader_is_not_limited
if !uploader.can_upload?
@ -262,9 +263,9 @@ class Upload < ApplicationRecord
end
end
def source_is_whitelisted
return true if source_url.nil?
valid, reason = UploadWhitelist.is_whitelisted?(source_url)
def direct_url_is_whitelisted
return true if direct_url_parsed.nil?
valid, reason = UploadWhitelist.is_whitelisted?(direct_url_parsed)
if !valid
self.errors.add(:source, "is not whitelisted: #{reason}")
return false
@ -278,6 +279,13 @@ class Upload < ApplicationRecord
end
end
def fixup_source
if direct_url_parsed.present?
canonical = Sources::Strategies.find(direct_url_parsed, referer_url).canonical_url
self.source += "\n#{canonical}" if canonical
end
end
def presenter
@presenter ||= UploadPresenter.new(self)
end

20
app/views/posts/partials/show/_edit.html.erb
View File

@ -4,8 +4,6 @@
</div>
<% end %>
<%= render "sources/info" %>
<%= form_for(post, :html => {:class => "simple_form", :id => "form"}) do |f| %>
<%= hidden_field_tag :tags_query, params[:q] %>
<%= hidden_field_tag :pool_id, params[:pool_id] %>
@ -53,10 +51,6 @@
<%= f.label :is_rating_locked, "Rating" %>
<% if CurrentUser.is_admin? %>
<%= f.check_box :hide_from_anonymous %>
<%= f.label :hide_from_anonymous, "Hide from Anon." %>
<%= f.check_box :hide_from_search_engines %>
<%= f.label :hide_from_search_engines, "Hide from search engines" %>
<%= f.check_box :is_status_locked %>
<%= f.label :is_status_locked, "Status" %>
<% end %>
@ -64,13 +58,25 @@
</div>
<% end %>
<% if CurrentUser.is_admin? %>
<div class="input">
<%= f.label :blank, "Limits" %>
<fieldset class="limits">
<%= f.check_box :hide_from_anonymous %>
<%= f.label :hide_from_anonymous, "Hide from Anon." %>
<%= f.check_box :hide_from_search_engines %>
<%= f.label :hide_from_search_engines, "Hide from search engines" %>
</fieldset>
</div>
<% end %>
<div class="input">
<%= f.label :parent_id, "Parent" %>
<%= f.text_field :parent_id, :size => 5 %>
</div>
<div class="input">
<%= f.label :source %>
<%= f.label :source, "Sources" %>
<%= f.text_area :source, size: '60x5', spellcheck: false %>
</div>

9
app/views/uploads/new.html.erb
View File

@ -36,13 +36,18 @@
</div>
<div class="input">
<%= f.label :source %>
<%= f.text_field :source, :size => 50, :value => params[:url] %>
<%= f.label :direct_url %>
<%= f.text_field :direct_url, size: 50, value: params[:url] %>
<%= button_tag "Similar", :id => "similar-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub" %>
<div id="whitelist-warning"></div>
<span class="hint">You can enter a URL to have <%= Danbooru.config.app_name %> automatically download and process it</span>
</div>
<div class="input">
<%= f.label :source, "Sources" %>
<%= f.text_area :source, size: "60x5", value: params[:url] %>
</div>
<div class="input">
<%= f.label :rating %>

2
test/models/upload_service_test.rb
View File

@ -47,7 +47,7 @@ class UploadServiceTest < ActiveSupport::TestCase
setup do
@source = "https://raikou1.donmai.us/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
@mock_upload = mock("upload")
@mock_upload.stubs(:source_url).returns(@source)
@mock_upload.stubs(:direct_url_parsed).returns(@source)
@mock_upload.stubs(:referer_url).returns(nil)
@bad_file = File.open("#{Rails.root}/test/files/test-corrupt.jpg", "rb")
Downloads::File.any_instance.stubs(:download!).returns([@bad_file, nil])