From ca8be10ab9b64900a49781cd68c720d1e15b36ec Mon Sep 17 00:00:00 2001 From: albert Date: Fri, 12 Mar 2010 19:27:54 -0500 Subject: [PATCH] more work on post uploads --- .gitignore | 1 + app/controllers/application_controller.rb | 10 +- app/controllers/uploads_controller.rb | 48 +---- app/logical/anonymous_user.rb | 10 +- app/models/job.rb | 76 +++----- app/models/post.rb | 10 +- app/models/upload.rb | 164 ++++++++++++------ app/models/user.rb | 4 + app/presenters/post_set_presenter.rb | 13 +- app/presenters/presenter.rb | 5 + app/views/sessions/new.html.erb | 6 +- app/views/uploads/new.html.erb | 2 +- app/views/uploads/show.html.erb | 15 ++ app/views/users/edit.html.erb | 5 + .../initializers/create_empty_directories.rb | 2 +- db/development_structure.sql | 10 +- db/migrate/20100204211522_create_users.rb | 1 + db/migrate/20100205224030_create_uploads.rb | 4 +- 18 files changed, 218 insertions(+), 168 deletions(-) create mode 100644 app/presenters/presenter.rb create mode 100644 app/views/uploads/show.html.erb diff --git a/.gitignore b/.gitignore index 3f39129eb..ecad907b5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ lib/danbooru_image_resizer/*.log db/*.sqlite3 log/*.log tmp/**/* +public/data \ No newline at end of file diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 9aaf078c1..dcd6faa0d 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -10,10 +10,10 @@ protected respond_to do |fmt| fmt.html do - if request.get? && Rails.environment != "test" - redirect_to new_sessions_path(:url => previous_url), :notice => "Access denied" + if request.get? && Rails.env.test? + redirect_to new_session_path(:url => previous_url), :notice => "Access denied" else - redirect_to new_sessions_path, :notice => "Access denied" + redirect_to new_session_path, :notice => "Access denied" end end fmt.xml do @@ -38,9 +38,11 @@ protected else @current_user = AnonymousUser.new end + + Time.zone = @current_user.time_zone end - %w(banned privileged contributor janitor moderator admin).each do |level| + %w(member banned privileged contributor janitor moderator admin).each do |level| define_method("#{level}_only") do if @current_user.__send__("is_#{level}?") true diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index 59d73a5cf..eb5409bfc 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -1,54 +1,20 @@ class UploadsController < ApplicationController + before_filter :member_only + respond_to :html, :xml, :json + def new - @upload = Upload.new + @upload = Upload.new(:rating => "q") if params[:url] @post = Post.find_by_source(params[:url]) end end def show + @upload = Upload.find(params[:id]) end def create - unless @current_user.can_upload? - respond_to_error("Daily limit exceeded", {:controller => "user", :action => "upload_limit"}, :status => 421) - return - end - - if @current_user.is_contributor_or_higher? - status = "active" - else - status = "pending" - end - - begin - @post = Post.new(params[:post].merge(:updater_user_id => @current_user.id, :updater_ip_addr => request.remote_ip)) - @post.user_id = @current_user.id - @post.status = status - @post.ip_addr = request.remote_ip - @post.save - rescue Errno::ENOENT - respond_to_error("Internal error. Try uploading again.", {:controller => "post", :action => "error"}) - return - end - - if @post.errors.empty? - if params[:md5] && @post.md5 != params[:md5].downcase - @post.destroy - respond_to_error("MD5 mismatch", {:action => "error"}, :status => 420) - else - respond_to_success("Post uploaded", {:controller => "post", :action => "show", :id => @post.id, :tag_title => @post.tag_title}, :api => {:post_id => @post.id, :location => url_for(:controller => "post", :action => "show", :id => @post.id)}) - end - elsif @post.errors.invalid?(:md5) - p = Post.find_by_md5(@post.md5) - - update = { :tags => p.cached_tags + " " + params[:post][:tags], :updater_user_id => session[:user_id], :updater_ip_addr => request.remote_ip } - update[:source] = @post.source if p.source.blank? && !@post.source.blank? - p.update_attributes(update) - - respond_to_error("Post already exists", {:controller => "post", :action => "show", :id => p.id, :tag_title => @post.tag_title}, :api => {:location => url_for(:controller => "post", :action => "show", :id => p.id)}, :status => 423) - else - respond_to_error(@post, :action => "error") - end + @upload = Upload.create(params[:upload].merge(:uploader_id => @current_user.id, :uploader_ip_addr => request.remote_ip)) + respond_with(@upload) end end diff --git a/app/logical/anonymous_user.rb b/app/logical/anonymous_user.rb index d60ab45bf..46cecbccf 100644 --- a/app/logical/anonymous_user.rb +++ b/app/logical/anonymous_user.rb @@ -99,11 +99,13 @@ class AnonymousUser def blacklisted_tags "" end - - %w(banned privileged contributor janitor moderator admin).each do |name, value| - normalized_name = name.downcase.gsub(/ /, "_") - define_method("is_#{normalized_name}?") do + def time_zone + "Eastern Time (US & Canada)" + end + + %w(member banned privileged contributor janitor moderator admin).each do |name| + define_method("is_#{name}?") do false end end diff --git a/app/models/job.rb b/app/models/job.rb index df6f1896c..c4e9e5843 100644 --- a/app/models/job.rb +++ b/app/models/job.rb @@ -1,5 +1,5 @@ class Job < ActiveRecord::Base - CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags calculate_post_count calculate_uploaded_tags s3_backup) + CATEGORIES = %w(mass_tag_edit approve_tag_alias approve_tag_implication calculate_tag_subscriptions calculate_related_tags s3_backup upload_processing) STATUSES = %w(pending processing finished error) validates_inclusion_of :category, :in => CATEGORIES @@ -37,6 +37,12 @@ class Job < ActiveRecord::Base end end + def execute_upload_processing + Upload.where("status = ?", "pending").each do |upload| + upload.process! + end + end + def execute_mass_tag_edit start_tags = data["start_tags"] result_tags = data["result_tags"] @@ -75,52 +81,32 @@ class Job < ActiveRecord::Base end end - def execute_calculate_post_count - Tag.recalculate_post_count(data["tag_name"]) - end - - def execute_calculate_uploaded_tags - tags = [] - user = User.find(data["id"]) - CONFIG["tag_types"].values.uniq.each do |tag_type| - tags += user.calculate_uploaded_tags(tag_type) - end - - user.update_attribute(:uploaded_tags, tags.join("\n")) - end - - def execute_bandwidth_throttle - bw = File.read("/proc/net/dev").split(/\n/).grep(/eth1/).first.scan(/\S+/)[8].to_i - if $danbooru_bandwidth_previous - diff = bw - $danbooru_bandwidth_previous - else - diff = 0 - end - $danbooru_bandwidth_previous = bw - Cache.put("db-bw", diff) - end - def execute_s3_backup last_id = data["last_id"].to_i begin - Post.find(:all, :conditions => ["id > ?", last_id], :limit => 200, :order => "id").each do |post| - AWS::S3::Base.establish_connection!(:access_key_id => CONFIG["amazon_s3_access_key_id"], :secret_access_key => CONFIG["amazon_s3_secret_access_key"]) + Post.where("id > ?", last_id).each do |post| + AWS::S3::Base.establish_connection!( + :access_key_id => Danbooru.config.amazon_s3_access_key_id, + :secret_access_key => Danbooru.config.amazon_s3_secret_access_key + ) if File.exists?(post.file_path) - base64_md5 = Base64.encode64(Digest::MD5.digest(File.read(post.file_path))) - AWS::S3::S3Object.store(post.file_name, open(post.file_path, "rb"), CONFIG["amazon_s3_bucket_name"], "Content-MD5" => base64_md5) + AWS::S3::S3Object.store( + post.file_name, + open(post.file_path, "rb"), + Danbooru.config.amazon_s3_bucket_name, + "Content-MD5" => Base64.encode64(post.md5) + ) end - if post.image? && File.exists?(post.preview_path) - AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_path, "rb"), CONFIG["amazon_s3_bucket_name"]) + AWS::S3::S3Object.store( + "preview/#{post.md5}.jpg", + open(post.preview_path, "rb"), + Danbooru.config.amazon_s3_bucket_name + ) end - if File.exists?(post.sample_path) - AWS::S3::S3Object.store("sample/" + CONFIG["sample_filename_prefix"] + "#{post.md5}.jpg", open(post.sample_path, "rb"), CONFIG["amazon_s3_bucket_name"]) - end - update_attributes(:data => {:last_id => post.id}) - base64_md5 = nil end rescue Exception => x @@ -156,19 +142,13 @@ class Job < ActiveRecord::Base else "tag:UNKNOWN" end - - when "calculate_post_count" - "tag:" + data["tag_name"] - - when "calculate_uploaded_tags" - "user:" + User.name(data["id"]) - + when "bandwidth_throttle" "" - + when "s3_backup" "last_id:" + data["last_id"].to_s - + end rescue Exception "ERROR" @@ -176,11 +156,11 @@ class Job < ActiveRecord::Base end def self.pending_count(task_type) - JobTask.count(:conditions => ["task_type = ? and status = 'pending'", task_type]) + where("task_type = ? and status = 'pending'", task_type).count end def self.execute_once - find(:all, :conditions => ["status = ?", "pending"], :order => "id desc").each do |task| + where("status = ?", "pending").each do |task| task.execute! sleep 1 end diff --git a/app/models/post.rb b/app/models/post.rb index 2959d2f60..2f296c7b4 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -23,7 +23,7 @@ class Post < ActiveRecord::Base FileUtils.rm_f(file_path) FileUtils.rm_f(medium_file_path) FileUtils.rm_f(large_file_path) - FileUtils.rm_f(thumb_file_path) + FileUtils.rm_f(preview_file_path) end def file_path_prefix @@ -42,8 +42,8 @@ class Post < ActiveRecord::Base "#{Rails.root}/public/data/large/#{file_path_prefix}#{md5}.jpg" end - def thumb_file_path - "#{Rails.root}/public/data/thumb/#{file_path_prefix}#{md5}.jpg" + def preview_file_path + "#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg" end def file_url @@ -58,8 +58,8 @@ class Post < ActiveRecord::Base "/data/large/#{file_path_prefix}#{md5}.jpg" end - def thumb_file_url - "/data/thumb/#{file_path_prefix}#{md5}.jpg" + def preview_file_url + "/data/preview/#{file_path_prefix}#{md5}.jpg" end def file_url_for(user) diff --git a/app/models/upload.rb b/app/models/upload.rb index b932b8a10..eb41d6951 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -2,62 +2,106 @@ require "danbooru_image_resizer/danbooru_image_resizer" require "tmpdir" class Upload < ActiveRecord::Base + class Error < Exception ; end + attr_accessor :file, :image_width, :image_height, :file_ext, :md5, :file_size belongs_to :uploader, :class_name => "User" belongs_to :post - before_save :convert_cgi_file + before_create :initialize_status + before_create :convert_cgi_file + validate :uploader_is_not_limited - def process! - update_attribute(:status, "processing") - if is_downloadable? - download_from_source(temp_file_path) + module ValidationMethods + def uploader_is_not_limited + if !uploader.can_upload? + update_attribute(:status, "error: uploader has reached their daily limit") + raise + end end - self.file_ext = content_type_to_file_ext(content_type) - calculate_hash(file_path) - calculate_file_size(file_path) - calculate_dimensions(file_path) if has_dimensions? - generate_resizes(file_path) - move_file - post = convert_to_post - if post.save - update_attributes(:status => "finished", :post_id => post.id) - else - update_attribute(:status, "error: " + post.errors.full_messages.join(", ")) + + def validate_file_exists + unless File.exists?(file_path) + update_attribute(:status, "error: file does not exist") + raise + end + end + + def validate_file_content_type + unless is_valid_content_type? + update_attribute(:status, "error: invalid content type (#{file_ext} not allowed)") + raise + end + end + + def validate_md5_confirmation + if !md5_confirmation.blank? && md5_confirmation != md5 + update_attribute(:status, "error: md5 mismatch") + raise + end + end + end + + module ConversionMethods + def process! + update_attribute(:status, "processing") + if is_downloadable? + download_from_source(temp_file_path) + end + validate_file_exists + self.file_ext = content_type_to_file_ext(content_type) + validate_file_content_type + calculate_hash(file_path) + validate_md5_confirmation + calculate_file_size(file_path) + calculate_dimensions(file_path) if has_dimensions? + generate_resizes(file_path) + move_file + post = convert_to_post + if post.save + update_attributes(:status => "completed", :post_id => post.id) + else + update_attribute(:status, "error: " + post.errors.full_messages.join(", ")) + end + rescue RuntimeError => x + end + + def convert_to_post + returning Post.new do |p| + p.tag_string = tag_string + p.md5 = md5 + p.file_ext = file_ext + p.image_width = image_width + p.image_height = image_height + p.uploader_id = uploader_id + p.uploader_ip_addr = uploader_ip_addr + p.updater_id = uploader_id + p.updater_ip_addr = uploader_ip_addr + p.rating = rating + p.source = source + p.file_size = file_size + + unless uploader.is_contributor? + p.is_pending = true + end + end + end + end + + module FileMethods + def move_file + FileUtils.mv(file_path, md5_file_path) + end + + def calculate_file_size(source_path) + self.file_size = File.size(source_path) + end + + # Calculates the MD5 based on whatever is in temp_file_path + def calculate_hash(source_path) + self.md5 = Digest::MD5.file(source_path).hexdigest end end - def convert_to_post - returning Post.new do |p| - p.tag_string = tag_string - p.md5 = md5 - p.file_ext = file_ext - p.image_width = image_width - p.image_height = image_height - p.uploader_id = uploader_id - p.uploader_ip_addr = uploader_ip_addr - p.updater_id = uploader_id - p.updater_ip_addr = uploader_ip_addr - p.rating = rating - p.source = source - p.file_size = file_size - end - end - - def move_file - FileUtils.mv(file_path, md5_file_path) - end - - def calculate_file_size(source_path) - self.file_size = File.size(source_path) - end - - # Calculates the MD5 based on whatever is in temp_file_path - def calculate_hash(source_path) - self.md5 = Digest::MD5.file(source_path).hexdigest - end - - class Error < Exception ; end - module ResizerMethods def generate_resizes(source_path) generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path) @@ -101,6 +145,10 @@ class Upload < ActiveRecord::Base end module ContentTypeMethods + def is_valid_content_type? + file_ext =~ /jpg|gif|png|swf/ + end + def content_type_to_file_ext(content_type) case content_type when "image/jpeg" @@ -152,7 +200,7 @@ class Upload < ActiveRecord::Base case width when Danbooru.config.small_image_width - "#{Rails.root}/public/data/thumb/#{prefix}#{md5}.jpg" + "#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg" when Danbooru.config.medium_image_width "#{Rails.root}/public/data/medium/#{prefix}#{md5}.jpg" @@ -202,10 +250,28 @@ class Upload < ActiveRecord::Base end end + module StatusMethods + def initialize_status + self.status = "pending" + end + + def is_pending? + status == "pending" + end + + def is_completed? + status == "completed" + end + end + + include ConversionMethods + include ValidationMethods + include FileMethods include ResizerMethods include DimensionMethods include ContentTypeMethods include DownloaderMethods include FilePathMethods include CgiFileMethods + include StatusMethods end \ No newline at end of file diff --git a/app/models/user.rb b/app/models/user.rb index 20fded626..0c2abb14c 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -135,6 +135,10 @@ class User < ActiveRecord::Base def is_anonymous? false end + + def is_member? + true + end end module EmailVerificationMethods diff --git a/app/presenters/post_set_presenter.rb b/app/presenters/post_set_presenter.rb index b04c6b9b8..0a2f993c2 100644 --- a/app/presenters/post_set_presenter.rb +++ b/app/presenters/post_set_presenter.rb @@ -1,4 +1,4 @@ -class PostSetPresenter +class PostSetPresenter < Presenter attr_accessor :post_set def initialize(post_set) @@ -24,13 +24,18 @@ class PostSetPresenter html = "" posts.each do |post| - html << %{
} + flags = [] + flags << "pending" if post.is_pending? + flags << "flagged" if post.is_flagged? + flags << "deleted" if post.is_deleted? + + html << %{} end - html + html.html_safe end end diff --git a/app/presenters/presenter.rb b/app/presenters/presenter.rb new file mode 100644 index 000000000..08eeaf636 --- /dev/null +++ b/app/presenters/presenter.rb @@ -0,0 +1,5 @@ +class Presenter + def h(s) + CGI.escapeHTML(s) + end +end diff --git a/app/views/sessions/new.html.erb b/app/views/sessions/new.html.erb index a10eadbae..ba0c40b38 100644 --- a/app/views/sessions/new.html.erb +++ b/app/views/sessions/new.html.erb @@ -28,9 +28,5 @@ <% content_for(:page_title) do %> -login -<% end %> - -<% content_for(:page_header) do %> - / login + / login <% end %> diff --git a/app/views/uploads/new.html.erb b/app/views/uploads/new.html.erb index 1780cc2c7..9340d57ee 100644 --- a/app/views/uploads/new.html.erb +++ b/app/views/uploads/new.html.erb @@ -2,7 +2,7 @@

Before uploading, please read the <%= link_to "how to upload guide", wiki_page_path("howto:upload") %>. It explains how to tag and what ratings are.

-<% form_for @upload do |f| %> +<% form_for(@upload, :html => {:multipart => true}) do |f| %> <% if params[:url] %>
<%= image_tag(params[:url], :title => "Preview") %> diff --git a/app/views/uploads/show.html.erb b/app/views/uploads/show.html.erb new file mode 100644 index 000000000..bff822c9b --- /dev/null +++ b/app/views/uploads/show.html.erb @@ -0,0 +1,15 @@ +

Upload #<%= @upload.id %>

+ +
    +
  • Date: <%= @upload.created_at %>
    • +
  • Source: <%= @upload.source %>
    • +
  • Tags: <%= @upload.tag_string %>
    • +
+ +<% if @upload.is_completed? %> +

This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.

+<% elsif @upload.is_pending? %> +

This upload is waiting to be processed. Please wait a few seconds.

+<% else %> +

An error occurred: <%= @upload.status %>

+<% end %> diff --git a/app/views/users/edit.html.erb b/app/views/users/edit.html.erb index 3bd612c4a..343ccb968 100644 --- a/app/views/users/edit.html.erb +++ b/app/views/users/edit.html.erb @@ -13,6 +13,11 @@ <% end %>

+

+ <%= f.label :time_zone %> + <%= f.time_zone_select :time_zone %> +

+

<%= f.label :receive_email_notifications, "Email notifications", :title => "Enable to receive email notification when you receive a DMail" %> <%= f.check_box :receive_email_notifications %> diff --git a/config/initializers/create_empty_directories.rb b/config/initializers/create_empty_directories.rb index f3380d06b..c87ab6509 100644 --- a/config/initializers/create_empty_directories.rb +++ b/config/initializers/create_empty_directories.rb @@ -1,6 +1,6 @@ require 'fileutils' -FileUtils.mkdir_p("#{Rails.root}/public/data/thumb") +FileUtils.mkdir_p("#{Rails.root}/public/data/preview") FileUtils.mkdir_p("#{Rails.root}/public/data/medium") FileUtils.mkdir_p("#{Rails.root}/public/data/large") FileUtils.mkdir_p("#{Rails.root}/public/data/original") diff --git a/db/development_structure.sql b/db/development_structure.sql index 62b939e0a..a250ff0a6 100644 --- a/db/development_structure.sql +++ b/db/development_structure.sql @@ -1324,8 +1324,6 @@ ALTER SEQUENCE unapprovals_id_seq OWNED BY unapprovals.id; CREATE TABLE uploads ( id integer NOT NULL, - created_at timestamp without time zone, - updated_at timestamp without time zone, source character varying(255), file_path character varying(255), content_type character varying(255), @@ -1334,7 +1332,10 @@ CREATE TABLE uploads ( uploader_ip_addr inet NOT NULL, tag_string text NOT NULL, status character varying(255) DEFAULT 'pending'::character varying NOT NULL, - post_id integer + post_id integer, + md5_confirmation character varying(255), + created_at timestamp without time zone, + updated_at timestamp without time zone ); @@ -1419,7 +1420,8 @@ CREATE TABLE users ( always_resize_images boolean DEFAULT false NOT NULL, default_image_size character varying(255) DEFAULT 'medium'::character varying NOT NULL, favorite_tags text, - blacklisted_tags text + blacklisted_tags text, + time_zone character varying(255) DEFAULT 'Eastern Time (US & Canada)'::character varying NOT NULL ); diff --git a/db/migrate/20100204211522_create_users.rb b/db/migrate/20100204211522_create_users.rb index 02a828ece..fae1ec44f 100644 --- a/db/migrate/20100204211522_create_users.rb +++ b/db/migrate/20100204211522_create_users.rb @@ -28,6 +28,7 @@ class CreateUsers < ActiveRecord::Migration t.column :default_image_size, :string, :null => false, :default => "medium" t.column :favorite_tags, :text t.column :blacklisted_tags, :text + t.column :time_zone, :string, :null => false, :default => "Eastern Time (US & Canada)" end execute "CREATE UNIQUE INDEX index_users_on_name ON users ((lower(name)))" diff --git a/db/migrate/20100205224030_create_uploads.rb b/db/migrate/20100205224030_create_uploads.rb index f54fa76f5..f888ba1a5 100644 --- a/db/migrate/20100205224030_create_uploads.rb +++ b/db/migrate/20100205224030_create_uploads.rb @@ -1,17 +1,17 @@ class CreateUploads < ActiveRecord::Migration def self.up create_table :uploads do |t| - t.timestamps t.column :source, :string t.column :file_path, :string t.column :content_type, :string - t.column :rating, :character, :null => false t.column :uploader_id, :integer, :null => false t.column :uploader_ip_addr, "inet", :null => false t.column :tag_string, :text, :null => false t.column :status, :string, :null => false, :default => "pending" t.column :post_id, :integer + t.column :md5_confirmation, :string + t.timestamps end end