diff --git a/Gemfile b/Gemfile index 6fc434b28..b9a21df6b 100644 --- a/Gemfile +++ b/Gemfile @@ -4,6 +4,7 @@ source 'http://gemcutter.org' gem "rails", "3.0.0.beta" gem "pg" gem "memcache-client", :require => "memcache" +gem "imagesize", :require => "image_size" group :test do gem "shoulda" diff --git a/app/models/post.rb b/app/models/post.rb index b31552f80..dc5f7e382 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -4,246 +4,251 @@ class Post < ActiveRecord::Base end class Pending < ActiveRecord::Base + class Error < Exception ; end + + class Download + class Error < Exception ; end + + attr_accessible :source, :content_type + + def initialize(source, file_path) + @source = source + @file_path = file_path + end + + # Downloads to @file_path + def download! + http_get_streaming(@source) do |response| + self.content_type = response["Content-Type"] + File.open(@file_path, "wb") do |out| + response.read_body(out) + end + end + @source = fix_image_board_sources(@source) + end + + private + def handle_pixiv(source, headers) + if source =~ /pixiv\.net/ + headers["Referer"] = "http://www.pixiv.net" + + # Don't download the small version + if source =~ %r!(/img/.+?/.+?)_m.+$! + match = $1 + source.sub!(match + "_m", match) + end + end + + source + end + + def http_get_streaming(source, options = {}) + max_size = options[:max_size] || Danbooru.config.max_file_size + max_size = nil if max_size == 0 # unlimited + limit = 4 + + while true + url = URI.parse(source) + + unless url.is_a?(URI::HTTP) + raise Error.new("URL must be HTTP") + end + + Net::HTTP.start(url.host, url.port) do |http| + http.read_timeout = 10 + headers = { + "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}" + } + source = handle_pixiv(source, headers) + http.request_get(url.request_uri, headers) do |res| + case res + when Net::HTTPSuccess then + if max_size + len = res["Content-Length"] + raise Error.new("File is too large (#{len} bytes)") if len && len.to_i > max_size + end + yield(res) + return + + when Net::HTTPRedirection then + if limit == 0 then + raise Error.new("Too many redirects") + end + source = res["location"] + limit -= 1 + + else + raise Error.new("HTTP error code: #{res.code} #{res.message}") + end + end # http.request_get + end # http.start + end # while + end # def + + def fix_image_board_sources(source) + if source =~ /\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/ + "Image board" + else + source + end + end + end # download + set_table_name "pending_posts" + belongs_to :post + attr_accessible :file, :image_width, :image_height def process! update_attribute(:status, "processing") + + if file + convert_cgi_file(temp_file_path) + elsif is_downloadable? + download_from_source(temp_file_path) + end + + calculate_hash(temp_file_path) move_file - calculate_hash calculate_dimensions generate_resizes convert_to_post update_attribute(:status, "finished") end - def move_file - # Download the file - # Move the tempfile into the data store - # Distribute to other servers - end - - def calculate_hash - # Calculate the MD5 hash of the file - end - - def calculate_dimensions - # Calculate the dimensions of the image - end - + private def generate_resizes - # Generate width=150 - # Generate width=1000 - # + generate_resize_for(Danbooru.config.small_image_width) + generate_resize_for(Danbooru.config.medium_image_width) + generate_resize_for(Danbooru.config.large_image_width) + end + + def create_resize_for(width) + return if width.nil? + return unless image_width > width + + unless File.exists?(final_file_path) + raise Error.new("file not found") + end + + size = Danbooru.reduce_to({:width => image_width, :height => image_height}, {:width => width}) + + # If we're not reducing the resolution, only reencode if the source image larger than + # 200 kilobytes. + if size[:width] == width && size[:height] == height && File.size?(path) > 200.kilobytes + return true + end + + begin + Danbooru.resize(file_ext, final_file_path, resize_file_path_for(width), size, 90) + rescue Exception => x + errors.add "sample", "couldn't be created: #{x}" + return false + end + + self.sample_width = size[:width] + self.sample_height = size[:height] + return true + end + + def resize_file_path_for(width) + case width + when Danbooru.config.small_image_width + "#{Rails.root}/public/data/preview" + + when Danbooru.config.medium_image_width + "#{Rails.root}/public/data/medium" + + when Danbooru.config.large_image_width + "#{Rails.root}/public/data/large" + end end def convert_to_post + returning Post.new do |p| + p.tag_string = tag_string + p.md5 = md5 + p.file_ext = file_ext + p.image_width = image_width + p.image_height = image_height + p.uploader_id = uploader_id + p.uploader_ip_addr = uploader_ip_addr + p.rating = rating + p.source = source + end end - - private - def download_from_source - self.source = "" if source.nil? - - return if source !~ /^http:\/\// || !file_ext.blank? - - begin - Danbooru.http_get_streaming(source) do |response| - File.open(tempfile_path, "wb") do |out| - response.read_body do |block| - out.write(block) - end - end - end - - if source.to_s =~ /\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/ - self.source = "Image board" - end - - return true - rescue SocketError, URI::Error, SystemCallError => x - delete_tempfile - errors.add "source", "couldn't be opened: #{x}" - return false + + def calculate_dimensions(post) + if has_dimensions? + image_size = ImageSize.new(File.open(final_file_path, "rb")) + self.image_width = image_size.get_width + self.image_hegiht = image_hegiht.get_height end end - def move_tempfile + def has_dimensions? + %w(jpg gif png swf).include?(file_ext) end - def distribute_file - end - - def generate_resize_for(width) - end - end - - class Version < ActiveRecord::Base - set_table_name "post_versions" - end - - module FileMethods - def self.included(m) - m.before_validation_on_create :download_source - m.before_validation_on_create :validate_tempfile_exists - m.before_validation_on_create :determine_content_type - m.before_validation_on_create :validate_content_type - m.before_validation_on_create :generate_hash - m.before_validation_on_create :set_image_dimensions - m.before_validation_on_create :generate_sample - m.before_validation_on_create :generate_preview - m.before_validation_on_create :move_file + def move_file + FileUtils.mv(temp_file_path, final_file_path) end - def validate_tempfile_exists - unless File.exists?(tempfile_path) - errors.add :file, "not found, try uploading again" - return false - end + def final_file_path + "#{Rails.root}/public/data/original/#{md5}.#{file_ext}" end - def validate_content_type - unless %w(jpg png gif swf).include?(file_ext.downcase) - errors.add(:file, "is an invalid content type: " + file_ext.downcase) - return false - end + # Calculates the MD5 based on whatever is in temp_file_path + def calculate_hash(file_path) + self.md5 = Digest::MD5.file(file_path).hexdigest end - def file_name - md5 + "." + file_ext - end + # Moves the cgi file to file_path + def convert_cgi_file(file_path) + return if file.blank? || file.size == 0 - def delete_tempfile - FileUtils.rm_f(tempfile_path) - FileUtils.rm_f(tempfile_preview_path) - FileUtils.rm_f(tempfile_sample_path) - end - - def tempfile_path - "#{RAILS_ROOT}/public/data/#{$PROCESS_ID}.upload" - end - - def tempfile_preview_path - "#{RAILS_ROOT}/public/data/#{$PROCESS_ID}-preview.jpg" - end - - # def file_size - # File.size(file_path) rescue 0 - # end - - # Generate an MD5 hash for the file. - def generate_hash - unless File.exists?(tempfile_path) - errors.add(:file, "not found") - return false - end - - self.md5 = File.open(tempfile_path, 'rb') {|fp| Digest::MD5.hexdigest(fp.read)} - self.file_size = File.size(tempfile_path) - - if Post.exists?(["md5 = ?", md5]) - delete_tempfile - errors.add "md5", "already exists" - return false + if file.local_path + FileUtils.mv(file.local_path, file_path) else - return true - end - end - - def generate_preview - return true unless image? && width && height - - unless File.exists?(tempfile_path) - errors.add(:file, "not found") - return false - end - - size = Danbooru.reduce_to({:width=>width, :height=>height}, {:width=>150, :height=>150}) - - # Generate the preview from the new sample if we have one to save CPU, otherwise from the image. - if File.exists?(tempfile_sample_path) - path, ext = tempfile_sample_path, "jpg" - else - path, ext = tempfile_path, file_ext - end - - begin - Danbooru.resize(ext, path, tempfile_preview_path, size, 95) - rescue Exception => x - errors.add "preview", "couldn't be generated (#{x})" - return false - end - - return true - end - - # Automatically download from the source if it's a URL. - def download_source - self.source = "" if source.nil? - - return if source !~ /^http:\/\// || !file_ext.blank? - - begin - Danbooru.http_get_streaming(source) do |response| - File.open(tempfile_path, "wb") do |out| - response.read_body do |block| - out.write(block) - end - end + File.open(file_path, 'wb') do |out| + out.write(file.read) end - - if source.to_s =~ /\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/ - self.source = "Image board" - end - - return true - rescue SocketError, URI::Error, SystemCallError => x - delete_tempfile - errors.add "source", "couldn't be opened: #{x}" - return false - end - end - - def determine_content_type - imgsize = ImageSize.new(File.open(tempfile_path, "rb")) - - unless imgsize.get_width.nil? - self.file_ext = imgsize.get_type.gsub(/JPEG/, "JPG").downcase end + self.file_ext = content_type_to_file_ext(file.content_type) || find_ext(file.original_filename) end - # Assigns a CGI file to the post. This writes the file to disk and generates a unique file name. - def file=(f) - return if f.nil? || f.size == 0 + # Determines whether the source is downloadable + def is_downloadable? + source =~ /^http:\/\// && file.blank? + end - self.file_ext = content_type_to_file_ext(f.content_type) || find_ext(f.original_filename) + # Downloads the file to file_path + def download_from_source(file_path) + download = Download.new(source, file_path) + download.download! + self.file_ext = content_type_to_file_ext(download.content_type) || find_ext(source) + end + + # Converts a content type string to a file extension + def content_type_to_file_ext(content_type) + case content_type + when /jpeg/ + return "jpg" + + when /gif/ + return "gif" + + when /png/ + return "png" + + when /x-shockwave-flash/ + return "swf" - if f.local_path - # Large files are stored in the temp directory, so instead of - # reading/rewriting through Ruby, just rely on system calls to - # copy the file to danbooru's directory. - FileUtils.cp(f.local_path, tempfile_path) else - File.open(tempfile_path, 'wb') {|nf| nf.write(f.read)} + nil end end - def set_image_dimensions - if image? or flash? - imgsize = ImageSize.new(File.open(tempfile_path, "rb")) - self.width = imgsize.get_width - self.height = imgsize.get_height - end - end - - # Returns true if the post is an image format that GD can handle. - def image? - %w(jpg jpeg gif png).include?(file_ext.downcase) - end - - # Returns true if the post is a Flash movie. - def flash? - file_ext == "swf" - end - + # Determines the file extention based on a path, normalizing if necessary def find_ext(file_path) ext = File.extname(file_path) if ext.blank? @@ -254,137 +259,11 @@ class Post < ActiveRecord::Base return ext end end - - def content_type_to_file_ext(content_type) - case content_type.chomp - when "image/jpeg" - return "jpg" - - when "image/gif" - return "gif" - - when "image/png" - return "png" - - when "application/x-shockwave-flash" - return "swf" - - else - nil - end - end - - def preview_dimensions - if image? - dim = Danbooru.reduce_to({:width => width, :height => height}, {:width => 150, :height => 150}) - return [dim[:width], dim[:height]] - else - return [150, 150] - end - end - - def tempfile_sample_path - "#{RAILS_ROOT}/public/data/#{$PROCESS_ID}-sample.jpg" - end - - def regenerate_sample - return false unless image? - - if generate_sample && File.exists?(tempfile_sample_path) - FileUtils.mkdir_p(File.dirname(sample_path), :mode => 0775) - FileUtils.mv(tempfile_sample_path, sample_path) - FileUtils.chmod(0775, sample_path) - puts "Fixed sample for #{id}" - return true - else - puts "Error generating sample for #{id}" - return false - end - end - - def generate_sample - return true unless image? - return true unless CONFIG["image_samples"] - return true unless (width && height) - return true if (file_ext.downcase == "gif") - - size = Danbooru.reduce_to({:width => width, :height => height}, {:width => CONFIG["sample_width"], :height => CONFIG["sample_height"]}, CONFIG["sample_ratio"]) - - # We can generate the sample image during upload or offline. Use tempfile_path - # if it exists, otherwise use file_path. - path = tempfile_path - path = file_path unless File.exists?(path) - unless File.exists?(path) - errors.add(:file, "not found") - return false - end - - # If we're not reducing the resolution for the sample image, only reencode if the - # source image is above the reencode threshold. Anything smaller won't be reduced - # enough by the reencode to bother, so don't reencode it and save disk space. - if size[:width] == width && size[:height] == height && File.size?(path) < CONFIG["sample_always_generate_size"] - return true - end - - # If we already have a sample image, and the parameters havn't changed, - # don't regenerate it. - if size[:width] == sample_width && size[:height] == sample_height - return true - end - - size = Danbooru.reduce_to({:width => width, :height => height}, {:width => CONFIG["sample_width"], :height => CONFIG["sample_height"]}) - begin - Danbooru.resize(file_ext, path, tempfile_sample_path, size, 90) - rescue Exception => x - errors.add "sample", "couldn't be created: #{x}" - return false - end - - self.sample_width = size[:width] - self.sample_height = size[:height] - return true - end - - # Returns true if the post has a sample image. - def has_sample? - sample_width.is_a?(Integer) - end - - # Returns true if the post has a sample image, and we're going to use it. - def use_sample?(user = nil) - if user && !user.show_samples? - false - else - CONFIG["image_samples"] && has_sample? - end - end - - def sample_url(user = nil) - if use_sample?(user) - store_sample_url - else - file_url - end - end - - def get_sample_width(user = nil) - if use_sample?(user) - sample_width - else - width - end - end - - def get_sample_height(user = nil) - if use_sample?(user) - sample_height - else - height - end - end - - def sample_percentage - 100 * get_sample_width.to_f / width + + # Path to a temporary file + def temp_file_path + @temp_file ||= Tempfile.new("danbooru-upload-#{$PROCESS_ID}") + @temp_file.path end end end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 1cfa13522..171ec5b15 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -10,6 +10,11 @@ module Danbooru "Danbooru" end + # Stripped of any special characters. + def safe_app_name + app_name.gsub(/[^a-zA-Z0-9_-]/, "_") + end + # The default name to use for anyone who isn't logged in. def default_guest_name "Anonymous" @@ -47,14 +52,14 @@ module Danbooru 150 end - # Medium resize image width + # Medium resize image width. Set to nil to disable. def medium_image_width - 500 + 480 end - # Large resize image width + # Large resize image width. Set to nil to disable. def large_image_width - 1024 + 1280 end # When calculating statistics based on the posts table, gather this many posts to sample from. @@ -107,11 +112,23 @@ module Danbooru 5 end + # Maximum size of an upload. + def max_file_size + 5.megabytes + end + # The name of the server the app is hosted on. def server_host Socket.gethostname end + # Names of other Danbooru servers which serve out of the same common database. + # Used in conjunction with load balancing to distribute files from one server to + # the others. This should match whatever gethostname returns on the other servers. + def other_server_hosts + [] + end + # Returns a hash mapping various tag categories to a numerical value. # Be sure to update the reverse_tag_category_mapping also. def tag_category_mapping diff --git a/config/initializers/create_empty_directories.rb b/config/initializers/create_empty_directories.rb new file mode 100644 index 000000000..fbc896521 --- /dev/null +++ b/config/initializers/create_empty_directories.rb @@ -0,0 +1,6 @@ +require 'fileutils' + +FileUtils.mkdir_p("#{Rails.root}/public/data/size_thumbnail") +FileUtils.mkdir_p("#{Rails.root}/public/data/size_medium") +FileUtils.mkdir_p("#{Rails.root}/public/data/size_large") +FileUtils.mkdir_p("#{Rails.root}/public/data/size_original") diff --git a/db/development_structure.sql b/db/development_structure.sql index a6584013f..2bba008b0 100644 --- a/db/development_structure.sql +++ b/db/development_structure.sql @@ -11,10 +11,161 @@ SET escape_string_warning = off; SET search_path = public, pg_catalog; +-- +-- Name: testprs_end(internal); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION testprs_end(internal) RETURNS void + LANGUAGE c STRICT + AS '$libdir/test_parser', 'testprs_end'; + + +-- +-- Name: testprs_getlexeme(internal, internal, internal); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION testprs_getlexeme(internal, internal, internal) RETURNS internal + LANGUAGE c STRICT + AS '$libdir/test_parser', 'testprs_getlexeme'; + + +-- +-- Name: testprs_lextype(internal); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION testprs_lextype(internal) RETURNS internal + LANGUAGE c STRICT + AS '$libdir/test_parser', 'testprs_lextype'; + + +-- +-- Name: testprs_start(internal, integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION testprs_start(internal, integer) RETURNS internal + LANGUAGE c STRICT + AS '$libdir/test_parser', 'testprs_start'; + + +-- +-- Name: testparser; Type: TEXT SEARCH PARSER; Schema: public; Owner: - +-- + +CREATE TEXT SEARCH PARSER testparser ( + START = testprs_start, + GETTOKEN = testprs_getlexeme, + END = testprs_end, + HEADLINE = prsd_headline, + LEXTYPES = testprs_lextype ); + + +-- +-- Name: danbooru; Type: TEXT SEARCH CONFIGURATION; Schema: public; Owner: - +-- + +CREATE TEXT SEARCH CONFIGURATION danbooru ( + PARSER = testparser ); + +ALTER TEXT SEARCH CONFIGURATION danbooru + ADD MAPPING FOR word WITH simple; + + SET default_tablespace = ''; SET default_with_oids = false; +-- +-- Name: pending_posts; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- + +CREATE TABLE pending_posts ( + id integer NOT NULL, + created_at timestamp without time zone, + updated_at timestamp without time zone, + source character varying(255), + rating character(1) NOT NULL, + uploader_id integer NOT NULL, + uploader_ip_addr inet NOT NULL, + tag_string text NOT NULL, + status character varying(255) DEFAULT 'pending'::character varying NOT NULL, + post_id integer +); + + +-- +-- Name: pending_posts_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE pending_posts_id_seq + START WITH 1 + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +-- +-- Name: pending_posts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE pending_posts_id_seq OWNED BY pending_posts.id; + + +-- +-- Name: posts; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- + +CREATE TABLE posts ( + id integer NOT NULL, + created_at timestamp without time zone, + updated_at timestamp without time zone, + score integer DEFAULT 0 NOT NULL, + source character varying(255), + md5 character varying(255) NOT NULL, + rating character(1) DEFAULT 'q'::bpchar NOT NULL, + is_note_locked boolean DEFAULT false NOT NULL, + is_rating_locked boolean DEFAULT false NOT NULL, + is_pending boolean DEFAULT false NOT NULL, + is_flagged boolean DEFAULT false NOT NULL, + approver_id integer, + change_seq integer DEFAULT 0, + uploader_id integer NOT NULL, + uploader_ip_addr inet NOT NULL, + last_noted_at timestamp without time zone, + last_commented_at timestamp without time zone, + tag_string text NOT NULL, + tag_index tsvector, + tag_count integer DEFAULT 0 NOT NULL, + tag_count_general integer DEFAULT 0 NOT NULL, + tag_count_artist integer DEFAULT 0 NOT NULL, + tag_count_character integer DEFAULT 0 NOT NULL, + tag_count_copyright integer DEFAULT 0 NOT NULL, + file_ext character varying(255) NOT NULL, + image_width integer NOT NULL, + image_height integer NOT NULL, + file_size integer NOT NULL +); + + +-- +-- Name: posts_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE posts_id_seq + START WITH 1 + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +-- +-- Name: posts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE posts_id_seq OWNED BY posts.id; + + -- -- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -; Tablespace: -- @@ -106,6 +257,20 @@ CREATE SEQUENCE users_id_seq ALTER SEQUENCE users_id_seq OWNED BY users.id; +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE pending_posts ALTER COLUMN id SET DEFAULT nextval('pending_posts_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE posts ALTER COLUMN id SET DEFAULT nextval('posts_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: public; Owner: - -- @@ -120,6 +285,22 @@ ALTER TABLE tags ALTER COLUMN id SET DEFAULT nextval('tags_id_seq'::regclass); ALTER TABLE users ALTER COLUMN id SET DEFAULT nextval('users_id_seq'::regclass); +-- +-- Name: pending_posts_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- + +ALTER TABLE ONLY pending_posts + ADD CONSTRAINT pending_posts_pkey PRIMARY KEY (id); + + +-- +-- Name: posts_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- + +ALTER TABLE ONLY posts + ADD CONSTRAINT posts_pkey PRIMARY KEY (id); + + -- -- Name: tags_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- @@ -136,6 +317,97 @@ ALTER TABLE ONLY users ADD CONSTRAINT users_pkey PRIMARY KEY (id); +-- +-- Name: index_posts_on_approver_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_approver_id ON posts USING btree (approver_id); + + +-- +-- Name: index_posts_on_change_seq; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_change_seq ON posts USING btree (change_seq); + + +-- +-- Name: index_posts_on_created_at; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_created_at ON posts USING btree (created_at); + + +-- +-- Name: index_posts_on_file_size; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_file_size ON posts USING btree (file_size); + + +-- +-- Name: index_posts_on_image_height; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_image_height ON posts USING btree (image_height); + + +-- +-- Name: index_posts_on_image_width; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_image_width ON posts USING btree (image_width); + + +-- +-- Name: index_posts_on_last_commented_at; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_last_commented_at ON posts USING btree (last_commented_at); + + +-- +-- Name: index_posts_on_last_noted_at; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_last_noted_at ON posts USING btree (last_noted_at); + + +-- +-- Name: index_posts_on_md5; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE UNIQUE INDEX index_posts_on_md5 ON posts USING btree (md5); + + +-- +-- Name: index_posts_on_mpixels; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_mpixels ON posts USING btree (((((image_width * image_height))::numeric / 1000000.0))); + + +-- +-- Name: index_posts_on_source; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_source ON posts USING btree (source); + + +-- +-- Name: index_posts_on_tags_index; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_tags_index ON posts USING gin (tag_index); + + +-- +-- Name: index_posts_on_uploader_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_posts_on_uploader_id ON posts USING btree (uploader_id); + + -- -- Name: index_tags_on_name; Type: INDEX; Schema: public; Owner: -; Tablespace: -- @@ -164,10 +436,24 @@ CREATE UNIQUE INDEX index_users_on_name ON users USING btree (lower((name)::text CREATE UNIQUE INDEX unique_schema_migrations ON schema_migrations USING btree (version); +-- +-- Name: trigger_posts_on_tag_index_update; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER trigger_posts_on_tag_index_update + BEFORE INSERT OR UPDATE ON posts + FOR EACH ROW + EXECUTE PROCEDURE tsvector_update_trigger('tag_index', 'public.danbooru', 'tag_string'); + + -- -- PostgreSQL database dump complete -- INSERT INTO schema_migrations (version) VALUES ('20100204211522'); -INSERT INTO schema_migrations (version) VALUES ('20100205162521'); \ No newline at end of file +INSERT INTO schema_migrations (version) VALUES ('20100205162521'); + +INSERT INTO schema_migrations (version) VALUES ('20100204214746'); + +INSERT INTO schema_migrations (version) VALUES ('20100205224030'); \ No newline at end of file diff --git a/db/migrate/20100205224030_create_pending_posts.rb b/db/migrate/20100205224030_create_pending_posts.rb new file mode 100644 index 000000000..cdc7ea310 --- /dev/null +++ b/db/migrate/20100205224030_create_pending_posts.rb @@ -0,0 +1,18 @@ +class CreatePendingPosts < ActiveRecord::Migration + def self.up + create_table :pending_posts do |t| + t.timestamps + t.column :source, :string + t.column :rating, :character, :null => false + t.column :uploader_id, :integer, :null => false + t.column :uploader_ip_addr, "inet", :null => false + t.column :tag_string, :text, :null => false + t.column :status, :string, :null => false, :default => "pending" + t.column :post_id, :integer + end + end + + def self.down + drop_table :pending_posts + end +end diff --git a/lib/danbooru_image_resizer/GIFReader.cpp b/lib/danbooru_image_resizer/GIFReader.cpp new file mode 100644 index 000000000..f02744e78 --- /dev/null +++ b/lib/danbooru_image_resizer/GIFReader.cpp @@ -0,0 +1,58 @@ +#include +#include +#include "GIFReader.h" +#include "RowBuffer.h" +#include "Resize.h" + +bool GIF::Read(FILE *f, Resizer *resizer, char error[1024]) +{ + RowBuffer Rows; + bool Ret = false; + gdImage *image = gdImageCreateFromGif(f); + + if(!image) + { + strcpy(error, "couldn't read GIF"); + return false; + } + + if(!Rows.Init(image->sx, image->sy, 3)) + { + strcpy(error, "out of memory"); + goto cleanup; + } + + resizer->SetSource(image->sx, image->sy, 3); + for(int y = 0; y < image->sy; ++y) + { + uint8_t *p = Rows.GetRow(y); + if(p == NULL) + { + strcpy(error, "out of memory"); + goto cleanup; + } + + for(int x = 0; x < image->sx; ++x) + { + int c = gdImageGetTrueColorPixel(image, x, y); + (*p++) = gdTrueColorGetRed(c); + (*p++) = gdTrueColorGetGreen(c); + (*p++) = gdTrueColorGetBlue(c); + } + + int DiscardRow; + if(!resizer->Run(Rows.GetRows(), Rows.GetStartRow(), Rows.GetEndRow(), DiscardRow)) + { + strcpy(error, resizer->GetError()); + goto cleanup; + } + + Rows.DiscardRows(DiscardRow); + } + + Ret = true; + +cleanup: + gdImageDestroy(image); + return Ret; +} diff --git a/lib/danbooru_image_resizer/GIFReader.h b/lib/danbooru_image_resizer/GIFReader.h new file mode 100644 index 000000000..61c200fa5 --- /dev/null +++ b/lib/danbooru_image_resizer/GIFReader.h @@ -0,0 +1,11 @@ +#ifndef GIF_READER_H +#define GIF_READER_H + +#include "Reader.h" +class GIF: public Reader +{ +public: + bool Read(FILE *f, Resizer *resizer, char error[1024]); +}; + +#endif diff --git a/lib/danbooru_image_resizer/JPEGReader.cpp b/lib/danbooru_image_resizer/JPEGReader.cpp new file mode 100644 index 000000000..e3d61eab9 --- /dev/null +++ b/lib/danbooru_image_resizer/JPEGReader.cpp @@ -0,0 +1,155 @@ +#include +#include "JPEGReader.h" +#include "RowBuffer.h" +#include "Resize.h" +#include +using namespace std; + +static void jpeg_error_exit(j_common_ptr CInfo) +{ + jpeg_error *myerr = (jpeg_error *) CInfo->err; + (*CInfo->err->format_message) (CInfo, myerr->buffer); + longjmp(myerr->setjmp_buffer, 1); +} + +static void jpeg_warning(j_common_ptr cinfo, int msg_level) +{ +} + +JPEGCompressor::JPEGCompressor(FILE *f) +{ + m_File = f; + memset(&m_CInfo, 0, sizeof(m_CInfo)); +} + +JPEGCompressor::~JPEGCompressor() +{ + jpeg_destroy_compress(&m_CInfo); +} + +const char *JPEGCompressor::GetError() const +{ + return m_JErr.buffer; +} + + +bool JPEGCompressor::Init(int width, int height, int quality) +{ + m_CInfo.err = jpeg_std_error(&m_JErr.pub); + + m_JErr.pub.error_exit = jpeg_error_exit; + m_JErr.pub.emit_message = jpeg_warning; + + if(setjmp(m_JErr.setjmp_buffer)) + return false; + + jpeg_create_compress(&m_CInfo); + + jpeg_stdio_dest(&m_CInfo, m_File); + + m_CInfo.image_width = width; + m_CInfo.image_height = height; + m_CInfo.input_components = 3; /* # of color components per pixel */ + m_CInfo.in_color_space = JCS_RGB; /* colorspace of input image */ + + jpeg_set_defaults(&m_CInfo); + jpeg_simple_progression(&m_CInfo); + jpeg_set_quality(&m_CInfo, quality, TRUE); // limit to baseline-JPEG values + + jpeg_start_compress(&m_CInfo, TRUE); + + return true; +} + +int JPEGCompressor::GetWidth() const +{ + return m_CInfo.image_width; +} + +int JPEGCompressor::GetHeight() const +{ + return m_CInfo.image_height; +} + +bool JPEGCompressor::WriteRow(uint8_t *row) +{ + if(setjmp(m_JErr.setjmp_buffer)) + return false; + + jpeg_write_scanlines(&m_CInfo, (JSAMPLE **) &row, 1); + return true; +} + +bool JPEGCompressor::Finish() +{ + if(setjmp(m_JErr.setjmp_buffer)) + return false; + + jpeg_finish_compress(&m_CInfo); + return true; +} + +bool JPEG::Read(FILE *f, Resizer *resizer, char error[1024]) +{ + // JMSG_LENGTH_MAX <= sizeof(error) + m_JErr.buffer = error; + RowBuffer Rows; + + m_Resizer = resizer; + + struct jpeg_decompress_struct CInfo; + CInfo.err = jpeg_std_error(&m_JErr.pub); + m_JErr.pub.error_exit = jpeg_error_exit; + m_JErr.pub.emit_message = jpeg_warning; + + bool Ret = false; + if(setjmp(m_JErr.setjmp_buffer)) + goto cleanup; + + jpeg_create_decompress(&CInfo); + + jpeg_stdio_src(&CInfo, f); + jpeg_read_header(&CInfo, TRUE); + CInfo.out_color_space = JCS_RGB; + + jpeg_start_decompress(&CInfo); + + if(!Rows.Init(CInfo.output_width, CInfo.output_height, 3)) + { + strcpy(error, "out of memory"); + goto cleanup; + } + + m_Resizer->SetSource(CInfo.output_width, CInfo.output_height, 3); + + while(CInfo.output_scanline < CInfo.output_height) + { + uint8_t *p = Rows.GetRow(CInfo.output_scanline); + if(p == NULL) + { + strcpy(error, "out of memory"); + goto cleanup; + } + + jpeg_read_scanlines(&CInfo, &p, 1); + + int DiscardRow; + if(!m_Resizer->Run(Rows.GetRows(), Rows.GetStartRow(), min(Rows.GetEndRow(), (int) CInfo.output_scanline+1), DiscardRow)) + { + strcpy(error, m_Resizer->GetError()); + goto cleanup; + } + + Rows.DiscardRows(DiscardRow); + } + + jpeg_finish_decompress(&CInfo); + + Ret = true; + +cleanup: + jpeg_destroy_decompress(&CInfo); + + return Ret; +} + diff --git a/lib/danbooru_image_resizer/JPEGReader.h b/lib/danbooru_image_resizer/JPEGReader.h new file mode 100644 index 000000000..14479979e --- /dev/null +++ b/lib/danbooru_image_resizer/JPEGReader.h @@ -0,0 +1,47 @@ +#ifndef JPEG_READER_H +#define JPEG_READER_H + +#include +#include +#include +#include "jpeglib-extern.h" +#include "Reader.h" + +struct jpeg_error +{ + struct jpeg_error_mgr pub; + jmp_buf setjmp_buffer; + char *buffer; +}; + +class JPEG: public Reader +{ +public: + bool Read(FILE *f, Resizer *resizer, char error[1024]); + +private: + Resizer *m_Resizer; + struct jpeg_error m_JErr; +}; + +class JPEGCompressor +{ +public: + JPEGCompressor(FILE *f); + ~JPEGCompressor(); + + bool Init(int width, int height, int quality); + bool WriteRow(uint8_t *row); + bool Finish(); + + int GetWidth() const; + int GetHeight() const; + const char *GetError() const; + +private: + FILE *m_File; + struct jpeg_compress_struct m_CInfo; + struct jpeg_error m_JErr; +}; + +#endif diff --git a/lib/danbooru_image_resizer/Makefile b/lib/danbooru_image_resizer/Makefile new file mode 100644 index 000000000..695e67980 --- /dev/null +++ b/lib/danbooru_image_resizer/Makefile @@ -0,0 +1,157 @@ + +SHELL = /bin/sh + +#### Start of system configuration section. #### + +srcdir = . +topdir = /opt/local/lib/ruby/1.8/i686-darwin10 +hdrdir = $(topdir) +VPATH = $(srcdir):$(topdir):$(hdrdir) +exec_prefix = $(prefix) +prefix = $(DESTDIR)/opt/local +sharedstatedir = $(prefix)/com +mandir = $(DESTDIR)/opt/local/share/man +psdir = $(docdir) +oldincludedir = $(DESTDIR)/usr/include +localedir = $(datarootdir)/locale +bindir = $(exec_prefix)/bin +libexecdir = $(exec_prefix)/libexec +sitedir = $(libdir)/ruby/site_ruby +htmldir = $(docdir) +vendorarchdir = $(vendorlibdir)/$(sitearch) +includedir = $(prefix)/include +infodir = $(datarootdir)/info +vendorlibdir = $(vendordir)/$(ruby_version) +sysconfdir = $(prefix)/etc +libdir = $(exec_prefix)/lib +sbindir = $(exec_prefix)/sbin +rubylibdir = $(libdir)/ruby/$(ruby_version) +docdir = $(datarootdir)/doc/$(PACKAGE) +dvidir = $(docdir) +vendordir = $(DESTDIR)/opt/local/lib/ruby/vendor_ruby +datarootdir = $(prefix)/share +pdfdir = $(docdir) +archdir = $(rubylibdir)/$(arch) +sitearchdir = $(sitelibdir)/$(sitearch) +datadir = $(datarootdir) +localstatedir = $(prefix)/var +sitelibdir = $(sitedir)/$(ruby_version) + +CC = g++ +LIBRUBY = $(LIBRUBY_SO) +LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a +LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME) +LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static + +RUBY_EXTCONF_H = +CFLAGS = -fno-common -O2 -fno-exceptions -Wall -arch x86_64 +INCFLAGS = -I. -I. -I/opt/local/lib/ruby/1.8/i686-darwin10 -I. +DEFS = +CPPFLAGS = -DHAVE_GD_H -DHAVE_GDIMAGECREATEFROMGIF -DHAVE_GDIMAGEJPEG -DHAVE_JPEG_SET_QUALITY -DHAVE_PNG_SET_EXPAND_GRAY_1_2_4_TO_8 -I/opt/local/include -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE -I/opt/local/include +CXXFLAGS = $(CFLAGS) +ldflags = -L. -L/opt/local/lib +dldflags = +archflag = -arch x86_64 +DLDFLAGS = $(ldflags) $(dldflags) $(archflag) +LDSHARED = $(CC) -dynamic -bundle -undefined suppress -flat_namespace +AR = ar +EXEEXT = + +RUBY_INSTALL_NAME = ruby +RUBY_SO_NAME = ruby +arch = i686-darwin10 +sitearch = i686-darwin10 +ruby_version = 1.8 +ruby = /opt/local/bin/ruby +RUBY = $(ruby) +RM = rm -f +MAKEDIRS = mkdir -p +INSTALL = /usr/bin/install -c +INSTALL_PROG = $(INSTALL) -m 0755 +INSTALL_DATA = $(INSTALL) -m 644 +COPY = cp + +#### End of system configuration section. #### + +preload = + +libpath = . $(libdir) +LIBPATH = -L. -L$(libdir) +DEFFILE = + +CLEANFILES = mkmf.log +DISTCLEANFILES = + +extout = +extout_prefix = +target_prefix = +LOCAL_LIBS = +LIBS = $(LIBRUBYARG_SHARED) -lpng -ljpeg -lgd -lpthread -ldl -lobjc +SRCS = danbooru_image_resizer.cpp GIFReader.cpp JPEGReader.cpp PNGReader.cpp Resize.cpp RowBuffer.cpp +OBJS = danbooru_image_resizer.o GIFReader.o JPEGReader.o PNGReader.o Resize.o RowBuffer.o +TARGET = danbooru_image_resizer +DLLIB = $(TARGET).bundle +EXTSTATIC = +STATIC_LIB = + +BINDIR = $(bindir) +RUBYCOMMONDIR = $(sitedir)$(target_prefix) +RUBYLIBDIR = $(sitelibdir)$(target_prefix) +RUBYARCHDIR = $(sitearchdir)$(target_prefix) + +TARGET_SO = $(DLLIB) +CLEANLIBS = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map +CLEANOBJS = *.o *.a *.s[ol] *.pdb *.exp *.bak + +all: $(DLLIB) +static: $(STATIC_LIB) + +clean: + @-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) + +distclean: clean + @-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log + @-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES) + +realclean: distclean +install: install-so install-rb + +install-so: $(RUBYARCHDIR) +install-so: $(RUBYARCHDIR)/$(DLLIB) +$(RUBYARCHDIR)/$(DLLIB): $(DLLIB) + $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR) +install-rb: pre-install-rb install-rb-default +install-rb-default: pre-install-rb-default +pre-install-rb: Makefile +pre-install-rb-default: Makefile +$(RUBYARCHDIR): + $(MAKEDIRS) $@ + +site-install: site-install-so site-install-rb +site-install-so: install-so +site-install-rb: install-rb + +.SUFFIXES: .c .m .cc .cxx .cpp .C .o + +.cc.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.cxx.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.cpp.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.C.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.c.o: + $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $< + +$(DLLIB): $(OBJS) Makefile + @-$(RM) $@ + $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS) + + + +$(OBJS): ruby.h defines.h diff --git a/lib/danbooru_image_resizer/PNGReader.cpp b/lib/danbooru_image_resizer/PNGReader.cpp new file mode 100644 index 000000000..d65886f19 --- /dev/null +++ b/lib/danbooru_image_resizer/PNGReader.cpp @@ -0,0 +1,138 @@ +#include +#include +#include "PNGReader.h" +#include "Resize.h" +#include +using namespace std; + +void PNG::Error(png_struct *png, const char *error) +{ + png_error_info *info = (png_error_info *) png->error_ptr; + strncpy(info->err, error, 1024); + info->err[1023] = 0; + longjmp(png->jmpbuf, 1); +} + +void PNG::Warning(png_struct *png, const char *warning) +{ +} + +void PNG::InfoCallback(png_struct *png, png_info *info_ptr) +{ + PNG *data = (PNG *) png_get_progressive_ptr(png); + + png_uint_32 width, height; + int bit_depth, color_type; + png_get_IHDR(png, info_ptr, &width, &height, &bit_depth, &color_type, NULL, NULL, NULL); + + png_set_palette_to_rgb(png); + png_set_tRNS_to_alpha(png); + png_set_filler(png, 0xFF, PNG_FILLER_AFTER); + if(bit_depth < 8) + png_set_packing(png); + if(color_type == PNG_COLOR_TYPE_GRAY && bit_depth < 8) + png_set_expand_gray_1_2_4_to_8(png); + if(bit_depth == 16) + png_set_strip_16(png); + data->m_Passes = png_set_interlace_handling(png); + + if (color_type == PNG_COLOR_TYPE_GRAY || color_type == PNG_COLOR_TYPE_GRAY_ALPHA) + png_set_gray_to_rgb(png); + + if(!data->m_Rows.Init(width, height, 4)) + Error(png, "out of memory"); + + png_read_update_info(png, info_ptr); + + data->m_Resizer->SetSource(width, height, 4); +} + +void PNG::RowCallback(png_struct *png, png_byte *new_row, png_uint_32 row_num, int pass) +{ + PNG *data = (PNG *) png_get_progressive_ptr(png); + + uint8_t *p = data->m_Rows.GetRow(row_num); + if(p == NULL) + Error(png, "out of memory"); + + png_progressive_combine_row(png, p, new_row); + + if(pass != data->m_Passes - 1) + return; + + /* We've allocated data->m_RowsAllocated, but if we're doing multiple passes, only + * rows 0 to row_num will actually have usable data. */ + int DiscardRow; + int LastRow = min(data->m_Rows.GetEndRow(), (int) row_num+1); + if(!data->m_Resizer->Run(data->m_Rows.GetRows(), data->m_Rows.GetStartRow(), LastRow, DiscardRow)) + Error(png, data->m_Resizer->GetError()); + + /* If we're interlaced, never discard rows. */ + if(data->m_Passes == 1) + data->m_Rows.DiscardRows(DiscardRow); +} + +void PNG::EndCallback(png_struct *png, png_info *info) +{ + PNG *data = (PNG *) png_get_progressive_ptr(png); + data->m_Done = true; +} + + +bool PNG::Read(FILE *f, Resizer *resizer, char error[1024]) +{ + m_Resizer = resizer; + + png_error_info err; + err.err = error; + + png_struct *png = png_create_read_struct(PNG_LIBPNG_VER_STRING, &err, Error, Warning); + if(png == NULL) + { + sprintf(error, "creating png_create_read_struct failed"); + return false; + } + + png_info *info_ptr = png_create_info_struct(png); + if(info_ptr == NULL) + { + png_destroy_read_struct(&png, NULL, NULL); + sprintf(error, "creating png_create_info_struct failed"); + return false; + } + + if(setjmp(png->jmpbuf)) + { + png_destroy_read_struct(&png, &info_ptr, NULL); + return false; + } + + png_set_progressive_read_fn(png, this, InfoCallback, RowCallback, EndCallback); + + while(1) + { + png_byte buf[1024*16]; + int ret = fread(buf, 1, sizeof(buf), f); + if(ret == 0) + break; + if(ferror(f)) + { + strcpy(error, strerror(errno)); + png_destroy_read_struct(&png, &info_ptr, NULL); + return false; + } + + png_process_data(png, info_ptr, buf, ret); + } + + if(!m_Done) + { + strcpy(error, "incomplete file"); + png_destroy_read_struct(&png, &info_ptr, NULL); + return false; + } + + png_destroy_read_struct(&png, &info_ptr, NULL); + return true; +} + diff --git a/lib/danbooru_image_resizer/PNGReader.h b/lib/danbooru_image_resizer/PNGReader.h new file mode 100644 index 000000000..5eeabc847 --- /dev/null +++ b/lib/danbooru_image_resizer/PNGReader.h @@ -0,0 +1,37 @@ +#ifndef PNG_READER_H +#define PNG_READER_H + +#include +#include "Reader.h" +#include "RowBuffer.h" + +struct png_error_info +{ + char *err; +}; + +class PNG: public Reader +{ +public: + PNG() + { + m_Done = false; + } + + bool Read(FILE *f, Resizer *resizer, char error[1024]); + +private: + RowBuffer m_Rows; + Resizer *m_Resizer; + + bool m_Done; + int m_Passes; + + static void Error(png_struct *png, const char *error); + static void Warning(png_struct *png, const char *warning); + static void InfoCallback(png_struct *png, png_info *info_ptr); + static void RowCallback(png_struct *png, png_byte *new_row, png_uint_32 row_num, int pass); + static void EndCallback(png_struct *png, png_info *info); +}; + +#endif diff --git a/lib/danbooru_image_resizer/Reader.h b/lib/danbooru_image_resizer/Reader.h new file mode 100644 index 000000000..c7b6892f6 --- /dev/null +++ b/lib/danbooru_image_resizer/Reader.h @@ -0,0 +1,14 @@ +#ifndef READER_H +#define READER_H + +#include + +class Resizer; +class Reader +{ +public: + virtual ~Reader() { } + virtual bool Read(FILE *f, Resizer *rp, char errorbuf[1024]) = 0; +}; + +#endif diff --git a/lib/danbooru_image_resizer/Resize.cpp b/lib/danbooru_image_resizer/Resize.cpp new file mode 100644 index 000000000..a6ad03fef --- /dev/null +++ b/lib/danbooru_image_resizer/Resize.cpp @@ -0,0 +1,184 @@ +#include +#include +#include +#include +#include "Resize.h" +#include "JPEGReader.h" +#include +using namespace std; + +Resizer::Resizer(JPEGCompressor *Compressor) +{ + m_Compressor = Compressor; + m_CurrentY = 0; + m_OutBuf = NULL; +} + +Resizer::~Resizer() +{ + if(m_OutBuf) + free(m_OutBuf); +} + +const char *Resizer::GetError() const +{ + return m_Compressor->GetError(); +} + +void Resizer::SetSource(int Width, int Height, int BPP) +{ + m_SourceWidth = Width; + m_SourceHeight = Height; + m_SourceBPP = BPP; +} + +bool Resizer::SetDest(int Width, int Height, int Quality) +{ + m_DestWidth = Width; + m_DestHeight = Height; + m_OutBuf = (uint8_t *) malloc(Width*3); + + return m_Compressor->Init(Width, Height, Quality); +} + +#define scale(x, l1, h1, l2, h2) (((x)-(l1))*((h2)-(l2))/((h1)-(l1))+(l2)) + +static void Average(const uint8_t *const *src, float Colors[3], float SourceXStart, float SourceXEnd, float SourceYStart, float SourceYEnd, int SourceBPP) +{ + float Total = 0.0f; + for(float y = SourceYStart; y < SourceYEnd; ++y) + { + float YCoverage = 1.0f; + if(int(y) == int(SourceYStart)) + YCoverage -= y - int(y); + if(int(y) == int(SourceYEnd)) + YCoverage -= 1.0f - (SourceYEnd - int(SourceYEnd)); + + const uint8_t *xsrc=src[(int) y]+(int)SourceXStart*SourceBPP; + + /* The two conditionals can only be true on the first and last iteration of the loop, + * so unfold those iterations and pull the conditionals out of the inner loop. */ +/* while(x < SourceXEnd) + { + float XCoverage = 1.0f; + if(int(x) == int(SourceXStart)) + XCoverage -= x - int(x); + if(int(x) == int(SourceXEnd)) + XCoverage -= 1.0f - (SourceXEnd - int(SourceXEnd)); + + Colors[0] += xsrc[0] * XCoverage * YCoverage; + Colors[1] += xsrc[1] * XCoverage * YCoverage; + Colors[2] += xsrc[2] * XCoverage * YCoverage; + if(SourceBPP == 4) + Colors[3] += xsrc[3] * XCoverage * YCoverage; + xsrc += SourceBPP; + + Total += XCoverage * YCoverage; + ++x; + } +*/ + float x = int(SourceXStart); + if(x < SourceXEnd) + { + float XCoverage = 1.0f; + if(int(x) == int(SourceXStart)) + XCoverage -= x - int(x); + if(int(x) == int(SourceXEnd)) + XCoverage -= 1.0f - (SourceXEnd - int(SourceXEnd)); + + Colors[0] += xsrc[0] * XCoverage * YCoverage; + Colors[1] += xsrc[1] * XCoverage * YCoverage; + Colors[2] += xsrc[2] * XCoverage * YCoverage; + if(SourceBPP == 4) + Colors[3] += xsrc[3] * XCoverage * YCoverage; + xsrc += SourceBPP; + + Total += XCoverage * YCoverage; + ++x; + } + + while(x < SourceXEnd-1) + { + Colors[0] += xsrc[0] * YCoverage; + Colors[1] += xsrc[1] * YCoverage; + Colors[2] += xsrc[2] * YCoverage; + if(SourceBPP == 4) + Colors[3] += xsrc[3] * YCoverage; + xsrc += SourceBPP; + + Total += YCoverage; + ++x; + } + + if(x < SourceXEnd) + { + float XCoverage = 1.0f; + if(int(x) == int(SourceXStart)) + XCoverage -= x - int(x); + if(int(x) == int(SourceXEnd)) + XCoverage -= 1.0f - (SourceXEnd - int(SourceXEnd)); + + Colors[0] += xsrc[0] * XCoverage * YCoverage; + Colors[1] += xsrc[1] * XCoverage * YCoverage; + Colors[2] += xsrc[2] * XCoverage * YCoverage; + if(SourceBPP == 4) + Colors[3] += xsrc[3] * XCoverage * YCoverage; + xsrc += SourceBPP; + + Total += XCoverage * YCoverage; + } + } + + if(Total != 0.0f) + for(int i = 0; i < 4; ++i) + Colors[i] /= Total; +} + +bool Resizer::Run(const uint8_t *const *Source, int StartRow, int EndRow, int &DiscardRow) +{ + while(m_CurrentY < m_DestHeight) + { + float SourceYStart = scale((float) m_CurrentY, 0.0f, (float) m_DestHeight, 0.0f, (float) m_SourceHeight); + float SourceYEnd = scale((float) m_CurrentY + 1, 0.0f, (float) m_DestHeight, 0.0f, (float) m_SourceHeight); + DiscardRow = int(SourceYStart)-1; + + if(EndRow != m_SourceHeight && int(SourceYEnd)+1 > EndRow-1) + return true; + assert(SourceYStart>=StartRow); + + uint8_t *Output = m_OutBuf; + for(int x = 0; x < m_DestWidth; ++x) + { + float SourceXStart = scale((float) x, 0.0f, (float) m_DestWidth, 0.0f, (float) m_SourceWidth); + float SourceXEnd = scale((float) x + 1, 0.0f, (float) m_DestWidth, 0.0f, (float) m_SourceWidth); + + float Colors[4] = { 0.0 }; + Average(Source, Colors, SourceXStart, SourceXEnd, SourceYStart, SourceYEnd, m_SourceBPP); + + if(m_SourceBPP == 4) + { + for(int i = 0; i < 3; ++i) + Colors[i] *= Colors[3]/255.0f; + } + + Output[0] = (uint8_t) min(255, int(Colors[0])); + Output[1] = (uint8_t) min(255, int(Colors[1])); + Output[2] = (uint8_t) min(255, int(Colors[2])); + + Output += 3; + } + + if(!m_Compressor->WriteRow((JSAMPLE *) m_OutBuf)) + return false; + ++m_CurrentY; + } + + if(m_CurrentY == m_DestHeight) + { + if(!m_Compressor->Finish()) + return false; + } + + return true; +} + diff --git a/lib/danbooru_image_resizer/Resize.h b/lib/danbooru_image_resizer/Resize.h new file mode 100644 index 000000000..fb1090a8e --- /dev/null +++ b/lib/danbooru_image_resizer/Resize.h @@ -0,0 +1,41 @@ +#ifndef RESIZE_H +#define RESIZE_H + +#include + +class JPEGCompressor; + +class Resizer +{ +public: + Resizer(JPEGCompressor *Compressor); + ~Resizer(); + + // BPP is 3 or 4, indicating RGB or RGBA. + void SetSource(int Width, int Height, int BPP); + bool SetDest(int Width, int Height, int Quality); + + /* + * Resize part of an image. + * + * [FirstRow,LastRow) is a range indicating which elements in src[] are available. + * On return, any rows in [0,DiscardRow) are no longer needed and can be deleted. + */ + bool Run(const uint8_t *const *src, int FirstRow, int LastRow, int &DiscardRow); + const char *GetError() const; + +private: + JPEGCompressor *m_Compressor; + uint8_t *m_OutBuf; + + int m_SourceWidth; + int m_SourceHeight; + int m_SourceBPP; + + int m_DestWidth; + int m_DestHeight; + + float m_CurrentY; +}; + +#endif diff --git a/lib/danbooru_image_resizer/RowBuffer.cpp b/lib/danbooru_image_resizer/RowBuffer.cpp new file mode 100644 index 000000000..e09179990 --- /dev/null +++ b/lib/danbooru_image_resizer/RowBuffer.cpp @@ -0,0 +1,81 @@ +#include +#include +#include +#include "RowBuffer.h" +#include +using namespace std; + +RowBuffer::RowBuffer() +{ + m_Rows = NULL; + m_StartRow = 0; + m_EndRow = 0; + m_BPP = 0; + m_Height = 0; +} + +RowBuffer::~RowBuffer() +{ + for(int i = 0; i < m_Height; ++i) + delete [] m_Rows[i]; + + delete [] m_Rows; +} + +bool RowBuffer::Init(int Width, int Height, int BPP) +{ + m_Width = Width; + m_Height = Height; + m_BPP = BPP; + + m_Rows = new uint8_t *[Height]; + if(m_Rows == NULL) + return false; + memset(m_Rows, 0, sizeof(uint8_t *) * Height); + + return true; +} + +uint8_t *RowBuffer::GetRow(int Row) +{ + assert(m_BPP > 0); + + if(m_Rows[Row] == NULL) + { + m_Rows[Row] = new uint8_t[m_Width*m_BPP]; + if(m_Rows[Row] == NULL) + return NULL; + if(m_StartRow == m_EndRow) + { + m_StartRow = Row; + m_EndRow = m_StartRow + 1; + } + } + + if(int(Row) == m_StartRow+1) + { + while(m_StartRow != 0 && m_Rows[m_StartRow-1]) + --m_StartRow; + } + + if(int(Row) == m_EndRow) + { + while(m_EndRow < m_Height && m_Rows[m_EndRow]) + ++m_EndRow; + } + return m_Rows[Row]; +} + +void RowBuffer::DiscardRows(int DiscardRow) +{ + assert(m_BPP > 0); + + for(int i = m_StartRow; i < DiscardRow; ++i) + { + delete [] m_Rows[i]; + m_Rows[i] = NULL; + } + + m_StartRow = max(m_StartRow, DiscardRow); + m_EndRow = max(m_EndRow, DiscardRow); +} diff --git a/lib/danbooru_image_resizer/RowBuffer.h b/lib/danbooru_image_resizer/RowBuffer.h new file mode 100644 index 000000000..2c792236f --- /dev/null +++ b/lib/danbooru_image_resizer/RowBuffer.h @@ -0,0 +1,40 @@ +#ifndef ROW_BUFFER_H +#define ROW_BUFFER_H + +#include + +class RowBuffer +{ +public: + RowBuffer(); + ~RowBuffer(); + + bool Init(int Width, int Height, int BPP); + + /* Return row, allocating if necessary. */ + uint8_t *GetRow(int row); + + // Free rows [0,DiscardRow). + void DiscardRows(int DiscardRow); + + /* Get a range of rows allocated in m_Rows: [m_StartRow,m_EndRow). If + * more than one allocated range exists, which range is returned is undefined. */ + int GetStartRow() const { return m_StartRow; } + int GetEndRow() const { return m_EndRow; } + const uint8_t *const *GetRows() const { return m_Rows; } + +private: + /* Array of image rows. These are allocated as needed. */ + uint8_t **m_Rows; + + /* in m_Rows is allocated: */ + int m_StartRow; + int m_EndRow; + + int m_Width; + int m_Height; + int m_BPP; +}; + +#endif + diff --git a/lib/danbooru_image_resizer/danbooru_image_resizer.cpp b/lib/danbooru_image_resizer/danbooru_image_resizer.cpp new file mode 100644 index 000000000..8c502450a --- /dev/null +++ b/lib/danbooru_image_resizer/danbooru_image_resizer.cpp @@ -0,0 +1,84 @@ +#include +#include +#include + +#include "PNGReader.h" +#include "GIFReader.h" +#include "JPEGReader.h" +#include "Resize.h" + +static VALUE danbooru_module; + +static VALUE danbooru_resize_image(VALUE module, VALUE file_ext_val, VALUE read_path_val, VALUE write_path_val, VALUE output_width_val, VALUE output_height_val, VALUE output_quality_val) +{ + const char * file_ext = StringValueCStr(file_ext_val); + const char * read_path = StringValueCStr(read_path_val); + const char * write_path = StringValueCStr(write_path_val); + int output_width = NUM2INT(output_width_val); + int output_height = NUM2INT(output_height_val); + int output_quality = NUM2INT(output_quality_val); + + FILE *read_file = fopen(read_path, "rb"); + if(read_file == NULL) + rb_raise(rb_eIOError, "can't open %s\n", read_path); + + FILE *write_file = fopen(write_path, "wb"); + if(write_file == NULL) + { + fclose(read_file); + rb_raise(rb_eIOError, "can't open %s\n", write_path); + } + + bool ret = false; + char error[1024]; + JPEGCompressor *Compressor = NULL; + Resizer *resizer = NULL; + Reader *Reader = NULL; + + if (!strcmp(file_ext, "jpg") || !strcmp(file_ext, "jpeg")) + Reader = new JPEG; + else if (!strcmp(file_ext, "gif")) + Reader = new GIF; + else if (!strcmp(file_ext, "png")) + Reader = new PNG; + else + { + strcpy(error, "unknown filetype"); + goto cleanup; + } + + Compressor = new JPEGCompressor(write_file); + if(Compressor == NULL) + { + strcpy(error, "out of memory"); + goto cleanup; + } + + resizer = new Resizer(Compressor); + if(resizer == NULL || Reader == NULL) + { + strcpy(error, "out of memory"); + goto cleanup; + } + + resizer->SetDest(output_width, output_height, output_quality); + ret = Reader->Read(read_file, resizer, error); + +cleanup: + delete Reader; + delete resizer; + delete Compressor; + + fclose(read_file); + fclose(write_file); + + if(!ret) + rb_raise(rb_eException, "%s", error); + + return INT2FIX(0); +} + +extern "C" void Init_danbooru_image_resizer() { + danbooru_module = rb_define_module("Danbooru"); + rb_define_module_function(danbooru_module, "resize_image", (VALUE(*)(...))danbooru_resize_image, 6); +} diff --git a/lib/danbooru_image_resizer/danbooru_image_resizer.rb b/lib/danbooru_image_resizer/danbooru_image_resizer.rb new file mode 100644 index 000000000..14d8dff56 --- /dev/null +++ b/lib/danbooru_image_resizer/danbooru_image_resizer.rb @@ -0,0 +1,29 @@ +require 'danbooru_image_resizer/danbooru_image_resizer.so' + +module Danbooru + def resize(file_ext, read_path, write_path, output_size, output_quality) + Danbooru.resize_image(file_ext, read_path, write_path, output_size[:width], output_size[:height], output_quality) + end + + def reduce_to(size, max_size) + returning size.dup do |new_size| + if new_size[:width] > max_size[:width] + scale = max_size[:width].to_f / new_size[:width].to_f + new_size[:width] = new_size[:width] * scale + new_size[:height] = new_size[:height] * scale + end + + if max_size[:height] && (new_size[:height] > max_size[:height]) + scale = max_size[:height].to_f / new_size[:height].to_f + new_size[:width] = new_size[:width] * scale + new_size[:height] = new_size[:height] * scale + end + + new_size[:width] = new_size[:width].to_i + new_size[:height] = new_size[:height].to_i + end + end + + module_function :resize + module_function :reduce_to +end diff --git a/lib/danbooru_image_resizer/extconf.rb b/lib/danbooru_image_resizer/extconf.rb new file mode 100644 index 000000000..5ffbbfea3 --- /dev/null +++ b/lib/danbooru_image_resizer/extconf.rb @@ -0,0 +1,26 @@ +#!/bin/env ruby + +require 'mkmf' + +CONFIG['CC'] = "g++" +CONFIG['LDSHARED'] = CONFIG['LDSHARED'].sub(/^cc /,'g++ ') # otherwise we would not link with the C++ runtime + +dir_config("gd") +dir_config("jpeg") +dir_config("png") + +have_header("gd.h") + +have_library("gd") +have_library("jpeg") +have_library("png") + +have_func("gdImageCreateFromGif", "gd.h") +have_func("gdImageJpeg", "gd.h") +have_func("jpeg_set_quality", ["stdlib.h", "stdio.h", "jpeglib-extern.h"]) +have_func("png_set_expand_gray_1_2_4_to_8", "png.h") + +with_cflags("-O2 -fno-exceptions -Wall") {true} +#with_cflags("-O0 -g -fno-exceptions -Wall") {true} + +create_makefile("danbooru_image_resizer") diff --git a/lib/danbooru_image_resizer/jpeglib-extern.h b/lib/danbooru_image_resizer/jpeglib-extern.h new file mode 100644 index 000000000..de92d4009 --- /dev/null +++ b/lib/danbooru_image_resizer/jpeglib-extern.h @@ -0,0 +1,16 @@ +// Needed for OS X + +#ifndef JPEGLIB_EXTERN_H +#define JPEGLIB_EXTERN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/test_helper.rb b/test/test_helper.rb index 59ef83a93..541f24366 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -9,4 +9,20 @@ require 'rails/test_help' Dir[File.expand_path(File.dirname(__FILE__) + "/factories/*.rb")].each {|file| require file} class ActiveSupport::TestCase +protected + def upload_file(path, content_type, filename) + tempfile = Tempfile.new(filename) + FileUtils.copy_file(path, tempfile.path) + (class << tempfile; self; end).class_eval do + alias local_path path + define_method(:original_filename) {filename} + define_method(:content_type) {content_type} + end + + tempfile + end + + def upload_jpeg(path) + upload_file(path, "image/jpeg", File.basename(path)) + end end diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index 8afe8cc42..7d1623cf9 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -1,8 +1,41 @@ -require 'test_helper' +require File.dirname(__FILE__) + '/../test_helper' class PostTest < ActiveSupport::TestCase - # Replace this with your real tests. - test "the truth" do - assert true + context "A post download" do + setup do + @source = "http://www.google.com/intl/en_ALL/images/logo.gif" + @tempfile = Tempfile.new("danbooru-test") + @download = ::Post::Pending::Download.new(@source, @tempfile.path) + end + + teardown do + @tempfile.close + end + + should "stream a file from an HTTP source" do + @download.http_get_streaming(@download.source) do |resp| + assert_equal("200", resp.code) + assert(resp["Content-Length"].to_i > 0, "File should be larger than 0 bytes") + end + end + + should "throw an exception when the file is larger than the maximum" do + assert_raise(Post::Pending::Download::Error) do + @download.http_get_streaming(@download.source, :max_size => 1) do |resp| + end + end + end + + should "store the file in the tempfile path" do + @download.download! + assert_equal(@source, @download.source) + assert(File.exists?(@tempfile.path), "temp file should exist") + assert(File.size(@tempfile.path) > 0, "should have data") + end + + should "initialize the content type" do + @download.download! + assert_match(/text\/html/, @download.content_type) + end end end