edshot99/eBooru
1
0
Fork 0
forked from e621ng/e621ng
Code Issues 11 Releases 1 Activity

[DB] Remove usage of dedicated index columns

Browse Source 
The index for posts is also getting added, but will get used at a later date
If this turns out to work fine the materialized index colums can be dropped

This contains a semantic change where dmail and wiki page searches for the body
no longer includes matches for the title
This commit is contained in:
Earlopain 2023-02-19 20:01:54 +01:00
parent f629e394ca
commit 5215876862
No known key found for this signature in database
GPG Key ID: 6CFB948E15246897
11 changed files with 90 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/models/application_record.rb
View File

@ -69,16 +69,14 @@ class ApplicationRecord < ActiveRecord::Base
PostQueryBuilder.new(nil).add_range_relation(parsed_range, qualified_column, self)
end
def text_attribute_matches(attribute, value, index_column: nil, ts_config: "english")
def text_attribute_matches(attribute, value)
column = column_for_attribute(attribute)
qualified_column = "#{table_name}.#{column.name}"
if value =~ /\*/
where("lower(#{qualified_column}) LIKE :value ESCAPE E'\\\\'", value: value.downcase.to_escaped_for_sql_like)
elsif index_column.present?
where("#{table_name}.#{index_column} @@ plainto_tsquery(:ts_config, :value)", ts_config: ts_config, value: value)
else
where("to_tsvector(:ts_config, #{qualified_column}) @@ plainto_tsquery(:ts_config, :value)", ts_config: ts_config, value: value)
where("to_tsvector(:ts_config, #{qualified_column}) @@ websearch_to_tsquery(:ts_config, :value)", ts_config: "english", value: value)
end
end

2
app/models/blip.rb
View File

@ -84,7 +84,7 @@ class Blip < ApplicationRecord
q = q.includes(:creator).includes(:responses).includes(:parent)
q = q.attribute_matches(:body, params[:body_matches], index_column: :body_index)
q = q.attribute_matches(:body, params[:body_matches])
if params[:response_to].present?
q = q.where('response_to = ?', params[:response_to].to_i)

2
app/models/comment.rb
View File

@ -75,7 +75,7 @@ class Comment < ApplicationRecord
def search(params)
q = super.includes(:creator).includes(:updater).includes(:post)
q = q.attribute_matches(:body, params[:body_matches], index_column: :body_index)
q = q.attribute_matches(:body, params[:body_matches])
if params[:post_id].present?
q = q.where("post_id in (?)", params[:post_id].split(",").map(&:to_i))

2
app/models/dmail.rb
View File

@ -128,7 +128,7 @@ class Dmail < ApplicationRecord
q = super
q = q.attribute_matches(:title, params[:title_matches])
q = q.attribute_matches(:body, params[:message_matches], index_column: :message_index)
q = q.attribute_matches(:body, params[:message_matches])
if params[:to_name].present?
q = q.to_name_matches(params[:to_name])

2
app/models/forum_post.rb
View File

@ -73,7 +73,7 @@ class ForumPost < ApplicationRecord
q = q.topic_title_matches(params[:topic_title_matches])
end
q = q.attribute_matches(:body, params[:body_matches], index_column: :text_index)
q = q.attribute_matches(:body, params[:body_matches])
if params[:creator_name].present?
q = q.creator_name(params[:creator_name].tr(" ", "_"))

2
app/models/forum_topic.rb
View File

@ -65,7 +65,7 @@ class ForumTopic < ApplicationRecord
q = super
q = q.permitted
q = q.attribute_matches(:title, params[:title_matches], index_column: :text_index)
q = q.attribute_matches(:title, params[:title_matches])
if params[:category_id].present?
q = q.for_category_id(params[:category_id])

2
app/models/note.rb
View File

@ -34,7 +34,7 @@ class Note < ApplicationRecord
def search(params)
q = super
q = q.attribute_matches(:body, params[:body_matches], index_column: :body_index)
q = q.attribute_matches(:body, params[:body_matches])
q = q.attribute_matches(:is_active, params[:is_active])
if params[:post_id].present?

2
app/models/wiki_page.rb
View File

@ -83,7 +83,7 @@ class WikiPage < ApplicationRecord
q = q.where("creator_id = ?", params[:creator_id])
end
q = q.attribute_matches(:body, params[:body_matches], index_column: :body_index, ts_config: "danbooru")
q = q.attribute_matches(:body, params[:body_matches])
if params[:other_names_match].present?
q = q.other_names_match(params[:other_names_match])

21
db/migrate/20230219115601_add_text_indices.rb Normal file
View File

@ -0,0 +1,21 @@
class AddTextIndices < ActiveRecord::Migration[7.0]
disable_ddl_transaction!
def change
add_gin_index :blips, "to_tsvector('english', body)"
add_gin_index :comments, "to_tsvector('english', body)"
add_gin_index :dmails, "to_tsvector('english', body)"
add_gin_index :forum_posts, "to_tsvector('english', body)"
add_gin_index :forum_topics, "to_tsvector('english', title)"
add_gin_index :notes, "to_tsvector('english', body)"
add_gin_index :wiki_pages, "to_tsvector('english', body)"
add_gin_index :posts, "string_to_array(tag_string, ' ')"
up_only do
execute "ALTER INDEX index_posts_on_string_to_array_tag_string ALTER COLUMN 1 SET STATISTICS 3000"
end
end
def add_gin_index(table, index)
add_index table, "(#{index})", using: :gin, algorithm: :concurrently
end
end

60
db/structure.sql
View File

@ -3497,6 +3497,13 @@ CREATE INDEX index_bans_on_user_id ON public.bans USING btree (user_id);
CREATE INDEX index_blips_on_body_index ON public.blips USING gin (body_index);
--
-- Name: index_blips_on_to_tsvector_english_body; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_blips_on_to_tsvector_english_body ON public.blips USING gin (to_tsvector('english'::regconfig, (body)::text));
--
-- Name: index_bulk_update_requests_on_forum_post_id; Type: INDEX; Schema: public; Owner: -
--
@ -3567,6 +3574,13 @@ CREATE INDEX index_comments_on_creator_ip_addr ON public.comments USING btree (c
CREATE INDEX index_comments_on_post_id ON public.comments USING btree (post_id);
--
-- Name: index_comments_on_to_tsvector_english_body; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_comments_on_to_tsvector_english_body ON public.comments USING gin (to_tsvector('english'::regconfig, body));
--
-- Name: index_dmail_filters_on_user_id; Type: INDEX; Schema: public; Owner: -
--
@ -3609,6 +3623,13 @@ CREATE INDEX index_dmails_on_message_index ON public.dmails USING gin (message_i
CREATE INDEX index_dmails_on_owner_id ON public.dmails USING btree (owner_id);
--
-- Name: index_dmails_on_to_tsvector_english_body; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_dmails_on_to_tsvector_english_body ON public.dmails USING gin (to_tsvector('english'::regconfig, body));
--
-- Name: index_edit_histories_on_user_id; Type: INDEX; Schema: public; Owner: -
--
@ -3672,6 +3693,13 @@ CREATE INDEX index_forum_posts_on_creator_id ON public.forum_posts USING btree (
CREATE INDEX index_forum_posts_on_text_index ON public.forum_posts USING gin (text_index);
--
-- Name: index_forum_posts_on_to_tsvector_english_body; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_forum_posts_on_to_tsvector_english_body ON public.forum_posts USING gin (to_tsvector('english'::regconfig, body));
--
-- Name: index_forum_posts_on_topic_id; Type: INDEX; Schema: public; Owner: -
--
@ -3735,6 +3763,13 @@ CREATE INDEX index_forum_topics_on_is_sticky_and_updated_at ON public.forum_topi
CREATE INDEX index_forum_topics_on_text_index ON public.forum_topics USING gin (text_index);
--
-- Name: index_forum_topics_on_to_tsvector_english_title; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_forum_topics_on_to_tsvector_english_title ON public.forum_topics USING gin (to_tsvector('english'::regconfig, (title)::text));
--
-- Name: index_forum_topics_on_updated_at; Type: INDEX; Schema: public; Owner: -
--
@ -3833,6 +3868,13 @@ CREATE INDEX index_notes_on_creator_id_and_post_id ON public.notes USING btree (
CREATE INDEX index_notes_on_post_id ON public.notes USING btree (post_id);
--
-- Name: index_notes_on_to_tsvector_english_body; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_notes_on_to_tsvector_english_body ON public.notes USING gin (to_tsvector('english'::regconfig, body));
--
-- Name: index_pool_versions_on_pool_id; Type: INDEX; Schema: public; Owner: -
--
@ -4071,6 +4113,14 @@ CREATE UNIQUE INDEX index_posts_on_md5 ON public.posts USING btree (md5);
CREATE INDEX index_posts_on_parent_id ON public.posts USING btree (parent_id);
--
-- Name: index_posts_on_string_to_array_tag_string; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_posts_on_string_to_array_tag_string ON public.posts USING gin (string_to_array(tag_string, ' '::text));
ALTER INDEX public.index_posts_on_string_to_array_tag_string ALTER COLUMN 1 SET STATISTICS 3000;
--
-- Name: index_posts_on_tags_index; Type: INDEX; Schema: public; Owner: -
--
@ -4365,6 +4415,13 @@ CREATE UNIQUE INDEX index_wiki_pages_on_title ON public.wiki_pages USING btree (
CREATE INDEX index_wiki_pages_on_title_pattern ON public.wiki_pages USING btree (title text_pattern_ops);
--
-- Name: index_wiki_pages_on_to_tsvector_english_body; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_wiki_pages_on_to_tsvector_english_body ON public.wiki_pages USING gin (to_tsvector('english'::regconfig, body));
--
-- Name: index_wiki_pages_on_updated_at; Type: INDEX; Schema: public; Owner: -
--
@ -4737,6 +4794,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20221014085948'),
('20230203162010'),
('20230204141325'),
('20230210092829');
('20230210092829'),
('20230219115601');

5
test/unit/dmail_test.rb
View File

@ -69,7 +69,7 @@ class DmailTest < ActiveSupport::TestCase
context "search" do
should "return results based on title contents" do
dmail = create(:dmail, title: "xxx", owner: @user)
dmail = create(:dmail, title: "xxx", body: "bbb", owner: @user)
matches = Dmail.search(title_matches: "x*")
assert_equal([dmail.id], matches.map(&:id))
@ -77,9 +77,6 @@ class DmailTest < ActiveSupport::TestCase
matches = Dmail.search(title_matches: "X*")
assert_equal([dmail.id], matches.map(&:id))
matches = Dmail.search(message_matches: "xxx")
assert_equal([dmail.id], matches.map(&:id))
matches = Dmail.search(message_matches: "aaa")
assert(matches.empty?)
end