forked from e621ng/e621ng
101 lines
2.3 KiB
Plaintext
101 lines
2.3 KiB
Plaintext
User-agent: *
|
|
Crawl-delay: 1
|
|
Disallow: /admin
|
|
Disallow: /advertisements
|
|
Disallow: /artists
|
|
Disallow: /artist_commentaries
|
|
Disallow: /artist_commentary_versions
|
|
Disallow: /artist_versions
|
|
Disallow: /bans
|
|
Disallow: /comment_votes
|
|
Disallow: /comment/
|
|
Disallow: /comments
|
|
Disallow: /comments.atom
|
|
Disallow: /comments/*.atom
|
|
Disallow: /counts
|
|
Disallow: /data/
|
|
Disallow: /delayed_jobs
|
|
Disallow: /deleted_posts
|
|
Disallow: /dmails
|
|
Disallow: /explore
|
|
Disallow: /favorite
|
|
Disallow: /forum_topics.atom
|
|
Disallow: /forum_posts
|
|
Disallow: /forum_posts.atom
|
|
Disallow: /forum_topics/*.atom
|
|
Disallow: /forum_posts/*.atom
|
|
Disallow: /iqdb_queries
|
|
Disallow: /ip_bans
|
|
Disallow: /janitor_trials
|
|
Disallow: /maintenance
|
|
Disallow: /m/
|
|
Disallow: /mod_actions
|
|
Disallow: /moderator
|
|
Disallow: /news_updates
|
|
Disallow: /note_versions
|
|
Disallow: /notes
|
|
Disallow: /pool_elements
|
|
Disallow: /pool_orders
|
|
Disallow: /pool_versions
|
|
Disallow: /popular
|
|
Disallow: /post_appeals
|
|
Disallow: /post_flags
|
|
Disallow: /post_sets
|
|
Disallow: /post_versions
|
|
Disallow: /post_votes
|
|
Disallow: /post_events
|
|
Disallow: /post_replacements
|
|
Disallow: /post/
|
|
Disallow: /posts.atom
|
|
Disallow: /posts/random
|
|
Disallow: /posts/copy_notes
|
|
Disallow: /posts/revert
|
|
Disallow: /posts/show_seq
|
|
Disallow: /posts*?*tags=*
|
|
Disallow: /posts*?*q=*
|
|
Disallow: /posts*?*md5*
|
|
Disallow: /posts*?*limit*
|
|
Disallow: /posts*?*random*
|
|
Disallow: /posts*?*pool_id*
|
|
Disallow: /posts*?*post_set_id*
|
|
Disallow: /posts.json
|
|
Disallow: /posts.xml
|
|
Disallow: /posts/*/favorites
|
|
Disallow: /reports
|
|
Disallow: /session
|
|
Disallow: /sources
|
|
Disallow: /static
|
|
Disallow: /tag_alias_corrections
|
|
Disallow: /tag_alias_requests
|
|
Disallow: /tag_aliases
|
|
Disallow: /tag_corrections
|
|
Disallow: /tag_implication_requests
|
|
Disallow: /tag_implications
|
|
Disallow: /tags
|
|
Disallow: /tickets
|
|
Disallow: /takedowns
|
|
Disallow: /uploads
|
|
Disallow: /user_feedback
|
|
Disallow: /user/
|
|
Disallow: /users
|
|
Disallow: /wiki/
|
|
Disallow: /wiki_page_versions
|
|
Disallow: /wiki_pages*?*title=*
|
|
Disallow: /wiki_pages/new
|
|
Disallow: /wiki_pages/revert
|
|
Disallow: /wiki_pages/show_or_new
|
|
|
|
User-agent: MJ12bot
|
|
User-agent: dotbot
|
|
User-agent: coccocbot-web
|
|
User-agent: BLEXBot
|
|
User-agent: HTTrack
|
|
User-agent: Slurp
|
|
User-agent: Baiduspider
|
|
User-agent: Yandex
|
|
User-agent: magpie-crawler
|
|
User-agent: SemrushBot
|
|
User-agent: SemrushBot-SA
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|