User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

# Feeds + sitemap are explicitly allowed for every agent so a defensive
# wildcard above never accidentally blocks discovery.
Allow: /sitemap.xml
Allow: /feed.xml
Allow: /feeds/pinterest.xml
Allow: /feeds/google.xml
Allow: /feeds/openai.jsonl

# AI shopping + answer-engine crawlers — explicitly allowed so Standard Blanks
# can surface in LLM product recommendations and answer-engine results.
# Each block mirrors the wildcard exclusions (/admin/, /api/, /account).
# A crawler obeys ONLY its most specific group, so a group that forgot one of
# them would open that path to that crawler.
#
# Search and user-fetch agents are listed explicitly, not left to the wildcard:
# OAI-SearchBot / ChatGPT-User (ChatGPT search + browsing), Claude-SearchBot /
# Claude-User (Claude search + user fetches), PerplexityBot / Perplexity-User,
# Googlebot and Bingbot (which also feed Gemini/AI Overviews and Copilot).

User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Claude-Web
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account
Disallow: /quote

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: GoogleOther
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account
Disallow: /quote

User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

User-agent: Bytespider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account

# /quote is a priced page an agent opens for a shopper (llms.txt points
# agents at it); it answers noindex. Every product page links one, so bulk
# crawlers fetched thousands of one-off quote URLs: Amazonbot ~1,070 a day,
# plus PerplexityBot and SemrushBot (2026-11-02). Those three crawlers skip
# it. User-fetch agents (ChatGPT-User, Claude-User, Perplexity-User) and
# everyone else keep it.

# SEO-tool crawlers: welcome, but slowly. They index us for keyword and
# backlink tools, not for shoppers, and one of them (DataForSeoBot, from
# Germany) piled load onto the database just before the 2026-10-26 stall.
# Each honors Crawl-delay (seconds between requests). Search and AI crawlers
# above get none: we want them to pick up changes quickly.

User-agent: DataForSeoBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account
Crawl-delay: 30

User-agent: AhrefsBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account
Crawl-delay: 30

User-agent: SemrushBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account
Disallow: /quote
Crawl-delay: 30

User-agent: MJ12bot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account
Crawl-delay: 30

Sitemap: https://standardblanks.com/sitemap.xml