# Crimes & Myths — crawl policy
# https://crimesandmyths.com

User-agent: *
Allow: /
Disallow: /admin

# AI answer and search crawlers — allowed explicitly rather than by the default
# wildcard group, so retrieval, citation and answer-engine traffic is a stated
# decision instead of an accident. Each group repeats the /admin disallow: a bot
# that matches a named group uses only that group, and admin tools stay closed.
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin

User-agent: ChatGPT-User
Allow: /
Disallow: /admin

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin

User-agent: PerplexityBot
Allow: /
Disallow: /admin

User-agent: Perplexity-User
Allow: /
Disallow: /admin

User-agent: Applebot
Allow: /
Disallow: /admin

User-agent: Amzn-SearchBot
Allow: /
Disallow: /admin

# Training crawlers — allowed deliberately. The measurement experiment includes
# model presence, so being in the training corpus is a goal, not a leak.
User-agent: GPTBot
Allow: /
Disallow: /admin

User-agent: ClaudeBot
Allow: /
Disallow: /admin

User-agent: Google-Extended
Allow: /
Disallow: /admin

# Two declarations, one index: /sitemap.xml is the sitemap file the platform
# serves, and the function URL is that same index. Both resolve to the same three
# sub-sitemaps (articles, hubs, images). The fallback declaration stays until the
# file-based one is confirmed in Search Console.
Sitemap: https://crimesandmyths.com/sitemap.xml
Sitemap: https://crimesandmyths.com/functions/generate_sitemap