# robots.txt for 0xArchive — maximize AI discovery + citation. Training welcome.

# AI search / retrieval crawlers (these produce AI-answer citations)
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Allow: /
# These signals apply to public content that Archive Labs Inc. owns or is authorized to license.
# They do not change account access, the Terms of Service, the Data License, or third-party rights.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
# Private app and operator bridge routes (require auth anyway)
Disallow: /admin
Disallow: /cms
Disallow: /settings
Disallow: /account/connections
Disallow: /oauth/consent
Disallow: /charts

# User-triggered fetchers (live answers that cite us)
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
Allow: /
# These signals apply to public content that Archive Labs Inc. owns or is authorized to license.
# They do not change account access, the Terms of Service, the Data License, or third-party rights.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
# Private app and operator bridge routes (require auth anyway)
Disallow: /admin
Disallow: /cms
Disallow: /settings
Disallow: /account/connections
Disallow: /oauth/consent
Disallow: /charts

# Training crawlers (we want models trained on our public surface)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: Applebot
Allow: /
# These signals apply to public content that Archive Labs Inc. owns or is authorized to license.
# They do not change account access, the Terms of Service, the Data License, or third-party rights.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
# Private app and operator bridge routes (require auth anyway)
Disallow: /admin
Disallow: /cms
Disallow: /settings
Disallow: /account/connections
Disallow: /oauth/consent
Disallow: /charts

# Note: do NOT add Google-Extended: Disallow, which would opt out of Gemini training (we want it). These Content-Signal values explicitly advertise the permitted AI uses for public content.

# Default policy (all other crawlers)
User-agent: *
Allow: /
# These signals apply to public content that Archive Labs Inc. owns or is authorized to license.
# They do not change account access, the Terms of Service, the Data License, or third-party rights.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
# Private app and operator bridge routes (require auth anyway)
Disallow: /admin
Disallow: /cms
Disallow: /settings
Disallow: /account/connections
Disallow: /oauth/consent
Disallow: /charts

# Sitemaps
Sitemap: https://0xarchive.io/sitemap.xml
Sitemap: https://docs.0xarchive.io/sitemap.xml