# Athanor Market — Robots & AI Crawler Policy
# Updated: 2026-02-09

User-agent: *

# Public content
Allow: /$
Allow: /terms$
Allow: /privacy$
Allow: /tokenized-promise$
Allow: /contact-us$
Allow: /troubleshooting$
Allow: /demo/gemscore$
Allow: /marketplace$
Allow: /abyss$
Allow: /abyss/ideas/
Allow: /coa$

# Blog (all posts)
Allow: /blog
Allow: /blog/

# Enterprise solutions (including sub-paths)
Allow: /solutions/vc
Allow: /solutions/syndicate
Allow: /solutions/accelerator
Allow: /solutions/platform
Allow: /solutions/corporate
Allow: /solutions/ma

# Unified landing pages
Allow: /enterprise
Allow: /alumni

# Investor resources
Allow: /investors/ai-discovery

# Static assets
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ico$

# AI-readable files
Allow: /llms.txt
Allow: /skill.md
Allow: /legal/agent-marketplace-tos
Allow: /api/v1/agent/docs
Allow: /api/v1/agent/health

# Block everything else
Disallow: /

# ─────────────────────────────────────────────
# AI Crawler Policies
# We welcome AI crawlers on public content.
# ─────────────────────────────────────────────

# OpenAI (ChatGPT Search, SearchGPT)
User-agent: GPTBot
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

User-agent: ChatGPT-User
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

User-agent: anthropic-ai
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

# Perplexity
User-agent: PerplexityBot
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

# Meta AI
User-agent: FacebookBot
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /$
Disallow: /

# Bing/Copilot
User-agent: Bingbot
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

# Apple Intelligence / Applebot
User-agent: Applebot
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /$
Disallow: /

# xAI (Grok)
User-agent: Grok
Allow: /blog
Allow: /blog/
Allow: /enterprise
Allow: /alumni
Allow: /solutions/
Allow: /demo/gemscore
Allow: /investors/ai-discovery
Allow: /tokenized-promise
Allow: /marketplace
Allow: /abyss
Allow: /abyss/ideas/
Allow: /llms.txt
Allow: /$
Disallow: /

# Sitemap & Feed
Sitemap: https://athanor.market/sitemap.xml
RSS: https://athanor.market/blog/feed