# stockfloc.com robots.txt
# Default: open to all crawlers.
User-agent: *
Allow: /

# --- AI answer / search engines (explicitly welcomed: these drive citations) ---
# OpenAI
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: GPTBot
Allow: /

# Anthropic
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: ClaudeBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# Google-Extended = Gemini training and grounding use. It does NOT affect Googlebot
# or Google AI Overviews (those use standard Googlebot and cannot be opted out of
# without leaving Search). Allowed here for maximum AI reach.
User-agent: Google-Extended
Allow: /

# Apple, Amazon, Common Crawl
User-agent: Applebot-Extended
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: CCBot
Allow: /

# Mistral (Le Chat answers with cited sources)
User-agent: MistralAI-User
Allow: /
User-agent: MistralAI-Index
Allow: /

# DuckDuckGo (DuckAssist answers), You.com AI search
User-agent: DuckAssistBot
Allow: /
User-agent: YouBot
Allow: /

# Anthropic on-demand retrieval (legacy/alternate tokens), Meta AI, Cohere, Diffbot
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: meta-externalagent
Allow: /
User-agent: cohere-ai
Allow: /
User-agent: Diffbot
Allow: /

# Google's newer named AI agents (NotebookLM, Vertex grounding)
User-agent: Google-NotebookLM
Allow: /
User-agent: Google-CloudVertexBot
Allow: /

Sitemap: https://stockfloc.com/sitemap-index.xml