# ScoreSage AI — robots.txt
# https://www.scoresageai.com
# Last updated: 2026-09-08

# Default: allow all crawlers to index public content
User-agent: *
Allow: /

# Sitemap endpoints live under /api/ — allow them explicitly (longest match wins)
Allow: /api/sitemap-leagues
Allow: /api/sitemap-blog

# Private / authenticated / system areas — keep out of the index
Disallow: /api/
Disallow: /admin
Disallow: /auth/
Disallow: /verify-email
Disallow: /settings
Disallow: /subscription-success
Disallow: /data/
Disallow: /dev/
Disallow: /following
Disallow: /notifications

# Auth-gated hubs (/players, /leagues, /matches, …) are NOT disallowed: they
# carry an X-Robots-Tag: noindex header (vercel.json), which Google can only
# honour if it is allowed to fetch the URL.

# AI / LLM crawlers — full access to public content
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

# Social / preview bots — allow everything for rich link previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Google AdSense crawler — reads public pages to match ads to their content
User-agent: Mediapartners-Google
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /auth/
Disallow: /settings
Disallow: /dev/

# Sitemap (index references the static, league-hub and blog sitemaps)
Sitemap: https://www.scoresageai.com/sitemap-index.xml

# Host directive
Host: https://www.scoresageai.com