# robots.txt — thetoolsverse.com
# LLM summary:     https://thetoolsverse.com/llms.txt
# Full AI catalog: https://thetoolsverse.com/llms-full.txt
# AI usage policy: https://thetoolsverse.com/ai.txt

# ─────────────────────────────────────────────────────────────
# DEFAULT — all crawlers
# ─────────────────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /alternatives/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*gclid
Disallow: /*?*fbclid
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?q=

# ─────────────────────────────────────────────────────────────
# GOOGLE — full crawl + AI training (Google-Extended)
# ─────────────────────────────────────────────────────────────
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /alternatives/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*gclid
Disallow: /*?*fbclid
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?q=

# ─────────────────────────────────────────────────────────────
# BING / MICROSOFT
# ─────────────────────────────────────────────────────────────
User-agent: Bingbot
User-agent: msnbot
User-agent: msnbot-media
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /*?*utm_
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?q=

# ─────────────────────────────────────────────────────────────
# OPENAI — ChatGPT search + training
# ─────────────────────────────────────────────────────────────
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/

# ─────────────────────────────────────────────────────────────
# ANTHROPIC — Claude search + training
# ─────────────────────────────────────────────────────────────
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /admin/

# ─────────────────────────────────────────────────────────────
# PERPLEXITY
# ─────────────────────────────────────────────────────────────
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /admin/

# ─────────────────────────────────────────────────────────────
# XAI — Grok
# ─────────────────────────────────────────────────────────────
User-agent: xAI-Bot
User-agent: Grok-Bot
Allow: /
Disallow: /api/
Disallow: /admin/

# ─────────────────────────────────────────────────────────────
# OTHER AI SYSTEMS
# ─────────────────────────────────────────────────────────────
User-agent: CCBot
User-agent: cohere-ai
User-agent: Cohere-AI
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Kagibot
User-agent: Mistral-AI
User-agent: MistralAI-User
User-agent: Diffbot
Allow: /
Disallow: /api/
Disallow: /admin/

# ─────────────────────────────────────────────────────────────
# APPLE
# ─────────────────────────────────────────────────────────────
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin/

# ─────────────────────────────────────────────────────────────
# META
# ─────────────────────────────────────────────────────────────
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: facebookexternalhit
Allow: /
Disallow: /api/
Disallow: /admin/

# ─────────────────────────────────────────────────────────────
# SEO CRAWLERS — allow with crawl delay
# ─────────────────────────────────────────────────────────────
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: rogerbot
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 5

# ─────────────────────────────────────────────────────────────
# BLOCKED — traffic-wasting scrapers that contribute nothing
# ─────────────────────────────────────────────────────────────
User-agent: PetalBot
User-agent: Bytespider
Disallow: /

Sitemap: https://thetoolsverse.com/sitemap.xml
Host: thetoolsverse.com