# robots.txt for jongto.net
#
# 2026-06-16: AI/LLM crawlers reverted to Disallow. 4/29 commit 666d879
# enabled them for AEO/GEO citation, but post-mortem (6/14-16) showed:
# - AI bots took 48% of detail-route traffic (meta-externalagent alone 43%)
# - Page views collapsed 30K → 4.2K (-86%) on the same timeline
# - AdSense revenue tracked the same path ($13/day → $0.46/day)
# Search bots (Googlebot/Yeti/Bingbot) and AdSense bots stay allowed.

# Default policy for all crawlers
User-agent: *
Allow: /
Allow: /api/og
Disallow: /api/
Allow: /_next/static/
Allow: /_next/image
Disallow: /_next/
Disallow: /login
Disallow: /signup
Disallow: /admin
Disallow: /profile

# Sitemap declaration — root sitemapindex serves all sub-sitemaps
Sitemap: https://jongto.net/sitemap.xml

# Google AdSense - content matching crawler. Must be explicit; AdsBot-Google
# ignores wildcard rules per Google docs.
User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

# Google Search
User-agent: Googlebot
Allow: /
Allow: /api/og
Disallow: /api/
Allow: /_next/static/
Allow: /_next/image
Disallow: /_next/
Disallow: /login
Disallow: /signup
Disallow: /admin
Disallow: /profile

# Naver
User-agent: Yeti
Allow: /
Allow: /api/og
Disallow: /api/
Allow: /_next/static/
Allow: /_next/image
Disallow: /_next/
Disallow: /login
Disallow: /signup
Disallow: /admin
Disallow: /profile

# Bing
User-agent: Bingbot
Allow: /
Allow: /api/og
Disallow: /api/
Allow: /_next/static/
Allow: /_next/image
Disallow: /_next/
Disallow: /login
Disallow: /signup
Disallow: /admin
Disallow: /profile

# ============================================================
# AI / LLM crawlers — BLOCKED
# ============================================================

# OpenAI
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

# Anthropic
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Google AI (Gemini / Bard / AI Overviews — separate from Googlebot)
User-agent: Google-Extended
Disallow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

# Meta — biggest single AI traffic source (43% of detail-route requests)
User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

# Common Crawl — feeds many AI training datasets
User-agent: CCBot
Disallow: /

# Apple Intelligence (Applebot for search remains allowed by default *)
User-agent: Applebot-Extended
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# DuckDuckGo Assist
User-agent: DuckAssistBot
Disallow: /

# Mistral
User-agent: MistralAI-User
Disallow: /

# ByteDance / TikTok AI
User-agent: Bytespider
Disallow: /

# Amazon AI
User-agent: Amazonbot
Disallow: /