# =====================================================================
# robots.txt — AI 爬虫友好版
# 目的: 放行所有主流 AI / LLM 检索爬虫,仅禁止后台与登录页
#
# 说明:
#   - 公开发布页 (/) → 全部允许
#   - 后台 / 登录 / API 内部端点 → Disallow
#   - sitemap 指向动态生成的 /sitemap.xml
# =====================================================================

User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /login
Disallow: /login/*
Disallow: /api/
Disallow: /private/
Disallow: /private/*

# ---------------------------------------------------------------------
# 主流 AI / LLM 检索爬虫 (显式 Allow,优先级高于上面的 *)
# ---------------------------------------------------------------------

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Google (AI 训练 + 检索)
User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

User-agent: GoogleOther
Allow: /

# 字节跳动
User-agent: Bytespider
Allow: /

User-agent: ByteDance
Allow: /

# Microsoft / Bing
User-agent: Bingbot
Allow: /

User-agent: MSNBot-Media
Allow: /

# 通用 LLM 检索
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Cohere-AI
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: SemanticScholarBot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: TimpiBot
Allow: /

User-agent: TurnitinBot
Allow: /

User-agent: PetalBot
Disallow: /

# ---------------------------------------------------------------------
# Sitemap 位置 (所有爬虫都会读这一行)
# ---------------------------------------------------------------------
Sitemap: http://localhost:3000/sitemap.xml

# ---------------------------------------------------------------------
# 爬虫速率提示 (可选,部分爬虫会读)
# ---------------------------------------------------------------------
Crawl-delay: 1