# robots.txt — skill.bongshai.com
# Copying/scraping this site is prohibited: https://skill.bongshai.com/terms.html#no-copying
# /staff-archive/ is a trap for scrapers that ignore this file. Every group keeps that Disallow line,
# because a crawler reads only its own group (and ignores "*" once it has one). The Disallow line
# comes BEFORE "Allow: /" so crawlers that use first-match rules (not just longest-match) obey it.
# One crawler per group on purpose: moving or removing one bot never changes another.

# --- Search engines: ALLOWED ---
User-agent: Googlebot
Disallow: /staff-archive/
Allow: /

User-agent: Bingbot
Disallow: /staff-archive/
Allow: /

User-agent: Applebot
Disallow: /staff-archive/
Allow: /

# --- AI search / answer engines: ALLOWED (they cite the site in AI answers - GEO/AEO, owner rule 2026-10-04) ---
# [OWNER: block or allow] To block one, change its "Allow: /" to "Disallow: /" (keep the trap line).
User-agent: GPTBot
Disallow: /staff-archive/
Allow: /

User-agent: OAI-SearchBot
Disallow: /staff-archive/
Allow: /

User-agent: ChatGPT-User
Disallow: /staff-archive/
Allow: /

User-agent: ClaudeBot
Disallow: /staff-archive/
Allow: /

User-agent: Claude-SearchBot
Disallow: /staff-archive/
Allow: /

User-agent: Claude-User
Disallow: /staff-archive/
Allow: /

User-agent: PerplexityBot
Disallow: /staff-archive/
Allow: /

User-agent: Perplexity-User
Disallow: /staff-archive/
Allow: /

User-agent: Google-Extended
Disallow: /staff-archive/
Allow: /

User-agent: Applebot-Extended
Disallow: /staff-archive/
Allow: /

User-agent: DuckAssistBot
Disallow: /staff-archive/
Allow: /

User-agent: MistralAI-User
Disallow: /staff-archive/
Allow: /

User-agent: YouBot
Disallow: /staff-archive/
Allow: /

User-agent: Meta-ExternalFetcher
Disallow: /staff-archive/
Allow: /

# --- AI training-only crawlers: BLOCKED (no search or citation value) ---
# [OWNER: block or allow]
User-agent: CCBot
User-agent: anthropic-ai
User-agent: GoogleOther
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: AI2Bot
Disallow: /

# --- Site copiers / bulk downloaders ---
User-agent: HTTrack
User-agent: WebCopier
User-agent: WebZIP
User-agent: Offline Explorer
User-agent: SiteSnagger
User-agent: Teleport
User-agent: TeleportPro
User-agent: wget
User-agent: Wget
Disallow: /

# --- Everyone else ---
User-agent: *
# Bot trap: never linked visibly; anything that requests it is blocked for a while.
Disallow: /staff-archive/
Allow: /

Sitemap: https://skill.bongshai.com/sitemap.xml