# Default: allow every crawler. We have nothing private behind a path.
# The only exclusions are the faceted-search query strings — see below.
User-agent: *
Disallow: /en/search?
Disallow: /ja/search?
Disallow: /ko/search?
Disallow: /es/buscar?
Disallow: /pt/pesquisar?
Disallow: /de/suche?

# AI training & retrieval crawlers — explicit allow. The default rule above
# already permits them, but several of these bots ignore wildcard rules and
# only obey their own user-agent block, so we list them explicitly.
# Source for the canonical names: each vendor's published docs.
#
# They share ONE group (multiple User-agent lines, one rule set) rather than a
# block each: per RFC 9309 a crawler obeys only the single most specific group
# matching its name and ignores every other group — including `*`. A named bot
# with its own block would therefore never see the Disallow rules above, so the
# exclusions have to be repeated in the group that actually applies to it.

# OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Google AI training (separate from Googlebot)
User-agent: Google-Extended
# Apple Intelligence (relevant: gota is iOS-first)
User-agent: Applebot-Extended
# Microsoft Copilot / Bing
User-agent: Bingbot
# Meta AI
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
# Amazon (Alexa / generative)
User-agent: Amazonbot
# DuckDuckGo / DuckAssist
User-agent: DuckAssistBot
# Cohere
User-agent: cohere-ai
# ByteDance (relevant: ja/pt locales)
User-agent: Bytespider
# Common Crawl (powers many downstream LLM training corpora)
User-agent: CCBot
# Baidu — China's dominant search engine (general crawler, not AI); listed so the
# faceted-search Disallow reaches it even though it may not honour the wildcard.
User-agent: Baiduspider
# SEO / backlink tools. Not search engines: they crawl to resell the map, and
# the faceted space is infinite and noindex. Listed so the Disallow below
# reaches them; `Allow: /` keeps the rest of the site crawlable, which is what
# keeps gota.cafe visible in their backlink indexes. The owner-driven audit
# bots (AhrefsSiteAudit, SiteAuditBot, DataForSeoBot) are deliberately absent —
# those are ours to run.
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: rogerbot
User-agent: BLEXBot
Allow: /
# Faceted search: every filter combination is a distinct URL, so the crawl
# space is combinatorial and infinite in practice. These pages already carry
# `noindex` when filtered (resources/views/search.blade.php), but noindex does
# not stop the fetch — only a Disallow does. The unfiltered landing pages
# (/de/suche and friends, no query string) stay crawlable: the `?` is literal,
# so it only matches URLs that actually carry a query string.
Disallow: /en/search?
Disallow: /ja/search?
Disallow: /ko/search?
Disallow: /es/buscar?
Disallow: /pt/pesquisar?
Disallow: /de/suche?

Sitemap: https://gota.cafe/sitemap.xml