# --- Default ---
User-agent: *
Allow: /
Disallow: /checkout
Disallow: /confirmation
Disallow: /become-a-guide/apply
Disallow: /csr
Disallow: /*?_rsc=
Disallow: /*?cnt=
Disallow: /*?tap_a=

# --- AI / LLM real-time-retrieval bots: explicitly allowed ---
# Disallows repeated — a named bot obeys only its most-specific group, never merges with `*`.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Google-Extended
Allow: /
Disallow: /checkout
Disallow: /confirmation
Disallow: /become-a-guide/apply
Disallow: /csr
Disallow: /*?_rsc=
Disallow: /*?cnt=
Disallow: /*?tap_a=

# --- AI bots: fully blocked ---
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: meta-externalagent
Disallow: /

# --- Sitemaps ---
Sitemap: https://www.walksdevour.com/sitemap.xml
Sitemap: https://www.walksdevour.com/blog/sitemap_index.xml