# robots.txt for questone.com
# Last updated: 2026-06-23

# -----------------------------------------------
# All crawlers – general rules
# -----------------------------------------------
User-agent: *
Allow: /en
Allow: /de

# Block low-value / internal pages
Disallow: /cdn-cgi/
Disallow: /.well-known/

# Block Webflow-internal paths
Disallow: /admin
Disallow: /.webflow/

# Block thank-you / form submission pages
Disallow: /de/erfolgreich/form-submission
Disallow: /en/success/form-submission

# -----------------------------------------------
# Google
# -----------------------------------------------
User-agent: Googlebot
Allow: /en
Allow: /de
Disallow: /cdn-cgi/

# -----------------------------------------------
# Bing
# -----------------------------------------------
User-agent: Bingbot
Allow: /en
Allow: /de
Disallow: /cdn-cgi/

# -----------------------------------------------
# AI Crawlers – explicitly allowed
# (llms.txt provides structured content guidance)
# -----------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

# -----------------------------------------------
# Block unwanted scrapers & bad bots
# -----------------------------------------------

User-agent: YandexBot
Disallow: /

User-agent: Googlebot
Disallow: /

User-agent: Bingbot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: DuckDuckBot
Disallow: /

User-agent: MojeekBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: SBIntuitionsBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Amazon Q
Disallow: /

User-agent: Claude-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: YandexGPT
Disallow: /

User-agent: MistralBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: youBot
Disallow: /

User-agent: NeevaAI
Disallow: /

Sitemap: https://questone.com/sitemap.xml