# CleenUI — robots.txt # Generated by scripts/optimize-ai.mjs — do not edit by hand. # Allow-list approach: every documented AI crawler is explicitly # permitted; everything else falls under the wildcard rule. # Disallow rules apply to every user-agent below. # # ── AI usage policy ───────────────────────────────────────── # CleenUI permits BOTH citation (live retrieval / RAG fetches) # AND training-data ingestion of public content on this site. # This is intentional, not an oversight: the site positions # itself as AI-agent-first, ships a /llms.txt convention file, # publishes a /.well-known/ai-plugin.json discovery manifest, # and explicitly allow-lists the 33 AI crawlers below. # # There are no , # noimageai, or similar opt-out signals anywhere on the site. # The machine-readable version of this policy lives at # /.well-known/ai-plugin.json under ai_usage_policy. # ──────────────────────────────────────────────────────────── # Default: allow all generic web crawlers, except internal routes. User-agent: * Disallow: /cubetest Allow: / User-agent: GPTBot Disallow: /cubetest Allow: / User-agent: ChatGPT-User Disallow: /cubetest Allow: / User-agent: OAI-SearchBot Disallow: /cubetest Allow: / User-agent: ClaudeBot Disallow: /cubetest Allow: / User-agent: Claude-Web Disallow: /cubetest Allow: / User-agent: Claude-User Disallow: /cubetest Allow: / User-agent: Claude-SearchBot Disallow: /cubetest Allow: / User-agent: anthropic-ai Disallow: /cubetest Allow: / User-agent: PerplexityBot Disallow: /cubetest Allow: / User-agent: Perplexity-User Disallow: /cubetest Allow: / User-agent: Google-Extended Disallow: /cubetest Allow: / User-agent: GoogleOther Disallow: /cubetest Allow: / User-agent: GoogleOther-Image Disallow: /cubetest Allow: / User-agent: Applebot Disallow: /cubetest Allow: / User-agent: Applebot-Extended Disallow: /cubetest Allow: / User-agent: bingbot Disallow: /cubetest Allow: / User-agent: Meta-ExternalAgent Disallow: /cubetest Allow: / User-agent: Meta-ExternalFetcher Disallow: /cubetest Allow: / User-agent: FacebookBot Disallow: /cubetest Allow: / User-agent: DuckAssistBot Disallow: /cubetest Allow: / User-agent: cohere-ai Disallow: /cubetest Allow: / User-agent: cohere-training-data-crawler Disallow: /cubetest Allow: / User-agent: xAI-Bot Disallow: /cubetest Allow: / User-agent: CCBot Disallow: /cubetest Allow: / User-agent: Diffbot Disallow: /cubetest Allow: / User-agent: omgili Disallow: /cubetest Allow: / User-agent: omgilibot Disallow: /cubetest Allow: / User-agent: ImagesiftBot Disallow: /cubetest Allow: / User-agent: Timpibot Disallow: /cubetest Allow: / User-agent: YouBot Disallow: /cubetest Allow: / User-agent: Kagibot Disallow: /cubetest Allow: / User-agent: Webzio-Extended Disallow: /cubetest Allow: / User-agent: Bytespider Disallow: /cubetest Allow: / Sitemap: https://cleenui.com/sitemap.xml