{"domain":"developers.cloudflare.com","name":"Cloudflare Docs","category":"developer","subcategory":"docs","country":"US","tags":[],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) has no rules for any AI bot we track. They are covered by its catch-all rule (<code>User-agent: *</code>), which lets them into the site.","computed_at":"2026-10-20T03:43:15.598Z","last_checked":"2026-10-19T13:38:31.305Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://developers.cloudflare.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://developers.cloudflare.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://developers.cloudflare.com/robots.txt","hops":[],"blob":"1d01a7ba34d49f51765541bf3391598f747c4c00be48bc79ddc088281ee01197","text":"# As a condition of accessing this website, you agree to\n# abide by the following content signals:\n\n# (a)  If a content-signal = yes, you may collect content\n# for the corresponding use.\n# (b)  If a content-signal = no, you may not collect content\n# for the corresponding use.\n# (c)  If the website operator does not include a content\n# signal for a corresponding use, the website operator\n# neither grants nor restricts permission via content signal\n# with respect to the corresponding use.\n\n# The content signals and their meanings are:\n\n# search: building a search index and providing search\n# results (e.g., returning hyperlinks and short excerpts\n# from your website's contents).  Search does not include\n# providing AI-generated search summaries.\n# ai-input: inputting content into one or more AI models\n# (e.g., retrieval augmented generation, grounding, or other\n# real-time taking of content for generative AI search\n# answers).\n# ai-train: training or fine-tuning AI models.\n\n# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS\n# RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN\n# UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS\n# IN THE DIGITAL SINGLE MARKET.\n\nUser-agent: *\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\n\nAllow: /\nDisallow: /client-ip-geolocation\nDisallow: /constellation\nDisallow: /cdn-cgi/\nDisallow: /email-security/\n\nSitemap: https://developers.cloudflare.com/sitemap-index.xml\n","refused":null,"warnings":[],"contentSignals":[{"value":"ai-train=yes, search=yes, ai-input=yes","signals":{"ai-train":"yes","search":"yes","ai-input":"yes"},"line":31,"agents":["*"]}],"contentUsage":[],"licenses":[],"aiComments":[{"line":18,"text":"providing AI-generated search summaries."},{"line":19,"text":"ai-input: inputting content into one or more AI models"},{"line":21,"text":"real-time taking of content for generative AI search"},{"line":23,"text":"ai-train: training or fine-tuning AI models."}],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*"],"lines":40},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":33},"any":{"allowed":true,"line":33},"search":{"allowed":true,"line":33},"cart":{"allowed":true,"line":33},"checkout":{"allowed":true,"line":33},"account":{"allowed":true,"line":33}},"disallows":4,"lines":[30,37],"groupRanges":[[30,37]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":["Cloudflare"],"botManagement":[{"vendor":"Cloudflare Bot Management","cookies":["__cf_bm"]}]},"llms":{"state":"present","http":200,"bytes":16787,"parsed":{"title":"Cloudflare Developer Documentation","summary":"Each product below links to its own llms.txt, which contains a full index of that product's documentation pages and is the recommended way to explore a specific product's content.","links":112,"sections":["Application performance","Application security","Cloudflare One","Consumer services","Core platform","Developer platform","Docs collections","Network security","Other"],"wellFormed":true},"url":"https://developers.cloudflare.com/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://developers.cloudflare.com/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://developers.cloudflare.com/.well-known/tdmrep.json"},"meta":{"title":"Cloudflare Developer Docs | Cloudflare Docs","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":["WebPage","Organization","ImageObject","PostalAddress","ContactPoint","WebSite"],"platform":null,"textRatio":0.046,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://www.cloudflare.com/policies/terms/","source":"discovered","skipped":"ToS hosted on another domain; not fetched in v0.1"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":44,"first":"2023-01-01","archive_points":43,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0}]},"events":[{"domain":"developers.cloudflare.com","name":"Cloudflare Docs","category":"developer","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-10-01"],"summary":"Added <code>Content-Signal: ai-train=yes, search=yes, ai-input=yes</code>.","dir":"other","up":[],"down":[],"major":false,"signals":true,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250801010335/https://developers.cloudflare.com/robots.txt","https://web.archive.org/web/20251001021308/https://developers.cloudflare.com/robots.txt"],"diff":[{"t":"add","s":"# As a condition of accessing this website, you agree to"},{"t":"add","s":"# abide by the following content signals:"},{"t":"add","s":""},{"t":"add","s":"# (a)  If a content-signal = yes, you may collect content"},{"t":"add","s":"# for the corresponding use."},{"t":"add","s":"# (b)  If a content-signal = no, you may not collect content"},{"t":"add","s":"# for the corresponding use."},{"t":"add","s":"# (c)  If the website operator does not include a content"},{"t":"add","s":"# signal for a corresponding use, the website operator"},{"t":"add","s":"# neither grants nor restricts permission via content signal"},{"t":"add","s":"# with respect to the corresponding use."},{"t":"add","s":""},{"t":"add","s":"# The content signals and their meanings are:"},{"t":"add","s":""},{"t":"add","s":"# search: building a search index and providing search"},{"t":"add","s":"# results (e.g., returning hyperlinks and short excerpts"},{"t":"add","s":"# from your website's contents).  Search does not include"},{"t":"add","s":"# providing AI-generated search summaries."},{"t":"add","s":"# ai-input: inputting content into one or more AI models"},{"t":"add","s":"# (e.g., retrieval augmented generation, grounding, or other"},{"t":"add","s":"# real-time taking of content for generative AI search"},{"t":"add","s":"# answers)."},{"t":"add","s":"# ai-train: training or fine-tuning AI models."},{"t":"add","s":""},{"t":"add","s":"# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS"},{"t":"add","s":"# RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN"},{"t":"add","s":"# UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS"},{"t":"add","s":"# IN THE DIGITAL SINGLE MARKET."},{"t":"add","s":""},{"t":"ctx","s":"User-agent: *"},{"t":"add","s":"Content-Signal: ai-train=yes, search=yes, ai-input=yes"},{"t":"add","s":""},{"t":"add","s":"Allow: /"},{"t":"ctx","s":"Disallow: /client-ip-geolocation"},{"t":"ctx","s":"Disallow: /plans/"},{"t":"add","s":"Disallow: /constellation"},{"t":"add","s":"Disallow: /cdn-cgi/"},{"t":"add","s":"Disallow: /email-security/"},{"t":"ctx","s":""},{"t":"del","s":"Sitemap: https://developers.cloudflare.com/sitemap.xml"},{"t":"add","s":"Sitemap: https://developers.cloudflare.com/sitemap-index.xml"},{"t":"ctx","s":""}]}]}