{"generated_at":"2026-10-20T03:43:15.760Z","run":{"id":"2026-10-19","started_at":"2026-10-19T13:23:06.249Z","finished_at":"2026-10-19T13:46:48.818Z","vantage":"residential connection, United States (manual run)","user_agent":"AgentAccessIndexBot/0.1 (+https://agent-access-bot.netlify.app/)","requests":1055,"sites_failed":7,"checker_version":"0.1.0"},"runs":["2026-10-19"],"history":{"from":"2023-01","to":"2026-10","source":"Internet Archive Wayback Machine, first capture per month"},"categories":{"retail":"Retail and marketplaces","travel":"Travel","finance":"Finance","local":"Local, listings and reviews","news":"News and publishing","social":"Social and forums","developer":"Developer and docs","reference":"Reference","government":"Government","healthcare":"Healthcare"},"agents":[{"token":"GPTBot","operator":"OpenAI","purpose":"training","major":true,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://developers.openai.com/api/docs/bots","robots_note":"\"Disallowing GPTBot indicates a site's content should not be used in training generative AI foundation models.\"","confidence":"high"},{"token":"ClaudeBot","operator":"Anthropic","purpose":"training","major":true,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","robots_note":"\"Anthropic's Bots respect 'do not crawl' signals by honoring industry standard directives in robots.txt.\"","confidence":"high"},{"token":"Google-Extended","operator":"Google","purpose":"training","major":true,"obeys_robots":"yes","control_token":true,"deprecated":false,"docs_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","robots_note":"A control token with no crawler of its own. Covers Gemini training and grounding. \"Google-Extended does not impact a site's inclusion in Google Search.\" It does not control AI Overviews or AI Mode.","confidence":"high"},{"token":"Applebot-Extended","operator":"Apple","purpose":"training","major":true,"obeys_robots":"yes","control_token":true,"deprecated":false,"docs_url":"https://support.apple.com/en-us/119829","robots_note":"\"Applebot-Extended does not crawl webpages.\" It only decides whether pages Applebot crawled may train Apple's models.","confidence":"high"},{"token":"CCBot","operator":"Common Crawl","purpose":"training","major":true,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://commoncrawl.org/ccbot","robots_note":"Builds an open web archive that is widely used downstream to train language models. Its FAQ says it checks robots.txt first.","confidence":"high"},{"token":"Meta-ExternalAgent","operator":"Meta","purpose":"training","major":true,"obeys_robots":"implied","control_token":false,"deprecated":false,"docs_url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","robots_note":"Crawls \"for use cases such as training foundation AI models\". Meta says to block it in robots.txt; it has no explicit compliance sentence.","confidence":"high"},{"token":"Bytespider","operator":"ByteDance","purpose":"training","major":false,"obeys_robots":"unverified","control_token":false,"deprecated":false,"docs_url":"","robots_note":"No operator documentation found. Widely seen in server logs; independent reports dispute that it follows robots.txt.","confidence":"low"},{"token":"MistralAI-Training","operator":"Mistral AI","purpose":"training","major":false,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://docs.mistral.ai/robots","robots_note":"Newly documented token. \"Webmasters can disallow this user agent in their robots.txt file.\"","confidence":"high"},{"token":"AI2Bot","operator":"Allen Institute for AI","purpose":"training","major":false,"obeys_robots":"unstated","control_token":false,"deprecated":false,"docs_url":"https://allenai.org/crawler","robots_note":"\"This web content is used to train open language models.\" No robots.txt statement.","confidence":"medium"},{"token":"anthropic-ai","operator":"Anthropic (legacy)","purpose":"training","major":false,"obeys_robots":"unknown","control_token":false,"deprecated":true,"docs_url":"","robots_note":"Legacy token no longer on Anthropic's crawler page. Tracked because many robots.txt files still name it.","confidence":"low"},{"token":"OAI-SearchBot","operator":"OpenAI","purpose":"search","major":true,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://developers.openai.com/api/docs/bots","robots_note":"Controls inclusion in ChatGPT search results; changes take about 24 hours.","confidence":"high"},{"token":"Claude-SearchBot","operator":"Anthropic","purpose":"search","major":true,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","robots_note":"\"Disabling Claude-SearchBot on your site prevents our system from indexing your content for search optimization.\"","confidence":"high"},{"token":"PerplexityBot","operator":"Perplexity","purpose":"search","major":true,"obeys_robots":"implied","control_token":false,"deprecated":false,"docs_url":"https://docs.perplexity.ai/guides/bots","robots_note":"\"It is not used to crawl content for AI foundation models.\"","confidence":"high"},{"token":"Amazonbot","operator":"Amazon","purpose":"search","major":false,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://developer.amazon.com/amazonbot","robots_note":"Mixed use: improves Amazon products, and content \"may be used to train Amazon AI models.\"","confidence":"high"},{"token":"Amzn-SearchBot","operator":"Amazon","purpose":"search","major":false,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://developer.amazon.com/amazonbot","robots_note":"\"Amzn-SearchBot does not crawl content for generative AI model training.\"","confidence":"high"},{"token":"DuckAssistBot","operator":"DuckDuckGo","purpose":"search","major":false,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot","robots_note":"Fetches pages for AI-assisted answers; \"not used in any way to train AI models.\"","confidence":"high"},{"token":"MistralAI-Index","operator":"Mistral AI","purpose":"search","major":false,"obeys_robots":"implied","control_token":false,"deprecated":false,"docs_url":"https://docs.mistral.ai/robots","robots_note":"Newly documented token for Mistral's search index.","confidence":"high"},{"token":"Meta-WebIndexer","operator":"Meta","purpose":"search","major":false,"obeys_robots":"implied","control_token":false,"deprecated":false,"docs_url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","robots_note":"\"Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.\"","confidence":"high"},{"token":"ChatGPT-User","operator":"OpenAI","purpose":"user_directed","major":true,"obeys_robots":"not_guaranteed","control_token":false,"deprecated":false,"docs_url":"https://developers.openai.com/api/docs/bots","robots_note":"\"Because these actions are initiated by a user, robots.txt rules may not apply.\"","confidence":"high"},{"token":"Claude-User","operator":"Anthropic","purpose":"user_directed","major":true,"obeys_robots":"yes","control_token":false,"deprecated":false,"docs_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","robots_note":"\"Claude-User allows site owners to control which sites can be accessed through these user-initiated requests.\"","confidence":"high"},{"token":"Perplexity-User","operator":"Perplexity","purpose":"user_directed","major":true,"obeys_robots":"no","control_token":false,"deprecated":false,"docs_url":"https://docs.perplexity.ai/guides/bots","robots_note":"\"Since a user requested the fetch, this fetcher generally ignores robots.txt rules.\"","confidence":"high"},{"token":"Google-Agent","operator":"Google","purpose":"user_directed","major":false,"obeys_robots":"no","control_token":false,"deprecated":false,"docs_url":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","robots_note":"\"Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.\" Google is testing Web Bot Auth signatures for it.","confidence":"high"},{"token":"Meta-ExternalFetcher","operator":"Meta","purpose":"user_directed","major":false,"obeys_robots":"not_guaranteed","control_token":false,"deprecated":false,"docs_url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","robots_note":"\"This crawler may bypass robots.txt rules.\"","confidence":"high"},{"token":"Amzn-User","operator":"Amazon","purpose":"user_directed","major":false,"obeys_robots":"not_guaranteed","control_token":false,"deprecated":false,"docs_url":"https://developer.amazon.com/amazonbot","robots_note":"\"Because actions taken by Amzn-User can be initiated by a user, it may not follow all robots.txt directives.\"","confidence":"high"},{"token":"MistralAI-User","operator":"Mistral AI","purpose":"user_directed","major":false,"obeys_robots":"implied","control_token":false,"deprecated":false,"docs_url":"https://docs.mistral.ai/robots","robots_note":"\"MistralAI-User governs which sites these user requests can be made to.\"","confidence":"high"}],"purposes":{"training":{"label":"AI training","short":"Train","blurb":"Collects pages to train AI models like ChatGPT and Claude. Blocking these keeps a site out of future training. It does not stop AI from reading the site to answer questions."},"search":{"label":"AI search","short":"Search","blurb":"Fetches pages so AI tools can quote and link to them in answers. Blocking these usually means the site stops showing up as a source."},"user_directed":{"label":"AI assistants","short":"Agents","blurb":"Visits a page the moment a person asks an AI to, for example to check a price or book a table. Several AI companies say these visits may ignore robots.txt."}},"unidentified":[{"name":"Grok (xAI)","note":"xAI publishes no crawler or agent token. Its hosted browser uses egress IP ranges that xAI discloses only through account teams."},{"name":"ChatGPT agent","note":"Uses a generic Chrome user agent; requests are signed with HTTP Message Signatures (Web Bot Auth) under signature agent https://chatgpt.com. No robots.txt token."},{"name":"Browser-based agents (Claude in Chrome, Comet, others)","note":"Run in the user's own browser, so sites see the user's network identity. robots.txt cannot address them."}],"deals":[{"operator":"OpenAI","org":"News Corp","domains":["wsj.com"],"deal":true,"announced":"2024-05-22","what":"Multi-year content partnership with News Corp (WSJ and others).","source_url":"https://newscorp.com/2024/05/22/news-corp-and-openai-sign-landmark-multi-year-global-partnership/","source_publisher":"News Corp","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"The Atlantic","domains":["theatlantic.com"],"deal":true,"announced":"2024-05-29","what":"Content and product partnership with The Atlantic.","source_url":"https://www.hollywoodreporter.com/business/business-news/vox-media-atlantic-strike-deals-with-openai-1235910777/","source_publisher":"The Hollywood Reporter","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"Financial Times","domains":["ft.com"],"deal":true,"announced":"2024-04-29","what":"Strategic partnership and licensing agreement with the FT.","source_url":"https://aboutus.ft.com/press_release/openai","source_publisher":"Financial Times","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"Vox Media","domains":["theverge.com"],"deal":true,"announced":"2024-05-29","what":"Content and product partnership with Vox Media (The Verge and others).","source_url":"https://www.hollywoodreporter.com/business/business-news/vox-media-atlantic-strike-deals-with-openai-1235910777/","source_publisher":"The Hollywood Reporter","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"Condé Nast","domains":["newyorker.com","wired.com"],"deal":true,"announced":"2024-08-20","what":"Multi-year partnership with Condé Nast (The New Yorker, WIRED and others).","source_url":"https://www.inpublishing.co.uk/articles/conde-nast-announces-openai-partnership-24468","source_publisher":"InPublishing","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"Axios","domains":["axios.com"],"deal":true,"announced":"2025-01-15","what":"Three-year partnership with Axios.","source_url":"https://cjr.org/feature/axios-wants-to-save-local-news-with-ai-jim-vandehei-tech-newsletters.php","source_publisher":"Columbia Journalism Review","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"The Washington Post","domains":["washingtonpost.com"],"deal":true,"announced":"2025-04-22","what":"Strategic partnership with The Washington Post.","source_url":"https://www.editorandpublisher.com/stories/the-washington-post-partners-with-openai-on-search-content,255404","source_publisher":"Editor & Publisher","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"Stack Overflow","domains":["stackoverflow.com"],"deal":true,"announced":"2024-05-06","what":"API partnership with Stack Overflow (OverflowAPI).","source_url":"https://stackoverflow.co/company/press/archive/openai-partnership","source_publisher":"Stack Overflow","checked_on":"2026-10-19"},{"operator":"OpenAI","org":"Reddit","domains":["reddit.com"],"deal":true,"announced":"2024-05-16","what":"Partnership giving OpenAI access to Reddit's Data API.","source_url":"https://redditinc.com/blog/reddit-and-oai-partner","source_publisher":"Reddit","checked_on":"2026-10-19","note":"Access is through Reddit's data API, not crawling; robots.txt still blocks every unnamed crawler."},{"operator":"OpenAI","org":"Automattic","domains":["tumblr.com"],"deal":"unverified","what":"Reported, not confirmed: 404 Media said Automattic (Tumblr and WordPress.com) was preparing to sell user data to OpenAI and Midjourney.","source_url":"https://www.404media.co/tumblr-and-wordpress-to-sell-users-data-to-train-ai-tools/","source_publisher":"404 Media","checked_on":"2026-10-19","reported":"2024-02-27"}],"deal_timeline":[{"domain":"wsj.com","name":"The Wall Street Journal","operator":"OpenAI","token":"GPTBot","org":"News Corp","announced":"2024-05-22","what":"Multi-year content partnership with News Corp (WSJ and others).","source_url":"https://newscorp.com/2024/05/22/news-corp-and-openai-sign-landmark-multi-year-global-partnership/","source_publisher":"News Corp","note":null,"lifted":{"date":"2024-06","precision":"month","between":["2024-05-01","2024-06-01"],"source":"archive"},"timing":"around","days_from_announcement":10,"now":{"access":"full","matched":"named"},"still_blocks":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"]},{"domain":"theatlantic.com","name":"The Atlantic","operator":"OpenAI","token":"GPTBot","org":"The Atlantic","announced":"2024-05-29","what":"Content and product partnership with The Atlantic.","source_url":"https://www.hollywoodreporter.com/business/business-news/vox-media-atlantic-strike-deals-with-openai-1235910777/","source_publisher":"The Hollywood Reporter","note":null,"lifted":{"date":"2024-07","precision":"month","between":["2024-05-01","2024-07-01"],"source":"archive"},"timing":"around","days_from_announcement":33,"now":{"access":"full","matched":"named"},"still_blocks":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","Claude-SearchBot","PerplexityBot","Amazonbot","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"]},{"domain":"ft.com","name":"Financial Times","operator":"OpenAI","token":"GPTBot","org":"Financial Times","announced":"2024-04-29","what":"Strategic partnership and licensing agreement with the FT.","source_url":"https://aboutus.ft.com/press_release/openai","source_publisher":"Financial Times","note":null,"lifted":{"date":"2024-08","precision":"month","between":["2024-07-01","2024-08-01"],"source":"archive"},"timing":"after","days_from_announcement":94,"now":{"access":"full","matched":"wildcard"},"still_blocks":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","PerplexityBot","Perplexity-User","Meta-ExternalFetcher"]},{"domain":"theverge.com","name":"The Verge","operator":"OpenAI","token":"GPTBot","org":"Vox Media","announced":"2024-05-29","what":"Content and product partnership with Vox Media (The Verge and others).","source_url":"https://www.hollywoodreporter.com/business/business-news/vox-media-atlantic-strike-deals-with-openai-1235910777/","source_publisher":"The Hollywood Reporter","note":null,"lifted":{"date":"2024-07","precision":"month","between":["2024-06-01","2024-07-01"],"source":"archive"},"timing":"after","days_from_announcement":33,"now":{"access":"full","matched":"named"},"still_blocks":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","AI2Bot","Claude-SearchBot","PerplexityBot","DuckAssistBot","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"]},{"domain":"newyorker.com","name":"The New Yorker","operator":"OpenAI","token":"GPTBot","org":"Condé Nast","announced":"2024-08-20","what":"Multi-year partnership with Condé Nast (The New Yorker, WIRED and others).","source_url":"https://www.inpublishing.co.uk/articles/conde-nast-announces-openai-partnership-24468","source_publisher":"InPublishing","note":null,"lifted":{"date":"2024-09","precision":"month","between":["2024-05-01","2024-09-01"],"source":"archive"},"timing":"around","days_from_announcement":12,"now":{"access":"full","matched":"wildcard"},"still_blocks":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","Claude-SearchBot","PerplexityBot","Amazonbot","DuckAssistBot","Meta-WebIndexer","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"]},{"domain":"wired.com","name":"WIRED","operator":"OpenAI","token":"GPTBot","org":"Condé Nast","announced":"2024-08-20","what":"Multi-year partnership with Condé Nast (The New Yorker, WIRED and others).","source_url":"https://www.inpublishing.co.uk/articles/conde-nast-announces-openai-partnership-24468","source_publisher":"InPublishing","note":null,"lifted":{"date":"2024-09","precision":"month","between":["2024-05-01","2024-09-01"],"source":"archive"},"timing":"around","days_from_announcement":12,"now":{"access":"full","matched":"wildcard"},"still_blocks":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","Claude-SearchBot","PerplexityBot","Amazonbot","DuckAssistBot","Meta-WebIndexer","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"]},{"domain":"axios.com","name":"Axios","operator":"OpenAI","token":"GPTBot","org":"Axios","announced":"2025-01-15","what":"Three-year partnership with Axios.","source_url":"https://cjr.org/feature/axios-wants-to-save-local-news-with-ai-jim-vandehei-tech-newsletters.php","source_publisher":"Columbia Journalism Review","note":null,"lifted":{"date":"2025-02","precision":"month","between":["2025-01-01","2025-02-01"],"source":"archive"},"timing":"around","days_from_announcement":17,"now":{"access":"full","matched":"wildcard"},"still_blocks":["CCBot","Bytespider","Amazonbot"]},{"domain":"washingtonpost.com","name":"The Washington Post","operator":"OpenAI","token":"GPTBot","org":"The Washington Post","announced":"2025-04-22","what":"Strategic partnership with The Washington Post.","source_url":"https://www.editorandpublisher.com/stories/the-washington-post-partners-with-openai-on-search-content,255404","source_publisher":"Editor & Publisher","note":null,"lifted":{"date":"2025-04","precision":"month","between":["2025-03-01","2025-04-01"],"source":"archive"},"timing":"before","days_from_announcement":-21,"now":{"access":"full","matched":"wildcard"},"still_blocks":["ClaudeBot","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","PerplexityBot","Amazonbot","Meta-ExternalFetcher"]},{"domain":"stackoverflow.com","name":"Stack Overflow","operator":"OpenAI","token":"GPTBot","org":"Stack Overflow","announced":"2024-05-06","what":"API partnership with Stack Overflow (OverflowAPI).","source_url":"https://stackoverflow.co/company/press/archive/openai-partnership","source_publisher":"Stack Overflow","note":null,"lifted":{"date":"2024-04","precision":"month","between":["2024-03-01","2024-04-01"],"source":"archive"},"timing":"before","days_from_announcement":-35,"now":{"access":"full","matched":"named"},"still_blocks":[]},{"domain":"reddit.com","name":"Reddit","operator":"OpenAI","token":"GPTBot","org":"Reddit","announced":"2024-05-16","what":"Partnership giving OpenAI access to Reddit's Data API.","source_url":"https://redditinc.com/blog/reddit-and-oai-partner","source_publisher":"Reddit","note":"Access is through Reddit's data API, not crawling; robots.txt still blocks every unnamed crawler.","lifted":null,"timing":null,"days_from_announcement":null,"now":{"access":"none","matched":"wildcard"},"still_blocks":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"]}],"counts":{"partial":61,"allowed":85,"unknown":10,"blocked":13,"paid":1},"lane_counts":{"training":{"mixed":35,"full":112,"unknown":11,"none":12},"search":{"mixed":34,"full":118,"unknown":11,"none":7},"user_directed":{"mixed":23,"full":126,"unknown":11,"none":10}},"sites":[{"domain":"amazon.com","name":"Amazon","category":"retail","subcategory":"marketplace","tags":["named-claim"],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":11,"named_ai":17,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:23:46.494Z","last_change":{"date":"2026-06","precision":"month","between":["2026-05-01","2026-06-07"],"dir":"up","summary":"Now blocked: Meta-WebIndexer.","provisional":false},"claim":true,"selective":null,"agents":["n1","n1","n1","f0","n1","n1","n1","f0","n1","f0","n1","n1","n1","f0","f0","n1","f0","n1","n1","n1","n1","f0","n1","f0","n1"]},{"domain":"walmart.com","name":"Walmart","category":"retail","subcategory":"big-box","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:24:08.913Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"target.com","name":"Target","category":"retail","subcategory":"big-box","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":["HUMAN (PerimeterX)"],"last_checked":"2026-10-19T13:23:43.332Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"bestbuy.com","name":"Best Buy","category":"retail","subcategory":"big-box","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":12,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:24:42.244Z","last_change":{"date":"2026-10","precision":"month","between":["2026-05-01","2026-10-19"],"dir":"other","summary":"Named GPTBot, ClaudeBot, Google-Extended, CCBot, Bytespider, anthropic-ai and 6 more explicitly, without changing what they may reach.","provisional":true},"claim":false,"selective":null,"agents":["f1","f1","f1","f0","f1","f0","f1","f0","f0","f1","f1","f0","f1","f1","f0","f0","f0","f0","f1","f1","f1","f0","f0","f0","f0"]},{"domain":"ebay.com","name":"eBay","category":"retail","subcategory":"marketplace","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":6,"named_ai":13,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:24:04.637Z","last_change":{"date":"2026-10","precision":"month","between":["2026-05-01","2026-10-19"],"dir":"other","summary":"Named OAI-SearchBot, Claude-SearchBot, ChatGPT-User and Claude-User explicitly, without changing what they may reach.","provisional":true},"claim":false,"selective":null,"agents":["n1","n1","f0","n1","n1","n1","n1","f0","f0","n1","f1","f1","n1","n1","f0","f0","f0","f0","f1","f1","f0","f0","f0","f0","f0"]},{"domain":"etsy.com","name":"Etsy","category":"retail","subcategory":"marketplace","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"DataDome challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["DataDome"],"last_checked":"2026-10-19T13:23:45.055Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"costco.com","name":"Costco","category":"retail","subcategory":"big-box","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:24:36.062Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"homedepot.com","name":"The Home Depot","category":"retail","subcategory":"big-box","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:24:31.533Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"lowes.com","name":"Lowe's","category":"retail","subcategory":"big-box","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"refused","door_detail":"a refusal page sent with HTTP 200 instead of an error code","door_reason":null,"robots_kind":"refused","robots_via":null,"checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:24:12.965Z","last_change":null,"claim":false,"selective":null,"agents":null},{"domain":"wayfair.com","name":"Wayfair","category":"retail","subcategory":"specialty","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["HUMAN (PerimeterX)"],"last_checked":"2026-10-19T13:24:43.244Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"macys.com","name":"Macy's","category":"retail","subcategory":"department","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":1,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:24:34.654Z","last_change":{"date":"2026-05","precision":"month","between":["2026-04-01","2026-05-01"],"dir":"other","summary":"Named ClaudeBot explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":["f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"nordstrom.com","name":"Nordstrom","category":"retail","subcategory":"department","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":2,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:24:52.835Z","last_change":{"date":"2026-10","precision":"month","between":["2026-05-01","2026-10-19"],"dir":"other","summary":"Named Meta-ExternalAgent and Meta-WebIndexer explicitly, without changing what they may reach.","provisional":true},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"ikea.com","name":"IKEA","category":"retail","subcategory":"specialty","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:25:20.292Z","last_change":{"date":"2025-09","precision":"month","between":["2025-08-01","2025-09-16"],"dir":"down","summary":"Now allowed: GPTBot, Google-Extended and CCBot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"shein.com","name":"Shein","category":"retail","subcategory":"marketplace","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:25:30.869Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"temu.com","name":"Temu","category":"retail","subcategory":"marketplace","tags":[],"status":"allowed","lanes":{"training":"mixed","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":9,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:25:24.252Z","last_change":{"date":"2025-09","precision":"month","between":["2025-08-01","2025-09-01"],"dir":"other","summary":"Named GPTBot, ClaudeBot, Google-Extended, OAI-SearchBot, Claude-SearchBot, PerplexityBot and 2 more explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":["f1","f1","f1","f0","f0","f0","n1","f0","f0","f0","f1","f1","f1","f0","f0","f0","f0","f0","f1","f1","f0","f0","f0","f0","f0"]},{"domain":"aliexpress.com","name":"AliExpress","category":"retail","subcategory":"marketplace","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":4,"named_ai":6,"door":"error","door_detail":"too many redirects","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:29.563Z","last_change":{"date":"2026-10","precision":"month","between":["2026-09-10","2026-10-19"],"dir":"up","summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, CCBot, Bytespider and Amazonbot.","provisional":true},"claim":false,"selective":null,"agents":["n1","n1","n1","f0","n1","f0","n1","f0","f0","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"chewy.com","name":"Chewy","category":"retail","subcategory":"specialty","tags":[],"status":"partial","lanes":{"training":"full","search":"mixed","user_directed":"full"},"blocked_major":0,"named_ai":1,"door":"refused","door_detail":"HTTP 429","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Kasada"],"last_checked":"2026-10-19T13:25:26.595Z","last_change":{"date":"2025-09","precision":"month","between":["2025-08-02","2025-09-16"],"dir":"up","summary":"Now blocked: Amazonbot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"nike.com","name":"Nike","category":"retail","subcategory":"brand","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:25:36.932Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"instacart.com","name":"Instacart","category":"retail","subcategory":"grocery","tags":[],"status":"partial","lanes":{"training":"none","search":"mixed","user_directed":"mixed"},"blocked_major":10,"named_ai":5,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:25:35.455Z","last_change":{"date":"2026-05","precision":"month","between":["2026-04-01","2026-05-01"],"dir":"other","summary":"Named Meta-ExternalAgent explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":["n0","n0","n0","n0","n0","n1","n0","n0","n0","n0","f1","n0","n1","n0","n0","n0","n0","n0","f1","n0","n1","n0","n0","n0","n0"]},{"domain":"kroger.com","name":"Kroger","category":"retail","subcategory":"grocery","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"unreachable","robots_via":null,"checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:16.767Z","last_change":null,"claim":false,"selective":null,"agents":null},{"domain":"sephora.com","name":"Sephora","category":"retail","subcategory":"specialty","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"refused","robots_via":null,"checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:25:52.831Z","last_change":null,"claim":false,"selective":null,"agents":null},{"domain":"zappos.com","name":"Zappos","category":"retail","subcategory":"specialty","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:15.426Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"allbirds.com","name":"Allbirds","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Shopify","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:13.271Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"gymshark.com","name":"Gymshark","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Shopify","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:14.096Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"brooklinen.com","name":"Brooklinen","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Shopify","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:39.770Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"rothys.com","name":"Rothy's","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Shopify","robots_platform":"Shopify default robots.txt","bot_mgmt":[],"last_checked":"2026-10-19T13:26:47.763Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"deathwishcoffee.com","name":"Death Wish Coffee","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Shopify","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:55.741Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"colourpop.com","name":"ColourPop","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Shopify","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:26:56.923Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"bombas.com","name":"Bombas","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 429","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":"Shopify default robots.txt","bot_mgmt":[],"last_checked":"2026-10-19T13:26:30.807Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"fashionnova.com","name":"Fashion Nova","category":"retail","subcategory":"independent","tags":["independent-store"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Shopify","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:27:16.927Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"delta.com","name":"Delta Air Lines","category":"travel","subcategory":"airline","tags":["named-claim"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:45:24.644Z","last_change":null,"claim":true,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"united.com","name":"United Airlines","category":"travel","subcategory":"airline","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"unreachable","robots_via":"archive","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:27:30.313Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"aa.com","name":"American Airlines","category":"travel","subcategory":"airline","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:27:23.858Z","last_change":{"date":"2025-08","precision":"month","between":["2025-07-03","2025-08-03"],"dir":"down","summary":"Now allowed: GPTBot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"southwest.com","name":"Southwest Airlines","category":"travel","subcategory":"airline","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:27:25.160Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"alaskaair.com","name":"Alaska Airlines","category":"travel","subcategory":"airline","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:27:52.775Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"jetblue.com","name":"JetBlue","category":"travel","subcategory":"airline","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:27:36.603Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"britishairways.com","name":"British Airways","category":"travel","subcategory":"airline","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"skipped","door_detail":null,"door_reason":"robots.txt unreachable; RFC 9309 says assume everything is disallowed","robots_kind":"unreachable","robots_via":"archive","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:46:05.933Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"lufthansa.com","name":"Lufthansa","category":"travel","subcategory":"airline","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:27:57.019Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"aircanada.com","name":"Air Canada","category":"travel","subcategory":"airline","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"absent","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:27:53.434Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"marriott.com","name":"Marriott","category":"travel","subcategory":"hotel","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:28:14.036Z","last_change":{"date":"2025-02","precision":"month","between":["2025-01-01","2025-02-01"],"dir":"down","summary":"Now allowed: GPTBot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"hilton.com","name":"Hilton","category":"travel","subcategory":"hotel","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:28:47.869Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"hyatt.com","name":"Hyatt","category":"travel","subcategory":"hotel","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"refused","robots_via":null,"checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:28:16.171Z","last_change":null,"claim":false,"selective":null,"agents":null},{"domain":"ihg.com","name":"IHG Hotels","category":"travel","subcategory":"hotel","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:28:04.466Z","last_change":{"date":"2025-11","precision":"month","between":["2025-10-09","2025-11-12"],"dir":"other","summary":"Stopped naming OAI-SearchBot, PerplexityBot and ChatGPT-User; access unchanged.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"airbnb.com","name":"Airbnb","category":"travel","subcategory":"lodging-marketplace","tags":[],"status":"partial","lanes":{"training":"mixed","search":"full","user_directed":"full"},"blocked_major":3,"named_ai":10,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:29:07.148Z","last_change":{"date":"2026-10","precision":"month","between":["2026-08-25","2026-10-01"],"dir":"up","summary":"Now blocked: GPTBot, ClaudeBot, Applebot-Extended, MistralAI-Training and AI2Bot.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"Meta","tokens":["Meta-ExternalAgent"]},{"operator":"Perplexity","tokens":["PerplexityBot"]}],"blocked_operators":["OpenAI","Anthropic","Apple","Mistral AI","Allen Institute for AI"],"blocked_agents":["GPTBot","ClaudeBot","Applebot-Extended","MistralAI-Training","AI2Bot"],"opened":null},"agents":["n1","n1","f0","n1","f0","f1","f0","n1","n1","f1","f1","f0","f1","f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0"]},{"domain":"booking.com","name":"Booking.com","category":"travel","subcategory":"ota","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 202","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:28:58.250Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"expedia.com","name":"Expedia","category":"travel","subcategory":"ota","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":6,"door":"refused","door_detail":"HTTP 429","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:28:42.838Z","last_change":{"date":"2025-09","precision":"month","between":["2025-08-01","2025-09-08"],"dir":"other","summary":"Named OAI-SearchBot, Claude-SearchBot, PerplexityBot, Claude-User and Perplexity-User explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f1","f1","f1","f0","f0","f0","f0","f0","f1","f1","f1","f0","f0","f0","f0"]},{"domain":"kayak.com","name":"KAYAK","category":"travel","subcategory":"metasearch","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":1,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:29:11.982Z","last_change":{"date":"2023-04","precision":"month","between":["2023-03-01","2023-04-01"],"dir":"other","summary":"Named ChatGPT-User explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0"]},{"domain":"tripadvisor.com","name":"Tripadvisor","category":"travel","subcategory":"reviews","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":6,"named_ai":11,"door":"challenged","door_detail":"DataDome challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["DataDome"],"last_checked":"2026-10-19T13:28:55.715Z","last_change":{"date":"2026-02","precision":"month","between":["2026-01-03","2026-02-01"],"dir":"up","summary":"Now blocked: CCBot, Meta-ExternalAgent and Amazonbot.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"Perplexity","tokens":["PerplexityBot"]}],"blocked_operators":["OpenAI","Anthropic","Google","Apple","Common Crawl","Meta","ByteDance","Amazon"],"blocked_agents":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","Amazonbot"],"opened":null},"agents":["n1","n1","n1","n1","n1","n1","n1","f0","f0","f0","f1","f0","f1","n1","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0"]},{"domain":"vrbo.com","name":"Vrbo","category":"travel","subcategory":"lodging-marketplace","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":6,"door":"refused","door_detail":"HTTP 429","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:29:22.549Z","last_change":{"date":"2025-09","precision":"month","between":["2025-08-01","2025-09-17"],"dir":"other","summary":"Named OAI-SearchBot, Claude-SearchBot, PerplexityBot, ChatGPT-User, Claude-User and Perplexity-User explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f1","f1","f1","f0","f0","f0","f0","f0","f1","f1","f1","f0","f0","f0","f0"]},{"domain":"hotels.com","name":"Hotels.com","category":"travel","subcategory":"ota","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"skipped","door_detail":null,"door_reason":"robots.txt unreachable; RFC 9309 says assume everything is disallowed","robots_kind":"unreachable","robots_via":null,"checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:46:08.377Z","last_change":{"date":"2026-03","precision":"month","between":["2026-02-01","2026-03-01"],"dir":"other","summary":"Named Perplexity-User explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":null},{"domain":"amtrak.com","name":"Amtrak","category":"travel","subcategory":"rail","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:29:48.302Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"chase.com","name":"JPMorgan Chase","category":"finance","subcategory":"bank","tags":["named-claim"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:29:43.977Z","last_change":null,"claim":true,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"schwab.com","name":"Charles Schwab","category":"finance","subcategory":"brokerage","tags":["named-claim"],"status":"blocked","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"refused","door_detail":"a refusal page sent with HTTP 200 instead of an error code","door_reason":null,"robots_kind":"refused","robots_via":null,"checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:46:48.818Z","last_change":null,"claim":true,"selective":null,"agents":null},{"domain":"bankofamerica.com","name":"Bank of America","category":"finance","subcategory":"bank","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:30:19.736Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"wellsfargo.com","name":"Wells Fargo","category":"finance","subcategory":"bank","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:30:00.179Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"citi.com","name":"Citi","category":"finance","subcategory":"bank","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:30:03.973Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"capitalone.com","name":"Capital One","category":"finance","subcategory":"bank","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:30:22.169Z","last_change":{"date":"2026-10","precision":"month","between":["2026-05-01","2026-10-19"],"dir":"other","summary":"Stopped naming ClaudeBot and Amazonbot; access unchanged.","provisional":true},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"americanexpress.com","name":"American Express","category":"finance","subcategory":"card","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:45:38.484Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"usbank.com","name":"U.S. Bank","category":"finance","subcategory":"bank","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:30:30.967Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"discover.com","name":"Discover","category":"finance","subcategory":"card","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:30:47.618Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"fidelity.com","name":"Fidelity","category":"finance","subcategory":"brokerage","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:30:51.546Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"vanguard.com","name":"Vanguard","category":"finance","subcategory":"brokerage","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:31:00.009Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"etrade.com","name":"E*TRADE","category":"finance","subcategory":"brokerage","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"unreachable","robots_via":null,"checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:31:10.783Z","last_change":null,"claim":false,"selective":null,"agents":null},{"domain":"robinhood.com","name":"Robinhood","category":"finance","subcategory":"brokerage","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:31:14.603Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"sofi.com","name":"SoFi","category":"finance","subcategory":"bank","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:31:11.047Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"paypal.com","name":"PayPal","category":"finance","subcategory":"payments","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:32:00.089Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"venmo.com","name":"Venmo","category":"finance","subcategory":"payments","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:31:30.582Z","last_change":{"date":"2024-07","precision":"month","between":["2024-06-01","2024-07-01"],"dir":"down","summary":"Now allowed: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"coinbase.com","name":"Coinbase","category":"finance","subcategory":"crypto","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:31:38.384Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"yelp.com","name":"Yelp","category":"local","subcategory":"reviews","tags":["named-claim"],"status":"paid","lanes":{"training":"none","search":"none","user_directed":"none"},"blocked_major":12,"named_ai":13,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:31:24.246Z","last_change":null,"claim":true,"selective":null,"agents":["n1","n1","n1","n0","n1","n1","n0","n0","n0","n1","n1","n1","n1","n0","n0","n0","n0","n0","n1","n1","n1","n0","n1","n0","n0"]},{"domain":"opentable.com","name":"OpenTable","category":"local","subcategory":"reservations","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"unreachable","robots_via":null,"checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:32:03.034Z","last_change":{"date":"2025-08","precision":"month","between":["2025-07-01","2025-08-01"],"dir":"up","summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 10 more.","provisional":false},"claim":false,"selective":null,"agents":null},{"domain":"doordash.com","name":"DoorDash","category":"local","subcategory":"delivery","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:31:40.736Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"ubereats.com","name":"Uber Eats","category":"local","subcategory":"delivery","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:31:50.712Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"grubhub.com","name":"Grubhub","category":"local","subcategory":"delivery","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:32:08.907Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"angi.com","name":"Angi","category":"local","subcategory":"services","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Generic bot challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:32:19.306Z","last_change":{"date":"2024-08","precision":"month","between":["2024-07-01","2024-08-01"],"dir":"down","summary":"Now allowed: CCBot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"thumbtack.com","name":"Thumbtack","category":"local","subcategory":"services","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:32:29.445Z","last_change":{"date":"2024-04","precision":"month","between":["2024-03-04","2024-04-01"],"dir":"down","summary":"Now allowed: Bytespider.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"nextdoor.com","name":"Nextdoor","category":"local","subcategory":"community","tags":[],"status":"blocked","lanes":{"training":"none","search":"none","user_directed":"none"},"blocked_major":12,"named_ai":1,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:31:57.259Z","last_change":{"date":"2023-09","precision":"month","between":["2023-08-01","2023-09-01"],"dir":"other","summary":"Named GPTBot explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":["n1","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0"]},{"domain":"zillow.com","name":"Zillow","category":"local","subcategory":"real-estate","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":2,"named_ai":4,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:32:47.220Z","last_change":{"date":"2026-04","precision":"month","between":["2026-03-01","2026-04-01"],"dir":"up","summary":"Now blocked: Applebot-Extended, Meta-ExternalAgent, AI2Bot and Amazonbot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","n1","f0","n1","f0","f0","n1","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"realtor.com","name":"Realtor.com","category":"local","subcategory":"real-estate","tags":[],"status":"partial","lanes":{"training":"mixed","search":"full","user_directed":"mixed"},"blocked_major":1,"named_ai":2,"door":"refused","door_detail":"HTTP 429","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Kasada"],"last_checked":"2026-10-19T13:32:42.332Z","last_change":{"date":"2024-11","precision":"month","between":["2024-10-01","2024-11-01"],"dir":"up","summary":"Now blocked: Meta-ExternalAgent and Meta-ExternalFetcher.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","n1","f0","f0"]},{"domain":"redfin.com","name":"Redfin","category":"local","subcategory":"real-estate","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:32:53.615Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"craigslist.org","name":"Craigslist","category":"local","subcategory":"classifieds","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:33:27.684Z","last_change":{"date":"2026-07","precision":"month","between":["2025-12-11","2026-07-04"],"dir":"down","summary":"Now allowed: Google-Extended, Meta-ExternalAgent and PerplexityBot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"ticketmaster.com","name":"Ticketmaster","category":"local","subcategory":"tickets","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:33:23.762Z","last_change":{"date":"2025-07","precision":"month","between":["2025-06-01","2025-07-02"],"dir":"down","summary":"Now allowed: GPTBot.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"eventbrite.com","name":"Eventbrite","category":"local","subcategory":"tickets","tags":[],"status":"partial","lanes":{"training":"mixed","search":"full","user_directed":"full"},"blocked_major":1,"named_ai":1,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:33:48.183Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"indeed.com","name":"Indeed","category":"local","subcategory":"jobs","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":20,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:33:24.994Z","last_change":{"date":"2026-09","precision":"month","between":["2026-04-03","2026-09-10"],"dir":"other","summary":"Named ClaudeBot, Meta-ExternalAgent, AI2Bot, Claude-SearchBot, PerplexityBot, DuckAssistBot and 5 more explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":["f1","f1","f1","f1","f1","f1","f1","f0","f1","f1","f1","f1","f1","f1","f0","f1","f0","f1","f1","f1","f1","f1","f1","f0","f0"]},{"domain":"glassdoor.com","name":"Glassdoor","category":"local","subcategory":"jobs","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":5,"named_ai":8,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:33:29.985Z","last_change":{"date":"2026-10","precision":"month","between":["2026-05-09","2026-10-19"],"dir":"up","summary":"Now blocked: ClaudeBot.","provisional":true},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","f0","n1","f0","f0","n1","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","subcategory":"national","tags":[],"status":"blocked","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":12,"named_ai":18,"door":"challenged","door_detail":"DataDome challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["DataDome"],"last_checked":"2026-10-19T13:33:35.820Z","last_change":{"date":"2026-04","precision":"month","between":["2026-03-01","2026-04-01"],"dir":"up","summary":"Now blocked: Meta-WebIndexer.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"Amazon","tokens":["Amazonbot"]}],"blocked_operators":["OpenAI","Anthropic","Google","Apple","Common Crawl","Meta","ByteDance","Perplexity","DuckDuckGo"],"blocked_agents":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","OAI-SearchBot","Claude-SearchBot","PerplexityBot","DuckAssistBot","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Meta-ExternalFetcher"],"opened":{"date":"2025-10","precision":"month","between":["2025-09-01","2025-10-01"],"source":"archive"}},"agents":["n1","n1","n1","n1","n1","n1","n1","f0","f0","n1","n1","n1","n1","f1","f0","n1","f0","n1","n1","n1","n1","f0","n1","f0","f0"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","subcategory":"national","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":5,"named_ai":9,"door":"error","door_detail":"TimeoutError","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:35:55.977Z","last_change":{"date":"2025-08","precision":"month","between":["2025-07-01","2025-08-03"],"dir":"down","summary":"Now allowed: Google-Extended.","provisional":false},"claim":false,"selective":null,"agents":["f0","n1","f0","n1","n1","n1","n1","f0","f0","n1","f0","f0","n1","n1","f0","f0","f0","f0","f0","f0","f0","f0","n1","f0","f0"]},{"domain":"wsj.com","name":"The Wall Street Journal","category":"news","subcategory":"business","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":9,"named_ai":3,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:33:39.337Z","last_change":{"date":"2026-06","precision":"month","between":["2026-05-01","2026-06-02"],"dir":"up","summary":"Now blocked: MistralAI-Training, AI2Bot, Claude-SearchBot, Amazonbot, Amzn-SearchBot, DuckAssistBot, MistralAI-Index, Meta-WebIndexer and 5 more.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"OpenAI","tokens":["GPTBot","OAI-SearchBot","ChatGPT-User"]}],"blocked_operators":["Anthropic","Google","Apple","Common Crawl","Meta","ByteDance","Mistral AI","Allen Institute for AI","Perplexity","Amazon","DuckDuckGo"],"blocked_agents":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"opened":{"date":"2024-06","precision":"month","between":["2024-05-01","2024-06-01"],"source":"archive"}},"agents":["f1","n0","n0","n0","n0","n0","n0","n0","n0","n0","f1","n0","n0","n0","n0","n0","n0","n0","f1","n0","n0","n0","n0","n0","n0"]},{"domain":"theguardian.com","name":"The Guardian","category":"news","subcategory":"national","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":7,"named_ai":12,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":true,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:34:20.753Z","last_change":null,"claim":false,"selective":null,"agents":["f0","n1","f0","n1","n1","n1","n1","f0","f0","n1","f0","n1","n1","n1","n1","n1","f0","f0","f0","n1","f0","f0","f0","f0","f0"]},{"domain":"bbc.com","name":"BBC","category":"news","subcategory":"broadcaster","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":10,"named_ai":15,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:34:17.699Z","last_change":{"date":"2026-01","precision":"month","between":["2025-12-01","2026-01-01"],"dir":"up","summary":"Now blocked: Amzn-SearchBot and Amzn-User.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","n1","n1","f0","f0","n1","n1","f0","n1","n1","n1","f0","f0","f0","n1","f0","n1","f0","f0","n1","f0"]},{"domain":"cnn.com","name":"CNN","category":"news","subcategory":"broadcaster","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":10,"named_ai":18,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:34:16.305Z","last_change":{"date":"2026-10","precision":"month","between":["2026-10-01","2026-10-19"],"dir":"down","summary":"Now allowed: Google-Extended.","provisional":true},"claim":false,"selective":null,"agents":["n1","n1","f0","n1","n1","f0","n1","f0","n1","n1","n1","n1","n1","n1","n1","n1","f0","f0","n1","n1","n1","f0","f0","n1","n1"]},{"domain":"reuters.com","name":"Reuters","category":"news","subcategory":"wire","tags":[],"status":"partial","lanes":{"training":"none","search":"mixed","user_directed":"mixed"},"blocked_major":10,"named_ai":5,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:33:53.304Z","last_change":{"date":"2026-09","precision":"month","between":["2026-08-05","2026-09-01"],"dir":"other","summary":"Named Applebot-Extended explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":["n0","n0","n0","n1","n0","n0","n0","n0","n0","n0","f1","n0","n0","n0","n0","n0","n0","f1","f1","n0","n0","n0","n0","f1","n0"]},{"domain":"apnews.com","name":"Associated Press","category":"news","subcategory":"wire","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":7,"named_ai":9,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:34:19.572Z","last_change":{"date":"2025-08","precision":"month","between":["2025-07-01","2025-08-01"],"dir":"up","summary":"Now blocked: Claude-SearchBot and Claude-User.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","f0","n1","n1","f0","f0","f0","f0","n1","f0","n1","n1","n1","f0","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0"]},{"domain":"bloomberg.com","name":"Bloomberg","category":"news","subcategory":"business","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":7,"named_ai":16,"door":"challenged","door_detail":"Generic bot challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["HUMAN (PerimeterX)"],"last_checked":"2026-10-19T13:34:31.057Z","last_change":{"date":"2026-10","precision":"month","between":["2026-09-11","2026-10-01"],"dir":"down","summary":"Now partly blocked: OAI-SearchBot, Claude-SearchBot, ChatGPT-User and Claude-User.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","n1","n1","f0","f0","n1","r1","r1","n1","n1","f0","n1","f0","f0","r1","r1","f0","f0","n1","f0","f0"]},{"domain":"ft.com","name":"Financial Times","category":"news","subcategory":"business","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":7,"named_ai":10,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:34:49.183Z","last_change":{"date":"2025-05","precision":"month","between":["2025-04-01","2025-05-01"],"dir":"up","summary":"Now blocked: Perplexity-User and Meta-ExternalFetcher.","provisional":false},"claim":false,"selective":null,"agents":["f0","n1","n1","n1","n1","n1","n1","f0","f0","n1","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","n1","f0","n1","f0","f0"]},{"domain":"theatlantic.com","name":"The Atlantic","category":"news","subcategory":"magazine","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":9,"named_ai":18,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":"search=yes, ai-input=no, ai-train=no","license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:34:50.496Z","last_change":{"date":"2026-04","precision":"month","between":["2026-02-01","2026-04-01"],"dir":"up","summary":"Now blocked: Meta-ExternalAgent, Claude-SearchBot, Claude-User, Meta-ExternalFetcher and MistralAI-User. Added <code>Content-Signal: search=yes, ai-input=no, ai-train=no</code>.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"OpenAI","tokens":["GPTBot","OAI-SearchBot","ChatGPT-User"]},{"operator":"DuckDuckGo","tokens":["DuckAssistBot"]}],"blocked_operators":["Anthropic","Google","Apple","Common Crawl","Meta","ByteDance","Mistral AI","Perplexity","Amazon"],"blocked_agents":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","Claude-SearchBot","PerplexityBot","Amazonbot","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"],"opened":{"date":"2024-07","precision":"month","between":["2024-05-01","2024-07-01"],"source":"archive"}},"agents":["f1","n1","n1","n1","n1","n1","n1","f0","f0","n1","f1","n1","n1","n1","f0","f1","f0","f0","f1","n1","n1","f0","n1","f0","n1"]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","subcategory":"magazine","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":9,"named_ai":15,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:34:54.039Z","last_change":{"date":"2025-10","precision":"month","between":["2025-09-01","2025-10-01"],"dir":"mixed","summary":"Now blocked: ClaudeBot, Claude-SearchBot, DuckAssistBot, Meta-WebIndexer, Claude-User, Perplexity-User, Meta-ExternalFetcher and MistralAI-User. Now allowed: anthropic-ai.","provisional":false},"claim":false,"selective":null,"agents":["f0","n1","n1","n1","n1","n1","n1","f0","f0","f0","f0","n1","n1","n1","f0","n1","f0","n1","f0","n1","n1","f0","n1","f0","n1"]},{"domain":"npr.org","name":"NPR","category":"news","subcategory":"broadcaster","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":10,"named_ai":12,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"unreachable","robots_via":"archive","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:35:11.010Z","last_change":{"date":"2026-10","precision":"month","between":["2026-09-01","2026-10-01"],"dir":"up","summary":"Now blocked: Claude-SearchBot and Claude-User.","provisional":true},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","f0","n1","f0","f0","n1","n1","n1","n1","f0","f0","f0","f0","f0","n1","n1","f0","f0","f0","f0","f0"]},{"domain":"usatoday.com","name":"USA Today","category":"news","subcategory":"national","tags":[],"status":"blocked","lanes":{"training":"mixed","search":"mixed","user_directed":"none"},"blocked_major":12,"named_ai":23,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:35:31.141Z","last_change":{"date":"2026-06","precision":"month","between":["2026-04-01","2026-06-30"],"dir":"up","summary":"Now blocked: Amzn-SearchBot, Meta-WebIndexer, Perplexity-User, Google-Agent and Amzn-User.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","n1","n1","f0","n1","n1","n1","n1","n1","n1","n1","n1","f0","n1","n1","n1","n1","n1","n1","n1","n1"]},{"domain":"foxnews.com","name":"Fox News","category":"news","subcategory":"broadcaster","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:35:28.772Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"axios.com","name":"Axios","category":"news","subcategory":"digital","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":1,"named_ai":4,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:35:27.350Z","last_change":{"date":"2026-07","precision":"month","between":["2026-05-01","2026-07-08"],"dir":"other","summary":"Named Google-Extended explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"Google","tokens":["Google-Extended"]}],"blocked_operators":["Common Crawl","ByteDance","Amazon"],"blocked_agents":["CCBot","Bytespider","Amazonbot"],"opened":{"date":"2026-03","precision":"month","between":["2026-02-01","2026-03-01"],"source":"archive"}},"agents":["f0","f0","f1","f0","n1","f0","n1","f0","f0","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"politico.com","name":"Politico","category":"news","subcategory":"digital","tags":[],"status":"blocked","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":12,"named_ai":19,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:35:14.736Z","last_change":{"date":"2025-09","precision":"month","between":["2025-08-01","2025-09-02"],"dir":"up","summary":"Now blocked: GPTBot, Google-Extended, Applebot-Extended, Meta-ExternalAgent, Bytespider, AI2Bot, anthropic-ai, OAI-SearchBot and 9 more.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","n1","n1","f0","n1","n1","n1","n1","n1","n1","f0","n1","f0","f0","n1","n1","n1","f0","n1","f0","n1"]},{"domain":"theverge.com","name":"The Verge","category":"news","subcategory":"digital","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":10,"named_ai":18,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"WordPress","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:09.735Z","last_change":{"date":"2026-06","precision":"month","between":["2026-05-01","2026-06-04"],"dir":"up","summary":"Now blocked: Meta-ExternalAgent, AI2Bot, Claude-SearchBot, DuckAssistBot, Claude-User, Perplexity-User, Meta-ExternalFetcher and MistralAI-User.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"Amazon","tokens":["Amazonbot"]}],"blocked_operators":["OpenAI","Anthropic","Google","Apple","Common Crawl","Meta","ByteDance","Mistral AI","Allen Institute for AI","Perplexity","DuckDuckGo"],"blocked_agents":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","AI2Bot","Claude-SearchBot","PerplexityBot","DuckAssistBot","ChatGPT-User","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"],"opened":{"date":"2025-05","precision":"month","between":["2025-04-01","2025-05-01"],"source":"archive"}},"agents":["f1","n1","n1","n1","n1","n1","n1","f0","n1","n1","f0","n1","n1","f1","f0","n1","f0","f0","n1","n1","n1","f0","n1","f0","n1"]},{"domain":"wired.com","name":"WIRED","category":"news","subcategory":"magazine","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":9,"named_ai":15,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:35:52.224Z","last_change":{"date":"2025-10","precision":"month","between":["2025-09-01","2025-10-01"],"dir":"mixed","summary":"Now blocked: ClaudeBot, Claude-SearchBot, DuckAssistBot, Meta-WebIndexer, Claude-User, Perplexity-User, Meta-ExternalFetcher and MistralAI-User. Now allowed: anthropic-ai.","provisional":false},"claim":false,"selective":null,"agents":["f0","n1","n1","n1","n1","n1","n1","f0","f0","f0","f0","n1","n1","n1","f0","n1","f0","n1","f0","n1","n1","f0","n1","f0","n1"]},{"domain":"economist.com","name":"The Economist","category":"news","subcategory":"magazine","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":11,"named_ai":13,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:36:07.042Z","last_change":{"date":"2026-08","precision":"month","between":["2026-07-12","2026-08-07"],"dir":"up","summary":"Now blocked: ClaudeBot, OAI-SearchBot, Claude-SearchBot and Claude-User.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","f0","n1","f0","f0","n1","n1","n1","n1","f0","f0","f0","f0","f0","n1","n1","n1","f0","f0","f0","f0"]},{"domain":"latimes.com","name":"Los Angeles Times","category":"news","subcategory":"regional","tags":[],"status":"partial","lanes":{"training":"mixed","search":"full","user_directed":"full"},"blocked_major":2,"named_ai":3,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:15.733Z","last_change":{"date":"2025-08","precision":"month","between":["2025-07-01","2025-08-01"],"dir":"up","summary":"Now blocked: ClaudeBot and Bytespider.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"medium.com","name":"Medium","category":"news","subcategory":"platform","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":4,"named_ai":6,"door":"challenged","door_detail":"Generic bot challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":true,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:04.444Z","last_change":{"date":"2025-09","precision":"month","between":["2025-08-01","2025-09-01"],"dir":"up","summary":"Now blocked: Meta-ExternalAgent and Amazonbot.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","f0","n1","f0","n1","n1","f0","f0","f0","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"substack.com","name":"Substack","category":"news","subcategory":"platform","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:36:27.626Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"reddit.com","name":"Reddit","category":"social","subcategory":"forum","tags":[],"status":"blocked","lanes":{"training":"none","search":"none","user_directed":"none"},"blocked_major":12,"named_ai":0,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:10.817Z","last_change":{"date":"2026-02","precision":"month","between":["2026-01-05","2026-02-01"],"dir":"up","summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","provisional":false},"claim":false,"selective":null,"agents":["n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0"]},{"domain":"x.com","name":"X","category":"social","subcategory":"network","tags":[],"status":"blocked","lanes":{"training":"none","search":"none","user_directed":"none"},"blocked_major":12,"named_ai":4,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:36:11.927Z","last_change":{"date":"2026-09","precision":"month","between":["2026-08-06","2026-09-03"],"dir":"other","summary":"Named Meta-ExternalAgent, Meta-WebIndexer and Meta-ExternalFetcher explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":["n0","n0","n1","n0","n0","n1","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n1","n0","n0","n0","n0","n1","n0","n0"]},{"domain":"facebook.com","name":"Facebook","category":"social","subcategory":"network","tags":[],"status":"blocked","lanes":{"training":"none","search":"none","user_directed":"none"},"blocked_major":12,"named_ai":6,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:14.355Z","last_change":{"date":"2026-10","precision":"month","between":["2026-10-01","2026-10-19"],"dir":"up","summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, PerplexityBot and Amazonbot.","provisional":true},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n0","n0","n0","n0","n0","n0","n0","n0","n1","n1","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0"]},{"domain":"instagram.com","name":"Instagram","category":"social","subcategory":"network","tags":[],"status":"blocked","lanes":{"training":"none","search":"none","user_directed":"none"},"blocked_major":12,"named_ai":6,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:23.998Z","last_change":{"date":"2026-04","precision":"month","between":["2025-01-01","2026-04-01"],"dir":"other","summary":"Named PerplexityBot explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n0","n0","n0","n0","n0","n0","n0","n0","n1","n1","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0"]},{"domain":"linkedin.com","name":"LinkedIn","category":"social","subcategory":"network","tags":[],"status":"partial","lanes":{"training":"none","search":"mixed","user_directed":"none"},"blocked_major":10,"named_ai":15,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:36:25.055Z","last_change":{"date":"2026-07","precision":"month","between":["2026-03-01","2026-07-02"],"dir":"mixed","summary":"Now blocked: Bytespider. Now allowed: Claude-SearchBot.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n0","n1","n1","n1","n0","n0","n1","f1","f1","n1","n0","n0","n1","n0","n0","n1","n1","n1","n0","n1","n0","n0"]},{"domain":"tiktok.com","name":"TikTok","category":"social","subcategory":"video","tags":[],"status":"blocked","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":12,"named_ai":17,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:37:16.218Z","last_change":{"date":"2025-10","precision":"month","between":["2025-09-02","2025-10-01"],"dir":"up","summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, AI2Bot, anthropic-ai and 8 more.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n1","n1","n1","f0","n1","n1","n1","n1","n1","f0","f0","n1","f0","f0","n1","n1","n1","f0","f0","f0","n1"]},{"domain":"youtube.com","name":"YouTube","category":"social","subcategory":"video","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:57.347Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"pinterest.com","name":"Pinterest","category":"social","subcategory":"network","tags":[],"status":"blocked","lanes":{"training":"none","search":"mixed","user_directed":"none"},"blocked_major":12,"named_ai":1,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:29.455Z","last_change":{"date":"2026-10","precision":"month","between":["2026-10-01","2026-10-19"],"dir":"up","summary":"Now blocked: Meta-ExternalFetcher.","provisional":true},"claim":false,"selective":{"favored":[{"operator":"DuckDuckGo","tokens":["DuckAssistBot"]}],"blocked_operators":["OpenAI","Anthropic","Google","Apple","Common Crawl","Meta","ByteDance","Mistral AI","Allen Institute for AI","Perplexity","Amazon"],"blocked_agents":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"opened":null},"agents":["n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","f1","n0","n0","n0","n0","n0","n0","n0","n0","n0"]},{"domain":"quora.com","name":"Quora","category":"social","subcategory":"forum","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"mixed"},"blocked_major":8,"named_ai":11,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:45.158Z","last_change":null,"claim":false,"selective":{"favored":[{"operator":"Google","tokens":["Google-Extended"]},{"operator":"Amazon","tokens":["Amazonbot"]}],"blocked_operators":["OpenAI","Anthropic","Apple","ByteDance","Perplexity"],"blocked_agents":["GPTBot","ClaudeBot","Applebot-Extended","Bytespider","OAI-SearchBot","Claude-SearchBot","PerplexityBot","ChatGPT-User","Claude-User"],"opened":null},"agents":["n1","n1","f1","n1","r0","r0","n1","r0","r0","r0","n1","n1","n1","f1","r0","r0","r0","r0","n1","n1","r0","r0","r0","r0","r0"]},{"domain":"tumblr.com","name":"Tumblr","category":"social","subcategory":"network","tags":[],"status":"partial","lanes":{"training":"mixed","search":"mixed","user_directed":"full"},"blocked_major":5,"named_ai":7,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"WordPress","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:37:22.105Z","last_change":{"date":"2024-09","precision":"month","between":["2024-08-01","2024-09-01"],"dir":"up","summary":"Now blocked: Meta-ExternalAgent.","provisional":false},"claim":false,"selective":null,"agents":["f0","n1","n1","n1","n1","n1","f0","f0","f0","n1","f0","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"discord.com","name":"Discord","category":"social","subcategory":"chat","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:37:10.012Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"threads.com","name":"Threads","category":"social","subcategory":"network","tags":[],"status":"blocked","lanes":{"training":"none","search":"none","user_directed":"none"},"blocked_major":12,"named_ai":6,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:36:44.033Z","last_change":{"date":"2025-05","precision":"month","between":["2024-09-01","2025-05-01"],"dir":"up","summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","n1","n1","n0","n0","n0","n0","n0","n0","n0","n0","n1","n1","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0"]},{"domain":"bsky.app","name":"Bluesky","category":"social","subcategory":"network","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:37:19.819Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","subcategory":"forum","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":4,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":"search=yes","license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:37:20.930Z","last_change":{"date":"2026-02","precision":"month","between":["2026-01-10","2026-02-13"],"dir":"other","summary":"Named GPTBot, Google-Extended, Amazonbot and ChatGPT-User explicitly, without changing what they may reach. Removed its RSL <code>License:</code> line.","provisional":false},"claim":false,"selective":null,"agents":["f1","f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0"]},{"domain":"twitch.tv","name":"Twitch","category":"social","subcategory":"video","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":3,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:37:40.229Z","last_change":{"date":"2026-07","precision":"month","between":["2026-05-02","2026-07-29"],"dir":"other","summary":"Named Amazonbot, Amzn-SearchBot and Amzn-User explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f1","f1","f0","f0","f0","f0","f0","f0","f0","f0","f1","f0"]},{"domain":"github.com","name":"GitHub","category":"developer","subcategory":"code-hosting","tags":[],"status":"allowed","lanes":{"training":"mixed","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":6,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:06.697Z","last_change":{"date":"2026-10","precision":"month","between":["2026-09-05","2026-10-01"],"dir":"up","summary":"Now blocked: Bytespider.","provisional":false},"claim":false,"selective":null,"agents":["f1","f1","f0","f0","f0","f0","n1","f0","f0","f1","f1","f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"gitlab.com","name":"GitLab","category":"developer","subcategory":"code-hosting","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:32.529Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"developer.mozilla.org","name":"MDN Web Docs","category":"developer","subcategory":"docs","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:01.581Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"npmjs.com","name":"npm","category":"developer","subcategory":"registry","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"refused","robots_via":null,"checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:37:49.045Z","last_change":null,"claim":false,"selective":null,"agents":null},{"domain":"pypi.org","name":"PyPI","category":"developer","subcategory":"registry","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:37:57.114Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"vercel.com","name":"Vercel","category":"developer","subcategory":"platform","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":"search=yes, ai-input=yes, ai-train=no","license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:14.263Z","last_change":{"date":"2026-05","precision":"month","between":["2026-04-01","2026-05-01"],"dir":"other","summary":"Added <code>Content-Signal: search=yes, ai-input=yes, ai-train=no</code>.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"docs.stripe.com","name":"Stripe Docs","category":"developer","subcategory":"docs","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":"ai-train=yes, search=yes, ai-input=yes","license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:23.570Z","last_change":{"date":"2026-05","precision":"month","between":["2026-04-02","2026-05-04"],"dir":"other","summary":"Added <code>Content-Signal: ai-train=yes, search=yes, ai-input=yes</code>.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"developers.cloudflare.com","name":"Cloudflare Docs","category":"developer","subcategory":"docs","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":"ai-train=yes, search=yes, ai-input=yes","license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:38:31.305Z","last_change":{"date":"2025-10","precision":"month","between":["2025-08-01","2025-10-01"],"dir":"other","summary":"Added <code>Content-Signal: ai-train=yes, search=yes, ai-input=yes</code>.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"supabase.com","name":"Supabase","category":"developer","subcategory":"platform","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":4,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":"ai-train=yes, search=yes, ai-input=yes","license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:36.231Z","last_change":{"date":"2026-05","precision":"month","between":["2026-04-01","2026-05-02"],"dir":"other","summary":"Named GPTBot, ClaudeBot, Google-Extended and PerplexityBot explicitly, without changing what they may reach. Added <code>Content-Signal: ai-train=yes, search=yes, ai-input=yes</code>.","provisional":false},"claim":false,"selective":null,"agents":["f1","f1","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"huggingface.co","name":"Hugging Face","category":"developer","subcategory":"platform","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:48.252Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"nextjs.org","name":"Next.js","category":"developer","subcategory":"docs","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:49.347Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"svelte.dev","name":"Svelte","category":"developer","subcategory":"docs","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:38:58.461Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"mintlify.com","name":"Mintlify","category":"developer","subcategory":"docs-platform","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:39:21.679Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"docker.com","name":"Docker","category":"developer","subcategory":"platform","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"WordPress","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:39:29.759Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"anthropic.com","name":"Anthropic","category":"developer","subcategory":"ai-lab","tags":["ai-operator"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:39:26.427Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"openai.com","name":"OpenAI","category":"developer","subcategory":"ai-lab","tags":["ai-operator"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:39:24.109Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"wikipedia.org","name":"Wikipedia","category":"reference","subcategory":"encyclopedia","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:39:46.816Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"britannica.com","name":"Britannica","category":"reference","subcategory":"encyclopedia","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:39:41.599Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"archive.org","name":"Internet Archive","category":"reference","subcategory":"library","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:39:57.511Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"imdb.com","name":"IMDb","category":"reference","subcategory":"entertainment","tags":[],"status":"blocked","lanes":{"training":"none","search":"mixed","user_directed":"mixed"},"blocked_major":12,"named_ai":4,"door":"skipped","door_detail":null,"door_reason":"Disallowed for AgentAccessIndexBot by robots.txt","robots_kind":"file","robots_via":"live","checker_allowed":false,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:39:38.069Z","last_change":{"date":"2026-09","precision":"month","between":["2026-05-01","2026-09-01"],"dir":"up","summary":"Now blocked: ClaudeBot, Applebot-Extended, Meta-ExternalAgent, Bytespider, MistralAI-Training, AI2Bot, OAI-SearchBot, Claude-SearchBot and 9 more.","provisional":false},"claim":false,"selective":{"favored":[{"operator":"Amazon","tokens":["Amazonbot","Amzn-SearchBot","Amzn-User"]},{"operator":"DuckDuckGo","tokens":["DuckAssistBot"]}],"blocked_operators":["OpenAI","Anthropic","Google","Apple","Common Crawl","Meta","ByteDance","Mistral AI","Allen Institute for AI","Perplexity"],"blocked_agents":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","OAI-SearchBot","Claude-SearchBot","PerplexityBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","MistralAI-User"],"opened":null},"agents":["n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","n0","f1","f1","f1","n0","n0","n0","n0","n0","n0","n0","f1","n0"]},{"domain":"merriam-webster.com","name":"Merriam-Webster","category":"reference","subcategory":"dictionary","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:40:08.294Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"fandom.com","name":"Fandom","category":"reference","subcategory":"wiki","tags":[],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"challenged","door_detail":"Cloudflare challenge page","door_reason":null,"robots_kind":"refused","robots_via":null,"checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:39:58.686Z","last_change":null,"claim":false,"selective":null,"agents":null},{"domain":"usa.gov","name":"USA.gov","category":"government","subcategory":"portal","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:40:21.681Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"irs.gov","name":"IRS","category":"government","subcategory":"tax","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:40:24.968Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"ssa.gov","name":"Social Security Administration","category":"government","subcategory":"benefits","tags":[],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:45:13.532Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"cdc.gov","name":"CDC","category":"government","subcategory":"health-agency","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:40:39.895Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"nih.gov","name":"NIH","category":"government","subcategory":"health-agency","tags":["healthcare"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Generic bot challenge page","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:40:27.509Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"medicare.gov","name":"Medicare.gov","category":"government","subcategory":"health-benefits","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:40:44.597Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"healthcare.gov","name":"HealthCare.gov","category":"government","subcategory":"health-benefits","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:40:50.852Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"gov.uk","name":"GOV.UK","category":"government","subcategory":"portal","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":1,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:41:11.266Z","last_change":{"date":"2026-05","precision":"month","between":["2026-04-01","2026-05-01"],"dir":"other","summary":"Named Meta-ExternalAgent explicitly, without changing what it may reach.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"canada.ca","name":"Canada.ca","category":"government","subcategory":"portal","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:41:53.251Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"ftc.gov","name":"FTC","category":"government","subcategory":"regulator","tags":[],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:41:10.041Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"mayoclinic.org","name":"Mayo Clinic","category":"healthcare","subcategory":"health-system","tags":["healthcare"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:41:08.575Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"clevelandclinic.org","name":"Cleveland Clinic","category":"healthcare","subcategory":"health-system","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:41:41.834Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"hopkinsmedicine.org","name":"Johns Hopkins Medicine","category":"healthcare","subcategory":"health-system","tags":["healthcare"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Generic bot challenge page","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:41:04.712Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"kp.org","name":"Kaiser Permanente","category":"healthcare","subcategory":"integrated","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:42:10.144Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"mountsinai.org","name":"Mount Sinai","category":"healthcare","subcategory":"health-system","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"html","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:45:44.049Z","last_change":{"date":"2026-10","precision":"month","between":["2026-09-29","2026-10-19"],"dir":"other","summary":"Stopped naming CCBot; access unchanged.","provisional":true},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"uhc.com","name":"UnitedHealthcare","category":"healthcare","subcategory":"insurer","tags":["healthcare"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"refused","door_detail":"HTTP 403","door_reason":null,"robots_kind":"refused","robots_via":"archive","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:41:37.503Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"aetna.com","name":"Aetna","category":"healthcare","subcategory":"insurer","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:42:00.919Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"cigna.com","name":"Cigna","category":"healthcare","subcategory":"insurer","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:42:10.253Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"anthem.com","name":"Anthem","category":"healthcare","subcategory":"insurer","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:42:26.669Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"humana.com","name":"Humana","category":"healthcare","subcategory":"insurer","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":3,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:45:46.268Z","last_change":{"date":"2026-10","precision":"month","between":["2026-10-06","2026-10-19"],"dir":"other","summary":"Named ClaudeBot, OAI-SearchBot and Claude-SearchBot explicitly, without changing what they may reach.","provisional":true},"claim":false,"selective":null,"agents":["f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f1","f1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"cvs.com","name":"CVS Health","category":"healthcare","subcategory":"pharmacy","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":6,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":true,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:42:41.970Z","last_change":{"date":"2024-05","precision":"month","between":["2024-04-01","2024-05-01"],"dir":"other","summary":"Named GPTBot, ClaudeBot, Google-Extended, CCBot, anthropic-ai and ChatGPT-User explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":["f1","f1","f1","f0","f1","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0","f0","f0","f1","f0","f0","f0","f0","f0","f0"]},{"domain":"walgreens.com","name":"Walgreens","category":"healthcare","subcategory":"pharmacy","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Akamai"],"last_checked":"2026-10-19T13:43:25.097Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","subcategory":"publisher","tags":["healthcare"],"status":"partial","lanes":{"training":"mixed","search":"full","user_directed":"full"},"blocked_major":3,"named_ai":3,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["Cloudflare Bot Management"],"last_checked":"2026-10-19T13:42:47.334Z","last_change":{"date":"2026-09","precision":"month","between":["2026-08-05","2026-09-03"],"dir":"down","summary":"Now allowed: ChatGPT-User.","provisional":false},"claim":false,"selective":null,"agents":["n1","n1","f0","f0","n1","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"goodrx.com","name":"GoodRx","category":"healthcare","subcategory":"pricing","tags":["healthcare"],"status":"partial","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"challenged","door_detail":"Generic bot challenge page","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:42:48.377Z","last_change":{"date":"2024-12","precision":"month","between":["2024-11-02","2024-12-02"],"dir":"down","summary":"Now allowed: GPTBot, Google-Extended, CCBot, Bytespider and anthropic-ai.","provisional":false},"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]},{"domain":"zocdoc.com","name":"Zocdoc","category":"healthcare","subcategory":"booking","tags":["healthcare"],"status":"unknown","lanes":{"training":"unknown","search":"unknown","user_directed":"unknown"},"blocked_major":0,"named_ai":null,"door":"challenged","door_detail":"DataDome challenge page","door_reason":null,"robots_kind":"refused","robots_via":null,"checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":null,"robots_platform":null,"bot_mgmt":["DataDome"],"last_checked":"2026-10-19T13:42:38.159Z","last_change":{"date":"2026-02","precision":"month","between":["2026-01-27","2026-02-25"],"dir":"other","summary":"Named ClaudeBot, Google-Extended, Claude-SearchBot, PerplexityBot, Claude-User and Perplexity-User explicitly, without changing what they may reach.","provisional":false},"claim":false,"selective":null,"agents":null},{"domain":"mychart.org","name":"MyChart (Epic)","category":"healthcare","subcategory":"portal","tags":["healthcare"],"status":"allowed","lanes":{"training":"full","search":"full","user_directed":"full"},"blocked_major":0,"named_ai":0,"door":"open","door_detail":"HTTP 200","door_reason":null,"robots_kind":"file","robots_via":"live","checker_allowed":true,"llms":false,"content_signal":null,"license":false,"platform":"Next.js","robots_platform":null,"bot_mgmt":[],"last_checked":"2026-10-19T13:42:58.152Z","last_change":null,"claim":false,"selective":null,"agents":["f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0","f0"]}]}