[{"domain":"airbnb.com","name":"Airbnb","category":"travel","date":"2026-10","precision":"month","source":"archive","provisional":false,"between":["2026-08-25","2026-10-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Applebot-Extended, MistralAI-Training and AI2Bot.","dir":"up","up":["GPTBot","ClaudeBot","Applebot-Extended","MistralAI-Training","AI2Bot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20260825094005/https://www.airbnb.com/robots.txt","https://web.archive.org/web/20261001122330/https://www.airbnb.com/robots.txt"]},{"domain":"aliexpress.com","name":"AliExpress","category":"retail","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-09-10","2026-10-19"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, CCBot, Bytespider and Amazonbot.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","CCBot","Bytespider","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20260910002748/https://www.aliexpress.com/robots.txt","https://www.aliexpress.com/robots.txt"]},{"domain":"bestbuy.com","name":"Best Buy","category":"retail","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-05-01","2026-10-19"],"summary":"Named GPTBot, ClaudeBot, Google-Extended, CCBot, Bytespider, anthropic-ai and 6 more explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260501111319/https://www.bestbuy.com/robots.txt","https://www.bestbuy.com/robots.txt"]},{"domain":"bloomberg.com","name":"Bloomberg","category":"news","date":"2026-10","precision":"month","source":"archive","provisional":false,"between":["2026-09-11","2026-10-01"],"summary":"Now partly blocked: OAI-SearchBot, Claude-SearchBot, ChatGPT-User and Claude-User.","dir":"down","up":[],"down":["OAI-SearchBot","Claude-SearchBot","ChatGPT-User","Claude-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260911092001/https://www.bloomberg.com/robots.txt","https://web.archive.org/web/20261001024331/https://www.bloomberg.com/robots.txt"]},{"domain":"capitalone.com","name":"Capital One","category":"finance","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-05-01","2026-10-19"],"summary":"Stopped naming ClaudeBot and Amazonbot; access unchanged.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260501221949/https://www.capitalone.com/robots.txt","https://www.capitalone.com/robots.txt"]},{"domain":"cnn.com","name":"CNN","category":"news","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-10-01","2026-10-19"],"summary":"Now allowed: Google-Extended.","dir":"down","up":[],"down":["Google-Extended"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20261001180945/https://www.cnn.com/robots.txt","https://www.cnn.com/robots.txt"]},{"domain":"ebay.com","name":"eBay","category":"retail","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-05-01","2026-10-19"],"summary":"Named OAI-SearchBot, Claude-SearchBot, ChatGPT-User and Claude-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260501163739/https://www.ebay.com/robots.txt","https://www.ebay.com/robots.txt"]},{"domain":"facebook.com","name":"Facebook","category":"social","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-10-01","2026-10-19"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, PerplexityBot and Amazonbot.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","PerplexityBot","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20261001201122/https://www.facebook.com/robots.txt","https://facebook.com/robots.txt"]},{"domain":"github.com","name":"GitHub","category":"developer","date":"2026-10","precision":"month","source":"archive","provisional":false,"between":["2026-09-05","2026-10-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260905135109/https://github.com/robots.txt","https://web.archive.org/web/20261001002701/https://github.com/robots.txt"]},{"domain":"glassdoor.com","name":"Glassdoor","category":"local","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-05-09","2026-10-19"],"summary":"Now blocked: ClaudeBot.","dir":"up","up":["ClaudeBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260509081437/https://www.glassdoor.com/robots.txt","https://www.glassdoor.com/robots.txt"]},{"domain":"humana.com","name":"Humana","category":"healthcare","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-10-06","2026-10-19"],"summary":"Named ClaudeBot, OAI-SearchBot and Claude-SearchBot explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20261006012446/https://www.humana.com/robots.txt","https://www.humana.com/robots.txt"]},{"domain":"mountsinai.org","name":"Mount Sinai","category":"healthcare","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-09-29","2026-10-19"],"summary":"Stopped naming CCBot; access unchanged.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260929031934/https://www.mountsinai.org/robots.txt","https://www.mountsinai.org/"]},{"domain":"nordstrom.com","name":"Nordstrom","category":"retail","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-05-01","2026-10-19"],"summary":"Named Meta-ExternalAgent and Meta-WebIndexer explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260501223326/https://www.nordstrom.com/robots.txt","https://www.nordstrom.com/robots.txt"]},{"domain":"npr.org","name":"NPR","category":"news","date":"2026-10","precision":"month","source":"archive","provisional":true,"between":["2026-09-01","2026-10-01"],"summary":"Now blocked: Claude-SearchBot and Claude-User.","dir":"up","up":["Claude-SearchBot","Claude-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260901065413/https://www.npr.org/robots.txt","https://web.archive.org/web/20261001001651/https://www.npr.org/robots.txt"]},{"domain":"pinterest.com","name":"Pinterest","category":"social","date":"2026-10","precision":"month","source":"archive+checker","provisional":true,"between":["2026-10-01","2026-10-19"],"summary":"Now blocked: Meta-ExternalFetcher.","dir":"up","up":["Meta-ExternalFetcher"],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20261001034116/https://www.pinterest.com/robots.txt","https://www.pinterest.com/robots.txt"]},{"domain":"imdb.com","name":"IMDb","category":"reference","date":"2026-09","precision":"month","source":"archive","provisional":false,"between":["2026-05-01","2026-09-01"],"summary":"Now blocked: ClaudeBot, Applebot-Extended, Meta-ExternalAgent, Bytespider, MistralAI-Training, AI2Bot, OAI-SearchBot, Claude-SearchBot and 9 more.","dir":"up","up":["ClaudeBot","Applebot-Extended","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","OAI-SearchBot","Claude-SearchBot","PerplexityBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20260501154213/https://www.imdb.com/robots.txt","https://web.archive.org/web/20260901222738/https://www.imdb.com/robots.txt"]},{"domain":"indeed.com","name":"Indeed","category":"local","date":"2026-09","precision":"month","source":"archive","provisional":false,"between":["2026-04-03","2026-09-10"],"summary":"Named ClaudeBot, Meta-ExternalAgent, AI2Bot, Claude-SearchBot, PerplexityBot, DuckAssistBot and 5 more explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260403031633/https://www.indeed.com/robots.txt","https://web.archive.org/web/20260910002742/https://www.indeed.com/robots.txt"]},{"domain":"reuters.com","name":"Reuters","category":"news","date":"2026-09","precision":"month","source":"archive","provisional":false,"between":["2026-08-05","2026-09-01"],"summary":"Named Applebot-Extended explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260805193147/https://www.reuters.com/robots.txt","https://web.archive.org/web/20260901092027/https://www.reuters.com/robots.txt"]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2026-09","precision":"month","source":"archive","provisional":false,"between":["2026-08-05","2026-09-03"],"summary":"Now allowed: ChatGPT-User.","dir":"down","up":[],"down":["ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260805083730/https://www.webmd.com/robots.txt","https://web.archive.org/web/20260903170921/https://www.webmd.com/robots.txt"]},{"domain":"x.com","name":"X","category":"social","date":"2026-09","precision":"month","source":"archive","provisional":false,"between":["2026-08-06","2026-09-03"],"summary":"Named Meta-ExternalAgent, Meta-WebIndexer and Meta-ExternalFetcher explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20260806020259/https://x.com/robots.txt","https://web.archive.org/web/20260903213435/https://x.com/robots.txt"]},{"domain":"economist.com","name":"The Economist","category":"news","date":"2026-08","precision":"month","source":"archive","provisional":false,"between":["2026-07-12","2026-08-07"],"summary":"Now blocked: ClaudeBot, OAI-SearchBot, Claude-SearchBot and Claude-User.","dir":"up","up":["ClaudeBot","OAI-SearchBot","Claude-SearchBot","Claude-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260712121601/https://www.economist.com/robots.txt","https://web.archive.org/web/20260807091347/https://www.economist.com/robots.txt"]},{"domain":"facebook.com","name":"Facebook","category":"social","date":"2026-08","precision":"month","source":"archive","provisional":false,"between":["2026-07-29","2026-08-03"],"summary":"Now allowed: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, PerplexityBot and Amazonbot.","dir":"down","up":[],"down":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","PerplexityBot","Amazonbot"],"major":true,"signals":false,"status_from":"blocked","status_to":"partial","evidence":["https://web.archive.org/web/20260729020032/https://www.facebook.com/robots.txt","https://web.archive.org/web/20260803020630/https://facebook.com/robots.txt"]},{"domain":"axios.com","name":"Axios","category":"news","date":"2026-07","precision":"month","source":"archive","provisional":false,"between":["2026-05-01","2026-07-08"],"summary":"Named Google-Extended explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260501023028/https://www.axios.com/robots.txt","https://web.archive.org/web/20260708091533/https://www.axios.com/robots.txt"]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2026-07","precision":"month","source":"archive","provisional":false,"between":["2025-12-11","2026-07-04"],"summary":"Now allowed: Google-Extended, Meta-ExternalAgent and PerplexityBot.","dir":"down","up":[],"down":["Google-Extended","Meta-ExternalAgent","PerplexityBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20251211144839/https://craigslist.org/robots.txt","https://web.archive.org/web/20260704193624/https://www.craigslist.org/robots.txt"]},{"domain":"linkedin.com","name":"LinkedIn","category":"social","date":"2026-07","precision":"month","source":"archive","provisional":false,"between":["2026-03-01","2026-07-02"],"summary":"Now blocked: Bytespider. Now allowed: Claude-SearchBot.","dir":"mixed","up":["Bytespider"],"down":["Claude-SearchBot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260301000310/https://www.linkedin.com/robots.txt","https://web.archive.org/web/20260702233947/https://www.linkedin.com/robots.txt"]},{"domain":"twitch.tv","name":"Twitch","category":"social","date":"2026-07","precision":"month","source":"archive","provisional":false,"between":["2026-05-02","2026-07-29"],"summary":"Named Amazonbot, Amzn-SearchBot and Amzn-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260502134429/https://www.twitch.tv/robots.txt","https://web.archive.org/web/20260729120413/https://www.twitch.tv/robots.txt"]},{"domain":"amazon.com","name":"Amazon","category":"retail","date":"2026-06","precision":"month","source":"archive","provisional":false,"between":["2026-05-01","2026-06-07"],"summary":"Now blocked: Meta-WebIndexer.","dir":"up","up":["Meta-WebIndexer"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260501162735/https://www.amazon.com/robots.txt","https://web.archive.org/web/20260607202731/https://www.amazon.com/robots.txt"]},{"domain":"bloomberg.com","name":"Bloomberg","category":"news","date":"2026-06","precision":"month","source":"archive","provisional":false,"between":["2026-05-01","2026-06-02"],"summary":"Now blocked: ClaudeBot, Applebot-Extended, OAI-SearchBot, Claude-SearchBot, DuckAssistBot, Claude-User and Meta-ExternalFetcher.","dir":"up","up":["ClaudeBot","Applebot-Extended","OAI-SearchBot","Claude-SearchBot","DuckAssistBot","Claude-User","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260501225650/https://www.bloomberg.com/robots.txt","https://web.archive.org/web/20260602050346/https://www.bloomberg.com/robots.txt"]},{"domain":"cnn.com","name":"CNN","category":"news","date":"2026-06","precision":"month","source":"archive","provisional":false,"between":["2026-03-01","2026-06-02"],"summary":"Now blocked: Claude-SearchBot, Claude-User, Perplexity-User and MistralAI-User.","dir":"up","up":["Claude-SearchBot","Claude-User","Perplexity-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260301162736/https://www.cnn.com/robots.txt","https://web.archive.org/web/20260602140226/https://www.cnn.com/robots.txt"]},{"domain":"reuters.com","name":"Reuters","category":"news","date":"2026-06","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-06-05"],"summary":"Now blocked: GPTBot, MistralAI-Training, Amazonbot, Amzn-SearchBot, DuckAssistBot, MistralAI-Index, Claude-User, Perplexity-User and 1 more.","dir":"up","up":["GPTBot","MistralAI-Training","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Claude-User","Perplexity-User","Google-Agent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260401002117/https://www.reuters.com/robots.txt","https://web.archive.org/web/20260605150558/https://www.reuters.com/robots.txt"]},{"domain":"theverge.com","name":"The Verge","category":"news","date":"2026-06","precision":"month","source":"archive","provisional":false,"between":["2026-05-01","2026-06-04"],"summary":"Now blocked: Meta-ExternalAgent, AI2Bot, Claude-SearchBot, DuckAssistBot, Claude-User, Perplexity-User, Meta-ExternalFetcher and MistralAI-User.","dir":"up","up":["Meta-ExternalAgent","AI2Bot","Claude-SearchBot","DuckAssistBot","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260501002446/https://www.theverge.com/robots.txt","https://web.archive.org/web/20260604075151/https://www.theverge.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2026-06","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-06-30"],"summary":"Now blocked: Amzn-SearchBot, Meta-WebIndexer, Perplexity-User, Google-Agent and Amzn-User.","dir":"up","up":["Amzn-SearchBot","Meta-WebIndexer","Perplexity-User","Google-Agent","Amzn-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20260401225033/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20260630204214/https://www.usatoday.com/robots.txt"]},{"domain":"wsj.com","name":"The Wall Street Journal","category":"news","date":"2026-06","precision":"month","source":"archive","provisional":false,"between":["2026-05-01","2026-06-02"],"summary":"Now blocked: MistralAI-Training, AI2Bot, Claude-SearchBot, Amazonbot, Amzn-SearchBot, DuckAssistBot, MistralAI-Index, Meta-WebIndexer and 5 more.","dir":"up","up":["MistralAI-Training","AI2Bot","Claude-SearchBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","Claude-User","Perplexity-User","Google-Agent","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260501000338/https://www.wsj.com/robots.txt","https://web.archive.org/web/20260602153104/https://www.wsj.com/robots.txt"]},{"domain":"docs.stripe.com","name":"Stripe Docs","category":"developer","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-04-02","2026-05-04"],"summary":"Added <code>Content-Signal: ai-train=yes, search=yes, ai-input=yes</code>.","dir":"other","up":[],"down":[],"major":false,"signals":true,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260402073054/https://docs.stripe.com/robots.txt","https://web.archive.org/web/20260504043220/https://docs.stripe.com/robots.txt"]},{"domain":"gov.uk","name":"GOV.UK","category":"government","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-05-01"],"summary":"Named Meta-ExternalAgent explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260401012736/https://www.gov.uk/robots.txt","https://web.archive.org/web/20260501154846/https://www.gov.uk/robots.txt"]},{"domain":"instacart.com","name":"Instacart","category":"retail","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-05-01"],"summary":"Named Meta-ExternalAgent explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260401020105/https://www.instacart.com/robots.txt","https://web.archive.org/web/20260501063612/https://www.instacart.com/robots.txt"]},{"domain":"macys.com","name":"Macy's","category":"retail","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-05-01"],"summary":"Named ClaudeBot explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260401000637/https://www.macys.com/robots.txt","https://web.archive.org/web/20260501001246/https://www.macys.com/robots.txt"]},{"domain":"pinterest.com","name":"Pinterest","category":"social","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-05-01"],"summary":"Now blocked: MistralAI-Training, AI2Bot, OAI-SearchBot, Claude-SearchBot, Amazonbot, Amzn-SearchBot, MistralAI-Index, Meta-WebIndexer and 5 more.","dir":"up","up":["MistralAI-Training","AI2Bot","OAI-SearchBot","Claude-SearchBot","Amazonbot","Amzn-SearchBot","MistralAI-Index","Meta-WebIndexer","Claude-User","Perplexity-User","Google-Agent","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20260201000239/https://www.pinterest.com/robots.txt","https://web.archive.org/web/20260501000210/https://www.pinterest.com/robots.txt"]},{"domain":"supabase.com","name":"Supabase","category":"developer","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-05-02"],"summary":"Named GPTBot, ClaudeBot, Google-Extended and PerplexityBot explicitly, without changing what they may reach. Added <code>Content-Signal: ai-train=yes, search=yes, ai-input=yes</code>.","dir":"other","up":[],"down":[],"major":false,"signals":true,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260401170539/https://supabase.com/robots.txt","https://web.archive.org/web/20260502005637/https://supabase.com/robots.txt"]},{"domain":"vercel.com","name":"Vercel","category":"developer","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-05-01"],"summary":"Added <code>Content-Signal: search=yes, ai-input=yes, ai-train=no</code>.","dir":"other","up":[],"down":[],"major":false,"signals":true,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260401003853/https://vercel.com/robots.txt","https://web.archive.org/web/20260501112444/https://vercel.com/robots.txt"]},{"domain":"ebay.com","name":"eBay","category":"retail","date":"2026-04","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-04-01"],"summary":"Now blocked: GPTBot, Applebot-Extended and Meta-ExternalAgent.","dir":"up","up":["GPTBot","Applebot-Extended","Meta-ExternalAgent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260201000400/https://www.ebay.com/robots.txt","https://web.archive.org/web/20260401000450/https://www.ebay.com/robots.txt"]},{"domain":"instagram.com","name":"Instagram","category":"social","date":"2026-04","precision":"month","source":"archive","provisional":false,"between":["2025-01-01","2026-04-01"],"summary":"Named PerplexityBot explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20250101000504/https://www.instagram.com/robots.txt","https://web.archive.org/web/20260401004226/https://www.instagram.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2026-04","precision":"month","source":"archive","provisional":false,"between":["2026-03-01","2026-04-01"],"summary":"Now blocked: Meta-WebIndexer.","dir":"up","up":["Meta-WebIndexer"],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20260301002328/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20260401001930/https://www.nytimes.com/robots.txt"]},{"domain":"theatlantic.com","name":"The Atlantic","category":"news","date":"2026-04","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-04-01"],"summary":"Now blocked: Meta-ExternalAgent, Claude-SearchBot, Claude-User, Meta-ExternalFetcher and MistralAI-User. Added <code>Content-Signal: search=yes, ai-input=no, ai-train=no</code>.","dir":"up","up":["Meta-ExternalAgent","Claude-SearchBot","Claude-User","Meta-ExternalFetcher","MistralAI-User"],"down":[],"major":true,"signals":true,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260201014820/https://www.theatlantic.com/robots.txt","https://web.archive.org/web/20260401001519/https://www.theatlantic.com/robots.txt"]},{"domain":"zillow.com","name":"Zillow","category":"local","date":"2026-04","precision":"month","source":"archive","provisional":false,"between":["2026-03-01","2026-04-01"],"summary":"Now blocked: Applebot-Extended, Meta-ExternalAgent, AI2Bot and Amazonbot.","dir":"up","up":["Applebot-Extended","Meta-ExternalAgent","AI2Bot","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20260301055249/https://www.zillow.com/robots.txt","https://web.archive.org/web/20260401045403/https://www.zillow.com/robots.txt"]},{"domain":"axios.com","name":"Axios","category":"news","date":"2026-03","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-03-01"],"summary":"Now allowed: ClaudeBot and Google-Extended.","dir":"down","up":[],"down":["ClaudeBot","Google-Extended"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260201030641/https://www.axios.com/robots.txt","https://web.archive.org/web/20260301010704/https://www.axios.com/robots.txt"]},{"domain":"hotels.com","name":"Hotels.com","category":"travel","date":"2026-03","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-03-01"],"summary":"Named Perplexity-User explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260201132410/https://www.hotels.com/robots.txt","https://web.archive.org/web/20260301174150/https://www.hotels.com/robots.txt"]},{"domain":"instacart.com","name":"Instacart","category":"retail","date":"2026-03","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-03-01"],"summary":"Now blocked: PerplexityBot.","dir":"up","up":["PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260201013900/https://www.instacart.com/robots.txt","https://web.archive.org/web/20260301222437/https://www.instacart.com/robots.txt"]},{"domain":"reuters.com","name":"Reuters","category":"news","date":"2026-03","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-03-01"],"summary":"Now blocked: AI2Bot and MistralAI-User. Now allowed: Claude-User.","dir":"mixed","up":["AI2Bot","MistralAI-User"],"down":["Claude-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260201001928/https://www.reuters.com/robots.txt","https://web.archive.org/web/20260301002752/https://www.reuters.com/robots.txt"]},{"domain":"reddit.com","name":"Reddit","category":"social","date":"2026-02","precision":"month","source":"archive","provisional":false,"between":["2026-01-05","2026-02-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20260105222836/https://www.reddit.com/robots.txt","https://web.archive.org/web/20260201182700/https://www.reddit.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2026-02","precision":"month","source":"archive","provisional":false,"between":["2026-01-10","2026-02-13"],"summary":"Named GPTBot, Google-Extended, Amazonbot and ChatGPT-User explicitly, without changing what they may reach. Removed its RSL <code>License:</code> line.","dir":"other","up":[],"down":[],"major":false,"signals":true,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260110004931/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20260213180442/https://stackoverflow.com/robots.txt"]},{"domain":"tripadvisor.com","name":"Tripadvisor","category":"travel","date":"2026-02","precision":"month","source":"archive","provisional":false,"between":["2026-01-03","2026-02-01"],"summary":"Now blocked: CCBot, Meta-ExternalAgent and Amazonbot.","dir":"up","up":["CCBot","Meta-ExternalAgent","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260103170415/https://www.tripadvisor.com/robots.txt","https://web.archive.org/web/20260201005843/https://www.tripadvisor.com/robots.txt"]},{"domain":"zocdoc.com","name":"Zocdoc","category":"healthcare","date":"2026-02","precision":"month","source":"archive","provisional":false,"between":["2026-01-27","2026-02-25"],"summary":"Named ClaudeBot, Google-Extended, Claude-SearchBot, PerplexityBot, Claude-User and Perplexity-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260127143309/https://www.zocdoc.com/robots.txt","https://web.archive.org/web/20260225234728/https://www.zocdoc.com/robots.txt"]},{"domain":"bbc.com","name":"BBC","category":"news","date":"2026-01","precision":"month","source":"archive","provisional":false,"between":["2025-12-01","2026-01-01"],"summary":"Now blocked: Amzn-SearchBot and Amzn-User.","dir":"up","up":["Amzn-SearchBot","Amzn-User"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20251201150544/https://www.bbc.com/robots.txt","https://web.archive.org/web/20260101012945/https://www.bbc.com/robots.txt"]},{"domain":"cnn.com","name":"CNN","category":"news","date":"2026-01","precision":"month","source":"archive","provisional":false,"between":["2025-10-01","2026-01-02"],"summary":"Now blocked: Amzn-SearchBot and Amzn-User. Now allowed: Meta-ExternalAgent and Meta-ExternalFetcher.","dir":"mixed","up":["Amzn-SearchBot","Amzn-User"],"down":["Meta-ExternalAgent","Meta-ExternalFetcher"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20251001121744/https://www.cnn.com/robots.txt","https://web.archive.org/web/20260102174350/https://www.cnn.com/robots.txt"]},{"domain":"ebay.com","name":"eBay","category":"retail","date":"2026-01","precision":"month","source":"archive","provisional":false,"between":["2025-12-02","2026-01-01"],"summary":"Now blocked: ClaudeBot, CCBot, anthropic-ai, PerplexityBot and Amazonbot.","dir":"up","up":["ClaudeBot","CCBot","anthropic-ai","PerplexityBot","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20251202053728/https://www.ebay.com/robots.txt","https://web.archive.org/web/20260101005600/https://www.ebay.com/robots.txt"]},{"domain":"amazon.com","name":"Amazon","category":"retail","date":"2025-12","precision":"month","source":"archive","provisional":false,"between":["2025-10-01","2025-12-01"],"summary":"Now blocked: OAI-SearchBot, Claude-SearchBot, ChatGPT-User, Claude-User, Perplexity-User and Meta-ExternalFetcher.","dir":"up","up":["OAI-SearchBot","Claude-SearchBot","ChatGPT-User","Claude-User","Perplexity-User","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20251001000703/https://www.amazon.com/robots.txt","https://web.archive.org/web/20251201130651/https://www.amazon.com/robots.txt"]},{"domain":"linkedin.com","name":"LinkedIn","category":"social","date":"2025-12","precision":"month","source":"archive","provisional":false,"between":["2025-11-01","2025-12-01"],"summary":"Now allowed: OAI-SearchBot.","dir":"down","up":[],"down":["OAI-SearchBot"],"major":true,"signals":false,"status_from":"blocked","status_to":"partial","evidence":["https://web.archive.org/web/20251101023240/https://www.linkedin.com/robots.txt","https://web.archive.org/web/20251201035433/https://www.linkedin.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2025-12","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-12-13"],"summary":"Added <code>Content-Signal: search=yes</code>. Added an RSL <code>License:</code> line pointing to licensing terms.","dir":"other","up":[],"down":[],"major":false,"signals":true,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250801163142/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20251213134544/https://stackoverflow.com/robots.txt"]},{"domain":"ihg.com","name":"IHG Hotels","category":"travel","date":"2025-11","precision":"month","source":"archive","provisional":false,"between":["2025-10-09","2025-11-12"],"summary":"Stopped naming OAI-SearchBot, PerplexityBot and ChatGPT-User; access unchanged.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20251009051056/https://www.ihg.com/robots.txt","https://web.archive.org/web/20251112165854/https://www.ihg.com/robots.txt"]},{"domain":"zocdoc.com","name":"Zocdoc","category":"healthcare","date":"2025-11","precision":"month","source":"archive","provisional":false,"between":["2025-08-02","2025-11-18"],"summary":"Named GPTBot, OAI-SearchBot and ChatGPT-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250802011953/https://www.zocdoc.com/robots.txt","https://web.archive.org/web/20251118231739/https://www.zocdoc.com/robots.txt"]},{"domain":"developers.cloudflare.com","name":"Cloudflare Docs","category":"developer","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-10-01"],"summary":"Added <code>Content-Signal: ai-train=yes, search=yes, ai-input=yes</code>.","dir":"other","up":[],"down":[],"major":false,"signals":true,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250801010335/https://developers.cloudflare.com/robots.txt","https://web.archive.org/web/20251001021308/https://developers.cloudflare.com/robots.txt"]},{"domain":"ebay.com","name":"eBay","category":"retail","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-09-01","2025-10-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250901081038/https://www.ebay.com/robots.txt","https://web.archive.org/web/20251001001455/https://www.ebay.com/robots.txt"]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-09-01","2025-10-01"],"summary":"Now blocked: ClaudeBot, Claude-SearchBot, DuckAssistBot, Meta-WebIndexer, Claude-User, Perplexity-User, Meta-ExternalFetcher and MistralAI-User. Now allowed: anthropic-ai.","dir":"mixed","up":["ClaudeBot","Claude-SearchBot","DuckAssistBot","Meta-WebIndexer","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"],"down":["anthropic-ai"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250901081731/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20251001000402/https://www.newyorker.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-09-01","2025-10-01"],"summary":"Now allowed: Amazonbot.","dir":"down","up":[],"down":["Amazonbot"],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20250901041400/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20251001000520/https://www.nytimes.com/robots.txt"]},{"domain":"tiktok.com","name":"TikTok","category":"social","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-09-02","2025-10-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, AI2Bot, anthropic-ai and 8 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","DuckAssistBot","ChatGPT-User","Claude-User","Perplexity-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"blocked","evidence":["https://web.archive.org/web/20250902231943/https://www.tiktok.com/robots.txt","https://web.archive.org/web/20251001141449/https://www.tiktok.com/robots.txt"]},{"domain":"tripadvisor.com","name":"Tripadvisor","category":"travel","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-10-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250801111323/https://www.tripadvisor.com/robots.txt","https://web.archive.org/web/20251001005613/https://www.tripadvisor.com/robots.txt"]},{"domain":"wired.com","name":"WIRED","category":"news","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-09-01","2025-10-01"],"summary":"Now blocked: ClaudeBot, Claude-SearchBot, DuckAssistBot, Meta-WebIndexer, Claude-User, Perplexity-User, Meta-ExternalFetcher and MistralAI-User. Now allowed: anthropic-ai.","dir":"mixed","up":["ClaudeBot","Claude-SearchBot","DuckAssistBot","Meta-WebIndexer","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"],"down":["anthropic-ai"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250901041749/https://www.wired.com/robots.txt","https://web.archive.org/web/20251001000357/https://www.wired.com/robots.txt"]},{"domain":"amazon.com","name":"Amazon","category":"retail","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-06-01","2025-09-03"],"summary":"Now blocked: ClaudeBot, Google-Extended, Meta-ExternalAgent, Bytespider, AI2Bot, DuckAssistBot and MistralAI-User.","dir":"up","up":["ClaudeBot","Google-Extended","Meta-ExternalAgent","Bytespider","AI2Bot","DuckAssistBot","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250601000120/https://www.amazon.com/robots.txt","https://web.archive.org/web/20250903165358/https://www.amazon.com/robots.txt"]},{"domain":"chewy.com","name":"Chewy","category":"retail","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-02","2025-09-16"],"summary":"Now blocked: Amazonbot.","dir":"up","up":["Amazonbot"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250802225029/https://www.chewy.com/robots.txt","https://web.archive.org/web/20250916074042/https://www.chewy.com/robots.txt"]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-02","2025-09-17"],"summary":"Now allowed: ClaudeBot, Applebot-Extended, CCBot, Bytespider, anthropic-ai and Amazonbot.","dir":"down","up":[],"down":["ClaudeBot","Applebot-Extended","CCBot","Bytespider","anthropic-ai","Amazonbot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250802231626/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20250917135505/https://www.craigslist.org/robots.txt"]},{"domain":"economist.com","name":"The Economist","category":"news","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-01"],"summary":"Now blocked: Perplexity-User.","dir":"up","up":["Perplexity-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250801023103/https://www.economist.com/robots.txt","https://web.archive.org/web/20250901053150/https://www.economist.com/robots.txt"]},{"domain":"expedia.com","name":"Expedia","category":"travel","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-08"],"summary":"Named OAI-SearchBot, Claude-SearchBot, PerplexityBot, Claude-User and Perplexity-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250801234526/https://www.expedia.com/robots.txt","https://web.archive.org/web/20250908002942/https://www.expedia.com/robots.txt"]},{"domain":"hotels.com","name":"Hotels.com","category":"travel","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-02","2025-09-14"],"summary":"Named OAI-SearchBot, Claude-SearchBot, PerplexityBot, ChatGPT-User and Claude-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250802111812/https://www.hotels.com/robots.txt","https://web.archive.org/web/20250914142001/https://www.hotels.com/robots.txt"]},{"domain":"ikea.com","name":"IKEA","category":"retail","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-16"],"summary":"Now allowed: GPTBot, Google-Extended and CCBot.","dir":"down","up":[],"down":["GPTBot","Google-Extended","CCBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20250801022108/https://www.ikea.com/robots.txt","https://web.archive.org/web/20250916081624/https://www.ikea.com/robots.txt"]},{"domain":"linkedin.com","name":"LinkedIn","category":"social","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-09-01"],"summary":"Named GPTBot, ClaudeBot, Google-Extended, CCBot, Meta-ExternalAgent, anthropic-ai and 7 more explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20250701004322/https://www.linkedin.com/robots.txt","https://web.archive.org/web/20250901032653/https://www.linkedin.com/robots.txt"]},{"domain":"medium.com","name":"Medium","category":"news","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-01"],"summary":"Now blocked: Meta-ExternalAgent and Amazonbot.","dir":"up","up":["Meta-ExternalAgent","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250801130117/https://medium.com/robots.txt","https://web.archive.org/web/20250901050621/https://medium.com/robots.txt"]},{"domain":"pinterest.com","name":"Pinterest","category":"social","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-01"],"summary":"Now blocked: Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, anthropic-ai, PerplexityBot and ChatGPT-User.","dir":"up","up":["Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","anthropic-ai","PerplexityBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250801044448/https://www.pinterest.com/robots.txt","https://web.archive.org/web/20250901000022/https://www.pinterest.com/robots.txt"]},{"domain":"politico.com","name":"Politico","category":"news","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-02"],"summary":"Now blocked: GPTBot, Google-Extended, Applebot-Extended, Meta-ExternalAgent, Bytespider, AI2Bot, anthropic-ai, OAI-SearchBot and 9 more.","dir":"up","up":["GPTBot","Google-Extended","Applebot-Extended","Meta-ExternalAgent","Bytespider","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","DuckAssistBot","ChatGPT-User","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20250801015710/https://www.politico.com/robots.txt","https://web.archive.org/web/20250902050836/https://www.politico.com/robots.txt"]},{"domain":"reddit.com","name":"Reddit","category":"social","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-09-02"],"summary":"Now partly blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"down","up":[],"down":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"major":true,"signals":false,"status_from":"blocked","status_to":"partial","evidence":["https://web.archive.org/web/20250701014234/https://www.reddit.com/robots.txt","https://web.archive.org/web/20250902201530/https://www.reddit.com/robots.txt"]},{"domain":"temu.com","name":"Temu","category":"retail","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-01"],"summary":"Named GPTBot, ClaudeBot, Google-Extended, OAI-SearchBot, Claude-SearchBot, PerplexityBot and 2 more explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250801203801/https://www.temu.com/robots.txt","https://web.archive.org/web/20250901060743/https://www.temu.com/robots.txt"]},{"domain":"theatlantic.com","name":"The Atlantic","category":"news","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-11"],"summary":"Now blocked: Perplexity-User.","dir":"up","up":["Perplexity-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250801110117/https://www.theatlantic.com/robots.txt","https://web.archive.org/web/20250911204856/https://www.theatlantic.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-10"],"summary":"Now blocked: MistralAI-User.","dir":"up","up":["MistralAI-User"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250801013743/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20250910211719/https://www.usatoday.com/robots.txt"]},{"domain":"vrbo.com","name":"Vrbo","category":"travel","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-17"],"summary":"Named OAI-SearchBot, Claude-SearchBot, PerplexityBot, ChatGPT-User, Claude-User and Perplexity-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250801161742/https://www.vrbo.com/robots.txt","https://web.archive.org/web/20250917144228/https://www.vrbo.com/robots.txt"]},{"domain":"aa.com","name":"American Airlines","category":"travel","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-03","2025-08-03"],"summary":"Now allowed: GPTBot.","dir":"down","up":[],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20250703231939/https://www.aa.com/robots.txt","https://web.archive.org/web/20250803065548/https://www.aa.com/robots.txt"]},{"domain":"apnews.com","name":"Associated Press","category":"news","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-08-01"],"summary":"Now blocked: Claude-SearchBot and Claude-User.","dir":"up","up":["Claude-SearchBot","Claude-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250701002414/https://www.apnews.com/robots.txt","https://web.archive.org/web/20250801003654/https://www.apnews.com/robots.txt"]},{"domain":"bbc.com","name":"BBC","category":"news","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-08-01"],"summary":"Now blocked: Amazonbot and Perplexity-User.","dir":"up","up":["Amazonbot","Perplexity-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250401011704/https://www.bbc.com/robots.txt","https://web.archive.org/web/20250801014616/https://www.bbc.com/robots.txt"]},{"domain":"bloomberg.com","name":"Bloomberg","category":"news","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-08-01"],"summary":"Now blocked: Meta-ExternalAgent, Bytespider and Amazonbot.","dir":"up","up":["Meta-ExternalAgent","Bytespider","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250701020735/https://www.bloomberg.com/robots.txt","https://web.archive.org/web/20250801055411/https://www.bloomberg.com/robots.txt"]},{"domain":"glassdoor.com","name":"Glassdoor","category":"local","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-06-05","2025-08-01"],"summary":"Now allowed: ClaudeBot, PerplexityBot and ChatGPT-User.","dir":"down","up":[],"down":["ClaudeBot","PerplexityBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250605080148/https://www.glassdoor.com/robots.txt","https://web.archive.org/web/20250801123523/https://www.glassdoor.com/robots.txt"]},{"domain":"latimes.com","name":"Los Angeles Times","category":"news","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-08-01"],"summary":"Now blocked: ClaudeBot and Bytespider.","dir":"up","up":["ClaudeBot","Bytespider"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250701020919/https://www.latimes.com/robots.txt","https://web.archive.org/web/20250801061405/https://www.latimes.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-08-01"],"summary":"Now blocked: Claude-SearchBot, Claude-User and Perplexity-User.","dir":"up","up":["Claude-SearchBot","Claude-User","Perplexity-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20250701001955/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20250801012859/https://www.nytimes.com/robots.txt"]},{"domain":"opentable.com","name":"OpenTable","category":"local","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-08-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 10 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20250701171403/https://www.opentable.com/robots.txt","https://web.archive.org/web/20250801122015/https://www.opentable.com/robots.txt"]},{"domain":"reuters.com","name":"Reuters","category":"news","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-08-01"],"summary":"Now blocked: Claude-SearchBot and Claude-User.","dir":"up","up":["Claude-SearchBot","Claude-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250701005843/https://www.reuters.com/robots.txt","https://web.archive.org/web/20250801185400/https://www.reuters.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2025-08","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-08-03"],"summary":"Now allowed: Google-Extended.","dir":"down","up":[],"down":["Google-Extended"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250701004129/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20250803231953/https://www.washingtonpost.com/robots.txt"]},{"domain":"imdb.com","name":"IMDb","category":"reference","date":"2025-07","precision":"month","source":"archive","provisional":false,"between":["2025-06-01","2025-07-01"],"summary":"Now blocked: GPTBot, Google-Extended, CCBot and anthropic-ai.","dir":"up","up":["GPTBot","Google-Extended","CCBot","anthropic-ai"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20250601000128/https://www.imdb.com/robots.txt","https://web.archive.org/web/20250701003403/https://www.imdb.com/robots.txt"]},{"domain":"medium.com","name":"Medium","category":"news","date":"2025-07","precision":"month","source":"archive","provisional":false,"between":["2025-06-01","2025-07-01"],"summary":"Now allowed: Google-Extended. Added an RSL <code>License:</code> line pointing to licensing terms.","dir":"down","up":[],"down":["Google-Extended"],"major":true,"signals":true,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250601001635/https://medium.com/robots.txt","https://web.archive.org/web/20250701005708/https://medium.com/robots.txt"]},{"domain":"npr.org","name":"NPR","category":"news","date":"2025-07","precision":"month","source":"archive","provisional":false,"between":["2025-05-01","2025-07-01"],"summary":"Now blocked: OAI-SearchBot.","dir":"up","up":["OAI-SearchBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250501011518/https://www.npr.org/robots.txt","https://web.archive.org/web/20250701012213/https://www.npr.org/robots.txt"]},{"domain":"ticketmaster.com","name":"Ticketmaster","category":"local","date":"2025-07","precision":"month","source":"archive","provisional":false,"between":["2025-06-01","2025-07-02"],"summary":"Now allowed: GPTBot.","dir":"down","up":[],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20250601114559/https://www.ticketmaster.com/robots.txt","https://web.archive.org/web/20250702114747/https://www.ticketmaster.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2025-07","precision":"month","source":"archive","provisional":false,"between":["2025-06-01","2025-07-01"],"summary":"Now blocked: AI2Bot, DuckAssistBot and Meta-ExternalFetcher.","dir":"up","up":["AI2Bot","DuckAssistBot","Meta-ExternalFetcher"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250601013730/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20250701024336/https://www.usatoday.com/robots.txt"]},{"domain":"zillow.com","name":"Zillow","category":"local","date":"2025-07","precision":"month","source":"archive","provisional":false,"between":["2025-06-02","2025-07-04"],"summary":"Now allowed: GPTBot.","dir":"down","up":[],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20250602135613/https://www.zillow.com/robots.txt","https://web.archive.org/web/20250704233130/https://www.zillow.com/robots.txt"]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2025-06","precision":"month","source":"archive","provisional":false,"between":["2025-05-22","2025-06-19"],"summary":"Now allowed: GPTBot, OAI-SearchBot and ChatGPT-User.","dir":"down","up":[],"down":["GPTBot","OAI-SearchBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250522235750/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20250619000726/https://www.craigslist.org/robots.txt"]},{"domain":"apnews.com","name":"Associated Press","category":"news","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now blocked: Applebot-Extended.","dir":"up","up":["Applebot-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250401001015/https://apnews.com/robots.txt","https://web.archive.org/web/20250501013832/https://apnews.com/robots.txt"]},{"domain":"ft.com","name":"Financial Times","category":"news","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now blocked: Perplexity-User and Meta-ExternalFetcher.","dir":"up","up":["Perplexity-User","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250401021020/https://www.ft.com/robots.txt","https://web.archive.org/web/20250501052735/https://www.ft.com/robots.txt"]},{"domain":"medium.com","name":"Medium","category":"news","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now blocked: ClaudeBot, Google-Extended, Applebot-Extended and Bytespider.","dir":"up","up":["ClaudeBot","Google-Extended","Applebot-Extended","Bytespider"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250401230316/https://medium.com/robots.txt","https://web.archive.org/web/20250501183543/https://medium.com/robots.txt"]},{"domain":"reddit.com","name":"Reddit","category":"social","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"blocked","evidence":["https://web.archive.org/web/20250401004524/https://www.reddit.com/robots.txt","https://web.archive.org/web/20250501151408/https://www.reddit.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-07","2025-05-04"],"summary":"Now allowed: ClaudeBot, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider and PerplexityBot.","dir":"down","up":[],"down":["ClaudeBot","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","PerplexityBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20250407014700/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20250504231302/https://stackoverflow.com/robots.txt"]},{"domain":"theverge.com","name":"The Verge","category":"news","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now allowed: Amazonbot.","dir":"down","up":[],"down":["Amazonbot"],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250401031222/https://www.theverge.com/robots.txt","https://web.archive.org/web/20250501054043/https://www.theverge.com/robots.txt"]},{"domain":"threads.com","name":"Threads","category":"social","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2024-09-01","2025-05-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"blocked","evidence":["https://web.archive.org/web/20240901191440/https://threads.com/robots.txt","https://web.archive.org/web/20250501035838/https://www.threads.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now blocked: Claude-SearchBot and Claude-User.","dir":"up","up":["Claude-SearchBot","Claude-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250401004320/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20250501152555/https://www.usatoday.com/robots.txt"]},{"domain":"cnn.com","name":"CNN","category":"news","date":"2025-04","precision":"month","source":"archive","provisional":false,"between":["2025-03-01","2025-04-01"],"summary":"Now blocked: Applebot-Extended, Meta-ExternalAgent, AI2Bot, OAI-SearchBot, Amazonbot, DuckAssistBot and Meta-ExternalFetcher.","dir":"up","up":["Applebot-Extended","Meta-ExternalAgent","AI2Bot","OAI-SearchBot","Amazonbot","DuckAssistBot","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250301011919/https://www.cnn.com/robots.txt","https://web.archive.org/web/20250401004714/https://www.cnn.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2025-04","precision":"month","source":"archive","provisional":false,"between":["2025-03-01","2025-04-01"],"summary":"Now blocked: DuckAssistBot and Meta-ExternalFetcher.","dir":"up","up":["DuckAssistBot","Meta-ExternalFetcher"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250301013739/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20250401005535/https://www.nytimes.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2025-04","precision":"month","source":"archive","provisional":false,"between":["2025-03-01","2025-04-01"],"summary":"Now allowed: GPTBot, OAI-SearchBot and ChatGPT-User.","dir":"down","up":[],"down":["GPTBot","OAI-SearchBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250301021544/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20250401005741/https://www.washingtonpost.com/robots.txt"]},{"domain":"bbc.com","name":"BBC","category":"news","date":"2025-03","precision":"month","source":"archive","provisional":false,"between":["2025-02-01","2025-03-02"],"summary":"Now blocked: Meta-ExternalAgent and OAI-SearchBot.","dir":"up","up":["Meta-ExternalAgent","OAI-SearchBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250201035627/https://www.bbc.com/robots.txt","https://web.archive.org/web/20250302094514/http://www.bbc.com/robots.txt"]},{"domain":"bloomberg.com","name":"Bloomberg","category":"news","date":"2025-03","precision":"month","source":"archive","provisional":false,"between":["2025-02-01","2025-03-01"],"summary":"Now blocked: anthropic-ai, PerplexityBot and ChatGPT-User.","dir":"up","up":["anthropic-ai","PerplexityBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250201081237/https://www.bloomberg.com/robots.txt","https://web.archive.org/web/20250301223848/https://www.bloomberg.com/robots.txt"]},{"domain":"capitalone.com","name":"Capital One","category":"finance","date":"2025-03","precision":"month","source":"archive","provisional":false,"between":["2025-02-01","2025-03-01"],"summary":"Named ClaudeBot and Amazonbot explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250201094947/https://www.capitalone.com/robots.txt","https://web.archive.org/web/20250301033034/https://www.capitalone.com/robots.txt"]},{"domain":"facebook.com","name":"Facebook","category":"social","date":"2025-03","precision":"month","source":"archive","provisional":false,"between":["2025-02-01","2025-03-01"],"summary":"Named PerplexityBot explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20250201005733/https://www.facebook.com/robots.txt","https://web.archive.org/web/20250301001510/https://www.facebook.com/robots.txt"]},{"domain":"ihg.com","name":"IHG Hotels","category":"travel","date":"2025-03","precision":"month","source":"archive","provisional":false,"between":["2025-02-01","2025-03-08"],"summary":"Named OAI-SearchBot, PerplexityBot and ChatGPT-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250201202235/https://www.ihg.com/robots.txt","https://web.archive.org/web/20250308134240/https://www.ihg.com/robots.txt"]},{"domain":"instacart.com","name":"Instacart","category":"retail","date":"2025-03","precision":"month","source":"archive","provisional":false,"between":["2025-02-01","2025-03-11"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 14 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","Claude-SearchBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20250201054115/https://www.instacart.com/robots.txt","https://web.archive.org/web/20250311021618/https://www.instacart.com/robots.txt"]},{"domain":"axios.com","name":"Axios","category":"news","date":"2025-02","precision":"month","source":"archive","provisional":false,"between":["2025-01-01","2025-02-01"],"summary":"Now allowed: GPTBot, OAI-SearchBot and ChatGPT-User.","dir":"down","up":[],"down":["GPTBot","OAI-SearchBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250101023137/https://www.axios.com/robots.txt","https://web.archive.org/web/20250201042227/https://www.axios.com/robots.txt"]},{"domain":"marriott.com","name":"Marriott","category":"travel","date":"2025-02","precision":"month","source":"archive","provisional":false,"between":["2025-01-01","2025-02-01"],"summary":"Now allowed: GPTBot.","dir":"down","up":[],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20250101030028/https://www.marriott.com/robots.txt","https://web.archive.org/web/20250201051550/https://www.marriott.com/robots.txt"]},{"domain":"theverge.com","name":"The Verge","category":"news","date":"2025-02","precision":"month","source":"archive","provisional":false,"between":["2025-01-01","2025-02-01"],"summary":"Named GPTBot explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250101001114/https://www.theverge.com/robots.txt","https://web.archive.org/web/20250201073817/https://www.theverge.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2025-02","precision":"month","source":"archive","provisional":false,"between":["2025-01-06","2025-02-14"],"summary":"Now blocked: OAI-SearchBot.","dir":"up","up":["OAI-SearchBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250106222839/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20250214064611/https://www.washingtonpost.com/robots.txt"]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2025-02","precision":"month","source":"archive","provisional":false,"between":["2025-01-01","2025-02-01"],"summary":"Now blocked: ChatGPT-User.","dir":"up","up":["ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250101003429/https://www.webmd.com/robots.txt","https://web.archive.org/web/20250201102653/https://www.webmd.com/robots.txt"]},{"domain":"amazon.com","name":"Amazon","category":"retail","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-01","2024-12-01"],"summary":"Now blocked: PerplexityBot.","dir":"up","up":["PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241101004322/https://www.amazon.com/robots.txt","https://web.archive.org/web/20241201001940/https://www.amazon.com/robots.txt"]},{"domain":"apnews.com","name":"Associated Press","category":"news","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-01","2024-12-01"],"summary":"Now blocked: Amazonbot.","dir":"up","up":["Amazonbot"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241101001416/https://apnews.com/robots.txt","https://web.archive.org/web/20241201010309/https://apnews.com/robots.txt"]},{"domain":"goodrx.com","name":"GoodRx","category":"healthcare","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-02","2024-12-02"],"summary":"Now allowed: GPTBot, Google-Extended, CCBot, Bytespider and anthropic-ai.","dir":"down","up":[],"down":["GPTBot","Google-Extended","CCBot","Bytespider","anthropic-ai"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20241102083510/https://www.goodrx.com/robots.txt","https://web.archive.org/web/20241202023714/https://www.goodrx.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-01","2024-12-01"],"summary":"Now allowed: Amazonbot.","dir":"down","up":[],"down":["Amazonbot"],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241101012522/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20241201113654/https://stackoverflow.com/robots.txt"]},{"domain":"tripadvisor.com","name":"Tripadvisor","category":"travel","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-12-01"],"summary":"Now blocked: Google-Extended and Applebot-Extended.","dir":"up","up":["Google-Extended","Applebot-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001024042/https://www.tripadvisor.com/robots.txt","https://web.archive.org/web/20241201010003/https://www.tripadvisor.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-01","2024-12-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241101004820/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20241201002444/https://www.usatoday.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-01","2024-12-01"],"summary":"Now blocked: Meta-ExternalAgent and Meta-ExternalFetcher.","dir":"up","up":["Meta-ExternalAgent","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241101005614/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20241201001451/https://www.washingtonpost.com/robots.txt"]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-01","2024-12-01"],"summary":"Now blocked: ClaudeBot and CCBot.","dir":"up","up":["ClaudeBot","CCBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241101121540/https://www.webmd.com/robots.txt","https://web.archive.org/web/20241201064835/https://www.webmd.com/robots.txt"]},{"domain":"airbnb.com","name":"Airbnb","category":"travel","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Named GPTBot, ClaudeBot, Meta-ExternalAgent, anthropic-ai, OAI-SearchBot, PerplexityBot and 1 more explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20241001021608/https://www.airbnb.com/robots.txt","https://web.archive.org/web/20241101011254/https://www.airbnb.com/robots.txt"]},{"domain":"axios.com","name":"Axios","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: OAI-SearchBot.","dir":"up","up":["OAI-SearchBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001000503/https://www.axios.com/robots.txt","https://web.archive.org/web/20241101001013/https://www.axios.com/robots.txt"]},{"domain":"economist.com","name":"The Economist","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001013710/https://www.economist.com/robots.txt","https://web.archive.org/web/20241101020741/https://www.economist.com/robots.txt"]},{"domain":"ft.com","name":"Financial Times","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: Meta-ExternalAgent.","dir":"up","up":["Meta-ExternalAgent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001015441/https://www.ft.com/robots.txt","https://web.archive.org/web/20241101004400/https://www.ft.com/robots.txt"]},{"domain":"indeed.com","name":"Indeed","category":"local","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-09-01","2024-11-01"],"summary":"Named Applebot-Extended, Bytespider, OAI-SearchBot, Amazonbot and ChatGPT-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20240901061716/https://www.indeed.com/robots.txt","https://web.archive.org/web/20241101024141/https://www.indeed.com/robots.txt"]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: Meta-ExternalAgent.","dir":"up","up":["Meta-ExternalAgent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001073502/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20241101004359/https://www.newyorker.com/robots.txt"]},{"domain":"realtor.com","name":"Realtor.com","category":"local","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: Meta-ExternalAgent and Meta-ExternalFetcher.","dir":"up","up":["Meta-ExternalAgent","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20241001015833/https://www.realtor.com/robots.txt","https://web.archive.org/web/20241101015513/https://www.realtor.com/robots.txt"]},{"domain":"reuters.com","name":"Reuters","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-08-01","2024-11-01"],"summary":"Now blocked: ClaudeBot, Applebot-Extended, Meta-ExternalAgent, Bytespider, PerplexityBot and Meta-ExternalFetcher.","dir":"up","up":["ClaudeBot","Applebot-Extended","Meta-ExternalAgent","Bytespider","PerplexityBot","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240801030613/https://www.reuters.com/robots.txt","https://web.archive.org/web/20241101004338/https://www.reuters.com/robots.txt"]},{"domain":"wired.com","name":"WIRED","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: Meta-ExternalAgent.","dir":"up","up":["Meta-ExternalAgent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001011203/https://www.wired.com/robots.txt","https://web.archive.org/web/20241101012720/https://www.wired.com/robots.txt"]},{"domain":"wsj.com","name":"The Wall Street Journal","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: Applebot-Extended, Meta-ExternalAgent and Meta-ExternalFetcher.","dir":"up","up":["Applebot-Extended","Meta-ExternalAgent","Meta-ExternalFetcher"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001000953/https://www.wsj.com/robots.txt","https://web.archive.org/web/20241101004723/https://www.wsj.com/robots.txt"]},{"domain":"bbc.com","name":"BBC","category":"news","date":"2024-10","precision":"month","source":"archive","provisional":false,"between":["2024-08-01","2024-10-01"],"summary":"Now blocked: ClaudeBot and Applebot-Extended.","dir":"up","up":["ClaudeBot","Applebot-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240801003016/https://www.bbc.com/robots.txt","https://web.archive.org/web/20241001000744/https://www.bbc.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2024-10","precision":"month","source":"archive","provisional":false,"between":["2024-08-01","2024-10-01"],"summary":"Now blocked: ClaudeBot, Applebot-Extended, CCBot and Meta-ExternalAgent.","dir":"up","up":["ClaudeBot","Applebot-Extended","CCBot","Meta-ExternalAgent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240801011925/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20241001000547/https://stackoverflow.com/robots.txt"]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2024-09","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-09-01"],"summary":"Now blocked: Applebot-Extended and Bytespider. Now allowed: GPTBot and ChatGPT-User.","dir":"mixed","up":["Applebot-Extended","Bytespider"],"down":["GPTBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501090606/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20240901034309/https://www.newyorker.com/robots.txt"]},{"domain":"ticketmaster.com","name":"Ticketmaster","category":"local","date":"2024-09","precision":"month","source":"archive","provisional":false,"between":["2024-08-01","2024-09-01"],"summary":"Now allowed: ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training, AI2Bot and 16 more.","dir":"down","up":[],"down":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"major":true,"signals":false,"status_from":"blocked","status_to":"partial","evidence":["https://web.archive.org/web/20240801011935/https://www1.ticketmaster.com/robots.txt","https://web.archive.org/web/20240901142458/https://www.ticketmaster.com/robots.txt"]},{"domain":"tumblr.com","name":"Tumblr","category":"social","date":"2024-09","precision":"month","source":"archive","provisional":false,"between":["2024-08-01","2024-09-01"],"summary":"Now blocked: Meta-ExternalAgent.","dir":"up","up":["Meta-ExternalAgent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240801000836/https://www.tumblr.com/robots.txt","https://web.archive.org/web/20240901000313/https://www.tumblr.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2024-09","precision":"month","source":"archive","provisional":false,"between":["2024-08-01","2024-09-01"],"summary":"Now blocked: Applebot-Extended, Meta-ExternalAgent and OAI-SearchBot.","dir":"up","up":["Applebot-Extended","Meta-ExternalAgent","OAI-SearchBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240801010507/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20240901013121/https://www.usatoday.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2024-09","precision":"month","source":"archive","provisional":false,"between":["2024-08-01","2024-09-01"],"summary":"Now blocked: Applebot-Extended.","dir":"up","up":["Applebot-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240801002032/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20240901092356/https://www.washingtonpost.com/robots.txt"]},{"domain":"wired.com","name":"WIRED","category":"news","date":"2024-09","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-09-01"],"summary":"Now blocked: Applebot-Extended and Bytespider. Now allowed: GPTBot and ChatGPT-User.","dir":"mixed","up":["Applebot-Extended","Bytespider"],"down":["GPTBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501020648/https://www.wired.com/robots.txt","https://web.archive.org/web/20240901032945/https://www.wired.com/robots.txt"]},{"domain":"angi.com","name":"Angi","category":"local","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-07-01","2024-08-01"],"summary":"Now allowed: CCBot.","dir":"down","up":[],"down":["CCBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20240701002634/https://www.angi.com/robots.txt","https://web.archive.org/web/20240801144834/https://www.angi.com/robots.txt"]},{"domain":"axios.com","name":"Axios","category":"news","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-08-01"],"summary":"Now blocked: ClaudeBot, Google-Extended, CCBot, Bytespider, Amazonbot and ChatGPT-User.","dir":"up","up":["ClaudeBot","Google-Extended","CCBot","Bytespider","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501153845/https://www.axios.com/robots.txt","https://web.archive.org/web/20240801055656/https://www.axios.com/robots.txt"]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-07-01","2024-08-04"],"summary":"Now blocked: GPTBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, anthropic-ai, OAI-SearchBot and 3 more.","dir":"up","up":["GPTBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","anthropic-ai","OAI-SearchBot","PerplexityBot","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240701204734/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20240804153250/https://craigslist.org/robots.txt"]},{"domain":"facebook.com","name":"Facebook","category":"social","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-07-01","2024-08-01"],"summary":"Named ClaudeBot, Applebot-Extended and Amazonbot explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20240701001135/https://www.facebook.com/robots.txt","https://web.archive.org/web/20240801000043/https://facebook.com/robots.txt"]},{"domain":"ft.com","name":"Financial Times","category":"news","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-07-01","2024-08-01"],"summary":"Now blocked: ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Bytespider, anthropic-ai and PerplexityBot. Now allowed: GPTBot.","dir":"mixed","up":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","anthropic-ai","PerplexityBot"],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240701005256/https://www.ft.com/robots.txt","https://web.archive.org/web/20240801004438/https://www.ft.com/robots.txt"]},{"domain":"glassdoor.com","name":"Glassdoor","category":"local","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-07-01","2024-08-01"],"summary":"Now blocked: ClaudeBot, Applebot-Extended and PerplexityBot.","dir":"up","up":["ClaudeBot","Applebot-Extended","PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240701014547/https://www.glassdoor.com/robots.txt","https://web.archive.org/web/20240801035134/https://www.glassdoor.com/robots.txt"]},{"domain":"instagram.com","name":"Instagram","category":"social","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-07-01","2024-08-01"],"summary":"Named ClaudeBot, Applebot-Extended and Amazonbot explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20240701000115/https://www.instagram.com/robots.txt","https://web.archive.org/web/20240801001531/https://www.instagram.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-06-01","2024-08-01"],"summary":"Now blocked: Applebot-Extended, Meta-ExternalAgent and OAI-SearchBot.","dir":"up","up":["Applebot-Extended","Meta-ExternalAgent","OAI-SearchBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240601000016/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20240801002227/https://www.nytimes.com/robots.txt"]},{"domain":"economist.com","name":"The Economist","category":"news","date":"2024-07","precision":"month","source":"archive","provisional":false,"between":["2024-06-01","2024-07-01"],"summary":"Now blocked: Applebot-Extended and PerplexityBot.","dir":"up","up":["Applebot-Extended","PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240601011915/https://www.economist.com/robots.txt","https://web.archive.org/web/20240701014007/https://www.economist.com/robots.txt"]},{"domain":"npr.org","name":"NPR","category":"news","date":"2024-07","precision":"month","source":"archive","provisional":false,"between":["2024-06-01","2024-07-01"],"summary":"Now blocked: ClaudeBot, Applebot-Extended, CCBot, Bytespider, anthropic-ai and PerplexityBot.","dir":"up","up":["ClaudeBot","Applebot-Extended","CCBot","Bytespider","anthropic-ai","PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240601000427/https://www.npr.org/robots.txt","https://web.archive.org/web/20240701014200/https://www.npr.org/robots.txt"]},{"domain":"theatlantic.com","name":"The Atlantic","category":"news","date":"2024-07","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-07-01"],"summary":"Now blocked: ClaudeBot, Applebot-Extended, Bytespider, anthropic-ai, PerplexityBot and Amazonbot. Now allowed: GPTBot and ChatGPT-User.","dir":"mixed","up":["ClaudeBot","Applebot-Extended","Bytespider","anthropic-ai","PerplexityBot","Amazonbot"],"down":["GPTBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501001750/https://www.theatlantic.com/robots.txt","https://web.archive.org/web/20240701014252/https://www.theatlantic.com/robots.txt"]},{"domain":"theverge.com","name":"The Verge","category":"news","date":"2024-07","precision":"month","source":"archive","provisional":false,"between":["2024-06-01","2024-07-01"],"summary":"Now blocked: Applebot-Extended. Now allowed: GPTBot.","dir":"mixed","up":["Applebot-Extended"],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240601001608/https://www.theverge.com/robots.txt","https://web.archive.org/web/20240701004615/https://www.theverge.com/robots.txt"]},{"domain":"tumblr.com","name":"Tumblr","category":"social","date":"2024-07","precision":"month","source":"archive","provisional":false,"between":["2024-06-01","2024-07-01"],"summary":"Now blocked: Applebot-Extended.","dir":"up","up":["Applebot-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240601000154/https://www.tumblr.com/robots.txt","https://web.archive.org/web/20240701004429/https://www.tumblr.com/robots.txt"]},{"domain":"venmo.com","name":"Venmo","category":"finance","date":"2024-07","precision":"month","source":"archive","provisional":false,"between":["2024-06-01","2024-07-01"],"summary":"Now allowed: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"down","up":[],"down":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"major":true,"signals":false,"status_from":"blocked","status_to":"allowed","evidence":["https://web.archive.org/web/20240601023141/https://venmo.com/robots.txt","https://web.archive.org/web/20240701132508/https://venmo.com/robots.txt"]},{"domain":"pinterest.com","name":"Pinterest","category":"social","date":"2024-06","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-06-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501000408/https://www.pinterest.com/robots.txt","https://web.archive.org/web/20240601000111/https://www.pinterest.com/robots.txt"]},{"domain":"politico.com","name":"Politico","category":"news","date":"2024-06","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-06-01"],"summary":"Now blocked: ClaudeBot.","dir":"up","up":["ClaudeBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501024436/https://www.politico.com/robots.txt","https://web.archive.org/web/20240601005552/https://www.politico.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2024-06","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-06-01"],"summary":"Now blocked: PerplexityBot.","dir":"up","up":["PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501052302/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20240601002441/https://www.usatoday.com/robots.txt"]},{"domain":"wsj.com","name":"The Wall Street Journal","category":"news","date":"2024-06","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-06-01"],"summary":"Now blocked: ClaudeBot, Bytespider and PerplexityBot. Now allowed: GPTBot and ChatGPT-User.","dir":"mixed","up":["ClaudeBot","Bytespider","PerplexityBot"],"down":["GPTBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501011459/https://www.wsj.com/robots.txt","https://web.archive.org/web/20240601000158/https://www.wsj.com/robots.txt"]},{"domain":"angi.com","name":"Angi","category":"local","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now allowed: GPTBot.","dir":"down","up":[],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240401065053/https://www.angi.com/robots.txt","https://web.archive.org/web/20240501015144/https://www.angi.com/robots.txt"]},{"domain":"apnews.com","name":"Associated Press","category":"news","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: ClaudeBot, anthropic-ai and PerplexityBot.","dir":"up","up":["ClaudeBot","anthropic-ai","PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240401000928/https://www.apnews.com/robots.txt","https://web.archive.org/web/20240501004555/https://www.apnews.com/robots.txt"]},{"domain":"bbc.com","name":"BBC","category":"news","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: Bytespider and PerplexityBot.","dir":"up","up":["Bytespider","PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240401000803/https://www.bbc.com/robots.txt","https://web.archive.org/web/20240501002309/http://www.bbc.com/robots.txt"]},{"domain":"cnn.com","name":"CNN","category":"news","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: ClaudeBot, CCBot, Bytespider, anthropic-ai, PerplexityBot and ChatGPT-User.","dir":"up","up":["ClaudeBot","CCBot","Bytespider","anthropic-ai","PerplexityBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240401010308/https://www.cnn.com/robots.txt","https://web.archive.org/web/20240501001430/https://www.cnn.com/robots.txt"]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: ClaudeBot.","dir":"up","up":["ClaudeBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20240401130409/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20240501200711/https://craigslist.org/robots.txt"]},{"domain":"cvs.com","name":"CVS Health","category":"healthcare","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Named GPTBot, ClaudeBot, Google-Extended, CCBot, anthropic-ai and ChatGPT-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20240401071910/https://www.cvs.com/robots.txt","https://web.archive.org/web/20240501015654/https://www.cvs.com/robots.txt"]},{"domain":"theverge.com","name":"The Verge","category":"news","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: ClaudeBot, CCBot, Bytespider, anthropic-ai, PerplexityBot, Amazonbot and ChatGPT-User.","dir":"up","up":["ClaudeBot","CCBot","Bytespider","anthropic-ai","PerplexityBot","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240401002248/https://www.theverge.com/robots.txt","https://web.archive.org/web/20240501102951/https://www.theverge.com/robots.txt"]},{"domain":"tiktok.com","name":"TikTok","category":"social","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20240401002034/https://www.tiktok.com/robots.txt","https://web.archive.org/web/20240501000424/https://www.tiktok.com/robots.txt"]},{"domain":"tripadvisor.com","name":"Tripadvisor","category":"travel","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: ClaudeBot.","dir":"up","up":["ClaudeBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20240401001656/https://www.tripadvisor.com/robots.txt","https://web.archive.org/web/20240501070106/https://www.tripadvisor.com/robots.txt"]},{"domain":"goodrx.com","name":"GoodRx","category":"healthcare","date":"2024-04","precision":"month","source":"archive","provisional":false,"between":["2024-03-06","2024-04-06"],"summary":"Now blocked: Google-Extended, CCBot, Bytespider and anthropic-ai.","dir":"up","up":["Google-Extended","CCBot","Bytespider","anthropic-ai"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240306205035/https://www.goodrx.com/robots.txt","https://web.archive.org/web/20240406221810/https://www.goodrx.com/robots.txt"]},{"domain":"ikea.com","name":"IKEA","category":"retail","date":"2024-04","precision":"month","source":"archive","provisional":false,"between":["2024-03-02","2024-04-01"],"summary":"Now blocked: CCBot.","dir":"up","up":["CCBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240302071101/https://www.ikea.com/robots.txt","https://web.archive.org/web/20240401053251/https://www.ikea.com/robots.txt"]},{"domain":"pinterest.com","name":"Pinterest","category":"social","date":"2024-04","precision":"month","source":"archive","provisional":false,"between":["2024-03-01","2024-04-01"],"summary":"Now blocked: ClaudeBot.","dir":"up","up":["ClaudeBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240301001424/https://www.pinterest.com/robots.txt","https://web.archive.org/web/20240401000121/https://www.pinterest.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2024-04","precision":"month","source":"archive","provisional":false,"between":["2024-03-01","2024-04-01"],"summary":"Now blocked: PerplexityBot. Now allowed: GPTBot.","dir":"mixed","up":["PerplexityBot"],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240301034635/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20240401024304/https://stackoverflow.com/robots.txt"]},{"domain":"thumbtack.com","name":"Thumbtack","category":"local","date":"2024-04","precision":"month","source":"archive","provisional":false,"between":["2024-03-04","2024-04-01"],"summary":"Now allowed: Bytespider.","dir":"down","up":[],"down":["Bytespider"],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20240304142423/https://www.thumbtack.com/robots.txt","https://web.archive.org/web/20240401010230/https://www.thumbtack.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2024-04","precision":"month","source":"archive","provisional":false,"between":["2024-03-01","2024-04-01"],"summary":"Now blocked: ClaudeBot, anthropic-ai, Amazonbot and ChatGPT-User.","dir":"up","up":["ClaudeBot","anthropic-ai","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240301000539/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20240401004158/https://www.usatoday.com/robots.txt"]},{"domain":"x.com","name":"X","category":"social","date":"2024-04","precision":"month","source":"archive","provisional":false,"between":["2024-03-01","2024-04-01"],"summary":"Named Google-Extended explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20240301002301/https://x.com/robots.txt","https://web.archive.org/web/20240401004311/https://x.com/robots.txt"]},{"domain":"airbnb.com","name":"Airbnb","category":"travel","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now allowed: GPTBot.","dir":"down","up":[],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20240201011022/https://www.airbnb.com/robots.txt","https://web.archive.org/web/20240301015734/https://www.airbnb.com/robots.txt"]},{"domain":"facebook.com","name":"Facebook","category":"social","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-01-01","2024-03-01"],"summary":"Named GPTBot and Google-Extended explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20240101000356/https://www.facebook.com/robots.txt","https://web.archive.org/web/20240301000033/https://www.facebook.com/robots.txt"]},{"domain":"glassdoor.com","name":"Glassdoor","category":"local","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-01-01","2024-03-01"],"summary":"Now blocked: CCBot, Bytespider, anthropic-ai, Amazonbot and ChatGPT-User.","dir":"up","up":["CCBot","Bytespider","anthropic-ai","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240101052543/https://www.glassdoor.com/robots.txt","https://web.archive.org/web/20240301070717/https://www.glassdoor.com/robots.txt"]},{"domain":"instagram.com","name":"Instagram","category":"social","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Named GPTBot and Google-Extended explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20240201000013/https://www.instagram.com/robots.txt","https://web.archive.org/web/20240301000747/https://www.instagram.com/robots.txt"]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now blocked: PerplexityBot.","dir":"up","up":["PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240201034902/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20240301072847/https://www.newyorker.com/robots.txt"]},{"domain":"npr.org","name":"NPR","category":"news","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240201003630/https://www.npr.org/robots.txt","https://web.archive.org/web/20240301001905/https://www.npr.org/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now blocked: ClaudeBot, anthropic-ai, PerplexityBot, Amazonbot and ChatGPT-User.","dir":"up","up":["ClaudeBot","anthropic-ai","PerplexityBot","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240201000030/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20240301002928/https://www.nytimes.com/robots.txt"]},{"domain":"ticketmaster.com","name":"Ticketmaster","category":"local","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now blocked: ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training, AI2Bot and 16 more.","dir":"up","up":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20240201074630/https://www.ticketmaster.com/robots.txt","https://web.archive.org/web/20240301182218/https://www1.ticketmaster.com/robots.txt"]},{"domain":"tumblr.com","name":"Tumblr","category":"social","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now blocked: ClaudeBot and anthropic-ai. Now allowed: GPTBot.","dir":"mixed","up":["ClaudeBot","anthropic-ai"],"down":["GPTBot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240201001731/https://www.tumblr.com/robots.txt","https://web.archive.org/web/20240301005930/https://www.tumblr.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-05"],"summary":"Now blocked: ClaudeBot, Bytespider, anthropic-ai, PerplexityBot and Amazonbot.","dir":"up","up":["ClaudeBot","Bytespider","anthropic-ai","PerplexityBot","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240201011720/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20240305223339/https://www.washingtonpost.com/robots.txt"]},{"domain":"wired.com","name":"WIRED","category":"news","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now blocked: PerplexityBot.","dir":"up","up":["PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240201001046/https://www.wired.com/robots.txt","https://web.archive.org/web/20240301034353/https://www.wired.com/robots.txt"]},{"domain":"bbc.com","name":"BBC","category":"news","date":"2024-02","precision":"month","source":"archive","provisional":false,"between":["2023-12-01","2024-02-01"],"summary":"Now blocked: Google-Extended and anthropic-ai.","dir":"up","up":["Google-Extended","anthropic-ai"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231201000213/http://www.bbc.com/robots.txt","https://web.archive.org/web/20240201000336/https://www.bbc.com/robots.txt"]},{"domain":"docs.stripe.com","name":"Stripe Docs","category":"developer","date":"2024-02","precision":"month","source":"archive","provisional":false,"between":["2024-01-07","2024-02-13"],"summary":"Now allowed: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"down","up":[],"down":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"major":true,"signals":false,"status_from":"blocked","status_to":"allowed","evidence":["https://web.archive.org/web/20240107003136/https://docs.stripe.com/robots.txt","https://web.archive.org/web/20240213234250/https://docs.stripe.com/robots.txt"]},{"domain":"indeed.com","name":"Indeed","category":"local","date":"2024-02","precision":"month","source":"archive","provisional":false,"between":["2024-01-01","2024-02-01"],"summary":"Named CCBot and anthropic-ai explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20240101000523/https://www.indeed.com/robots.txt","https://web.archive.org/web/20240201054137/https://www.indeed.com/robots.txt"]},{"domain":"opentable.com","name":"OpenTable","category":"local","date":"2024-02","precision":"month","source":"archive","provisional":false,"between":["2024-01-01","2024-02-01"],"summary":"Now allowed: GPTBot and ChatGPT-User.","dir":"down","up":[],"down":["GPTBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20240101003525/https://www.opentable.com/robots.txt","https://web.archive.org/web/20240201022530/https://www.opentable.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2024-02","precision":"month","source":"archive","provisional":false,"between":["2024-01-01","2024-02-01"],"summary":"Now allowed: Google-Extended.","dir":"down","up":[],"down":["Google-Extended"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240101000541/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20240201001637/https://stackoverflow.com/robots.txt"]},{"domain":"zillow.com","name":"Zillow","category":"local","date":"2024-02","precision":"month","source":"archive","provisional":false,"between":["2024-01-01","2024-02-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20240101002343/http://zillow.com/robots.txt","https://web.archive.org/web/20240201020538/https://www.zillow.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2024-01","precision":"month","source":"archive","provisional":false,"between":["2023-12-01","2024-01-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231201004027/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20240101000929/https://www.nytimes.com/robots.txt"]},{"domain":"politico.com","name":"Politico","category":"news","date":"2024-01","precision":"month","source":"archive","provisional":false,"between":["2023-12-01","2024-01-01"],"summary":"Now blocked: CCBot.","dir":"up","up":["CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20231201000722/https://www.politico.com/robots.txt","https://web.archive.org/web/20240101004219/https://www.politico.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2024-01","precision":"month","source":"archive","provisional":false,"between":["2023-12-01","2024-01-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231201000112/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20240101001154/https://www.usatoday.com/robots.txt"]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2023-12","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-12-01"],"summary":"Now blocked: Google-Extended, anthropic-ai and Amazonbot.","dir":"up","up":["Google-Extended","anthropic-ai","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001004822/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20231201002111/https://www.newyorker.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2023-12","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-12-01"],"summary":"Now blocked: Google-Extended and CCBot.","dir":"up","up":["Google-Extended","CCBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001001645/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20231201043917/https://www.washingtonpost.com/robots.txt"]},{"domain":"wired.com","name":"WIRED","category":"news","date":"2023-12","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-12-01"],"summary":"Now blocked: Google-Extended, anthropic-ai and Amazonbot.","dir":"up","up":["Google-Extended","anthropic-ai","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001005749/https://www.wired.com/robots.txt","https://web.archive.org/web/20231201052212/https://www.wired.com/robots.txt"]},{"domain":"wsj.com","name":"The Wall Street Journal","category":"news","date":"2023-12","precision":"month","source":"archive","provisional":false,"between":["2023-11-01","2023-12-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231101003040/https://www.wsj.com/robots.txt","https://web.archive.org/web/20231201001359/https://www.wsj.com/robots.txt"]},{"domain":"apnews.com","name":"Associated Press","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: GPTBot and CCBot.","dir":"up","up":["GPTBot","CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20231001005428/https://apnews.com/robots.txt","https://web.archive.org/web/20231101000433/https://apnews.com/robots.txt"]},{"domain":"bbc.com","name":"BBC","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: GPTBot, CCBot and ChatGPT-User.","dir":"up","up":["GPTBot","CCBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20231001002857/https://www.bbc.com/robots.txt","https://web.archive.org/web/20231101000948/https://www.bbc.com/robots.txt"]},{"domain":"bloomberg.com","name":"Bloomberg","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-11-01"],"summary":"Now blocked: GPTBot, Google-Extended and CCBot.","dir":"up","up":["GPTBot","Google-Extended","CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801001431/https://www.bloomberg.com/robots.txt","https://web.archive.org/web/20231101001611/https://www.bloomberg.com/robots.txt"]},{"domain":"cnn.com","name":"CNN","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001000140/https://www.cnn.com/robots.txt","https://web.archive.org/web/20231101001304/http://www.cnn.com/robots.txt"]},{"domain":"economist.com","name":"The Economist","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended, CCBot, anthropic-ai and ChatGPT-User.","dir":"up","up":["Google-Extended","CCBot","anthropic-ai","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001002307/https://www.economist.com/robots.txt","https://web.archive.org/web/20231101010450/https://www.economist.com/robots.txt"]},{"domain":"glassdoor.com","name":"Glassdoor","category":"local","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001014824/https://www.glassdoor.com/robots.txt","https://web.archive.org/web/20231101014034/https://www.glassdoor.com/robots.txt"]},{"domain":"ikea.com","name":"IKEA","category":"retail","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001182636/https://www.ikea.com/robots.txt","https://web.archive.org/web/20231101022123/https://www.ikea.com/robots.txt"]},{"domain":"indeed.com","name":"Indeed","category":"local","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Named Google-Extended explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20231001061852/https://www.indeed.com/robots.txt","https://web.archive.org/web/20231101002911/https://www.indeed.com/robots.txt"]},{"domain":"marriott.com","name":"Marriott","category":"travel","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20231001112057/https://www.marriott.com/robots.txt","https://web.archive.org/web/20231101013041/https://www.marriott.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001000841/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20231101000406/https://www.nytimes.com/robots.txt"]},{"domain":"reuters.com","name":"Reuters","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-11-01"],"summary":"Now blocked: Google-Extended, CCBot and anthropic-ai.","dir":"up","up":["Google-Extended","CCBot","anthropic-ai"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801011936/https://www.reuters.com/robots.txt","https://web.archive.org/web/20231101023513/https://www.reuters.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended and Amazonbot.","dir":"up","up":["Google-Extended","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001034346/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20231101001314/https://stackoverflow.com/robots.txt"]},{"domain":"temu.com","name":"Temu","category":"retail","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20231001131229/https://www.temu.com/robots.txt","https://web.archive.org/web/20231101001724/https://www.temu.com/robots.txt"]},{"domain":"theatlantic.com","name":"The Atlantic","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001011212/https://www.theatlantic.com/robots.txt","https://web.archive.org/web/20231101030644/https://www.theatlantic.com/robots.txt"]},{"domain":"theverge.com","name":"The Verge","category":"news","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-11-01"],"summary":"Now blocked: Google-Extended.","dir":"up","up":["Google-Extended"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001000320/https://www.theverge.com/robots.txt","https://web.archive.org/web/20231101003911/https://www.theverge.com/robots.txt"]},{"domain":"ticketmaster.com","name":"Ticketmaster","category":"local","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-09-01","2023-11-01"],"summary":"Now allowed: ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training, AI2Bot and 16 more.","dir":"down","up":[],"down":["ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"major":true,"signals":false,"status_from":"blocked","status_to":"partial","evidence":["https://web.archive.org/web/20230901014329/https://www1.ticketmaster.com/robots.txt","https://web.archive.org/web/20231101005337/https://www.ticketmaster.com/robots.txt"]},{"domain":"tumblr.com","name":"Tumblr","category":"social","date":"2023-11","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-11-01"],"summary":"Now blocked: GPTBot, Google-Extended, CCBot and Amazonbot.","dir":"up","up":["GPTBot","Google-Extended","CCBot","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801002934/https://www.tumblr.com/robots.txt","https://web.archive.org/web/20231101002158/https://www.tumblr.com/robots.txt"]},{"domain":"amazon.com","name":"Amazon","category":"retail","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-10-01"],"summary":"Now blocked: GPTBot and CCBot.","dir":"up","up":["GPTBot","CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801000123/https://www.amazon.com/robots.txt","https://web.archive.org/web/20231001000733/https://www.amazon.com/robots.txt"]},{"domain":"angi.com","name":"Angi","category":"local","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-09-01","2023-10-01"],"summary":"Now blocked: GPTBot and CCBot.","dir":"up","up":["GPTBot","CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230901210745/https://www.angi.com/robots.txt","https://web.archive.org/web/20231001001722/https://www.angi.com/robots.txt"]},{"domain":"ft.com","name":"Financial Times","category":"news","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-09-01","2023-10-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230901012747/https://www.ft.com/robots.txt","https://web.archive.org/web/20231001000230/https://www.ft.com/robots.txt"]},{"domain":"goodrx.com","name":"GoodRx","category":"healthcare","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-09-03","2023-10-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230903004506/https://www.goodrx.com/robots.txt","https://web.archive.org/web/20231001232124/https://www.goodrx.com/robots.txt"]},{"domain":"npr.org","name":"NPR","category":"news","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-09-01","2023-10-01"],"summary":"Now blocked: GPTBot and ChatGPT-User.","dir":"up","up":["GPTBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230901000300/https://www.npr.org/robots.txt","https://web.archive.org/web/20231001014508/https://www.npr.org/robots.txt"]},{"domain":"pinterest.com","name":"Pinterest","category":"social","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-09-01","2023-10-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230901002207/https://www.pinterest.com/robots.txt","https://web.archive.org/web/20231001001441/https://www.pinterest.com/robots.txt"]},{"domain":"thumbtack.com","name":"Thumbtack","category":"local","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-09-02","2023-10-02"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20230902183205/https://www.thumbtack.com/robots.txt","https://web.archive.org/web/20231002204827/https://www.thumbtack.com/robots.txt"]},{"domain":"usatoday.com","name":"USA Today","category":"news","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-10-01"],"summary":"Now blocked: GPTBot and CCBot.","dir":"up","up":["GPTBot","CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801000146/https://www.usatoday.com/robots.txt","https://web.archive.org/web/20231001001255/https://www.usatoday.com/robots.txt"]},{"domain":"wsj.com","name":"The Wall Street Journal","category":"news","date":"2023-10","precision":"month","source":"archive","provisional":false,"between":["2023-09-01","2023-10-01"],"summary":"Now blocked: GPTBot, CCBot, anthropic-ai and ChatGPT-User.","dir":"up","up":["GPTBot","CCBot","anthropic-ai","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230901000033/https://www.wsj.com/robots.txt","https://web.archive.org/web/20231001004238/https://www.wsj.com/robots.txt"]},{"domain":"aa.com","name":"American Airlines","category":"travel","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-02","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230802081839/https://www.aa.com/robots.txt","https://web.archive.org/web/20230901141345/https://www.aa.com/robots.txt"]},{"domain":"airbnb.com","name":"Airbnb","category":"travel","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801015834/https://www.airbnb.com/robots.txt","https://web.archive.org/web/20230901025526/https://www.airbnb.com/robots.txt"]},{"domain":"axios.com","name":"Axios","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801012224/https://www.axios.com/robots.txt","https://web.archive.org/web/20230901020806/https://www.axios.com/robots.txt"]},{"domain":"cnn.com","name":"CNN","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801014937/http://www.cnn.com/robots.txt","https://web.archive.org/web/20230901001844/https://www.cnn.com/robots.txt"]},{"domain":"economist.com","name":"The Economist","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801020419/https://www.economist.com/robots.txt","https://web.archive.org/web/20230901013757/https://www.economist.com/robots.txt"]},{"domain":"glassdoor.com","name":"Glassdoor","category":"local","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801000037/https://www.glassdoor.com/robots.txt","https://web.archive.org/web/20230901003306/https://www.glassdoor.com/robots.txt"]},{"domain":"ikea.com","name":"IKEA","category":"retail","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-02"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801023138/https://www.ikea.com/robots.txt","https://web.archive.org/web/20230902072743/https://www.ikea.com/robots.txt"]},{"domain":"indeed.com","name":"Indeed","category":"local","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Named GPTBot explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20230801015342/https://www.indeed.com/robots.txt","https://web.archive.org/web/20230901000836/https://www.indeed.com/robots.txt"]},{"domain":"latimes.com","name":"Los Angeles Times","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801012552/https://www.latimes.com/robots.txt","https://web.archive.org/web/20230901004036/https://www.latimes.com/robots.txt"]},{"domain":"medium.com","name":"Medium","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801010749/https://medium.com/robots.txt","https://web.archive.org/web/20230901021613/https://medium.com/robots.txt"]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot, CCBot and ChatGPT-User.","dir":"up","up":["GPTBot","CCBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801073834/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20230901002009/https://www.newyorker.com/robots.txt"]},{"domain":"nextdoor.com","name":"Nextdoor","category":"local","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Named GPTBot explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20230801034558/https://nextdoor.com/robots.txt","https://web.archive.org/web/20230901012557/https://nextdoor.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20230801000228/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20230901000459/https://www.nytimes.com/robots.txt"]},{"domain":"opentable.com","name":"OpenTable","category":"local","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20230801002028/https://www.opentable.com/robots.txt","https://web.archive.org/web/20230901011111/https://www.opentable.com/robots.txt"]},{"domain":"stackoverflow.com","name":"Stack Overflow","category":"social","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801012935/https://stackoverflow.com/robots.txt","https://web.archive.org/web/20230901004203/https://stackoverflow.com/robots.txt"]},{"domain":"theatlantic.com","name":"The Atlantic","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot, CCBot and ChatGPT-User.","dir":"up","up":["GPTBot","CCBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801002703/https://www.theatlantic.com/robots.txt","https://web.archive.org/web/20230901001836/https://www.theatlantic.com/robots.txt"]},{"domain":"theverge.com","name":"The Verge","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801013521/https://www.theverge.com/robots.txt","https://web.archive.org/web/20230901030045/https://www.theverge.com/robots.txt"]},{"domain":"washingtonpost.com","name":"The Washington Post","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot and ChatGPT-User.","dir":"up","up":["GPTBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801004136/https://www.washingtonpost.com/robots.txt","https://web.archive.org/web/20230901052156/https://www.washingtonpost.com/robots.txt"]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801014425/https://www.webmd.com/robots.txt","https://web.archive.org/web/20230901011205/https://www.webmd.com/robots.txt"]},{"domain":"wired.com","name":"WIRED","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot, CCBot and ChatGPT-User.","dir":"up","up":["GPTBot","CCBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801000612/https://www.wired.com/robots.txt","https://web.archive.org/web/20230901030624/https://www.wired.com/robots.txt"]},{"domain":"ticketmaster.com","name":"Ticketmaster","category":"local","date":"2023-08","precision":"month","source":"archive","provisional":false,"between":["2023-07-01","2023-08-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"blocked","evidence":["https://web.archive.org/web/20230701044605/https://www.ticketmaster.com/robots.txt","https://web.archive.org/web/20230801010853/https://www1.ticketmaster.com/robots.txt"]},{"domain":"indeed.com","name":"Indeed","category":"local","date":"2023-07","precision":"month","source":"archive","provisional":false,"between":["2023-06-01","2023-07-01"],"summary":"Now allowed: CCBot.","dir":"down","up":[],"down":["CCBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20230601000225/https://www.indeed.com/robots.txt","https://web.archive.org/web/20230701040238/https://www.indeed.com/robots.txt"]},{"domain":"opentable.com","name":"OpenTable","category":"local","date":"2023-06","precision":"month","source":"archive","provisional":false,"between":["2023-05-01","2023-06-01"],"summary":"Now blocked: ChatGPT-User.","dir":"up","up":["ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230501005619/https://www.opentable.com/robots.txt","https://web.archive.org/web/20230601025532/https://www.opentable.com/robots.txt"]},{"domain":"nytimes.com","name":"The New York Times","category":"news","date":"2023-05","precision":"month","source":"archive","provisional":false,"between":["2023-04-01","2023-05-01"],"summary":"Now blocked: CCBot.","dir":"up","up":["CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230401000237/https://www.nytimes.com/robots.txt","https://web.archive.org/web/20230501002921/https://www.nytimes.com/robots.txt"]},{"domain":"expedia.com","name":"Expedia","category":"travel","date":"2023-04","precision":"month","source":"archive","provisional":false,"between":["2023-03-02","2023-04-01"],"summary":"Named ChatGPT-User explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20230302222052/https://www.expedia.com/robots.txt","https://web.archive.org/web/20230401192444/https://www.expedia.com/robots.txt"]},{"domain":"instacart.com","name":"Instacart","category":"retail","date":"2023-04","precision":"month","source":"archive","provisional":false,"between":["2023-03-01","2023-04-01"],"summary":"Named ChatGPT-User explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20230301070906/https://www.instacart.com/robots.txt","https://web.archive.org/web/20230401080910/https://www.instacart.com/robots.txt"]},{"domain":"kayak.com","name":"KAYAK","category":"travel","date":"2023-04","precision":"month","source":"archive","provisional":false,"between":["2023-03-01","2023-04-01"],"summary":"Named ChatGPT-User explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20230301001423/https://www.kayak.com/robots.txt","https://web.archive.org/web/20230401013543/http://www.kayak.com/robots.txt"]},{"domain":"indeed.com","name":"Indeed","category":"local","date":"2023-03","precision":"month","source":"archive","provisional":false,"between":["2023-02-01","2023-03-01"],"summary":"Now blocked: CCBot.","dir":"up","up":["CCBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230201100836/https://www.indeed.com/robots.txt","https://web.archive.org/web/20230301002753/https://www.indeed.com/robots.txt"]}]