{"domain":"newyorker.com","name":"The New Yorker","category":"news","subcategory":"magazine","country":"US","tags":[],"notes":"","status":"partial","status_label":"Blocks some AI","rationale":"Its robots.txt (the public rules file for bots) treats AI bots differently from each other. It blocks 15 of the 24 AI bots we track from the whole site, including ClaudeBot, Google-Extended, Applebot-Extended, CCBot and Meta-ExternalAgent. GPTBot, OAI-SearchBot and ChatGPT-User can reach most of it.","computed_at":"2026-10-20T03:43:15.245Z","last_checked":"2026-10-19T13:34:54.039Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.newyorker.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.newyorker.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.newyorker.com/robots.txt","hops":[{"from":"https://newyorker.com/robots.txt","to":"https://www.newyorker.com/robots.txt","status":301}],"blob":"f75d421e6b5344b9995de7ea08e894ea5e80efd95333610c98b8812a1d044dad","text":"Sitemap: https://www.newyorker.com/sitemap.xml\nSitemap: https://www.newyorker.com/tagpages-sitemap.xml\nSitemap: https://www.newyorker.com/categories-sitemap.xml\nSitemap: https://www.newyorker.com/contributors-sitemap.xml\nSitemap: https://www.newyorker.com/bundles-sitemap.xml\nSitemap: https://www.newyorker.com/feed/google-news-sitemap-feed/sitemap-google-news\nSitemap: https://www.newyorker.com/feed/rss\n\nUser-agent: *\nDisallow: /*?\nAllow: /*?page\nAllow: /*.xml?\nAllow: /*.js\nAllow: /*.css\nAllow: /*.ico\nAllow: /verso/static\nAllow: /*.svg\nAllow: /*.png\nAllow: /*.jpg\nAllow: /*.jpeg\nAllow: /*rss?\nAllow: /*?id=\nDisallow: /auth/\nDisallow: /account/\nDisallow: /user/\nDisallow: /user-context\nDisallow: /preview/\nDisallow: /search\nDisallow: /product/\nDisallow: /cdn-cgi/\nDisallow: /services.min.js\nDisallow: /com.condenast/yv8\nDisallow: /reject-all\nDisallow: */c_limit/\n\n###\nAllow: /v2/offers/tnya\nAllow: /*?*utm_medium=social\nAllow: /*?*utm_medium=syndication\nDisallow: /cartoon/\nDisallow: /newsletters/\n###\n\nUser-agent: AmazonAdBot\nUser-agent: FacebookExternalHit\nUser-agent: Google-InspectionTool\nUser-agent: LinkedInBot\nUser-agent: Pinterestbot\nUser-agent: Storebot-Google\nUser-agent: TikTokSpider\nUser-agent: Twitterbot\nAllow: /\n\nUser-agent: Amazonbot\nUser-agent: Applebot-Extended\nUser-agent: archive.org_bot\nUser-agent: Bravebot\nUser-agent: Brightbot 1.0\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: Claude-SearchBot\nUser-agent: Claude-User\nUser-agent: ClaudeBot\nUser-agent: cohere-ai\nUser-agent: cohere-training-data-crawler\nUser-agent: Diffbot\nUser-agent: DuckAssistBot\nUser-agent: factset_spyderbot\nUser-agent: FirecrawlAgent\nUser-agent: Google-CloudVertexBot\nUser-agent: Google-Extended\nUser-agent: GoogleOther\nUser-agent: heritrix\nUser-agent: ia_archiver\nUser-agent: ia_archiver-web.archive.org\nUser-agent: meta-externalagent\nUser-agent: meta-externalfetcher\nUser-agent: meta-webindexer\nUser-agent: MistralAI-User\nUser-agent: PanguBot\nUser-agent: Perplexity-User\nUser-agent: PerplexityBot\nUser-agent: PetalBot\nUser-agent: Scrapy\nUser-agent: Shap-User\nUser-agent: ShapBot\nUser-agent: Timpibot\nUser-agent: Yisouspider\nUser-agent: YouBot\nUser-agent: Webzio\nUser-agent: Webzio-Extended\nDisallow: /","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*","AmazonAdBot","FacebookExternalHit","Google-InspectionTool","LinkedInBot","Pinterestbot","Storebot-Google","TikTokSpider","Twitterbot","Amazonbot","Applebot-Extended","archive.org_bot","Bravebot","Brightbot 1.0","Bytespider","CCBot","Claude-SearchBot","Claude-User","ClaudeBot","cohere-ai","cohere-training-data-crawler","Diffbot","DuckAssistBot","factset_spyderbot","FirecrawlAgent","Google-CloudVertexBot","Google-Extended","GoogleOther","heritrix","ia_archiver","ia_archiver-web.archive.org","meta-externalagent","meta-externalfetcher","meta-webindexer","MistralAI-User","PanguBot","Perplexity-User","PerplexityBot","PetalBot","Scrapy","Shap-User","ShapBot","Timpibot","Yisouspider","YouBot","Webzio","Webzio-Extended"],"lines":92},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"ClaudeBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Google-Extended","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Applebot-Extended","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"CCBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Bytespider","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"PerplexityBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Amazonbot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"DuckAssistBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"Claude-User","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Perplexity-User","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"MistralAI-User","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":92},"any":{"allowed":false,"line":92},"search":{"allowed":false,"line":92},"cart":{"allowed":false,"line":92},"checkout":{"allowed":false,"line":92},"account":{"allowed":false,"line":92}},"disallows":1,"lines":[54,92],"groupRanges":[[54,92]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":28},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[9,41],"groupRanges":[[9,41]]}],"lanes":{"training":{"level":"mixed","none":6,"full":3,"restricted":0,"n":9},"search":{"level":"mixed","none":5,"full":3,"restricted":0,"n":8},"user_directed":{"level":"mixed","none":4,"full":3,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":["Amazon CloudFront"],"botManagement":[]},"llms":{"state":"absent","http":404,"url":"https://www.newyorker.com/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://www.newyorker.com/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://www.newyorker.com/.well-known/tdmrep.json"},"meta":{"title":"The New Yorker","robotsMeta":[{"name":"robots","content":"index, follow, max-image-preview:large"}],"noai":[],"noindex":false,"jsonLdTypes":["Organization","ImageObject","WebSite","SearchAction","ItemList","ListItem","CollectionPage"],"platform":null,"textRatio":0.006,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://www.condenast.com/user-agreement/","source":"discovered","skipped":"ToS hosted on another domain; not fetched in v0.1"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":47,"first":"2023-01-01","archive_points":46,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":3},{"m":"2023-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":3},{"m":"2023-11","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2023-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":5},{"m":"2024-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":5},{"m":"2024-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":5},{"m":"2024-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":6},{"m":"2024-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":6},{"m":"2024-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":6},{"m":"2024-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":7},{"m":"2024-07","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":8},{"m":"2024-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":9},{"m":"2024-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2024-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2024-11","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2024-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-07","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":7},{"m":"2025-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2025-11","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2025-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-07","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15},{"m":"2026-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":15}]},"events":[{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-09-01","2025-10-01"],"summary":"Now blocked: ClaudeBot, Claude-SearchBot, DuckAssistBot, Meta-WebIndexer, Claude-User, Perplexity-User, Meta-ExternalFetcher and MistralAI-User. Now allowed: anthropic-ai.","dir":"mixed","up":["ClaudeBot","Claude-SearchBot","DuckAssistBot","Meta-WebIndexer","Claude-User","Perplexity-User","Meta-ExternalFetcher","MistralAI-User"],"down":["anthropic-ai"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250901081731/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20251001000402/https://www.newyorker.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /com.condenast/yv8*"},{"t":"add","s":"Disallow: /newsletters/"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /user-context?referrer"},{"t":"add","s":"Disallow: /newsletters/"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /services.min.js?_="},{"t":"add","s":"Disallow: /newsletters/"},{"t":"ctx","s":""},{"t":"ctx","s":"User-agent: Google-InspectionTool"},{"t":"add","s":"User-agent: Twitterbot"},{"t":"ctx","s":"Allow: /"},{"t":"ctx","s":""},{"t":"del","s":"User-agent: CCBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"ctx","s":"User-agent: Google-Extended"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"add","s":"User-agent: Google-CloudVertexBot"},{"t":"add","s":"User-agent: GoogleOther"},{"t":"add","s":"User-agent: Applebot-Extended"},{"t":"ctx","s":"User-agent: Amazonbot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: FacebookBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"add","s":"User-agent: meta-webindexer"},{"t":"ctx","s":"User-agent: meta-externalagent"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: anthropic-ai"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: Claude-Web"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: cohere-ai"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: Omgilibot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: omgili"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"add","s":"User-agent: meta-externalfetcher"},{"t":"add","s":"User-agent: ClaudeBot"},{"t":"add","s":"User-agent: Claude-SearchBot"},{"t":"add","s":"User-agent: Claude-User"},{"t":"ctx","s":"User-agent: PerplexityBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: YouBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"add","s":"User-agent: Perplexity-User"},{"t":"gap","s":"… 27 more lines"}]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: Meta-ExternalAgent.","dir":"up","up":["Meta-ExternalAgent"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001073502/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20241101004359/https://www.newyorker.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /services.min.js?_="},{"t":"add","s":"Disallow: /com.condenast/yv8*"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: meta-externalagent"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: anthropic-ai"}]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2024-09","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-09-01"],"summary":"Now blocked: Applebot-Extended and Bytespider. Now allowed: GPTBot and ChatGPT-User.","dir":"mixed","up":["Applebot-Extended","Bytespider"],"down":["GPTBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501090606/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20240901034309/https://www.newyorker.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"del","s":"User-agent: GPTBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: ChatGPT-User"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"ctx","s":"User-agent: Google-Extended"},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"del","s":"User-Agent: PerplexityBot"},{"t":"add","s":"User-agent: omgili"},{"t":"ctx","s":"Disallow: /"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: PerplexityBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: YouBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Bytespider"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Diffbot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Applebot-Extended"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Twitterbot"}]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2024-03","precision":"month","source":"archive","provisional":false,"between":["2024-02-01","2024-03-01"],"summary":"Now blocked: PerplexityBot.","dir":"up","up":["PerplexityBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240201034902/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20240301072847/https://www.newyorker.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /product/"},{"t":"add","s":"Disallow: /user-context?referrer"},{"t":"add","s":"Disallow: /services.min.js?_="},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /search"},{"t":"del","s":""},{"t":"add","s":"Disallow: /user-context?referrer"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /search"},{"t":"del","s":""},{"t":"add","s":"Disallow: /user-context?referrer"},{"t":"add","s":"Disallow: /services.min.js?_="},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"add","s":"User-Agent: PerplexityBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Twitterbot"},{"t":"gap","s":"…"},{"t":"ctx","s":"Sitemap: https://www.newyorker.com/categories-sitemap.xml"},{"t":"add","s":"Sitemap: https://www.newyorker.com/bundles-sitemap.xml"},{"t":"ctx","s":"Sitemap: https://www.newyorker.com/sitemap-archive-1.xml"}]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2023-12","precision":"month","source":"archive","provisional":false,"between":["2023-10-01","2023-12-01"],"summary":"Now blocked: Google-Extended, anthropic-ai and Amazonbot.","dir":"up","up":["Google-Extended","anthropic-ai","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20231001004822/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20231201002111/https://www.newyorker.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /product/"},{"t":"add","s":"Disallow: /search"},{"t":"ctx","s":""},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Bingbot"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /product/"},{"t":"add","s":"Disallow: /search"},{"t":"ctx","s":""},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Google-InspectionTool"},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: Google-Extended"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Amazonbot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: FacebookBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: anthropic-ai"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Claude-Web"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: cohere-ai"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Omgilibot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Twitterbot"}]},{"domain":"newyorker.com","name":"The New Yorker","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot, CCBot and ChatGPT-User.","dir":"up","up":["GPTBot","CCBot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801073834/https://www.newyorker.com/robots.txt","https://web.archive.org/web/20230901002009/https://www.newyorker.com/robots.txt"],"diff":[{"t":"del","s":""},{"t":"ctx","s":"User-agent: *"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /*?redirectURL="},{"t":"add","s":"Disallow: /*{{url}}"},{"t":"add","s":"Disallow: /*%7B%7Burl%7D%7D"},{"t":"add","s":"Disallow: /auth/*"},{"t":"add","s":"Disallow: /account/*"},{"t":"add","s":"Disallow: /*{{section}}"},{"t":"add","s":"Disallow: /*%7B%7Bsection%7D%7D"},{"t":"add","s":"Disallow: /cartoon/"},{"t":"add","s":"Disallow: /product/"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /projects"},{"t":"add","s":"Disallow: /*?redirectURL="},{"t":"add","s":"Disallow: /*{{url}}"},{"t":"add","s":"Disallow: /*%7B%7Burl%7D%7D"},{"t":"add","s":"Disallow: /*{{section}}"},{"t":"add","s":"Disallow: /*%7B%7Bsection%7D%7D"},{"t":"add","s":"Disallow: /auth/*"},{"t":"add","s":"Disallow: /account/*"},{"t":"add","s":"Disallow: /product/"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Allow: /*/amp"},{"t":"add","s":"Disallow: /cartoon/"},{"t":"add","s":"Disallow: /*?redirectURL="},{"t":"add","s":"Disallow: /*{{url}}"},{"t":"add","s":"Disallow: /*%7B%7Burl%7D%7D"},{"t":"add","s":"Disallow: /*{{section}}"},{"t":"add","s":"Disallow: /*%7B%7Bsection%7D%7D"},{"t":"add","s":"Disallow: /auth/*"},{"t":"add","s":"Disallow: /account/*"},{"t":"add","s":"Disallow: /product/"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: Google-InspectionTool"},{"t":"add","s":"Allow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: CCBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: GPTBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: ChatGPT-User"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Twitterbot"},{"t":"gap","s":"…"},{"t":"ctx","s":"Sitemap: https://www.newyorker.com/feed/google-news-sitemap-feed/sitemap-google-news"},{"t":"del","s":"            "},{"t":"add","s":"Sitemap: https://www.newyorker.com/contributors-sitemap.xml"},{"t":"add","s":"Sitemap: https://www.newyorker.com/categories-sitemap.xml"},{"t":"add","s":"Sitemap: https://www.newyorker.com/sitemap-archive-1.xml"},{"t":"add","s":"Sitemap: https://www.newyorker.com/sitemap-archive-2.xml"},{"t":"add","s":"Sitemap: https://www.newyorker.com/sitemap-archive-3.xml"},{"t":"add","s":"Sitemap: https://www.newyorker.com/sitemap-archive-4.xml"},{"t":"add","s":"Sitemap: https://www.newyorker.com/sitemap-archive-5.xml"}]}]}