{"domain":"axios.com","name":"Axios","category":"news","subcategory":"digital","country":"US","tags":[],"notes":"","status":"partial","status_label":"Blocks some AI","rationale":"Its robots.txt (the public rules file for bots) treats AI bots differently from each other. It blocks 3 of the 24 AI bots we track from the whole site, including CCBot, Bytespider and Amazonbot. GPTBot, ClaudeBot, Google-Extended, Applebot-Extended and 7 more can reach most of it. The homepage also showed our bot, which says who it is, a Cloudflare challenge page instead of the site.","computed_at":"2026-10-20T03:43:15.274Z","last_checked":"2026-10-19T13:35:27.350Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.axios.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.axios.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.axios.com/robots.txt","hops":[{"from":"https://axios.com/robots.txt","to":"https://www.axios.com/robots.txt","status":301}],"blob":"9dbf5ea2259b2cb8a0236f5208c2bab2e0e83985be508732d648ca27fd32f9b5","text":"\n# Disallow Rules\n\nUser-agent: Twitterbot/*\nDisallow:\nCrawl-delay: 1\n\nUser-agent: Google-Extended\nAllow: /\nDisallow: /core/*\nDisallow: /admin/\nDisallow: /login/\nDisallow: /sso/\nDisallow: /api/\n\nUser-agent: *\nDisallow: /core/*\nDisallow: /admin/\nDisallow: /login/\nDisallow: /sso/\nDisallow: /api/\n\n# Disallow Rules (bot-specific)\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\n# Sitemaps\n\nSitemap: https://www.axios.com/sitemap.xml\n\n# 8fkux4mqab196bbs\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["Twitterbot/*","Google-Extended","*","CCBot","Bytespider","Diffbot","FacebookBot","ImagesiftBot","Scrapy","Amazonbot"],"lines":51},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Google-Extended","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":9},"any":{"allowed":true,"line":9},"search":{"allowed":true,"line":9},"cart":{"allowed":true,"line":9},"checkout":{"allowed":true,"line":9},"account":{"allowed":true,"line":9}},"disallows":5,"lines":[8,14],"groupRanges":[[8,14]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"CCBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[25,26],"groupRanges":[[25,26]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Bytespider","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":29},"any":{"allowed":false,"line":29},"search":{"allowed":false,"line":29},"cart":{"allowed":false,"line":29},"checkout":{"allowed":false,"line":29},"account":{"allowed":false,"line":29}},"disallows":1,"lines":[28,29],"groupRanges":[[28,29]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Amazonbot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":44},"any":{"allowed":false,"line":44},"search":{"allowed":false,"line":44},"cart":{"allowed":false,"line":44},"checkout":{"allowed":false,"line":44},"account":{"allowed":false,"line":44}},"disallows":1,"lines":[43,44],"groupRanges":[[43,44]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":5,"lines":[16,21],"groupRanges":[[16,21]]}],"lanes":{"training":{"level":"mixed","none":2,"full":7,"restricted":0,"n":9},"search":{"level":"mixed","none":1,"full":7,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"challenged","http":403,"detail":"Cloudflare challenge page"},"edge":{"cdn":["Cloudflare"],"botManagement":[{"vendor":"Cloudflare Bot Management","cookies":["__cf_bm"]}]},"llms":{"state":"absent","http":404,"url":"https://www.axios.com/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://www.axios.com/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://www.axios.com/.well-known/tdmrep.json"},"meta":null,"x_robots_tag":null,"tos":{"url":null,"source":null,"skipped":"No ToS URL curated or discovered"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":45,"first":"2023-01-01","archive_points":44,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-11","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-12","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-01","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-02","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-03","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-04","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-05","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-06","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":2},{"m":"2024-07","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":6},{"m":"2024-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":7},{"m":"2024-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":7},{"m":"2024-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":7},{"m":"2024-11","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":8},{"m":"2024-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":8},{"m":"2025-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":8},{"m":"2025-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-07","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-11","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2025-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2026-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2026-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":5},{"m":"2026-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-07","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3}]},"events":[{"domain":"axios.com","name":"Axios","category":"news","date":"2026-07","precision":"month","source":"archive","provisional":false,"between":["2026-05-01","2026-07-08"],"summary":"Named Google-Extended explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260501023028/https://www.axios.com/robots.txt","https://web.archive.org/web/20260708091533/https://www.axios.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"add","s":"User-agent: Google-Extended"},{"t":"add","s":"Allow: /"},{"t":"add","s":"Disallow: /core/*"},{"t":"add","s":"Disallow: /admin/"},{"t":"add","s":"Disallow: /login/"},{"t":"add","s":"Disallow: /sso/"},{"t":"add","s":"Disallow: /api/"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: *"}]},{"domain":"axios.com","name":"Axios","category":"news","date":"2026-03","precision":"month","source":"archive","provisional":false,"between":["2026-02-01","2026-03-01"],"summary":"Now allowed: ClaudeBot and Google-Extended.","dir":"down","up":[],"down":["ClaudeBot","Google-Extended"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260201030641/https://www.axios.com/robots.txt","https://web.archive.org/web/20260301010704/https://www.axios.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"del","s":"User-agent: Google-Extended"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"ctx","s":"User-agent: CCBot"},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"del","s":"User-agent: ClaudeBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: Claude-Web"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"ctx","s":"User-agent: Diffbot"}]},{"domain":"axios.com","name":"Axios","category":"news","date":"2025-02","precision":"month","source":"archive","provisional":false,"between":["2025-01-01","2025-02-01"],"summary":"Now allowed: GPTBot, OAI-SearchBot and ChatGPT-User.","dir":"down","up":[],"down":["GPTBot","OAI-SearchBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250101023137/https://www.axios.com/robots.txt","https://web.archive.org/web/20250201042227/https://www.axios.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /sso/"},{"t":"add","s":"Disallow: /api/"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"del","s":"User-agent: GPTBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: ChatGPT-User"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-agent: OAI-SearchBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"ctx","s":"User-agent: Google-Extended"}]},{"domain":"axios.com","name":"Axios","category":"news","date":"2024-11","precision":"month","source":"archive","provisional":false,"between":["2024-10-01","2024-11-01"],"summary":"Now blocked: OAI-SearchBot.","dir":"up","up":["OAI-SearchBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241001000503/https://www.axios.com/robots.txt","https://web.archive.org/web/20241101001013/https://www.axios.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"add","s":"User-agent: OAI-SearchBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Google-Extended"}]},{"domain":"axios.com","name":"Axios","category":"news","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-05-01","2024-08-01"],"summary":"Now blocked: ClaudeBot, Google-Extended, CCBot, Bytespider, Amazonbot and ChatGPT-User.","dir":"up","up":["ClaudeBot","Google-Extended","CCBot","Bytespider","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240501153845/https://www.axios.com/robots.txt","https://web.archive.org/web/20240801055656/https://www.axios.com/robots.txt"],"diff":[{"t":"add","s":""},{"t":"add","s":"# Disallow Rules"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: Twitterbot/*"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /core/*"},{"t":"add","s":"Disallow: /admin/"},{"t":"add","s":"Disallow: /login/"},{"t":"add","s":"Disallow: /sso/"},{"t":"ctx","s":""},{"t":"add","s":"# Disallow Rules (bot-specific)"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: GPTBot"},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: ChatGPT-User"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Google-Extended"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: CCBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Bytespider"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: ClaudeBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Claude-Web"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Diffbot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: FacebookBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: ImagesiftBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Scrapy"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"User-agent: Amazonbot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":"# Sitemaps"},{"t":"add","s":""},{"t":"ctx","s":"Sitemap: https://www.axios.com/sitemap.xml"}]},{"domain":"axios.com","name":"Axios","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801012224/https://www.axios.com/robots.txt","https://web.archive.org/web/20230901020806/https://www.axios.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"add","s":"User-agent: GPTBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"Sitemap: https://www.axios.com/sitemap.xml"}]}]}