{"domain":"medium.com","name":"Medium","category":"news","subcategory":"platform","country":"US","tags":[],"notes":"","status":"partial","status_label":"Blocks some AI","rationale":"Its robots.txt (the public rules file for bots) treats AI bots differently from each other. It blocks 6 of the 24 AI bots we track from the whole site, including GPTBot, ClaudeBot, Applebot-Extended, Meta-ExternalAgent and Bytespider. Google-Extended, CCBot, OAI-SearchBot, Claude-SearchBot and 4 more can reach most of it. The homepage also showed our bot, which says who it is, a Generic bot challenge page instead of the site.","computed_at":"2026-10-20T03:43:15.322Z","last_checked":"2026-10-19T13:36:04.444Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://medium.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://medium.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://medium.com/robots.txt","hops":[],"blob":"fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74","text":"User-Agent: *\nDisallow: /m/\nDisallow: /me/\nDisallow: /@me$\nDisallow: /@me/\nDisallow: /*/edit$\nDisallow: /*/*/edit$\nDisallow: /media/\nDisallow: /p/*/share\nDisallow: /r/\nDisallow: /trending\nDisallow: /search?q$\nDisallow: /search?q=\nDisallow: /*/search?q=\nDisallow: /*/search/*?q=\nDisallow: /*/*source=\nAllow: /_/api/users/*/meta\nAllow: /_/api/users/*/profile/stream\nAllow: /_/api/posts/*/responses\nAllow: /_/api/posts/*/responsesStream\nAllow: /_/api/posts/*/related\nUser-Agent: Amazonbot\nUser-Agent: Applebot-Extended\nUser-Agent: Bytespider\nUser-Agent: ClaudeBot\nUser-Agent: FacebookBot\nUser-Agent: GoogleOther\nUser-Agent: GPTBot\nUser-Agent: meta-externalagent\nDisallow: /\nAllow: /about\nAllow: /business\nAllow: /earn\nAllow: /gift\nAllow: /membership\nAllow: /partner-program\nAllow: /verified-authors\nSitemap: https://medium.com/sitemap/sitemap.xml\nLicense: https://medium.com/license.xml\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[{"url":"https://medium.com/license.xml","line":39,"agents":["amazonbot","applebot-extended","bytespider","claudebot","facebookbot","googleother","gptbot","meta-externalagent"]}],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*","Amazonbot","Applebot-Extended","Bytespider","ClaudeBot","FacebookBot","GoogleOther","GPTBot","meta-externalagent"],"lines":40},"agents":[{"token":"GPTBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":30},"any":{"allowed":false,"line":30},"search":{"allowed":false,"line":30},"cart":{"allowed":false,"line":30},"checkout":{"allowed":false,"line":30},"account":{"allowed":false,"line":30}},"disallows":1,"lines":[22,39],"groupRanges":[[22,39]]},{"token":"ClaudeBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":30},"any":{"allowed":false,"line":30},"search":{"allowed":false,"line":30},"cart":{"allowed":false,"line":30},"checkout":{"allowed":false,"line":30},"account":{"allowed":false,"line":30}},"disallows":1,"lines":[22,39],"groupRanges":[[22,39]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Applebot-Extended","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":30},"any":{"allowed":false,"line":30},"search":{"allowed":false,"line":30},"cart":{"allowed":false,"line":30},"checkout":{"allowed":false,"line":30},"account":{"allowed":false,"line":30}},"disallows":1,"lines":[22,39],"groupRanges":[[22,39]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":30},"any":{"allowed":false,"line":30},"search":{"allowed":false,"line":30},"cart":{"allowed":false,"line":30},"checkout":{"allowed":false,"line":30},"account":{"allowed":false,"line":30}},"disallows":1,"lines":[22,39],"groupRanges":[[22,39]]},{"token":"Bytespider","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":30},"any":{"allowed":false,"line":30},"search":{"allowed":false,"line":30},"cart":{"allowed":false,"line":30},"checkout":{"allowed":false,"line":30},"account":{"allowed":false,"line":30}},"disallows":1,"lines":[22,39],"groupRanges":[[22,39]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Amazonbot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":30},"any":{"allowed":false,"line":30},"search":{"allowed":false,"line":30},"cart":{"allowed":false,"line":30},"checkout":{"allowed":false,"line":30},"account":{"allowed":false,"line":30}},"disallows":1,"lines":[22,39],"groupRanges":[[22,39]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":15,"lines":[1,21],"groupRanges":[[1,21]]}],"lanes":{"training":{"level":"mixed","none":5,"full":4,"restricted":0,"n":9},"search":{"level":"mixed","none":1,"full":7,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"challenged","http":403,"detail":"Generic bot challenge page"},"edge":{"cdn":["Cloudflare"],"botManagement":[]},"llms":{"state":"refused","http":403,"url":"https://medium.com/llms.txt"},"ai_txt":{"state":"refused","http":403,"url":"https://medium.com/ai.txt"},"tdmrep":{"state":"refused","http":403,"url":"https://medium.com/.well-known/tdmrep.json"},"meta":null,"x_robots_tag":null,"tos":{"url":null,"source":null,"skipped":"No ToS URL curated or discovered"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":45,"first":"2023-01-01","archive_points":44,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-11","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-12","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-01","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-02","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-03","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-04","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-05","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-06","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-07","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-08","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-11","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-12","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-01","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-02","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-03","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-04","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-05","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":5},{"m":"2025-06","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":5},{"m":"2025-07","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":4},{"m":"2025-08","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":4},{"m":"2025-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2025-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2025-11","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2025-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-07","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6},{"m":"2026-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":6}]},"events":[{"domain":"medium.com","name":"Medium","category":"news","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-01","2025-09-01"],"summary":"Now blocked: Meta-ExternalAgent and Amazonbot.","dir":"up","up":["Meta-ExternalAgent","Amazonbot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250801130117/https://medium.com/robots.txt","https://web.archive.org/web/20250901050621/https://medium.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /*/*source="},{"t":"del","s":"Allow: /_/"},{"t":"ctx","s":"Allow: /_/api/users/*/meta"},{"t":"gap","s":"…"},{"t":"ctx","s":"Allow: /_/api/posts/*/related"},{"t":"del","s":"Sitemap: https://medium.com/sitemap/sitemap.xml"},{"t":"add","s":"User-Agent: Amazonbot"},{"t":"ctx","s":"User-Agent: Applebot-Extended"},{"t":"gap","s":"…"},{"t":"ctx","s":"User-Agent: GPTBot"},{"t":"add","s":"User-Agent: meta-externalagent"},{"t":"ctx","s":"Disallow: /"},{"t":"gap","s":"…"},{"t":"ctx","s":"Allow: /verified-authors"},{"t":"add","s":"Sitemap: https://medium.com/sitemap/sitemap.xml"},{"t":"ctx","s":"License: https://medium.com/license.xml"}]},{"domain":"medium.com","name":"Medium","category":"news","date":"2025-07","precision":"month","source":"archive","provisional":false,"between":["2025-06-01","2025-07-01"],"summary":"Now allowed: Google-Extended. Added an RSL <code>License:</code> line pointing to licensing terms.","dir":"down","up":[],"down":["Google-Extended"],"major":true,"signals":true,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250601001635/https://medium.com/robots.txt","https://web.archive.org/web/20250701005708/https://medium.com/robots.txt"],"diff":[{"t":"ctx","s":"User-Agent: GoogleOther"},{"t":"del","s":"User-Agent: Google-Extended"},{"t":"ctx","s":"User-Agent: GPTBot"},{"t":"gap","s":"…"},{"t":"ctx","s":"Allow: /verified-authors"},{"t":"add","s":"License: https://medium.com/license.xml"},{"t":"ctx","s":""}]},{"domain":"medium.com","name":"Medium","category":"news","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now blocked: ClaudeBot, Google-Extended, Applebot-Extended and Bytespider.","dir":"up","up":["ClaudeBot","Google-Extended","Applebot-Extended","Bytespider"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250401230316/https://medium.com/robots.txt","https://web.archive.org/web/20250501183543/https://medium.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /search?q="},{"t":"add","s":"Disallow: /*/search?q="},{"t":"add","s":"Disallow: /*/search/*?q="},{"t":"ctx","s":"Disallow: /*/*source="},{"t":"gap","s":"…"},{"t":"ctx","s":"Sitemap: https://medium.com/sitemap/sitemap.xml"},{"t":"add","s":"User-Agent: Applebot-Extended"},{"t":"add","s":"User-Agent: Bytespider"},{"t":"add","s":"User-Agent: ClaudeBot"},{"t":"add","s":"User-Agent: FacebookBot"},{"t":"add","s":"User-Agent: GoogleOther"},{"t":"add","s":"User-Agent: Google-Extended"},{"t":"ctx","s":"User-Agent: GPTBot"},{"t":"ctx","s":"Disallow: /"},{"t":"add","s":"Allow: /about"},{"t":"add","s":"Allow: /business"},{"t":"add","s":"Allow: /earn"},{"t":"add","s":"Allow: /gift"},{"t":"add","s":"Allow: /membership"},{"t":"add","s":"Allow: /partner-program"},{"t":"add","s":"Allow: /verified-authors"},{"t":"ctx","s":""}]},{"domain":"medium.com","name":"Medium","category":"news","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801010749/https://medium.com/robots.txt","https://web.archive.org/web/20230901021613/https://medium.com/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /search?q="},{"t":"add","s":"Disallow: /*/*source="},{"t":"ctx","s":"Allow: /_/"},{"t":"gap","s":"…"},{"t":"ctx","s":"Sitemap: https://medium.com/sitemap/sitemap.xml"},{"t":"add","s":"User-Agent: GPTBot"},{"t":"add","s":"Disallow: /"},{"t":"ctx","s":""}]}]}