{"domain":"webmd.com","name":"WebMD","category":"healthcare","subcategory":"publisher","country":"US","tags":["healthcare"],"notes":"","status":"partial","status_label":"Blocks some AI","rationale":"Its robots.txt (the public rules file for bots) treats AI bots differently from each other. It blocks 3 of the 24 AI bots we track from the whole site, including GPTBot, ClaudeBot and CCBot. Google-Extended, Applebot-Extended, Meta-ExternalAgent, OAI-SearchBot and 5 more can reach most of it.","computed_at":"2026-10-20T03:43:15.734Z","last_checked":"2026-10-19T13:42:47.334Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.webmd.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.webmd.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.webmd.com/robots.txt","hops":[{"from":"https://webmd.com/robots.txt","to":"https://www.webmd.com/robots.txt","status":301}],"blob":"e7610682d0934ea2b4266992e2a88b3ca70c2e2ff9f8047223b03c2ca4fbce7a","text":"# Robots.txt file WebMD\r\n# Updated 8/31/2026 (CONSFE-458)\r\n\r\nUser-agent: *\r\nDisallow: */search/search_results/\r\nDisallow: /500\r\nDisallow: /aim/\r\nDisallow: /api/\r\nDisallow: /bipolar-disorder/video/bipolar-fatherhood\r\nDisallow: /bipolar-disorder/video/bipolar-toolbox\r\nDisallow: /breast-cancer/bc-treatment-21/provider-select\r\nDisallow: /click*\r\nDisallow: /dna\r\nDisallow: /drugs/2/search*\r\nDisallow: /drugs/ReportAbuse.aspx*\r\nDisallow: /kapi/\r\nDisallow: /mm/\r\nDisallow: /my-library\r\nDisallow: /news/header.php\r\nDisallow: /pill-identification/search-results*\r\nDisallow: /search/2/api/\r\nDisallow: /Share.aspx*\r\nDisallow: /sponsored/\r\nDisallow: /static/\r\nDisallow: /story/\r\nDisallow: /webmd_static_vue/\r\n\r\nUser-agent: AdsBot-Google\r\nDisallow: /mm/\r\n\r\nUser-agent: CCBot\r\nDisallow: /\r\n\r\nUser-agent: ClaudeBot\r\nDisallow: /\r\n\r\nUser-agent: Doximity\r\nDisallow: /\r\n\r\nUser-agent: GPTBot\r\nDisallow: /","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*","AdsBot-Google","CCBot","ClaudeBot","Doximity","GPTBot"],"lines":41},"agents":[{"token":"GPTBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":41},"any":{"allowed":false,"line":41},"search":{"allowed":false,"line":41},"cart":{"allowed":false,"line":41},"checkout":{"allowed":false,"line":41},"account":{"allowed":false,"line":41}},"disallows":1,"lines":[40,41],"groupRanges":[[40,41]]},{"token":"ClaudeBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":35},"any":{"allowed":false,"line":35},"search":{"allowed":false,"line":35},"cart":{"allowed":false,"line":35},"checkout":{"allowed":false,"line":35},"account":{"allowed":false,"line":35}},"disallows":1,"lines":[34,35],"groupRanges":[[34,35]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"CCBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":32},"any":{"allowed":false,"line":32},"search":{"allowed":false,"line":32},"cart":{"allowed":false,"line":32},"checkout":{"allowed":false,"line":32},"account":{"allowed":false,"line":32}},"disallows":1,"lines":[31,32],"groupRanges":[[31,32]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":22,"lines":[4,26],"groupRanges":[[4,26]]}],"lanes":{"training":{"level":"mixed","none":3,"full":6,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":["Cloudflare"],"botManagement":[{"vendor":"Cloudflare Bot Management","cookies":["__cf_bm"]}]},"llms":{"state":"absent","http":404,"url":"https://www.webmd.com/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://www.webmd.com/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://www.webmd.com/.well-known/tdmrep.json"},"meta":{"title":"WebMD - Better information. Better health.","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":[],"platform":null,"textRatio":0.015,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://www.webmd.com/about-webmd-policies/about-terms-and-conditions-of-use","source":"discovered","http":200,"hash":"0a06a0b8d8af6fe52ab7e4250e249bc46af211b9b72d3738a9b9cc2bc63289a6","chars":27082,"thin":false,"clauses":[{"keyword":"agent","prohibition":true,"excerpt":"… you have a good faith belief that the complained of use of the materials is not authorized by the copyright owner, its agent, or the law."},{"keyword":"scraping","prohibition":false,"excerpt":"Harvesting, scraping or otherwise collecting information about others, including email addresses, without their consent for posting or viewing comments;"},{"keyword":"agents","prohibition":false,"excerpt":"You agree to defend, indemnify, and hold WebMD, its officers, directors, employees, agents, licensors, and suppliers, harmless from and against any claims, actions or demands, liabilities and settlements, including, without limitation, reasonable legal and accounting fees, resulting from, or alleged…"}],"door":{"door":"open","http":200}},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":45,"first":"2023-01-01","archive_points":44,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-11","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2023-12","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-01","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-02","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-03","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-04","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-05","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-06","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-07","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-08","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-11","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-12","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":3},{"m":"2025-01","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":3},{"m":"2025-02","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-03","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-04","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-05","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-06","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-07","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-08","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-11","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2025-12","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-01","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-02","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-03","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-04","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-05","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-06","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-07","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-08","s":"partial","l":{"training":"mixed","search":"full","user_directed":"mixed"},"n":4},{"m":"2026-09","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":3},{"m":"2026-10","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":3}]},"events":[{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2026-09","precision":"month","source":"archive","provisional":false,"between":["2026-08-05","2026-09-03"],"summary":"Now allowed: ChatGPT-User.","dir":"down","up":[],"down":["ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20260805083730/https://www.webmd.com/robots.txt","https://web.archive.org/web/20260903170921/https://www.webmd.com/robots.txt"],"diff":[{"t":"ctx","s":"# Robots.txt file WebMD"},{"t":"del","s":"# Updated 1/8/2025"},{"t":"add","s":"# Updated 8/31/2026 (CONSFE-458)"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: */search/search_results/"},{"t":"del","s":"Disallow: /drugs/ReportAbuse.aspx*"},{"t":"del","s":"Disallow: */cm/"},{"t":"del","s":"Disallow: /Share.aspx*"},{"t":"del","s":"Disallow: /mm/"},{"t":"del","s":"Disallow: /kapi/"},{"t":"del","s":"Disallow: /click*"},{"t":"del","s":"Disallow: /drugs/2/search*"},{"t":"del","s":"Disallow: /sponsored/"},{"t":"del","s":"Disallow: /api/"},{"t":"del","s":"Disallow: /search/2/api/"},{"t":"del","s":"Disallow: /webmd_static_vue/"},{"t":"add","s":"Disallow: /500"},{"t":"ctx","s":"Disallow: /aim/"},{"t":"del","s":"Disallow: /6-tools-for-managing-your-medications"},{"t":"del","s":"Disallow: /attr-cm-warning-signs"},{"t":"del","s":"Disallow: /bipolar-disorder/bipolar-depression-management"},{"t":"add","s":"Disallow: /api/"},{"t":"ctx","s":"Disallow: /bipolar-disorder/video/bipolar-fatherhood"},{"t":"ctx","s":"Disallow: /bipolar-disorder/video/bipolar-toolbox"},{"t":"del","s":"Disallow: /bluechew-review"},{"t":"del","s":"Disallow: /brain/treating-gmg-video"},{"t":"del","s":"Disallow: /brain/video/treating-gmg-video"},{"t":"ctx","s":"Disallow: /breast-cancer/bc-treatment-21/provider-select"},{"t":"del","s":"Disallow: /cbd/about-cbd"},{"t":"del","s":"Disallow: /cbd/best-cbd-gummies-for-anxiety"},{"t":"del","s":"Disallow: /cbd/best-cbd-oil-for-anxiety"},{"t":"del","s":"Disallow: /cbd/best-cbd-oil-for-pain"},{"t":"del","s":"Disallow: /cbd/best-cbd-oil-of-2023"},{"t":"del","s":"Disallow: /cbd/best-full-spectrum-cbd-oils"},{"t":"del","s":"Disallow: /cbd/best-gummies-of-2023"},{"t":"del","s":"Disallow: /cbd/cbd-101-concentrated-cannabidiol"},{"t":"del","s":"Disallow: /cbd/cbd-for-dogs"},{"t":"del","s":"Disallow: /cbd/cbd-for-sleep"},{"t":"del","s":"Disallow: /cbd/cbdfx-review"},{"t":"del","s":"Disallow: /cbd/cbdistillery-review"},{"t":"del","s":"Disallow: /cbd/koi-cbd-review"},{"t":"del","s":"Disallow: /chemo-combo"},{"t":"del","s":"Disallow: /cholesterol-management/high-cholesterol-treatment"},{"t":"del","s":"Disallow: /choose-cancer-care"},{"t":"del","s":"Disallow: /diagnosed-with-glioblastoma"},{"t":"add","s":"Disallow: /click*"},{"t":"ctx","s":"Disallow: /dna"},{"t":"del","s":"Disallow: /dna/meet-your-match"},{"t":"del","s":"Disallow: /dna/meet-your-match-acne-prone-skin-routine"},{"t":"del","s":"Disallow: /dna/meet-your-match-combination-to-oily-skin-routine"},{"t":"del","s":"Disallow: /dna/meet-your-match-normal-to-dry-skin-routine"},{"t":"del","s":"Disallow: /dupuytrens/default.htm"},{"t":"del","s":"Disallow: /dupuytrens/treatment"},{"t":"del","s":"Disallow: /dupuytrens/what-to-expect"},{"t":"del","s":"Disallow: /ear-health-support/balance"},{"t":"del","s":"Disallow: /ear-health-support/daily-care"},{"t":"del","s":"Disallow: /ear-health-support/default.htm"},{"t":"del","s":"Disallow: /ear-health-support/hearing-loss"},{"t":"del","s":"Disallow: /epilepsy-seizures-rapid-rescue-single-treatment"},{"t":"gap","s":"… 63 more lines"}]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2025-02","precision":"month","source":"archive","provisional":false,"between":["2025-01-01","2025-02-01"],"summary":"Now blocked: ChatGPT-User.","dir":"up","up":["ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250101003429/https://www.webmd.com/robots.txt","https://web.archive.org/web/20250201102653/https://www.webmd.com/robots.txt"],"diff":[{"t":"ctx","s":"# Robots.txt file WebMD"},{"t":"del","s":"# Updated 11/8/2024"},{"t":"add","s":"# Updated 1/8/2025"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"del","s":"Sitemap: https://www.webmd.com/rx/sitemap.xml"},{"t":"del","s":"Sitemap: https://www.webmd.com/sitemapxml/sitemap.xml"},{"t":"del","s":"Sitemap: https://www.webmd.com/sitemapxml/news.xml"},{"t":"add","s":"User-agent: AdsBot-Google"},{"t":"add","s":"Disallow: /mm/"},{"t":"add","s":""},{"t":"add","s":"User-agent: ChatGPT-User"},{"t":"add","s":"Disallow: /"}]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2024-12","precision":"month","source":"archive","provisional":false,"between":["2024-11-01","2024-12-01"],"summary":"Now blocked: ClaudeBot and CCBot.","dir":"up","up":["ClaudeBot","CCBot"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20241101121540/https://www.webmd.com/robots.txt","https://web.archive.org/web/20241201064835/https://www.webmd.com/robots.txt"],"diff":[{"t":"ctx","s":"# Robots.txt file WebMD"},{"t":"del","s":"# Updated 8/24/2023"},{"t":"add","s":"# Updated 11/8/2024"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /aim/"},{"t":"add","s":"Disallow: /6-tools-for-managing-your-medications"},{"t":"add","s":"Disallow: /attr-cm-warning-signs"},{"t":"add","s":"Disallow: /bipolar-disorder/bipolar-depression-management"},{"t":"add","s":"Disallow: /bipolar-disorder/video/bipolar-fatherhood"},{"t":"add","s":"Disallow: /bipolar-disorder/video/bipolar-toolbox"},{"t":"add","s":"Disallow: /bluechew-review"},{"t":"add","s":"Disallow: /brain/treating-gmg-video"},{"t":"add","s":"Disallow: /brain/video/treating-gmg-video"},{"t":"add","s":"Disallow: /breast-cancer/bc-treatment-21/provider-select"},{"t":"add","s":"Disallow: /cbd/about-cbd"},{"t":"add","s":"Disallow: /cbd/best-cbd-gummies-for-anxiety"},{"t":"add","s":"Disallow: /cbd/best-cbd-oil-for-anxiety"},{"t":"add","s":"Disallow: /cbd/best-cbd-oil-for-pain"},{"t":"add","s":"Disallow: /cbd/best-cbd-oil-of-2023"},{"t":"add","s":"Disallow: /cbd/best-full-spectrum-cbd-oils"},{"t":"add","s":"Disallow: /cbd/best-gummies-of-2023"},{"t":"add","s":"Disallow: /cbd/cbd-101-concentrated-cannabidiol"},{"t":"add","s":"Disallow: /cbd/cbd-for-dogs"},{"t":"add","s":"Disallow: /cbd/cbd-for-sleep"},{"t":"add","s":"Disallow: /cbd/cbdfx-review"},{"t":"add","s":"Disallow: /cbd/cbdistillery-review"},{"t":"add","s":"Disallow: /cbd/koi-cbd-review"},{"t":"add","s":"Disallow: /chemo-combo"},{"t":"add","s":"Disallow: /cholesterol-management/high-cholesterol-treatment"},{"t":"add","s":"Disallow: /choose-cancer-care"},{"t":"add","s":"Disallow: /diagnosed-with-glioblastoma"},{"t":"add","s":"Disallow: /dna"},{"t":"add","s":"Disallow: /dna/meet-your-match"},{"t":"add","s":"Disallow: /dna/meet-your-match-acne-prone-skin-routine"},{"t":"add","s":"Disallow: /dna/meet-your-match-combination-to-oily-skin-routine"},{"t":"add","s":"Disallow: /dna/meet-your-match-normal-to-dry-skin-routine"},{"t":"add","s":"Disallow: /dupuytrens/default.htm"},{"t":"add","s":"Disallow: /dupuytrens/treatment"},{"t":"add","s":"Disallow: /dupuytrens/what-to-expect"},{"t":"add","s":"Disallow: /ear-health-support/balance"},{"t":"add","s":"Disallow: /ear-health-support/daily-care"},{"t":"add","s":"Disallow: /ear-health-support/default.htm"},{"t":"add","s":"Disallow: /ear-health-support/hearing-loss"},{"t":"add","s":"Disallow: /epilepsy-seizures-rapid-rescue-single-treatment"},{"t":"add","s":"Disallow: /friends-through-gvhd"},{"t":"add","s":"Disallow: /gum-disease-treatment"},{"t":"add","s":"Disallow: /hair-transplant-services"},{"t":"add","s":"Disallow: /hims-hard-mints"},{"t":"add","s":"Disallow: /how-caregivers-helped-loved-ones-with-attr-cm"},{"t":"add","s":"Disallow: /knee-osteoarthritis-clinical-trial"},{"t":"add","s":"Disallow: /lung/copd/treat-copd-21/copd-new-treatment"},{"t":"add","s":"Disallow: /mental-health-genetic-testing"},{"t":"add","s":"Disallow: /mother-to-baby-observational-research-studies"},{"t":"add","s":"Disallow: /neuromyelitis-optica-spectrum-disorder-nmosd-treatment"},{"t":"add","s":"Disallow: /nvaf-stroke-risk-treatment"},{"t":"add","s":"Disallow: /oral-and-maxillofacial-surgeon"},{"t":"add","s":"Disallow: /parenting/cpp-21/what-is-cpp"},{"t":"add","s":"Disallow: /peyronies-treatment-option/talk-to-doctor"},{"t":"add","s":"Disallow: /peyronies-treatment-option/treatment-option"},{"t":"gap","s":"… 33 more lines"}]},{"domain":"webmd.com","name":"WebMD","category":"healthcare","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Now blocked: GPTBot.","dir":"up","up":["GPTBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20230801014425/https://www.webmd.com/robots.txt","https://web.archive.org/web/20230901011205/https://www.webmd.com/robots.txt"],"diff":[{"t":"del","s":"#   Robots.txt file WebMD"},{"t":"del","s":"#   Updated: Oct 2022"},{"t":"add","s":"# Robots.txt file WebMD"},{"t":"add","s":"# Updated 8/24/2023"},{"t":"ctx","s":""},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /Share.aspx*"},{"t":"add","s":"Disallow: /mm/"},{"t":"add","s":"Disallow: /kapi/"},{"t":"add","s":"Disallow: /click*"},{"t":"add","s":"Disallow: /drugs/2/search*"},{"t":"add","s":"Disallow: /sponsored/"},{"t":"add","s":"Disallow: /api/"},{"t":"add","s":"Disallow: /search/2/api/"},{"t":"add","s":"Disallow: /webmd_static_vue/"},{"t":"add","s":"Disallow: /aim/"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: GPTBot"},{"t":"add","s":"Disallow: / "},{"t":"add","s":""}]}]}