{"domain":"walgreens.com","name":"Walgreens","category":"healthcare","subcategory":"pharmacy","country":"US","tags":["healthcare"],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) has no rules for any AI bot we track. They are covered by its catch-all rule (<code>User-agent: *</code>), which lets them into the site.","computed_at":"2026-10-20T03:43:15.724Z","last_checked":"2026-10-19T13:43:25.097Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.walgreens.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.walgreens.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.walgreens.com/robots.txt","hops":[{"from":"https://walgreens.com/robots.txt","to":"https://www.walgreens.com/robots.txt","status":301}],"blob":"f3b301ce426bc203f1fd60f2594383f743c766ee09b329e93614cff693dee835","text":"# Robots.txt exclusion for walgreens.com\r\n\r\nUser-agent: *\r\n\r\nAllow: /common/react/assets/*.js$\r\nDisallow: /emailsignup/\r\nDisallow: /includes/\r\nDisallow: /retailsearch/\r\nDisallow: /buyagain/product/\r\nDisallow: /productrectransformer/v1/\r\nDisallow: /digitalretailbff/v1/sdui/\r\nDisallow: /library/checkdrug/\r\nDisallow: /logout.jsp\r\nDisallow: /balancerewards/email_update.jsp\r\nDisallow: /balancerewards/v1/email_update.jsp\r\nDisallow: /cartsvc/\r\nDisallow: /checkout/\r\nDisallow: /tendercloud/\r\nDisallow: /tender/\r\nDisallow: /payment/\r\nDisallow: /epspayment/\r\nDisallow: /messaging/\r\nDisallow: /marketing/emailsignup/\r\nDisallow: /password/\r\nDisallow: /pharmacy/order/enterrxno.jsp\r\nDisallow: /pharmacy/history/\r\nDisallow: /pharmacy/overlays/\r\nDisallow: /pharmacy/order/enterprescription.jsp\r\nDisallow: /pharmacy/order/selectordermethod.jsp\r\nDisallow: /pharmacy/register/\r\nDisallow: /pharmacy/report/\r\nDisallow: /pharmacy/remind/\r\nDisallow: /pharmacy/rd_history.jsp\r\nDisallow: /pharmacy/thirdpartyrefill/\r\nDisallow: /popups/\r\nDisallow: /overlays/\r\nDisallow: /search/results.jsp?\r\nDisallow: /search/search_results.jsp?\r\nDisallow: /send/\r\nDisallow: /shoppinglist/\r\nDisallow: /store/browse/overlays/\r\nDisallow: /store/checkout/\r\nDisallow: /store/sscart.jsp\r\nDisallow: /register/\r\nDisallow: /reviews/\r\nDisallow: /youraccount/\r\nDisallow: /webpickup/\r\nDisallow: /iso/\r\nDisallow: /health/diet-and-fitness/a/what-your-bmi-does-not-tell-you/1317\r\nDisallow: /health/diet-and-fitness/a/6-ways-mindfulness-can-help-you-lose-weight/1318\r\nDisallow: /health/diet-and-fitness/a/10-Ways-to-Lighten-Up-Your-Favorite-Fatty-Foods/1315\r\nDisallow: /health/womens-health/a/nine-sun-safety-tips-for-your-skin/1319\r\nDisallow: /health/womens-health/a/the-411-on-skin-lighteners/1320\r\nDisallow: /health/womens-health/a/can-you-beat-cellulite/1321\r\nDisallow: /health/diet-and-fitness/a/Potassium-and-Your-Heart/1312\r\nDisallow: /health/diet-and-fitness/a/Exercise-for-a-Healthy-Heart/1311\r\nDisallow: /health/diet-and-fitness/a/Heart-Disease-and-Stress/1310\r\nDisallow: /health/diabetes/a/Diabetes and Alcohol/1322\r\nDisallow: /health/diabetes/a/Diabetes and Smoking/1323\r\nDisallow: /health/diet-and-fitness/a/can_you_drink_too_much_water/1324\r\nDisallow: /health/diet-and-fitness/a/how-to-manage-sore-muscles-and-joint-pain/1325\r\nDisallow: /health/diet-and-fitness/a/6-fixes-if-you-hate-exercise/1326\r\nDisallow: /health/diet-and-fitness/a/stay-focused-on-health-have-backup-plans/1327\r\nDisallow: /q/www.walgreens.com/\r\nDisallow: /search/q/\r\nAllow: /includes/home/homeBodyContent\r\n\r\nSitemap: https://www.walgreens.com/sitemap-index.xml","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*"],"lines":68},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":60,"lines":[3,66],"groupRanges":[[3,66]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":["Akamai"],"botManagement":[{"vendor":"Akamai","cookies":["bm_sz"]}]},"llms":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.walgreens.com/request_error.jsp?redirect=/llms.txt"},"ai_txt":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.walgreens.com/request_error.jsp?redirect=/ai.txt"},"tdmrep":{"state":"error","error":"TimeoutError","url":"https://www.walgreens.com/.well-known/tdmrep.json"},"meta":{"title":"Walgreens: Pharmacy, Health & Wellness, Photo & More for You","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":[],"platform":null,"textRatio":0.03,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://www.walgreens.com/topic/help/generalhelp/termsofuse.jsp?foot=terms","source":"discovered","http":200,"hash":"12975c0bba97514de2aa73d291bba167d6767ecc491a7e8ee0a96f4d27749c68","chars":42958,"thin":false,"clauses":[{"keyword":"robots","prohibition":true,"excerpt":"…any proprietary notices; and (viii) deploy or utilize any automated method of collecting content from our site, such as robots, crawlers, and scraping mechanisms (Sections (i) through (viii) collectively, \"Prohibited Use\")."}],"door":{"door":"open","http":200}},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":42,"first":"2023-01-01","archive_points":41,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0}]},"events":[]}