{"domain":"chewy.com","name":"Chewy","category":"retail","subcategory":"specialty","country":"US","tags":[],"notes":"","status":"partial","status_label":"Blocks some AI","rationale":"Its robots.txt (the public rules file for bots) mentions Amazonbot, but still lets all 12 major AI bots into the site. But the homepage turned our bot, which says who it is, away (HTTP 429). The written rules say come in; the front door disagrees.","computed_at":"2026-10-20T03:43:14.143Z","last_checked":"2026-10-19T13:25:26.595Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.chewy.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.chewy.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.chewy.com/robots.txt","hops":[{"from":"https://chewy.com/robots.txt","to":"https://www.chewy.com/robots.txt","status":301}],"blob":"222b0261ee4d989851592e7ede082076354b032c04bce9998052ba7cce54632f","text":"#\n#                   .#################\n#               ###########################\n#            #################################\n#         ,#####################################\n#        ##################      .################\n#      *##############                #############\n#     ,############(                   #############\n#     ############        /######     ###############\n#    /###########        (###########################\n#    ############        /###########################\n#    *###########         #########      ############\n#     ############           ###         *###########\n#      ############                     *###########\n#       ##############                #############\n#        ##################*    (#################\n#          #####################################\n#            #################################\n#               ##########################/\n#                    (###############/\n#\n#  Hello,\n#\n#  If you're a human and you're looking at this file, Chewy's SEO team is looking for folks like you to join our pack!\n#\n#  Check out our current openings at https://careers.chewy.com/us/en/search-results\n\nSitemap: https://www.chewy.com/app/sitemap/behijkmoqrrtttvvy-sitemap_index.xml\nUser-agent: *\nDisallow: /app/buy\nDisallow: /app/deals?\nDisallow: /app/account\nDisallow: /app/ajax\nDisallow: /app/checkout\nDisallow: /*/dp/*?features\nDisallow: /*/dp/*?hybrid\nDisallow: /app/catalog/product-dimensions\nDisallow: /app/we\nDisallow: /app/write-review\nDisallow: /app/review-purchases\nDisallow: /app/api\nDisallow: /app/subs\nDisallow: /rp-resources\nDisallow: *query=*\nDisallow: /cc/\nDisallow: /app/cc/\nDisallow: /*?*ref=*\nDisallow: /*lp=*\nDisallow: /app/dp/*\nDisallow: /api/login\nDisallow: /app/login\nDisallow: /app/register\nDisallow: /app/resetpassword/\nDisallow: /auth/\nDisallow: /ca/app/login\nDisallow: /ca/app/write-review\nDisallow: /ca/app/review-purchases\nDisallow: /ca/app/register\nDisallow: /ca/app/api\nDisallow: /ca/app/resetpassword\nDisallow: /ca/app/account\nDisallow: /ca/ca/\nDisallow: /ca/auth/\nDisallow: /ca/api/login\nDisallow: *adId=*\nDisallow: /vet-care/scheduler\nDisallow: /api/event/\nDisallow: login*targetUrl*\nDisallow: x_tr*\nDisallow: /shops/*\nDisallow: /pethealth/connect-with-a-vet/guest/\nDisallow: /pethealth/connect-with-a-vet/schedule-tele*\nDisallow: /pethealth/connect-with-a-vet/schedule-chat*\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: AdsBot-Google\nAllow: /*lp=*\nAllow: *query=*\n\nUser-agent: adidxbot\nAllow: /*lp=*\nAllow: *query=*\n","refused":null,"warnings":[{"line":68,"msg":"Path \"login*targetUrl*\" does not start with \"/\"; treated as \"/login*targetUrl*\"."},{"line":69,"msg":"Path \"x_tr*\" does not start with \"/\"; treated as \"/x_tr*\"."}],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*","Amazonbot","AdsBot-Google","adidxbot"],"lines":85},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Amazonbot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":76},"any":{"allowed":false,"line":76},"search":{"allowed":false,"line":76},"cart":{"allowed":false,"line":76},"checkout":{"allowed":false,"line":76},"account":{"allowed":false,"line":76}},"disallows":1,"lines":[75,76],"groupRanges":[[75,76]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":44,"lines":[29,73],"groupRanges":[[29,73]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"mixed","none":1,"full":7,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"refused","http":429},"edge":{"cdn":["Akamai","Kasada"],"botManagement":[{"vendor":"Kasada","cookies":["KP_UIDz-ssn","KP_UIDz"]}]},"llms":{"state":"absent","http":404,"url":"https://www.chewy.com/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://www.chewy.com/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://www.chewy.com/.well-known/tdmrep.json"},"meta":null,"x_robots_tag":null,"tos":{"url":null,"source":null,"skipped":"No ToS URL curated or discovered"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":40,"first":"2023-01-01","archive_points":39,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2025-10","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2025-11","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2025-12","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-01","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-02","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-03","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-04","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-05","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-06","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"mixed","user_directed":"full"},"n":1}]},"events":[{"domain":"chewy.com","name":"Chewy","category":"retail","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-02","2025-09-16"],"summary":"Now blocked: Amazonbot.","dir":"up","up":["Amazonbot"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20250802225029/https://www.chewy.com/robots.txt","https://web.archive.org/web/20250916074042/https://www.chewy.com/robots.txt"],"diff":[{"t":"ctx","s":"#"},{"t":"del","s":"#  Check out our current openings at https://careers.chewy.com/us/en/search-results?keywords=SEO"},{"t":"add","s":"#  Check out our current openings at https://careers.chewy.com/us/en/search-results"},{"t":"ctx","s":""},{"t":"add","s":"Sitemap: https://www.chewy.com/app/sitemap/behijkmoqrrtttvvy-sitemap_index.xml"},{"t":"ctx","s":"User-agent: *"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /app/deals?"},{"t":"del","s":"Disallow: /app/login?"},{"t":"ctx","s":"Disallow: /app/account"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /app/dp/*"},{"t":"add","s":"Disallow: /api/login"},{"t":"add","s":"Disallow: /app/login"},{"t":"add","s":"Disallow: /app/register"},{"t":"add","s":"Disallow: /app/resetpassword/"},{"t":"add","s":"Disallow: /auth/"},{"t":"add","s":"Disallow: /ca/app/login"},{"t":"add","s":"Disallow: /ca/app/write-review"},{"t":"add","s":"Disallow: /ca/app/register"},{"t":"add","s":"Disallow: /ca/app/api"},{"t":"add","s":"Disallow: /ca/app/resetpassword"},{"t":"add","s":"Disallow: /ca/app/account"},{"t":"add","s":"Disallow: /ca/ca/"},{"t":"add","s":"Disallow: /ca/auth/"},{"t":"add","s":"Disallow: /ca/api/login"},{"t":"add","s":"Disallow: *adId=*"},{"t":"add","s":"Disallow: /vet-care/scheduler"},{"t":"add","s":"Disallow: /api/event/"},{"t":"add","s":"Disallow: login*targetUrl*"},{"t":"add","s":"Disallow: x_tr*"},{"t":"add","s":"Disallow: /shops/*"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: Amazonbot"},{"t":"add","s":"Disallow: /"},{"t":"ctx","s":""}]}]}