{"domain":"twitch.tv","name":"Twitch","category":"social","subcategory":"video","country":"US","tags":[],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) mentions Amazonbot, Amzn-SearchBot and Amzn-User, but still lets all 12 major AI bots into the site.","computed_at":"2026-10-20T03:43:15.560Z","last_checked":"2026-10-19T13:37:40.229Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.twitch.tv"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.twitch.tv/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.twitch.tv/robots.txt","hops":[{"from":"https://twitch.tv/robots.txt","to":"https://www.twitch.tv/robots.txt","status":301}],"blob":"34a44188490e1ff02884c384cf6e546e0fa4b2af0c8525f280b32480620002c8","text":"User-agent: Amazonbot\nDisallow:\n\nUser-agent: Amazonbot-Video\nDisallow:\n\nUser-agent: Amzn-SearchBot\nDisallow:\n\nUser-agent: Amzn-User\nDisallow:\n\nUser-agent: *\nDisallow: /login$\nDisallow: /login?\nDisallow: /login/\nDisallow: /signup$\nDisallow: /signup?\nDisallow: /signup/\nDisallow: /auth/\nDisallow: /user/account-recovery\nDisallow: /user/password-reset\nDisallow: /user/not-me\nDisallow: /activate$\nDisallow: /activate?\nDisallow: /activate/\nDisallow: /admin/*\nDisallow: /wv/*\nDisallow: /user/*\nDisallow: /message/*\nDisallow: /settings/\nDisallow: /subscriptions$\nDisallow: /subscriptions?\nDisallow: /subscriptions/\nDisallow: /inventory$\nDisallow: /inventory?\nDisallow: /inventory/\nDisallow: /wallet$\nDisallow: /wallet?\nDisallow: /wallet/\nDisallow: /drops/\nDisallow: /payments/\nDisallow: /email-unsubscribe/\nDisallow: /search$\nDisallow: /search?\nDisallow: /search/\nDisallow: /bits-checkout\nDisallow: /redeem$\nDisallow: /redeem?\nDisallow: /redeem/\nDisallow: /claim$\nDisallow: /claim?\nDisallow: /claim/\nDisallow: /download-keys\nDisallow: /popout/\nDisallow: /embed/\nDisallow: /chat/\nDisallow: /moderator/\nDisallow: /stream$\nDisallow: /stream?\nDisallow: /stream/\nDisallow: /collab$\nDisallow: /collab?\nDisallow: /collab/\nDisallow: /guest-star\nDisallow: /sq$\nDisallow: /sq?\nDisallow: /sq/\nDisallow: /gl$\nDisallow: /gl?\nDisallow: /gl/\nDisallow: /sportradar-widgets\nDisallow: /beta/\nDisallow: /extensions/oauth-redirect\nDisallow: /copyright-claims\nDisallow: /report-appeal\nDisallow: /oss-attribution\nDisallow: /dj-signup\nDisallow: /r/\nDisallow: /stale/\nAllow: /directory\nAllow: /.well-known/assetlinks.json\n\nSitemap: https://www.twitch.tv/sitemapv2_index.xml.gz\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["Amazonbot","Amazonbot-Video","Amzn-SearchBot","Amzn-User","*"],"lines":85},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Amazonbot","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":0,"lines":[1,2],"groupRanges":[[1,2]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":0,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Amzn-User","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":0,"lines":[10,11],"groupRanges":[[10,11]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":44},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":67,"lines":[13,82],"groupRanges":[[13,82]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":[],"botManagement":[]},"llms":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.twitch.tv/llms.txt"},"ai_txt":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.twitch.tv/ai.txt"},"tdmrep":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.twitch.tv/.well-known/tdmrep.json"},"meta":{"title":"Twitch","robotsMeta":[{"name":"robots","content":"index"}],"noai":[],"noindex":false,"jsonLdTypes":["Organization"],"platform":null,"textRatio":0,"llmsLink":false},"x_robots_tag":null,"tos":{"url":null,"source":null,"skipped":"No ToS URL curated or discovered"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":44,"first":"2023-01-01","archive_points":43,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0}]},"events":[{"domain":"twitch.tv","name":"Twitch","category":"social","date":"2026-07","precision":"month","source":"archive","provisional":false,"between":["2026-05-02","2026-07-29"],"summary":"Named Amazonbot, Amzn-SearchBot and Amzn-User explicitly, without changing what they may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260502134429/https://www.twitch.tv/robots.txt","https://web.archive.org/web/20260729120413/https://www.twitch.tv/robots.txt"],"diff":[{"t":"del","s":"User-Agent: *"},{"t":"del","s":"Allow: /"},{"t":"del","s":"Allow: /directory"},{"t":"del","s":"Allow: /directory/all"},{"t":"del","s":"Allow: /directory/*"},{"t":"del","s":"Allow: /videos/week"},{"t":"del","s":"Allow: /.well-known/assetlinks.json"},{"t":"add","s":"User-agent: Amazonbot"},{"t":"add","s":"Disallow:"},{"t":"add","s":""},{"t":"add","s":"User-agent: Amazonbot-Video"},{"t":"add","s":"Disallow:"},{"t":"add","s":""},{"t":"add","s":"User-agent: Amzn-SearchBot"},{"t":"add","s":"Disallow:"},{"t":"add","s":""},{"t":"add","s":"User-agent: Amzn-User"},{"t":"add","s":"Disallow:"},{"t":"add","s":""},{"t":"add","s":"User-agent: *"},{"t":"add","s":"Disallow: /login"},{"t":"add","s":"Disallow: /signup"},{"t":"add","s":"Disallow: /auth/"},{"t":"add","s":"Disallow: /user/account-recovery"},{"t":"add","s":"Disallow: /user/password-reset"},{"t":"add","s":"Disallow: /user/not-me"},{"t":"add","s":"Disallow: /activate"},{"t":"ctx","s":"Disallow: /admin/*"},{"t":"del","s":"Disallow: /email-unsubscribe/"},{"t":"del","s":"Disallow: /login$"},{"t":"del","s":"Disallow: /message/*"},{"t":"del","s":"Disallow: /signup$"},{"t":"add","s":"Disallow: /wv/*"},{"t":"ctx","s":"Disallow: /user/*"},{"t":"add","s":"Disallow: /message/*"},{"t":"add","s":"Disallow: /settings/"},{"t":"add","s":"Disallow: /subscriptions"},{"t":"add","s":"Disallow: /inventory"},{"t":"add","s":"Disallow: /wallet"},{"t":"add","s":"Disallow: /drops/"},{"t":"add","s":"Disallow: /payments/"},{"t":"add","s":"Disallow: /email-unsubscribe/"},{"t":"add","s":"Disallow: /search"},{"t":"add","s":"Disallow: /bits-checkout"},{"t":"add","s":"Disallow: /redeem"},{"t":"add","s":"Disallow: /claim"},{"t":"add","s":"Disallow: /download-keys"},{"t":"add","s":"Disallow: /popout/"},{"t":"add","s":"Disallow: /embed/"},{"t":"add","s":"Disallow: /chat/"},{"t":"add","s":"Disallow: /moderator/"},{"t":"add","s":"Disallow: /stream"},{"t":"add","s":"Disallow: /collab"},{"t":"add","s":"Disallow: /guest-star"},{"t":"add","s":"Disallow: /sq"},{"t":"add","s":"Disallow: /gl"},{"t":"add","s":"Disallow: /sportradar-widgets"},{"t":"add","s":"Disallow: /beta/"},{"t":"add","s":"Disallow: /extensions/oauth-redirect"},{"t":"add","s":"Disallow: /copyright-claims"},{"t":"gap","s":"… 8 more lines"}]}]}