{"domain":"tiktok.com","name":"TikTok","category":"social","subcategory":"video","country":"SG","tags":[],"notes":"","status":"blocked","status_label":"Blocks AI","rationale":"Its robots.txt (the public rules file for bots) tells all 12 major AI bots we track to stay out of the whole site, and 16 of the 24 bots in total. It names every major AI bot one by one.","computed_at":"2026-10-20T03:43:15.428Z","last_checked":"2026-10-19T13:37:16.218Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.tiktok.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.tiktok.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.tiktok.com/robots.txt","hops":[{"from":"https://tiktok.com/robots.txt","to":"https://www.tiktok.com/robots.txt","status":301}],"blob":"835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802","text":"User-agent: Baiduspider\nUser-agent: 360Spider\nUser-agent: Sogouspider\nUser-agent: Yisouspider\nUser-agent: PetalBot\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: anthropic-ai\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: meta-externalagent\nUser-agent: DuckAssistBot\nUser-agent: ChatGPT-User\nUser-agent: Gemini-Deep-Research\nUser-agent: GoogleAgent-Mariner\nUser-agent: Google-NotebookLM\nUser-agent: CCBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nUser-agent: Perplexity-User\nUser-agent: AI2Bot\nUser-agent: MistralAI-User\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: *\nAllow: /foryou\nAllow: /discover\nAllow: /about\nAllow: /legal\nAllow: /safety\nAllow: /forgood\nAllow: /community-guidelines\nAllow: /tag\nAllow: /amp\nAllow: /transparency\nAllow: /share\nAllow: /music\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /search/video?\nDisallow: /search/user?q=\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /discover/trending/detail/\nDisallow: /search?\nDisallow: /search/live?\n\nUser-agent: Bingbot\nDisallow: /discover\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /search?\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["Baiduspider","360Spider","Sogouspider","Yisouspider","PetalBot","GPTBot","OAI-SearchBot","anthropic-ai","ClaudeBot","PerplexityBot","Google-Extended","Applebot-Extended","meta-externalagent","DuckAssistBot","ChatGPT-User","Gemini-Deep-Research","GoogleAgent-Mariner","Google-NotebookLM","CCBot","Claude-User","Claude-SearchBot","Perplexity-User","AI2Bot","MistralAI-User","Bytespider","*","Bingbot"],"lines":72},"agents":[{"token":"GPTBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"ClaudeBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Google-Extended","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Applebot-Extended","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"CCBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Bytespider","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"AI2Bot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"anthropic-ai","deprecated":true,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"PerplexityBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"DuckAssistBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"ChatGPT-User","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Claude-User","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Perplexity-User","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"MistralAI-User","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":26},"any":{"allowed":false,"line":26},"search":{"allowed":false,"line":26},"cart":{"allowed":false,"line":26},"checkout":{"allowed":false,"line":26},"account":{"allowed":false,"line":26}},"disallows":1,"lines":[1,26],"groupRanges":[[1,26]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":16,"lines":[28,56],"groupRanges":[[28,56]]},{"token":"Bingbot","matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":13,"lines":[58,71],"groupRanges":[[58,71]]}],"lanes":{"training":{"level":"mixed","none":8,"full":1,"restricted":0,"n":9},"search":{"level":"mixed","none":4,"full":4,"restricted":0,"n":8},"user_directed":{"level":"mixed","none":4,"full":3,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":[],"botManagement":[]},"llms":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.tiktok.com/404?fromUrl=/llms.txt"},"ai_txt":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.tiktok.com/404?fromUrl=/ai.txt"},"tdmrep":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.tiktok.com/404?fromUrl=/.well-known/tdmrep.json"},"meta":{"title":"TikTok - Make Your Day","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":[],"platform":null,"textRatio":0,"llmsLink":false},"x_robots_tag":null,"tos":{"url":null,"source":null,"skipped":"No ToS URL curated or discovered"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":46,"first":"2023-01-01","archive_points":45,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-06","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-07","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-08","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-09","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-10","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-11","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-12","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-01","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-02","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-03","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-04","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-05","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-06","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-07","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-08","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-09","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2025-10","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2025-11","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2025-12","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-01","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-02","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-03","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-04","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-05","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-06","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-07","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-08","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-09","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16},{"m":"2026-10","s":"blocked","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":16}]},"events":[{"domain":"tiktok.com","name":"TikTok","category":"social","date":"2025-10","precision":"month","source":"archive","provisional":false,"between":["2025-09-02","2025-10-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, AI2Bot, anthropic-ai and 8 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","DuckAssistBot","ChatGPT-User","Claude-User","Perplexity-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"blocked","evidence":["https://web.archive.org/web/20250902231943/https://www.tiktok.com/robots.txt","https://web.archive.org/web/20251001141449/https://www.tiktok.com/robots.txt"],"diff":[{"t":"ctx","s":"User-agent: Bytespider"},{"t":"add","s":"User-agent: GPTBot"},{"t":"add","s":"User-agent: OAI-SearchBot"},{"t":"add","s":"User-agent: anthropic-ai"},{"t":"add","s":"User-agent: ClaudeBot"},{"t":"add","s":"User-agent: PerplexityBot"},{"t":"add","s":"User-agent: Google-Extended"},{"t":"add","s":"User-agent: Applebot-Extended"},{"t":"add","s":"User-agent: meta-externalagent"},{"t":"add","s":"User-agent: DuckAssistBot"},{"t":"add","s":"User-agent: ChatGPT-User"},{"t":"add","s":"User-agent: Gemini-Deep-Research"},{"t":"add","s":"User-agent: GoogleAgent-Mariner"},{"t":"add","s":"User-agent: Google-NotebookLM"},{"t":"add","s":"User-agent: CCBot"},{"t":"add","s":"User-agent: Claude-User"},{"t":"add","s":"User-agent: Claude-SearchBot"},{"t":"add","s":"User-agent: Perplexity-User"},{"t":"add","s":"User-agent: AI2Bot"},{"t":"add","s":"User-agent: MistralAI-User"},{"t":"ctx","s":"Disallow: /"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: */directory/"},{"t":"add","s":"Disallow: /search/video?"},{"t":"add","s":"Disallow: /search/user?q="},{"t":"add","s":"Disallow: /shop/view/product/"},{"t":"add","s":"Disallow: /sgtm/g/collect"},{"t":"add","s":"Disallow: /api/share/settings"},{"t":"add","s":"Disallow: /api/recommend/embed_videos"},{"t":"add","s":"Disallow: /discover/trending/detail/"},{"t":"add","s":"Disallow: /search?"},{"t":"add","s":"Disallow: /search/live?"},{"t":"ctx","s":""}]},{"domain":"tiktok.com","name":"TikTok","category":"social","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20240401002034/https://www.tiktok.com/robots.txt","https://web.archive.org/web/20240501000424/https://www.tiktok.com/robots.txt"],"diff":[{"t":"ctx","s":"User-agent: Yisouspider"},{"t":"add","s":"User-agent: PetalBot"},{"t":"add","s":"User-agent: Bytespider"},{"t":"ctx","s":"Disallow: /"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /embed/curated"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_tag_jpagency_index_20200509_-1553821641.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_tag_jpagency_index_20200525_-1553821583.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_tag_jpagency_index_20200615_-1553820653.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_tag_jpagency_index_20200814_-1553818732.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_tag_jpagency_index_20200817_-1553818729.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_tag_jpagency_index_20200923_-1553817741.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_tag_jpagency_index_20201027_-1553796595.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_JP-sitemap-202011_20201118_500859092.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_JP-sitemap-202012_20210110_-1305701649.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_JP-sitemap-202101_20210110_-1098685040.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_JP-sitemap_20210204_1693176571.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_JP-sitemap-202103-SG_20210305_1559370417.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_JP-sitemap-202104_20210429_2068889086.xml"},{"t":"del","s":"Sitemap: https://www.tiktok.com/tos/node/sitemaps/sitemap_JP-sitemap-202106_20210611_-1546017957.xml"},{"t":"add","s":"Disallow: /link"},{"t":"add","s":"Disallow: */directory/"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: Bingbot"},{"t":"add","s":"Disallow: /discover"},{"t":"add","s":""}]}]}