{"domain":"reddit.com","name":"Reddit","category":"social","subcategory":"forum","country":"US","tags":[],"notes":"","status":"blocked","status_label":"Blocks AI","rationale":"Its robots.txt (the public rules file for bots) tells all 24 AI bots we track to stay out of the whole site. It does not name any of them. A catch-all rule (<code>User-agent: *</code>) keeps out every bot it does not list by name, AI or not. The same rules ask our bot to stay out, so we read only robots.txt.","computed_at":"2026-10-20T03:43:15.337Z","last_checked":"2026-10-19T13:36:10.817Z","checker":{"allowed":false,"vantage":"residential connection, United States (manual run)","origin":"https://www.reddit.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.reddit.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.reddit.com/robots.txt","hops":[{"from":"https://reddit.com/robots.txt","to":"https://www.reddit.com/robots.txt","status":301}],"blob":"41aea42110242175c7b4247f222fa95ac6caee62044af686b684785832781857","text":"# Welcome to Reddit's robots.txt\n# Reddit believes in an open internet, but not the misuse of public content.\n# See https://support.reddithelp.com/hc/en-us/articles/26410290525844-Public-Content-Policy Reddit's Public Content Policy for access and use restrictions to Reddit content.\n# See https://www.reddit.com/r/reddit4researchers/ for details on how Reddit continues to support research and non-commercial use.\n# policy: https://support.reddithelp.com/hc/en-us/articles/26410290525844-Public-Content-Policy\n\nUser-agent: *\nDisallow: /\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"none"},"named":["*"],"lines":9},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]},{"token":"Bingbot","matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":8},"any":{"allowed":false,"line":8},"search":{"allowed":false,"line":8},"cart":{"allowed":false,"line":8},"checkout":{"allowed":false,"line":8},"account":{"allowed":false,"line":8}},"disallows":1,"lines":[7,8],"groupRanges":[[7,8]]}],"lanes":{"training":{"level":"none","none":9,"full":0,"restricted":0,"n":9},"search":{"level":"none","none":8,"full":0,"restricted":0,"n":8},"user_directed":{"level":"none","none":7,"full":0,"restricted":0,"n":7}},"door":{"door":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"edge":{"cdn":["Fastly"],"botManagement":[]},"llms":{"state":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"ai_txt":{"state":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"tdmrep":{"state":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"meta":null,"x_robots_tag":null,"tos":{"url":null,"source":null,"skipped":"No ToS URL curated or discovered"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":44,"first":"2023-01-01","archive_points":43,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-06","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-07","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"partial","l":{"training":"restricted","search":"restricted","user_directed":"restricted"},"n":0},{"m":"2025-10","s":"partial","l":{"training":"restricted","search":"restricted","user_directed":"restricted"},"n":0},{"m":"2025-11","s":"partial","l":{"training":"restricted","search":"restricted","user_directed":"restricted"},"n":0},{"m":"2025-12","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-01","s":"partial","l":{"training":"restricted","search":"restricted","user_directed":"restricted"},"n":0},{"m":"2026-02","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-03","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-04","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-05","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-06","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-07","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-08","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-09","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-10","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24}]},"events":[{"domain":"reddit.com","name":"Reddit","category":"social","date":"2026-02","precision":"month","source":"archive","provisional":false,"between":["2026-01-05","2026-02-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"blocked","evidence":["https://web.archive.org/web/20260105222836/https://www.reddit.com/robots.txt","https://web.archive.org/web/20260201182700/https://www.reddit.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"del","s":"# Allowed for archiving"},{"t":"del","s":"# ====================="},{"t":"del","s":"User-Agent: *"},{"t":"del","s":"Allow: /$"},{"t":"add","s":"User-agent: *"},{"t":"ctx","s":"Disallow: /"}]},{"domain":"reddit.com","name":"Reddit","category":"social","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-07-01","2025-09-02"],"summary":"Now partly blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"down","up":[],"down":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"major":true,"signals":false,"status_from":"blocked","status_to":"partial","evidence":["https://web.archive.org/web/20250701014234/https://www.reddit.com/robots.txt","https://web.archive.org/web/20250902201530/https://www.reddit.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"del","s":"User-agent: *"},{"t":"add","s":"# Allowed for archiving"},{"t":"add","s":"# ====================="},{"t":"add","s":"User-Agent: *"},{"t":"add","s":"Allow: /$"},{"t":"ctx","s":"Disallow: /"}]},{"domain":"reddit.com","name":"Reddit","category":"social","date":"2025-05","precision":"month","source":"archive","provisional":false,"between":["2025-04-01","2025-05-01"],"summary":"Now blocked: GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, MistralAI-Training and 17 more.","dir":"up","up":["GPTBot","ClaudeBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","MistralAI-Training","AI2Bot","anthropic-ai","OAI-SearchBot","Claude-SearchBot","PerplexityBot","Amazonbot","Amzn-SearchBot","DuckAssistBot","MistralAI-Index","Meta-WebIndexer","ChatGPT-User","Claude-User","Perplexity-User","Google-Agent","Meta-ExternalFetcher","Amzn-User","MistralAI-User"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"blocked","evidence":["https://web.archive.org/web/20250401004524/https://www.reddit.com/robots.txt","https://web.archive.org/web/20250501151408/https://www.reddit.com/robots.txt"],"diff":[{"t":"del","s":"# 80legs"},{"t":"del","s":"User-agent: 008"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"# 80legs' new crawler"},{"t":"del","s":"User-agent: voltron"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"User-Agent: bender"},{"t":"del","s":"Disallow: /my_shiny_metal_ass"},{"t":"del","s":""},{"t":"del","s":"User-Agent: Gort"},{"t":"del","s":"Disallow: /earth"},{"t":"del","s":""},{"t":"del","s":"User-agent: MJ12bot"},{"t":"del","s":"Disallow: /"},{"t":"add","s":"# Welcome to Reddit's robots.txt"},{"t":"add","s":"# Reddit believes in an open internet, but not the misuse of public content."},{"t":"add","s":"# See https://support.reddithelp.com/hc/en-us/articles/26410290525844-Public-Content-Policy Reddit's Public Content Policy for access and use restrictions to Reddit content."},{"t":"add","s":"# See https://www.reddit.com/r/reddit4researchers/ for details on how Reddit continues to support research and non-commercial use."},{"t":"add","s":"# policy: https://support.reddithelp.com/hc/en-us/articles/26410290525844-Public-Content-Policy"},{"t":"ctx","s":""},{"t":"del","s":"User-agent: PiplBot"},{"t":"add","s":"User-agent: *"},{"t":"ctx","s":"Disallow: /"},{"t":"ctx","s":""},{"t":"del","s":"User-Agent: *"},{"t":"del","s":"Disallow: /*.json"},{"t":"del","s":"Disallow: /*.json-compact"},{"t":"del","s":"Disallow: /*.json-html"},{"t":"del","s":"Disallow: /*.xml"},{"t":"del","s":"Disallow: /*.rss"},{"t":"del","s":"Allow: /r/*.rss"},{"t":"del","s":"Disallow: /r/*/comments/*.rss"},{"t":"del","s":"Disallow: /r/*/config/*.rss"},{"t":"del","s":"Disallow: /r/*/wiki/*.rss"},{"t":"del","s":"Disallow: /*.i"},{"t":"del","s":"Disallow: /*.embed"},{"t":"del","s":"Disallow: /*/comments/*?*sort="},{"t":"del","s":"Disallow: */comment/*"},{"t":"del","s":"Disallow: /r/*/submit$"},{"t":"del","s":"Disallow: /r/*/submit/$"},{"t":"del","s":"Disallow: /message/compose*"},{"t":"del","s":"Disallow: /api"},{"t":"del","s":"Disallow: /post"},{"t":"del","s":"Disallow: /submit"},{"t":"del","s":"Disallow: /goto"},{"t":"del","s":"Disallow: /*after="},{"t":"del","s":"Disallow: /*before="},{"t":"del","s":"Disallow: /domain/*t="},{"t":"del","s":"Disallow: /login"},{"t":"del","s":"Disallow: /remove_email/t2_*"},{"t":"del","s":"Disallow: /r/*/user/"},{"t":"del","s":"Disallow: /gold?"},{"t":"del","s":"Disallow: /static/button/button1.js"},{"t":"del","s":"Disallow: /static/button/button1.html"},{"t":"del","s":"Disallow: /static/button/button2.html"},{"t":"del","s":"Disallow: /static/button/button3.html"},{"t":"del","s":"Disallow: /buttonlite.js"},{"t":"del","s":"Disallow: /timings/perf"},{"t":"gap","s":"… 7 more lines"}]}]}