{"domain":"nextdoor.com","name":"Nextdoor","category":"local","subcategory":"community","country":"US","tags":[],"notes":"","status":"blocked","status_label":"Blocks AI","rationale":"Its robots.txt (the public rules file for bots) tells all 24 AI bots we track to stay out of the whole site. It names GPTBot directly. A catch-all rule (<code>User-agent: *</code>) keeps the rest out too. The same rules ask our bot to stay out, so we read only robots.txt.","computed_at":"2026-10-20T03:43:14.907Z","last_checked":"2026-10-19T13:31:57.259Z","checker":{"allowed":false,"vantage":"residential connection, United States (manual run)","origin":"https://nextdoor.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://nextdoor.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://nextdoor.com/robots.txt","hops":[],"blob":"8b119e996e5bd07e794ff4bc47a05b4d3e56999fffcf59b423d22cce8863ee13","text":"User-agent: Googlebot\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: Slurp\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: msnbot\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: bingbot\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: DuckDuckBot\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: ia_archiver\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: Teoma\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: SemrushBot\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\nUser-agent: Seobility\nDisallow: /join/\nDisallow: /lidworden/\nDisallow: /businesses/\nDisallow: /reset_password/\nDisallow: /invitation/\nDisallow: /generic_unsub/\nDisallow: /inv_unsub/\nDisallow: /generic_nm_unsub/\nDisallow: /category_unsub/\nDisallow: /rso_database_match_new/\n\n# Twitter specifies this format here https://dev.twitter.com/cards/getting-started#crawling\nUser-agent: Twitterbot\nDisallow: *\nAllow: /events/\nAllow: /city/feed/\nAllow: /city/post/\nAllow: /agency/\nAllow: /pages/\nAllow: /link_preview_image/\n\nUser-agent: facebookexternalhit\nDisallow: /\nAllow: /rewind\nAllow: /?from_rewind=true\nAllow: /link_preview_image/\nAllow: /for_sale_and_free/\nAllow: /p/\nAllow: /g/\nAllow: /faves/\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: *\nDisallow: /\nAllow: /link_preview_image/\nAllow: /for_sale_and_free/\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"none"},"named":["Googlebot","Slurp","msnbot","bingbot","DuckDuckBot","ia_archiver","Teoma","SemrushBot","Seobility","Twitterbot","facebookexternalhit","GPTBot","*"],"lines":136},"agents":[{"token":"GPTBot","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":130},"any":{"allowed":false,"line":130},"search":{"allowed":false,"line":130},"cart":{"allowed":false,"line":130},"checkout":{"allowed":false,"line":130},"account":{"allowed":false,"line":130}},"disallows":1,"lines":[129,130],"groupRanges":[[129,130]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"none","probes":{"root":{"allowed":false,"line":133},"any":{"allowed":false,"line":133},"search":{"allowed":false,"line":133},"cart":{"allowed":false,"line":133},"checkout":{"allowed":false,"line":133},"account":{"allowed":false,"line":133}},"disallows":1,"lines":[132,135],"groupRanges":[[132,135]]}],"refs":[{"token":"Googlebot","matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":10,"lines":[1,11],"groupRanges":[[1,11]]},{"token":"Bingbot","matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":10,"lines":[37,47],"groupRanges":[[37,47]]}],"lanes":{"training":{"level":"none","none":9,"full":0,"restricted":0,"n":9},"search":{"level":"none","none":8,"full":0,"restricted":0,"n":8},"user_directed":{"level":"none","none":7,"full":0,"restricted":0,"n":7}},"door":{"door":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"edge":{"cdn":["Amazon CloudFront"],"botManagement":[]},"llms":{"state":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"ai_txt":{"state":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"tdmrep":{"state":"skipped","reason":"Disallowed for AgentAccessIndexBot by robots.txt"},"meta":null,"x_robots_tag":null,"tos":{"url":null,"source":null,"skipped":"No ToS URL curated or discovered"},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":42,"first":"2023-01-01","archive_points":41,"series":[{"m":"2023-01","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-02","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-03","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-04","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-05","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-06","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-07","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-08","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-09","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-10","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-11","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2023-12","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-01","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-02","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-03","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-04","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-05","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-06","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-07","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-08","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-09","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-10","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-11","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2024-12","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-01","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-02","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-03","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-04","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-05","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-06","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-07","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-08","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-09","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-10","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-11","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2025-12","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-01","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-02","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-03","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-04","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-05","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-06","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-07","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-08","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-09","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24},{"m":"2026-10","s":"blocked","l":{"training":"none","search":"none","user_directed":"none"},"n":24}]},"events":[{"domain":"nextdoor.com","name":"Nextdoor","category":"local","date":"2023-09","precision":"month","source":"archive","provisional":false,"between":["2023-08-01","2023-09-01"],"summary":"Named GPTBot explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"blocked","status_to":"blocked","evidence":["https://web.archive.org/web/20230801034558/https://nextdoor.com/robots.txt","https://web.archive.org/web/20230901012557/https://nextdoor.com/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"add","s":"User-agent: GPTBot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"ctx","s":"User-agent: *"},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"del","s":"Sitemap: https://nextdoor.com/sitemap-index.xml"},{"t":"ctx","s":""}]}]}