{"domain":"craigslist.org","name":"Craigslist","category":"local","subcategory":"classifieds","country":"US","tags":[],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) has no rules for any AI bot we track. They are covered by its catch-all rule (<code>User-agent: *</code>), which lets them into the site.","computed_at":"2026-10-20T03:43:15.003Z","last_checked":"2026-10-19T13:33:27.684Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://craigslist.org"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://craigslist.org/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://craigslist.org/robots.txt","hops":[],"blob":"5a9a92114f608593d9fe7d2ea52809032be3fbab3b0f019a980526acb695ee27","text":"User-agent: *\nDisallow: /reply\nDisallow: /fb/\nDisallow: /suggest\nDisallow: /flag\nDisallow: /mf\nDisallow: /mailflag\nDisallow: /eaf\n\nDisallow: /sitemap/\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*"],"lines":11},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[1,10],"groupRanges":[[1,10]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":[],"botManagement":[]},"llms":{"state":"absent","http":404,"url":"https://craigslist.org/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://craigslist.org/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://craigslist.org/.well-known/tdmrep.json"},"meta":{"title":"craigslist: ann arbor jobs, apartments, for sale, services, community, and events","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":["WebSite","SearchAction"],"platform":null,"textRatio":0.082,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://www.craigslist.org/about/terms.of.use","source":"discovered","http":200,"hash":"8e85ad1a1bcab3f6a49be36165af6043b21157564d4d36fd92c4617432e49ef6","chars":10086,"thin":false,"clauses":[{"keyword":"robots","prohibition":true,"excerpt":"You agree not to copy/collect CL content via robots, spiders, scripts, scrapers, crawlers, or any automated or manual equivalent (e.g., by hand)."},{"keyword":"agents","prohibition":true,"excerpt":"To the full extent permitted by law, craigslist, Inc., and its officers, directors, employees, agents, licensors, affiliates, and successors in interest (\"CL Entities\") (1) make no promises, warranties, or representations as to CL, including its completeness, accuracy, availability, timeliness, prop…"},{"keyword":"agents","prohibition":false,"excerpt":"…ur use of CL; and (3) be liable and responsible for any Claims we may have against your officers, directors, employees, agents, affiliates, or any other party, directly or indirectly, paid, directed or controlled by you, or acting for your benefit."}],"door":{"door":"open","http":200}},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":37,"first":"2023-01-01","archive_points":36,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-06","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-07","s":"partial","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1},{"m":"2024-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2024-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2024-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2024-11","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2024-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2025-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2025-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2025-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2025-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2025-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"mixed"},"n":11},{"m":"2025-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":8},{"m":"2025-07","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":8},{"m":"2025-08","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":8},{"m":"2025-09","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2025-10","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2025-11","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2025-12","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":3},{"m":"2026-01","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":2},{"m":"2026-02","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":2},{"m":"2026-03","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":2},{"m":"2026-04","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":2},{"m":"2026-05","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":2},{"m":"2026-06","s":"partial","l":{"training":"mixed","search":"mixed","user_directed":"full"},"n":2},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0}]},"events":[{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2026-07","precision":"month","source":"archive","provisional":false,"between":["2025-12-11","2026-07-04"],"summary":"Now allowed: Google-Extended, Meta-ExternalAgent and PerplexityBot.","dir":"down","up":[],"down":["Google-Extended","Meta-ExternalAgent","PerplexityBot"],"major":true,"signals":false,"status_from":"partial","status_to":"allowed","evidence":["https://web.archive.org/web/20251211144839/https://craigslist.org/robots.txt","https://web.archive.org/web/20260704193624/https://www.craigslist.org/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"del","s":"User-agent: Diffbot"},{"t":"del","s":"User-agent: FacebookBot"},{"t":"del","s":"User-agent: FriendlyCrawler"},{"t":"del","s":"User-agent: Google-Extended"},{"t":"del","s":"User-agent: ImagesiftBot"},{"t":"del","s":"User-agent: img2dataset"},{"t":"del","s":"User-agent: Meta-ExternalAgent"},{"t":"del","s":"User-agent: omgili"},{"t":"del","s":"User-agent: omgilibot"},{"t":"del","s":"User-agent: PerplexityBot"},{"t":"del","s":"User-agent: Timpibot"},{"t":"del","s":"User-agent: VelenPublicWebCrawler"},{"t":"del","s":"User-agent: YouBot"},{"t":"del","s":"Disallow: /"},{"t":"del","s":""},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-categories-and-hubs.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-00.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-01.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-02.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-03.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-04.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-05.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-06.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-07.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-08.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-09.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-10.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-11.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-12.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-13.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-14.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-15.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-16.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-17.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-18.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-19.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-20.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-21.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-22.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-23.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-24.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-25.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-26.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-27.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-28.xml"},{"t":"add","s":"Sitemap: https://www.craigslist.org/sitemap-index-postings-29.xml"},{"t":"ctx","s":""}]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2025-09","precision":"month","source":"archive","provisional":false,"between":["2025-08-02","2025-09-17"],"summary":"Now allowed: ClaudeBot, Applebot-Extended, CCBot, Bytespider, anthropic-ai and Amazonbot.","dir":"down","up":[],"down":["ClaudeBot","Applebot-Extended","CCBot","Bytespider","anthropic-ai","Amazonbot"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250802231626/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20250917135505/https://www.craigslist.org/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"del","s":"User-agent: Amazonbot"},{"t":"del","s":"User-agent: anthropic-ai"},{"t":"del","s":"User-agent: Applebot-Extended"},{"t":"del","s":"User-agent: Bytespider"},{"t":"del","s":"User-agent: CCBot"},{"t":"del","s":"User-agent: ClaudeBot"},{"t":"del","s":"User-agent: Claude-Web"},{"t":"del","s":"User-agent: cohere-ai"},{"t":"ctx","s":"User-agent: Diffbot"}]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2025-06","precision":"month","source":"archive","provisional":false,"between":["2025-05-22","2025-06-19"],"summary":"Now allowed: GPTBot, OAI-SearchBot and ChatGPT-User.","dir":"down","up":[],"down":["GPTBot","OAI-SearchBot","ChatGPT-User"],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20250522235750/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20250619000726/https://www.craigslist.org/robots.txt"],"diff":[{"t":"ctx","s":"User-agent: CCBot"},{"t":"del","s":"User-agent: ChatGPT-User"},{"t":"ctx","s":"User-agent: ClaudeBot"},{"t":"gap","s":"…"},{"t":"ctx","s":"User-agent: Google-Extended"},{"t":"del","s":"User-agent: GPTBot"},{"t":"ctx","s":"User-agent: ImagesiftBot"},{"t":"gap","s":"…"},{"t":"ctx","s":"User-agent: Meta-ExternalAgent"},{"t":"del","s":"User-agent: OAI-SearchBot"},{"t":"ctx","s":"User-agent: omgili"}]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2024-08","precision":"month","source":"archive","provisional":false,"between":["2024-07-01","2024-08-04"],"summary":"Now blocked: GPTBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent, Bytespider, anthropic-ai, OAI-SearchBot and 3 more.","dir":"up","up":["GPTBot","Google-Extended","Applebot-Extended","CCBot","Meta-ExternalAgent","Bytespider","anthropic-ai","OAI-SearchBot","PerplexityBot","Amazonbot","ChatGPT-User"],"down":[],"major":true,"signals":false,"status_from":"partial","status_to":"partial","evidence":["https://web.archive.org/web/20240701204734/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20240804153250/https://craigslist.org/robots.txt"],"diff":[{"t":"ctx","s":""},{"t":"del","s":"User-agent: claudebot"},{"t":"add","s":"User-agent: Amazonbot"},{"t":"add","s":"User-agent: anthropic-ai"},{"t":"add","s":"User-agent: Applebot-Extended"},{"t":"add","s":"User-agent: Bytespider"},{"t":"add","s":"User-agent: CCBot"},{"t":"add","s":"User-agent: ChatGPT-User"},{"t":"add","s":"User-agent: ClaudeBot"},{"t":"add","s":"User-agent: Claude-Web"},{"t":"add","s":"User-agent: cohere-ai"},{"t":"add","s":"User-agent: Diffbot"},{"t":"add","s":"User-agent: FacebookBot"},{"t":"add","s":"User-agent: FriendlyCrawler"},{"t":"add","s":"User-agent: Google-Extended"},{"t":"add","s":"User-agent: GPTBot"},{"t":"add","s":"User-agent: ImagesiftBot"},{"t":"add","s":"User-agent: img2dataset"},{"t":"add","s":"User-agent: Meta-ExternalAgent"},{"t":"add","s":"User-agent: OAI-SearchBot"},{"t":"add","s":"User-agent: omgili"},{"t":"add","s":"User-agent: omgilibot"},{"t":"add","s":"User-agent: PerplexityBot"},{"t":"add","s":"User-agent: Timpibot"},{"t":"add","s":"User-agent: VelenPublicWebCrawler"},{"t":"add","s":"User-agent: YouBot"},{"t":"ctx","s":"Disallow: /"}]},{"domain":"craigslist.org","name":"Craigslist","category":"local","date":"2024-05","precision":"month","source":"archive","provisional":false,"between":["2024-04-01","2024-05-01"],"summary":"Now blocked: ClaudeBot.","dir":"up","up":["ClaudeBot"],"down":[],"major":true,"signals":false,"status_from":"allowed","status_to":"partial","evidence":["https://web.archive.org/web/20240401130409/https://www.craigslist.org/robots.txt","https://web.archive.org/web/20240501200711/https://craigslist.org/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /eaf"},{"t":"del","s":"Disallow: /*.html?lang="},{"t":"del","s":"Sitemap: https://craigslist.org/sitemap_index.xml"},{"t":"ctx","s":""},{"t":"add","s":"User-agent: claudebot"},{"t":"add","s":"Disallow: /"},{"t":"add","s":""},{"t":"add","s":""}]}]}