{"domain":"clevelandclinic.org","name":"Cleveland Clinic","category":"healthcare","subcategory":"health-system","country":"US","tags":["healthcare"],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) has no rules for any AI bot we track. They are covered by its catch-all rule (<code>User-agent: *</code>), which lets them into the site.","computed_at":"2026-10-20T03:43:15.674Z","last_checked":"2026-10-19T13:41:41.834Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://my.clevelandclinic.org"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://my.clevelandclinic.org/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://my.clevelandclinic.org/robots.txt","hops":[{"from":"https://clevelandclinic.org/robots.txt","to":"https://www.clevelandclinic.org/robots.txt","status":302},{"from":"https://www.clevelandclinic.org/robots.txt","to":"https://my.clevelandclinic.org/robots.txt","status":301}],"blob":"a194b4ba24f77365a0a9dc31272ca4fab4b7efaa7f1a44e25c89ac2f79712419","text":"# Robots.txt file CCF\n# Updated 11/11/2025\n\nUser-agent: AmazonAdBot\n# Only allow crawling of areas where we show advertisements\nAllow: /health/\nAllow: /watch/\n\nDisallow: /\n \n# --- Default directives for all other crawlers ---\nUser-agent: *\nDisallow:\nCrawl-delay: 10\n\nDisallow: /Search\nDisallow: /search\nDisallow: /*/Search\nDisallow: /*/search\n\nDisallow: /*?date=*\nDisallow: /*?q=*\nDisallow: /*?view=*\n\nDisallow: /ccf/media/files/*guide.pdf\nDisallow: /cache/clearcache/\nDisallow: /atoz/healthinformationletterstatus/\nDisallow: /atoz/healthinformationpages/\nDisallow: /atoz/healthinformationribbonlinkvisibility/\nDisallow: /atoz/institutesdepartmentsletterstatus/\nDisallow: /atoz/institutesdepartmentspages/\nDisallow: /ribbon/locationspecialtylists/\nDisallow: /provider/physicianratings/\nDisallow: /provider/image/\nDisallow: /video/panel/\nDisallow: /clinicaltrial/jsonresults/\nDisallow: /patientstory/jsonresults/\nDisallow: /patientstory/forsearchindex/\nDisallow: /location/waittimes/\nDisallow: /treatment-guides/\n\nSitemap: https://my.clevelandclinic.org/sitemap.xml","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["AmazonAdBot","*"],"lines":42},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":17},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":23,"lines":[12,40],"groupRanges":[[12,40]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":["Cloudflare"],"botManagement":[{"vendor":"Cloudflare Bot Management","cookies":["__cf_bm"]}]},"llms":{"state":"present","http":200,"bytes":3242,"parsed":{"title":"Cleveland Clinic (my.clevelandclinic.org)","summary":null,"links":17,"sections":[],"wellFormed":true},"url":"https://my.clevelandclinic.org/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://my.clevelandclinic.org/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://my.clevelandclinic.org/.well-known/tdmrep.json"},"meta":{"title":"Cleveland Clinic: Every Life Deserves World Class Care","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":["MedicalOrganization","QuantitativeValue"],"platform":"Next.js","textRatio":0.024,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://my.clevelandclinic.org/about/website/terms-of-use","source":"discovered","http":200,"hash":"327da9dbe003eaf45d4306fae5c43a93aaae47cf10d2a0d94562cee0074bd457","chars":12660,"thin":false,"clauses":[{"keyword":"large language model","prohibition":true,"excerpt":"…s to develop, create, train, fine tune, or ground (including through a retrieval-augmented generation (RAG) model), any large language model, foundation model, deep machine learning, generative artificial intelligence model or algorithm, or any software or tool that incorporates artificial intellige…"},{"keyword":"bots","prohibition":true,"excerpt":"Use the Services for unauthorized framing of or linking to, or access via automated devices, bots, agents, scraping scripts, intelligent search or any similar means of access to the Services;"},{"keyword":"robot","prohibition":false,"excerpt":"Except as expressly permitted by Cleveland Clinic, use any robot, spider, crawler, scraper, script, or any automatic device or means to harvest, gather, access, copy, or otherwise use the materials and information on the Services;"},{"keyword":"agents","prohibition":false,"excerpt":"You agree to indemnify and hold harmless Cleveland Clinic and its respective directors, officers, agents, and employees, individually or collectively, from and against any and all claims, liabilities, losses, damages, obligations, costs and expenses (including reasonable attorneys' fees and costs) a…"}],"door":{"door":"open","http":200}},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":1,"first":"2026-10-19","archive_points":0,"series":[{"m":"2023-01","s":null},{"m":"2023-02","s":null},{"m":"2023-03","s":null},{"m":"2023-04","s":null},{"m":"2023-05","s":null},{"m":"2023-06","s":null},{"m":"2023-07","s":null},{"m":"2023-08","s":null},{"m":"2023-09","s":null},{"m":"2023-10","s":null},{"m":"2023-11","s":null},{"m":"2023-12","s":null},{"m":"2024-01","s":null},{"m":"2024-02","s":null},{"m":"2024-03","s":null},{"m":"2024-04","s":null},{"m":"2024-05","s":null},{"m":"2024-06","s":null},{"m":"2024-07","s":null},{"m":"2024-08","s":null},{"m":"2024-09","s":null},{"m":"2024-10","s":null},{"m":"2024-11","s":null},{"m":"2024-12","s":null},{"m":"2025-01","s":null},{"m":"2025-02","s":null},{"m":"2025-03","s":null},{"m":"2025-04","s":null},{"m":"2025-05","s":null},{"m":"2025-06","s":null},{"m":"2025-07","s":null},{"m":"2025-08","s":null},{"m":"2025-09","s":null},{"m":"2025-10","s":null},{"m":"2025-11","s":null},{"m":"2025-12","s":null},{"m":"2026-01","s":null},{"m":"2026-02","s":null},{"m":"2026-03","s":null},{"m":"2026-04","s":null},{"m":"2026-05","s":null},{"m":"2026-06","s":null},{"m":"2026-07","s":null},{"m":"2026-08","s":null},{"m":"2026-09","s":null},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0}]},"events":[]}