{"domain":"gov.uk","name":"GOV.UK","category":"government","subcategory":"portal","country":"GB","tags":[],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) mentions Meta-ExternalAgent, but still lets all 12 major AI bots into the site.","computed_at":"2026-10-20T03:43:15.662Z","last_checked":"2026-10-19T13:41:11.266Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.gov.uk"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.gov.uk/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.gov.uk/robots.txt","hops":[{"from":"https://gov.uk/robots.txt","to":"https://www.gov.uk/robots.txt","status":301}],"blob":"459835d29527b9c00f00f6af00f45c73a909af1944e81fefa153f585e1857305","text":"User-agent: *\nDisallow: /*/print$\n# Don't allow indexing of site search\nDisallow: /search/all*\nSitemap: https://www.gov.uk/sitemap.xml\n\n# The Meta-ExternalAgent crawler crawls the web for use cases such as training foundation AI models.\n# It results in timeouts from Vertex that back up requests from users making genuine searches\nUser-agent: meta-externalagent\nDisallow: /search/all*\n\n# https://ahrefs.com/robot/ crawls the site frequently\nUser-agent: AhrefsBot\nCrawl-delay: 10\n\n# https://www.deepcrawl.com/bot/ makes lots of requests. Ideally we'd slow it\n# down rather than blocking it but it doesn't mention whether or not it\n# supports crawl-delay.\nUser-agent: deepcrawl\nDisallow: /\n\n# Complaints of 429 'Too many requests' seem to be coming from SharePoint servers\n# (https://social.msdn.microsoft.com/Forums/en-US/3ea268ed-58a6-4166-ab40-d3f4fc55fef4)\n# The robot doesn't recognise its User-Agent string, see the MS support article:\n# https://support.microsoft.com/en-us/help/3019711/the-sharepoint-server-crawler-ignores-directives-in-robots-txt\nUser-agent: MS Search 6.0 Robot\nDisallow: /","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[{"line":7,"text":"The Meta-ExternalAgent crawler crawls the web for use cases such as training foundation AI models."},{"line":24,"text":"The robot doesn't recognise its User-Agent string, see the MS support article:"}],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*","meta-externalagent","AhrefsBot","deepcrawl","MS Search 6.0 Robot"],"lines":27},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":1,"lines":[9,10],"groupRanges":[[9,10]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":2,"lines":[1,4],"groupRanges":[[1,4]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":["Fastly"],"botManagement":[]},"llms":{"state":"absent","http":404,"url":"https://www.gov.uk/llms.txt"},"ai_txt":{"state":"absent","http":404,"url":"https://www.gov.uk/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://www.gov.uk/.well-known/tdmrep.json"},"meta":{"title":"Welcome to GOV.UK","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":[],"platform":null,"textRatio":0.064,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://www.gov.uk/help/terms-conditions","source":"discovered","http":200,"hash":"10907d13f3bd7798054322b35ceb73f4c362dbb10ca723148c6a1a0a238e5d52","chars":10801,"thin":false,"clauses":[{"keyword":"robots","prohibition":false,"excerpt":"This field is for robots only."}],"door":{"door":"open","http":200}},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":47,"first":"2023-01-01","archive_points":46,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0}]},"events":[{"domain":"gov.uk","name":"GOV.UK","category":"government","date":"2026-05","precision":"month","source":"archive","provisional":false,"between":["2026-04-01","2026-05-01"],"summary":"Named Meta-ExternalAgent explicitly, without changing what it may reach.","dir":"other","up":[],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260401012736/https://www.gov.uk/robots.txt","https://web.archive.org/web/20260501154846/https://www.gov.uk/robots.txt"],"diff":[{"t":"ctx","s":"Disallow: /*/print$"},{"t":"del","s":"# Don't allow indexing of user needs pages"},{"t":"del","s":"Disallow: /info/*"},{"t":"add","s":"# Don't allow indexing of site search"},{"t":"add","s":"Disallow: /search/all*"},{"t":"ctx","s":"Sitemap: https://www.gov.uk/sitemap.xml"},{"t":"ctx","s":""},{"t":"add","s":"# The Meta-ExternalAgent crawler crawls the web for use cases such as training foundation AI models."},{"t":"add","s":"# It results in timeouts from Vertex that back up requests from users making genuine searches"},{"t":"add","s":"User-agent: meta-externalagent"},{"t":"add","s":"Disallow: /search/all*"},{"t":"add","s":""},{"t":"ctx","s":"# https://ahrefs.com/robot/ crawls the site frequently"},{"t":"gap","s":"…"},{"t":"ctx","s":""},{"t":"del","s":"# https://www.deepcrawl.com/bot/ makes lots of requests. Ideally"},{"t":"del","s":"# we'd slow it down rather than blocking it but it doesn't mention"},{"t":"del","s":"# whether or not it supports crawl-delay."},{"t":"add","s":"# https://www.deepcrawl.com/bot/ makes lots of requests. Ideally we'd slow it"},{"t":"add","s":"# down rather than blocking it but it doesn't mention whether or not it"},{"t":"add","s":"# supports crawl-delay."},{"t":"ctx","s":"User-agent: deepcrawl"},{"t":"gap","s":"…"},{"t":"ctx","s":"Disallow: /"},{"t":"del","s":""},{"t":"del","s":"# Google's crawler was sending requests for each variation of query param for the sectors page of licence-finder"},{"t":"del","s":"# resulting in millions of requests a day."},{"t":"del","s":"User-agent: Googlebot"},{"t":"del","s":"Disallow: /licence-finder/*"},{"t":"del","s":""}]}]}