{"domain":"canada.ca","name":"Canada.ca","category":"government","subcategory":"portal","country":"CA","tags":[],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) has no rules for any AI bot we track. They are covered by its catch-all rule (<code>User-agent: *</code>), which lets them into the site.","computed_at":"2026-10-20T03:43:15.666Z","last_checked":"2026-10-19T13:41:53.251Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://www.canada.ca"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://www.canada.ca/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://www.canada.ca/robots.txt","hops":[{"from":"https://canada.ca/robots.txt","to":"https://www.canada.ca/robots.txt","status":302}],"blob":"5e6c293b04d4b15808595bf89bb179003f0e74ead9ca5219ec25944a25d78826","text":"#Government of Canada / Gouvernement du Canada\n#Block AEM folders for CRA\nUser-agent: *\nDisallow: /content/dam/cra-arc/formspubs/\nDisallow: /en/revenue-agency/web-services-test/\nDisallow: /fr/agence-revenu/test-web-services/\nDisallow: /content/dam/cra-arc/itavp\nDisallow: /content/dam/cra-arc/serv-info/tax/itavp\nDisallow: /content/dam/cra-arc/serv-info/tax/cvitp\n#Search pages do not need to be crawled\nDisallow: /en/sr/srb.html\nDisallow: /fr/sr/srb.html\nDisallow: /en/sr/srb/sra.html\nDisallow: /fr/sr/srb/sra.html\nDisallow: /en/*/search.html\nDisallow: /en/*/search/advanced-search.html\nDisallow: /fr/*/rechercher.html\nDisallow: /fr/*/rechercher/recherche-avancee.html\nDisallow: /en/*/menu/header.html\nDisallow: /fr/*/menu/header.html\nDisallow: /en/*/menu/footer.html\nDisallow: /fr/*/menu/footer.html\nDisallow: /en/*/menu.html\nDisallow: /fr/*/menu.html\nDisallow: /en/*/footer/contactinformation.html\nDisallow: /fr/*/footer/Coordonnees.html\nDisallow: /*/_jcr_content/par*\nDisallow: /en/service-canada/\nDisallow: /fr/service-canada/\nDisallow: /en/immigration-refugees-citizenship/services/reference-include/\nDisallow: /fr/immigration-refugies-citoyennete/services/reference-inclusion/\n#IRCC PDFs\nDisallow: /content/dam/ircc/documents/pdf/english/kits/guides/guide-0142-airlifted-afghanistan-pathway-pr.pdf\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/guides/guide-0142-avion-afghanistan-voie-acces-rp.pdf\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/imm0143e.pdf\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/imm0143f.pdf\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/imm0144e.pdf\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/imm0144f.pdf\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/imm5444/\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/imm5444/\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/imm5644/\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/imm5644/\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/imm5475/\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/imm5475/\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/imm5476/\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/imm5476/\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/irm0002/\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/irm0002/\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/irm0004/\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/irm0004/\nDisallow: /content/dam/ircc/documents/pdf/english/kits/forms/irm0005/\nDisallow: /content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["*"],"lines":52},"agents":[{"token":"GPTBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"ClaudeBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Bytespider","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"anthropic-ai","deprecated":true,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"PerplexityBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]},{"token":"Bingbot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":47,"lines":[3,52],"groupRanges":[[3,52]]}],"lanes":{"training":{"level":"full","none":0,"full":9,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":["Akamai"],"botManagement":[]},"llms":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.canada.ca/errors/404.html"},"ai_txt":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.canada.ca/errors/404.html"},"tdmrep":{"state":"absent","http":200,"note":"Served an HTML page, not a text file (soft 404 or redirect to homepage).","url":"https://www.canada.ca/errors/404.html"},"meta":{"title":"Canada.ca","robotsMeta":[{"name":"robots","content":"noindex, follow"}],"noai":[],"noindex":true,"jsonLdTypes":[],"platform":null,"textRatio":0.019,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://www.canada.ca/en/transparency/terms.html","source":"discovered","http":200,"hash":"01966bd6cb2e350d7e6113f9a18d9356f42bd764fd6366c31d14477aca276ec1","chars":14300,"thin":false,"clauses":[],"door":{"door":"open","http":200}},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":45,"first":"2023-01-01","archive_points":44,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0}]},"events":[]}