{"domain":"github.com","name":"GitHub","category":"developer","subcategory":"code-hosting","country":"US","tags":[],"notes":"","status":"allowed","status_label":"Allows AI","rationale":"Its robots.txt (the public rules file for bots) mentions GPTBot, ClaudeBot, Bytespider, anthropic-ai and 2 more, but still lets all 12 major AI bots into the site.","computed_at":"2026-10-20T03:43:15.572Z","last_checked":"2026-10-19T13:38:06.697Z","checker":{"allowed":true,"vantage":"residential connection, United States (manual run)","origin":"https://github.com"},"robots":{"via":{"type":"live","date":"2026-10-19","url":"https://github.com/robots.txt"},"kind":"file","http":200,"error":null,"url":"https://github.com/robots.txt","hops":[],"blob":"d2f2d319b7631f56e3223c5e0718a0dff762b9ad561a7b1e4ac71ca9678ecb48","text":"# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots\n# We also provide an extensive API: https://docs.github.com\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: ClaudeBot\nUser-agent: anthropic-ai\nUser-agent: PerplexityBot\nCrawl-delay: 1\nAllow: /*?tab=achievements&achievement=*\nAllow: /marketplace/*?tab=readme$\nAllow: /marketplace/*?tab=transparency$\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n\nUser-agent: bingbot\nUser-agent: adidxbot\nUser-agent: BingPreview\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /account-login\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\n\nUser-agent: baidu\ncrawl-delay: 1\n\nUser-agent: *\nAllow: /*?tab=achievements&achievement=*\nAllow: /marketplace/*?tab=readme$\nAllow: /marketplace/*?tab=transparency$\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n","refused":null,"warnings":[],"contentSignals":[],"contentUsage":[],"licenses":[],"aiComments":[],"platform":null,"wildcard":{"exists":true,"access":"full"},"named":["GPTBot","OAI-SearchBot","ClaudeBot","anthropic-ai","PerplexityBot","Bytespider","AdsBot-Google","AdsBot-Google-Mobile","bingbot","adidxbot","BingPreview","baidu","*"],"lines":524},"agents":[{"token":"GPTBot","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":20},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[3,173],"groupRanges":[[3,173]]},{"token":"ClaudeBot","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":20},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[3,173],"groupRanges":[[3,173]]},{"token":"Google-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Applebot-Extended","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"CCBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Meta-ExternalAgent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Bytespider","deprecated":false,"matched":"named","access":"none","probes":{"root":{"allowed":false,"line":176},"any":{"allowed":false,"line":176},"search":{"allowed":false,"line":176},"cart":{"allowed":false,"line":176},"checkout":{"allowed":false,"line":176},"account":{"allowed":false,"line":176}},"disallows":1,"lines":[175,176],"groupRanges":[[175,176]]},{"token":"MistralAI-Training","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"AI2Bot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"anthropic-ai","deprecated":true,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":20},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[3,173],"groupRanges":[[3,173]]},{"token":"OAI-SearchBot","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":20},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[3,173],"groupRanges":[[3,173]]},{"token":"Claude-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"PerplexityBot","deprecated":false,"matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":20},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[3,173],"groupRanges":[[3,173]]},{"token":"Amazonbot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Amzn-SearchBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"DuckAssistBot","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"MistralAI-Index","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Meta-WebIndexer","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"ChatGPT-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Claude-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Perplexity-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Google-Agent","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Meta-ExternalFetcher","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Amzn-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"MistralAI-User","deprecated":false,"matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]}],"refs":[{"token":"Googlebot","matched":"wildcard","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":false,"line":370},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":162,"lines":[355,523],"groupRanges":[[355,523]]},{"token":"Bingbot","matched":"named","access":"full","probes":{"root":{"allowed":true,"line":null},"any":{"allowed":true,"line":null},"search":{"allowed":true,"line":null},"cart":{"allowed":true,"line":null},"checkout":{"allowed":true,"line":null},"account":{"allowed":true,"line":null}},"disallows":8,"lines":[343,353],"groupRanges":[[343,353]]}],"lanes":{"training":{"level":"mixed","none":1,"full":8,"restricted":0,"n":9},"search":{"level":"full","none":0,"full":8,"restricted":0,"n":8},"user_directed":{"level":"full","none":0,"full":7,"restricted":0,"n":7}},"door":{"door":"open","http":200},"edge":{"cdn":[],"botManagement":[]},"llms":{"state":"present","http":200,"bytes":28658,"parsed":{"title":"GitHub","summary":"GitHub is a developer platform for building, shipping, and maintaining software. It provides cloud-based Git repository hosting, CI/CD via GitHub Actions, project management with Issues and Projects, code review via pull requests, AI-powere","links":117,"sections":["How to use","Building with GitHub, for coding agents and automation","GitHub Copilot","Authentication","Get started","Account and profile","GitHub Pages","GitHub Actions","Pull requests","Repositories","Apps","More pages"],"wellFormed":true},"url":"https://github.com/llms.txt"},"ai_txt":{"state":"refused","http":406,"url":"https://github.com/ai.txt"},"tdmrep":{"state":"absent","http":404,"url":"https://github.com/.well-known/tdmrep.json"},"meta":{"title":"GitHub · Change is constant. GitHub keeps you ahead. · GitHub · GitHub","robotsMeta":[],"noai":[],"noindex":false,"jsonLdTypes":[],"platform":null,"textRatio":0.014,"llmsLink":false},"x_robots_tag":null,"tos":{"url":"https://docs.github.com/site-policy/github-terms/github-terms-of-service","source":"discovered","http":200,"hash":"2963f70537a2bcbdc07dbd6f9c54533400805b0a1679c5da4a29b8982720e150","chars":49293,"thin":false,"clauses":[{"keyword":"artificial intelligence","prohibition":true,"excerpt":"For the avoidance of doubt, use of Your Content to develop, train, and improve artificial intelligence and machine learning models and technologies of GitHub and our Affiliates is within the scope of this license and does not constitute a sale or other restricted transfer of Your Content."},{"keyword":"automated means","prohibition":true,"excerpt":"By using automated means to access, collect, or otherwise use (“Access”) any publicly accessible Content from the Service for the purpose of developing or training any commercially available artificial intelligence model, machine learning system, or similar technology (a \"Commercial AI System\"), you…"},{"keyword":"AI","prohibition":true,"excerpt":"…itHub Customer Agreement or volume licensing agreement, those agreements govern the use of your data in connection with AI Features and Section J.3 does not apply to you."},{"keyword":"AI","prohibition":true,"excerpt":"This license does not, however, permit GitHub or its Affiliates to share your Inputs or Outputs with third-party AI model providers for their own independent model training purposes."},{"keyword":"AI","prohibition":true,"excerpt":"This license does not extend to Your Content stored in repositories that are not provided as Input to an AI Feature."},{"keyword":"bots","prohibition":true,"excerpt":"Accounts registered by \"bots\" or other automated methods are not permitted."},{"keyword":"agents","prohibition":true,"excerpt":"…mail or GitHub Support's messaging system will not constitute legal notice to GitHub or any of its officers, employees, agents or representatives in any situation where notice to GitHub is required by contract or any law or regulation."},{"keyword":"AI","prohibition":false,"excerpt":"AI Features, Training, and Your Data"}],"door":{"door":"open","http":200}},"paid":null,"program":null,"statements":[],"claim":null,"reports":[],"history":{"points":47,"first":"2023-01-01","archive_points":46,"series":[{"m":"2023-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2023-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2024-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-10","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-11","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2025-12","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-01","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-02","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-03","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-04","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-05","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-06","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-07","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-08","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-09","s":"allowed","l":{"training":"full","search":"full","user_directed":"full"},"n":0},{"m":"2026-10","s":"allowed","l":{"training":"mixed","search":"full","user_directed":"full"},"n":1}]},"events":[{"domain":"github.com","name":"GitHub","category":"developer","date":"2026-10","precision":"month","source":"archive","provisional":false,"between":["2026-09-05","2026-10-01"],"summary":"Now blocked: Bytespider.","dir":"up","up":["Bytespider"],"down":[],"major":false,"signals":false,"status_from":"allowed","status_to":"allowed","evidence":["https://web.archive.org/web/20260905135109/https://github.com/robots.txt","https://web.archive.org/web/20261001002701/https://github.com/robots.txt"],"diff":[{"t":"ctx","s":"# We also provide an extensive API: https://docs.github.com"},{"t":"add","s":"User-agent: GPTBot"},{"t":"add","s":"User-agent: OAI-SearchBot"},{"t":"add","s":"User-agent: ClaudeBot"},{"t":"add","s":"User-agent: anthropic-ai"},{"t":"add","s":"User-agent: PerplexityBot"},{"t":"add","s":"Crawl-delay: 1"},{"t":"add","s":"Allow: /$"},{"t":"add","s":"Allow: /about"},{"t":"add","s":"Allow: /customer-stories"},{"t":"add","s":"Allow: /enterprise"},{"t":"add","s":"Allow: /features"},{"t":"add","s":"Allow: /marketplace"},{"t":"add","s":"Allow: /mcp"},{"t":"add","s":"Allow: /newsroom"},{"t":"add","s":"Allow: /pricing"},{"t":"add","s":"Allow: /resources"},{"t":"add","s":"Allow: /security"},{"t":"add","s":"Allow: /solutions"},{"t":"add","s":"Allow: /topics/download*"},{"t":"add","s":"Allow: /topics/comments*"},{"t":"add","s":"Allow: /marketplace/actions/download*"},{"t":"add","s":"Disallow: /*/*/pulse"},{"t":"add","s":"Disallow: /*/*/projects"},{"t":"add","s":"Disallow: /*/*/forks"},{"t":"add","s":"Disallow: /*/*/issues/new"},{"t":"add","s":"Disallow: /*/*/milestones/new"},{"t":"add","s":"Disallow: /*/*/issues/search"},{"t":"add","s":"Disallow: /*/*/commits/"},{"t":"add","s":"Disallow: /*/*/branches"},{"t":"add","s":"Disallow: /*/*/contributors"},{"t":"add","s":"Disallow: /*/*/tags"},{"t":"add","s":"Disallow: /*/*/stargazers"},{"t":"add","s":"Disallow: /*/*/watchers"},{"t":"add","s":"Disallow: /*/*/network"},{"t":"add","s":"Disallow: /*/*/graphs"},{"t":"add","s":"Disallow: /*/*/compare"},{"t":"add","s":"Disallow: /*/tree/"},{"t":"add","s":"Disallow: /gist/"},{"t":"add","s":"Disallow: /*/download"},{"t":"add","s":"Disallow: /*/revisions"},{"t":"add","s":"Disallow: /*/commits/*?author"},{"t":"add","s":"Disallow: /*/commits/*?path"},{"t":"add","s":"Disallow: /*/comments"},{"t":"add","s":"Disallow: /*/archive/"},{"t":"add","s":"Disallow: /*/blame/"},{"t":"add","s":"Disallow: /*/raw/"},{"t":"add","s":"Disallow: /*/cache/"},{"t":"add","s":"Disallow: /.git/"},{"t":"add","s":"Disallow: */.git/"},{"t":"add","s":"Disallow: /*.git$"},{"t":"add","s":"Disallow: /search/advanced"},{"t":"add","s":"Disallow: /search$"},{"t":"add","s":"Disallow: /*.atom$"},{"t":"add","s":"Disallow: /ekansa/Open-Context-Data"},{"t":"add","s":"Disallow: /ekansa/opencontext-*"},{"t":"add","s":"Disallow: */tarball/"},{"t":"add","s":"Disallow: */zipball/"},{"t":"add","s":"Disallow: /*commits?author=*"},{"t":"add","s":"Disallow: /*report-abuse?report=*"},{"t":"gap","s":"… 438 more lines"}]}]}