{
  "edition": "2026-11",
  "snapshot": "data/census-2026-11-02.json",
  "universe": {
    "source": "Tranco",
    "listSize": 100000,
    "listDate": "2026-11-01"
  },
  "collectedAt": "2026-11-02",
  "publishedAt": "2026-11-10",
  "correctedAt": null,
  "methodologyVersion": "2.0",
  "coverage": {
    "domainsChecked": 99998,
    "denominator": 100000
  },
  "exclusions": [
    "Subdominios fuera de alcance (solo apex domains)",
    "Catch-alls excluidos con canary en DOS directorios: la raíz y /.well-known/ (v2.0)",
    "Respuesta HTML a rutas de texto contada como ausente",
    "Manifiestos (ai-catalog.json, ard.json) solo cuentan con un array `entries` no vacío (v2.0)",
    "Dominios propios fuera del Tranco top-100K no contados"
  ],
  "ownDomainsInScope": [],
  "notes": [
    "No 2026-10 edition was published. Its data was collected on 2026-10-01/02 and is archived, but it is not used as a comparison point: it recorded a one-month dip in robots.txt blocking (GPTBot fully blocked by 4,553 domains, against 8,307 in 2026-09 and 8,109 now) that this edition does not reproduce. Every change below is measured against 2026-09.",
    "ai-catalog.json: 126 domains with a valid catalog, against 37 in 2026-09. Most of that difference is not new publishing. Of the publishers counted this month, 27 serve a file whose content first appeared after the September edition; most of the rest serve the byte-identical file we recorded in July 2026, which our crawler did not see from August until mid-October. Nothing in our crawler changed in that window, most of those domains sit behind Cloudflare, and we have not established the cause. Read the 37 of 2026-09 as an undercount, not the 126 as growth.",
    "ard.json: 55 domains, against 4 in 2026-09. 13 serve content that first appeared after the September edition; the rest already existed in August and were not counted in September, for reasons we have not established. Same caveat as ai-catalog.json.",
    "robots.txt: blocking of AI crawlers is flat against 2026-09 (GPTBot 8,307 to 8,109). In a sample of 94 blocking domains re-fetched on 2026-11-10, 64 carry Cloudflare's managed robots.txt block: much of the AI-crawler blocking in the top 100K is written by the CDN, not by the site. A sample, not a census figure.",
    "New sections, measured for the first time: adoption by sector (Cloudflare content categories, multi-label), llms.txt written by the site versus served as a platform template per sector, and the media types of catalog entries against ARD v0.91 §3.3. A missing value in an earlier edition means not measured."
  ],
  "corrections": [],
  "domainsChecked": 99998,
  "tiers": [
    {
      "tier": "top 1K",
      "checked": 1000,
      "ai_catalog": 2,
      "llms_txt": 87,
      "agents_md": 2,
      "agent_card": 1,
      "mcp_well_known": 2,
      "ard_json": 2
    },
    {
      "tier": "1K–10K",
      "checked": 9000,
      "ai_catalog": 17,
      "llms_txt": 637,
      "agents_md": 24,
      "agent_card": 8,
      "mcp_well_known": 13,
      "ard_json": 8
    },
    {
      "tier": "10K–100K",
      "checked": 89998,
      "ai_catalog": 107,
      "llms_txt": 6669,
      "agents_md": 1015,
      "agent_card": 49,
      "mcp_well_known": 98,
      "ard_json": 45
    }
  ],
  "aiCatalogQuality": {
    "present": 137,
    "invalid_json": 3,
    "with_entries": 126
  },
  "catchAll": [
    {
      "kind": "robots_ai",
      "responding": 54189,
      "real": 54189,
      "catch_all": 0
    },
    {
      "kind": "llms_txt",
      "responding": 8685,
      "real": 7393,
      "catch_all": 1292
    },
    {
      "kind": "agents_md",
      "responding": 1369,
      "real": 1041,
      "catch_all": 328
    },
    {
      "kind": "ai_catalog",
      "responding": 542,
      "real": 137,
      "catch_all": 405
    },
    {
      "kind": "mcp_well_known",
      "responding": 498,
      "real": 113,
      "catch_all": 385
    },
    {
      "kind": "ard_json",
      "responding": 456,
      "real": 65,
      "catch_all": 391
    },
    {
      "kind": "agent_card",
      "responding": 435,
      "real": 58,
      "catch_all": 377
    },
    {
      "kind": "dns_aid",
      "responding": 42,
      "real": 42,
      "catch_all": 0
    }
  ],
  "manifestQuality": [
    {
      "kind": "ai_catalog",
      "present": 137,
      "invalid_json": 3,
      "with_entries": 126
    },
    {
      "kind": "ard_json",
      "present": 65,
      "invalid_json": 6,
      "with_entries": 55
    }
  ],
  "sectors": {
    "source": "cloudflare-intel",
    "scope": 99998,
    "classified": 99998,
    "withCategory": 93167,
    "multiLabel": true,
    "boilerplateMinDomains": 10,
    "rows": [
      {
        "sector": "Technology",
        "domains": 29665,
        "llms_txt": 2520,
        "agents_md": 150,
        "manifest": 74,
        "names_ai_bot": 2218,
        "blocks_gptbot": 1068
      },
      {
        "sector": "Entertainment",
        "domains": 16698,
        "llms_txt": 938,
        "agents_md": 45,
        "manifest": 6,
        "names_ai_bot": 4365,
        "blocks_gptbot": 3270
      },
      {
        "sector": "Business & Economy",
        "domains": 14496,
        "llms_txt": 1995,
        "agents_md": 119,
        "manifest": 55,
        "names_ai_bot": 1484,
        "blocks_gptbot": 608
      },
      {
        "sector": "CIPA",
        "domains": 8955,
        "llms_txt": 318,
        "agents_md": 13,
        "manifest": 3,
        "names_ai_bot": 1545,
        "blocks_gptbot": 1233
      },
      {
        "sector": "Education",
        "domains": 8074,
        "llms_txt": 430,
        "agents_md": 20,
        "manifest": 8,
        "names_ai_bot": 1135,
        "blocks_gptbot": 749
      },
      {
        "sector": "Shopping & Auctions",
        "domains": 7803,
        "llms_txt": 1371,
        "agents_md": 773,
        "manifest": 14,
        "names_ai_bot": 1369,
        "blocks_gptbot": 601
      },
      {
        "sector": "(not categorized by source)",
        "domains": 6831,
        "llms_txt": 138,
        "agents_md": 9,
        "manifest": 5,
        "names_ai_bot": 394,
        "blocks_gptbot": 307
      },
      {
        "sector": "Society & Lifestyle",
        "domains": 6384,
        "llms_txt": 992,
        "agents_md": 558,
        "manifest": 7,
        "names_ai_bot": 1128,
        "blocks_gptbot": 621
      },
      {
        "sector": "Government & Politics",
        "domains": 5995,
        "llms_txt": 233,
        "agents_md": 6,
        "manifest": 3,
        "names_ai_bot": 454,
        "blocks_gptbot": 301
      },
      {
        "sector": "Internet Communication",
        "domains": 4759,
        "llms_txt": 348,
        "agents_md": 8,
        "manifest": 8,
        "names_ai_bot": 831,
        "blocks_gptbot": 575
      },
      {
        "sector": "Adult Themes",
        "domains": 4065,
        "llms_txt": 133,
        "agents_md": 5,
        "manifest": 2,
        "names_ai_bot": 741,
        "blocks_gptbot": 573
      },
      {
        "sector": "Gambling",
        "domains": 2754,
        "llms_txt": 76,
        "agents_md": 0,
        "manifest": 1,
        "names_ai_bot": 454,
        "blocks_gptbot": 364
      },
      {
        "sector": "Health",
        "domains": 2460,
        "llms_txt": 267,
        "agents_md": 41,
        "manifest": 1,
        "names_ai_bot": 334,
        "blocks_gptbot": 177
      },
      {
        "sector": "Travel",
        "domains": 2340,
        "llms_txt": 200,
        "agents_md": 1,
        "manifest": 9,
        "names_ai_bot": 376,
        "blocks_gptbot": 155
      },
      {
        "sector": "Ads",
        "domains": 2171,
        "llms_txt": 253,
        "agents_md": 5,
        "manifest": 8,
        "names_ai_bot": 113,
        "blocks_gptbot": 30
      },
      {
        "sector": "Sports",
        "domains": 2023,
        "llms_txt": 117,
        "agents_md": 22,
        "manifest": 2,
        "names_ai_bot": 462,
        "blocks_gptbot": 259
      },
      {
        "sector": "Miscellaneous",
        "domains": 1543,
        "llms_txt": 32,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 77,
        "blocks_gptbot": 66
      },
      {
        "sector": "Vehicles",
        "domains": 1474,
        "llms_txt": 124,
        "agents_md": 11,
        "manifest": 4,
        "names_ai_bot": 241,
        "blocks_gptbot": 133
      },
      {
        "sector": "Questionable Content",
        "domains": 1061,
        "llms_txt": 45,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 281,
        "blocks_gptbot": 261
      },
      {
        "sector": "Job Search & Careers",
        "domains": 859,
        "llms_txt": 82,
        "agents_md": 2,
        "manifest": 2,
        "names_ai_bot": 134,
        "blocks_gptbot": 70
      },
      {
        "sector": "Real Estate",
        "domains": 761,
        "llms_txt": 69,
        "agents_md": 3,
        "manifest": 1,
        "names_ai_bot": 105,
        "blocks_gptbot": 40
      },
      {
        "sector": "Religion",
        "domains": 342,
        "llms_txt": 12,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 62,
        "blocks_gptbot": 40
      },
      {
        "sector": "Weather",
        "domains": 237,
        "llms_txt": 16,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 39,
        "blocks_gptbot": 22
      },
      {
        "sector": "Military & Weapons",
        "domains": 202,
        "llms_txt": 18,
        "agents_md": 2,
        "manifest": 0,
        "names_ai_bot": 22,
        "blocks_gptbot": 17
      },
      {
        "sector": "Safe for Kids",
        "domains": 178,
        "llms_txt": 12,
        "agents_md": 10,
        "manifest": 0,
        "names_ai_bot": 32,
        "blocks_gptbot": 22
      },
      {
        "sector": "Violence",
        "domains": 18,
        "llms_txt": 1,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 2,
        "blocks_gptbot": 2
      },
      {
        "sector": "Blocked by Default",
        "domains": 4,
        "llms_txt": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 1,
        "blocks_gptbot": 0
      }
    ],
    "byPlatform": [
      {
        "sector": "(all)",
        "kind": "agents_md",
        "domains": 99998,
        "present": 1041,
        "fingerprinted": 1024,
        "boilerplate": 879,
        "editorial": 145,
        "top_template": "v1:82463",
        "top_template_n": 852
      },
      {
        "sector": "(all)",
        "kind": "llms_txt",
        "domains": 99998,
        "present": 7393,
        "fingerprinted": 5737,
        "boilerplate": 836,
        "editorial": 4901,
        "top_template": "v1:33652",
        "top_template_n": 791
      },
      {
        "sector": "Technology",
        "kind": "agents_md",
        "domains": 29665,
        "present": 150,
        "fingerprinted": 144,
        "boilerplate": 95,
        "editorial": 49,
        "top_template": "v1:82463",
        "top_template_n": 91
      },
      {
        "sector": "Technology",
        "kind": "llms_txt",
        "domains": 29665,
        "present": 2520,
        "fingerprinted": 1873,
        "boilerplate": 91,
        "editorial": 1782,
        "top_template": "v1:33652",
        "top_template_n": 76
      },
      {
        "sector": "Entertainment",
        "kind": "agents_md",
        "domains": 16698,
        "present": 45,
        "fingerprinted": 44,
        "boilerplate": 33,
        "editorial": 11,
        "top_template": "v1:82463",
        "top_template_n": 32
      },
      {
        "sector": "Entertainment",
        "kind": "llms_txt",
        "domains": 16698,
        "present": 938,
        "fingerprinted": 676,
        "boilerplate": 37,
        "editorial": 639,
        "top_template": "v1:33652",
        "top_template_n": 32
      },
      {
        "sector": "Business & Economy",
        "kind": "agents_md",
        "domains": 14496,
        "present": 119,
        "fingerprinted": 114,
        "boilerplate": 79,
        "editorial": 35,
        "top_template": "v1:82463",
        "top_template_n": 73
      },
      {
        "sector": "Business & Economy",
        "kind": "llms_txt",
        "domains": 14496,
        "present": 1995,
        "fingerprinted": 1524,
        "boilerplate": 82,
        "editorial": 1442,
        "top_template": "v1:33652",
        "top_template_n": 70
      },
      {
        "sector": "CIPA",
        "kind": "agents_md",
        "domains": 8955,
        "present": 13,
        "fingerprinted": 13,
        "boilerplate": 11,
        "editorial": 2,
        "top_template": "v1:82463",
        "top_template_n": 10
      },
      {
        "sector": "CIPA",
        "kind": "llms_txt",
        "domains": 8955,
        "present": 318,
        "fingerprinted": 218,
        "boilerplate": 12,
        "editorial": 206,
        "top_template": "v1:33652",
        "top_template_n": 10
      },
      {
        "sector": "Education",
        "kind": "agents_md",
        "domains": 8074,
        "present": 20,
        "fingerprinted": 20,
        "boilerplate": 19,
        "editorial": 1,
        "top_template": "v1:82463",
        "top_template_n": 19
      },
      {
        "sector": "Education",
        "kind": "llms_txt",
        "domains": 8074,
        "present": 430,
        "fingerprinted": 333,
        "boilerplate": 19,
        "editorial": 314,
        "top_template": "v1:33652",
        "top_template_n": 18
      },
      {
        "sector": "Shopping & Auctions",
        "kind": "agents_md",
        "domains": 7803,
        "present": 773,
        "fingerprinted": 769,
        "boilerplate": 701,
        "editorial": 68,
        "top_template": "v1:82463",
        "top_template_n": 683
      },
      {
        "sector": "Shopping & Auctions",
        "kind": "llms_txt",
        "domains": 7803,
        "present": 1371,
        "fingerprinted": 1238,
        "boilerplate": 653,
        "editorial": 585,
        "top_template": "v1:33652",
        "top_template_n": 636
      },
      {
        "sector": "(not categorized by source)",
        "kind": "agents_md",
        "domains": 6831,
        "present": 9,
        "fingerprinted": 7,
        "boilerplate": 6,
        "editorial": 1,
        "top_template": "v1:82463",
        "top_template_n": 6
      },
      {
        "sector": "(not categorized by source)",
        "kind": "llms_txt",
        "domains": 6831,
        "present": 138,
        "fingerprinted": 104,
        "boilerplate": 5,
        "editorial": 99,
        "top_template": "v1:33652",
        "top_template_n": 4
      },
      {
        "sector": "Society & Lifestyle",
        "kind": "agents_md",
        "domains": 6384,
        "present": 558,
        "fingerprinted": 555,
        "boilerplate": 520,
        "editorial": 35,
        "top_template": "v1:82463",
        "top_template_n": 507
      },
      {
        "sector": "Society & Lifestyle",
        "kind": "llms_txt",
        "domains": 6384,
        "present": 992,
        "fingerprinted": 874,
        "boilerplate": 488,
        "editorial": 386,
        "top_template": "v1:33652",
        "top_template_n": 475
      },
      {
        "sector": "Government & Politics",
        "kind": "agents_md",
        "domains": 5995,
        "present": 6,
        "fingerprinted": 5,
        "boilerplate": 2,
        "editorial": 3,
        "top_template": "v1:82463",
        "top_template_n": 2
      },
      {
        "sector": "Government & Politics",
        "kind": "llms_txt",
        "domains": 5995,
        "present": 233,
        "fingerprinted": 167,
        "boilerplate": 2,
        "editorial": 165,
        "top_template": "v1:33652",
        "top_template_n": 2
      },
      {
        "sector": "Internet Communication",
        "kind": "agents_md",
        "domains": 4759,
        "present": 8,
        "fingerprinted": 8,
        "boilerplate": 6,
        "editorial": 2,
        "top_template": "v1:82463",
        "top_template_n": 6
      },
      {
        "sector": "Internet Communication",
        "kind": "llms_txt",
        "domains": 4759,
        "present": 348,
        "fingerprinted": 249,
        "boilerplate": 6,
        "editorial": 243,
        "top_template": "v1:33652",
        "top_template_n": 5
      },
      {
        "sector": "Adult Themes",
        "kind": "agents_md",
        "domains": 4065,
        "present": 5,
        "fingerprinted": 5,
        "boilerplate": 4,
        "editorial": 1,
        "top_template": "v1:82463",
        "top_template_n": 4
      },
      {
        "sector": "Adult Themes",
        "kind": "llms_txt",
        "domains": 4065,
        "present": 133,
        "fingerprinted": 72,
        "boilerplate": 5,
        "editorial": 67,
        "top_template": "v1:33652",
        "top_template_n": 4
      },
      {
        "sector": "Gambling",
        "kind": "llms_txt",
        "domains": 2754,
        "present": 76,
        "fingerprinted": 58,
        "boilerplate": 0,
        "editorial": 58,
        "top_template": "v1:f60cd",
        "top_template_n": 2
      },
      {
        "sector": "Health",
        "kind": "agents_md",
        "domains": 2460,
        "present": 41,
        "fingerprinted": 41,
        "boilerplate": 36,
        "editorial": 5,
        "top_template": "v1:82463",
        "top_template_n": 33
      },
      {
        "sector": "Health",
        "kind": "llms_txt",
        "domains": 2460,
        "present": 267,
        "fingerprinted": 218,
        "boilerplate": 29,
        "editorial": 189,
        "top_template": "v1:33652",
        "top_template_n": 25
      },
      {
        "sector": "Travel",
        "kind": "agents_md",
        "domains": 2340,
        "present": 1,
        "fingerprinted": 1,
        "boilerplate": 1,
        "editorial": 0,
        "top_template": "v1:82463",
        "top_template_n": 1
      },
      {
        "sector": "Travel",
        "kind": "llms_txt",
        "domains": 2340,
        "present": 200,
        "fingerprinted": 161,
        "boilerplate": 1,
        "editorial": 160,
        "top_template": "v1:01ada",
        "top_template_n": 1
      },
      {
        "sector": "Ads",
        "kind": "agents_md",
        "domains": 2171,
        "present": 5,
        "fingerprinted": 5,
        "boilerplate": 1,
        "editorial": 4,
        "top_template": "v1:10f0d",
        "top_template_n": 1
      },
      {
        "sector": "Ads",
        "kind": "llms_txt",
        "domains": 2171,
        "present": 253,
        "fingerprinted": 180,
        "boilerplate": 3,
        "editorial": 177,
        "top_template": "v1:f807b",
        "top_template_n": 2
      },
      {
        "sector": "Sports",
        "kind": "agents_md",
        "domains": 2023,
        "present": 22,
        "fingerprinted": 22,
        "boilerplate": 20,
        "editorial": 2,
        "top_template": "v1:82463",
        "top_template_n": 20
      },
      {
        "sector": "Sports",
        "kind": "llms_txt",
        "domains": 2023,
        "present": 117,
        "fingerprinted": 91,
        "boilerplate": 20,
        "editorial": 71,
        "top_template": "v1:33652",
        "top_template_n": 20
      },
      {
        "sector": "Miscellaneous",
        "kind": "agents_md",
        "domains": 1543,
        "present": 3,
        "fingerprinted": 3,
        "boilerplate": 3,
        "editorial": 0,
        "top_template": "v1:82463",
        "top_template_n": 3
      },
      {
        "sector": "Miscellaneous",
        "kind": "llms_txt",
        "domains": 1543,
        "present": 32,
        "fingerprinted": 28,
        "boilerplate": 3,
        "editorial": 25,
        "top_template": "v1:33652",
        "top_template_n": 3
      },
      {
        "sector": "Vehicles",
        "kind": "agents_md",
        "domains": 1474,
        "present": 11,
        "fingerprinted": 11,
        "boilerplate": 10,
        "editorial": 1,
        "top_template": "v1:82463",
        "top_template_n": 9
      },
      {
        "sector": "Vehicles",
        "kind": "llms_txt",
        "domains": 1474,
        "present": 124,
        "fingerprinted": 90,
        "boilerplate": 10,
        "editorial": 80,
        "top_template": "v1:33652",
        "top_template_n": 9
      },
      {
        "sector": "Questionable Content",
        "kind": "agents_md",
        "domains": 1061,
        "present": 3,
        "fingerprinted": 3,
        "boilerplate": 3,
        "editorial": 0,
        "top_template": "v1:82463",
        "top_template_n": 3
      },
      {
        "sector": "Questionable Content",
        "kind": "llms_txt",
        "domains": 1061,
        "present": 45,
        "fingerprinted": 33,
        "boilerplate": 3,
        "editorial": 30,
        "top_template": "v1:17dac",
        "top_template_n": 3
      },
      {
        "sector": "Job Search & Careers",
        "kind": "agents_md",
        "domains": 859,
        "present": 2,
        "fingerprinted": 1,
        "boilerplate": 1,
        "editorial": 0,
        "top_template": "v1:82463",
        "top_template_n": 1
      },
      {
        "sector": "Job Search & Careers",
        "kind": "llms_txt",
        "domains": 859,
        "present": 82,
        "fingerprinted": 60,
        "boilerplate": 1,
        "editorial": 59,
        "top_template": "v1:04dae",
        "top_template_n": 1
      },
      {
        "sector": "Real Estate",
        "kind": "agents_md",
        "domains": 761,
        "present": 3,
        "fingerprinted": 3,
        "boilerplate": 3,
        "editorial": 0,
        "top_template": "v1:82463",
        "top_template_n": 3
      },
      {
        "sector": "Real Estate",
        "kind": "llms_txt",
        "domains": 761,
        "present": 69,
        "fingerprinted": 62,
        "boilerplate": 3,
        "editorial": 59,
        "top_template": "v1:33652",
        "top_template_n": 3
      },
      {
        "sector": "Religion",
        "kind": "agents_md",
        "domains": 342,
        "present": 1,
        "fingerprinted": 1,
        "boilerplate": 1,
        "editorial": 0,
        "top_template": "v1:82463",
        "top_template_n": 1
      },
      {
        "sector": "Religion",
        "kind": "llms_txt",
        "domains": 342,
        "present": 12,
        "fingerprinted": 12,
        "boilerplate": 1,
        "editorial": 11,
        "top_template": "v1:038ad",
        "top_template_n": 1
      },
      {
        "sector": "Weather",
        "kind": "agents_md",
        "domains": 237,
        "present": 1,
        "fingerprinted": 1,
        "boilerplate": 0,
        "editorial": 1,
        "top_template": "v1:28b69",
        "top_template_n": 1
      },
      {
        "sector": "Weather",
        "kind": "llms_txt",
        "domains": 237,
        "present": 16,
        "fingerprinted": 14,
        "boilerplate": 0,
        "editorial": 14,
        "top_template": "v1:18781",
        "top_template_n": 1
      },
      {
        "sector": "Military & Weapons",
        "kind": "agents_md",
        "domains": 202,
        "present": 2,
        "fingerprinted": 2,
        "boilerplate": 2,
        "editorial": 0,
        "top_template": "v1:82463",
        "top_template_n": 2
      },
      {
        "sector": "Military & Weapons",
        "kind": "llms_txt",
        "domains": 202,
        "present": 18,
        "fingerprinted": 15,
        "boilerplate": 2,
        "editorial": 13,
        "top_template": "v1:33652",
        "top_template_n": 2
      },
      {
        "sector": "Safe for Kids",
        "kind": "agents_md",
        "domains": 178,
        "present": 10,
        "fingerprinted": 10,
        "boilerplate": 7,
        "editorial": 3,
        "top_template": "v1:82463",
        "top_template_n": 7
      },
      {
        "sector": "Safe for Kids",
        "kind": "llms_txt",
        "domains": 178,
        "present": 12,
        "fingerprinted": 12,
        "boilerplate": 7,
        "editorial": 5,
        "top_template": "v1:33652",
        "top_template_n": 7
      },
      {
        "sector": "Violence",
        "kind": "llms_txt",
        "domains": 18,
        "present": 1,
        "fingerprinted": 0,
        "boilerplate": 0,
        "editorial": 0,
        "top_template": null,
        "top_template_n": null
      }
    ],
    "byLeaf": [
      {
        "sector": "Technology",
        "super": "Technology",
        "domains": 11769,
        "llms_txt": 1637,
        "llms_editorial": 1142,
        "llms_boilerplate": 72,
        "agents_md": 117,
        "manifest": 47,
        "names_ai_bot": 1282,
        "blocks_gptbot": 571
      },
      {
        "sector": "Business",
        "super": "Business & Economy",
        "domains": 10639,
        "llms_txt": 1588,
        "llms_editorial": 1127,
        "llms_boilerplate": 81,
        "agents_md": 109,
        "manifest": 51,
        "names_ai_bot": 1083,
        "blocks_gptbot": 419
      },
      {
        "sector": "Content Servers",
        "super": "Technology",
        "domains": 9724,
        "llms_txt": 59,
        "llms_editorial": 37,
        "llms_boilerplate": 8,
        "agents_md": 8,
        "manifest": 2,
        "names_ai_bot": 154,
        "blocks_gptbot": 117
      },
      {
        "sector": "CIPA Filter",
        "super": "CIPA",
        "domains": 8955,
        "llms_txt": 318,
        "llms_editorial": 206,
        "llms_boilerplate": 12,
        "agents_md": 13,
        "manifest": 3,
        "names_ai_bot": 1545,
        "blocks_gptbot": 1233
      },
      {
        "sector": "News & Media",
        "super": "Entertainment",
        "domains": 6683,
        "llms_txt": 412,
        "llms_editorial": 290,
        "llms_boilerplate": 0,
        "agents_md": 4,
        "manifest": 1,
        "names_ai_bot": 2207,
        "blocks_gptbot": 1695
      },
      {
        "sector": "Ecommerce",
        "super": "Shopping & Auctions",
        "domains": 6150,
        "llms_txt": 1219,
        "llms_editorial": 482,
        "llms_boilerplate": 640,
        "agents_md": 757,
        "manifest": 12,
        "names_ai_bot": 950,
        "blocks_gptbot": 363
      },
      {
        "sector": "Information Technology",
        "super": "Technology",
        "domains": 5041,
        "llms_txt": 471,
        "llms_editorial": 336,
        "llms_boilerplate": 8,
        "agents_md": 18,
        "manifest": 10,
        "names_ai_bot": 302,
        "blocks_gptbot": 130
      },
      {
        "sector": "Politics, Advocacy, and Government-Related",
        "super": "Government & Politics",
        "domains": 4568,
        "llms_txt": 141,
        "llms_editorial": 89,
        "llms_boilerplate": 0,
        "agents_md": 2,
        "manifest": 2,
        "names_ai_bot": 306,
        "blocks_gptbot": 206
      },
      {
        "sector": "Education",
        "super": "Education",
        "domains": 3927,
        "llms_txt": 240,
        "llms_editorial": 164,
        "llms_boilerplate": 18,
        "agents_md": 18,
        "manifest": 5,
        "names_ai_bot": 637,
        "blocks_gptbot": 426
      },
      {
        "sector": "Pornography",
        "super": "Adult Themes",
        "domains": 3689,
        "llms_txt": 108,
        "llms_editorial": 54,
        "llms_boilerplate": 2,
        "agents_md": 2,
        "manifest": 0,
        "names_ai_bot": 686,
        "blocks_gptbot": 535
      },
      {
        "sector": "Gaming",
        "super": "Entertainment",
        "domains": 3422,
        "llms_txt": 152,
        "llms_editorial": 103,
        "llms_boilerplate": 9,
        "agents_md": 7,
        "manifest": 1,
        "names_ai_bot": 551,
        "blocks_gptbot": 441
      },
      {
        "sector": "Video Streaming",
        "super": "Entertainment",
        "domains": 3379,
        "llms_txt": 152,
        "llms_editorial": 105,
        "llms_boilerplate": 3,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 1106,
        "blocks_gptbot": 875
      },
      {
        "sector": "Educational Institutions",
        "super": "Education",
        "domains": 3308,
        "llms_txt": 145,
        "llms_editorial": 112,
        "llms_boilerplate": 0,
        "agents_md": 1,
        "manifest": 1,
        "names_ai_bot": 262,
        "blocks_gptbot": 146
      },
      {
        "sector": "Gambling",
        "super": "Gambling",
        "domains": 2754,
        "llms_txt": 76,
        "llms_editorial": 58,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 1,
        "names_ai_bot": 454,
        "blocks_gptbot": 364
      },
      {
        "sector": "Health & Fitness",
        "super": "Health",
        "domains": 2451,
        "llms_txt": 266,
        "llms_editorial": 188,
        "llms_boilerplate": 29,
        "agents_md": 41,
        "manifest": 1,
        "names_ai_bot": 333,
        "blocks_gptbot": 176
      },
      {
        "sector": "Travel",
        "super": "Travel",
        "domains": 2340,
        "llms_txt": 200,
        "llms_editorial": 160,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 9,
        "names_ai_bot": 376,
        "blocks_gptbot": 155
      },
      {
        "sector": "Sports",
        "super": "Sports",
        "domains": 2023,
        "llms_txt": 117,
        "llms_editorial": 71,
        "llms_boilerplate": 20,
        "agents_md": 22,
        "manifest": 2,
        "names_ai_bot": 462,
        "blocks_gptbot": 259
      },
      {
        "sector": "Personal Finance",
        "super": "Business & Economy",
        "domains": 1916,
        "llms_txt": 188,
        "llms_editorial": 140,
        "llms_boilerplate": 0,
        "agents_md": 4,
        "manifest": 1,
        "names_ai_bot": 137,
        "blocks_gptbot": 50
      },
      {
        "sector": "Trackers/Analytics",
        "super": "Ads",
        "domains": 1897,
        "llms_txt": 240,
        "llms_editorial": 169,
        "llms_boilerplate": 2,
        "agents_md": 5,
        "manifest": 8,
        "names_ai_bot": 99,
        "blocks_gptbot": 19
      },
      {
        "sector": "Economy & Finance",
        "super": "Business & Economy",
        "domains": 1710,
        "llms_txt": 179,
        "llms_editorial": 142,
        "llms_boilerplate": 1,
        "agents_md": 2,
        "manifest": 2,
        "names_ai_bot": 218,
        "blocks_gptbot": 116
      },
      {
        "sector": "Shareware/Freeware",
        "super": "Technology",
        "domains": 1676,
        "llms_txt": 184,
        "llms_editorial": 147,
        "llms_boilerplate": 6,
        "agents_md": 8,
        "manifest": 4,
        "names_ai_bot": 222,
        "blocks_gptbot": 115
      },
      {
        "sector": "Personal Blogs",
        "super": "Internet Communication",
        "domains": 1648,
        "llms_txt": 115,
        "llms_editorial": 73,
        "llms_boilerplate": 3,
        "agents_md": 3,
        "manifest": 1,
        "names_ai_bot": 283,
        "blocks_gptbot": 198
      },
      {
        "sector": "Vehicles",
        "super": "Vehicles",
        "domains": 1474,
        "llms_txt": 124,
        "llms_editorial": 80,
        "llms_boilerplate": 10,
        "agents_md": 11,
        "manifest": 4,
        "names_ai_bot": 241,
        "blocks_gptbot": 133
      },
      {
        "sector": "Fashion",
        "super": "Society & Lifestyle",
        "domains": 1360,
        "llms_txt": 393,
        "llms_editorial": 114,
        "llms_boilerplate": 259,
        "agents_md": 292,
        "manifest": 6,
        "names_ai_bot": 215,
        "blocks_gptbot": 82
      },
      {
        "sector": "Clothing",
        "super": "Society & Lifestyle",
        "domains": 1281,
        "llms_txt": 367,
        "llms_editorial": 80,
        "llms_boilerplate": 256,
        "agents_md": 283,
        "manifest": 1,
        "names_ai_bot": 183,
        "blocks_gptbot": 85
      },
      {
        "sector": "Entertainment",
        "super": "Entertainment",
        "domains": 1193,
        "llms_txt": 70,
        "llms_editorial": 53,
        "llms_boilerplate": 0,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 258,
        "blocks_gptbot": 154
      },
      {
        "sector": "Search Engines",
        "super": "Technology",
        "domains": 1173,
        "llms_txt": 63,
        "llms_editorial": 48,
        "llms_boilerplate": 1,
        "agents_md": 3,
        "manifest": 2,
        "names_ai_bot": 190,
        "blocks_gptbot": 104
      },
      {
        "sector": "Movies",
        "super": "Entertainment",
        "domains": 1169,
        "llms_txt": 43,
        "llms_editorial": 32,
        "llms_boilerplate": 1,
        "agents_md": 2,
        "manifest": 0,
        "names_ai_bot": 294,
        "blocks_gptbot": 250
      },
      {
        "sector": "Login Screens",
        "super": "Miscellaneous",
        "domains": 1155,
        "llms_txt": 24,
        "llms_editorial": 17,
        "llms_boilerplate": 3,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 69,
        "blocks_gptbot": 60
      },
      {
        "sector": "Forums",
        "super": "Internet Communication",
        "domains": 1149,
        "llms_txt": 43,
        "llms_editorial": 32,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 331,
        "blocks_gptbot": 286
      },
      {
        "sector": "Government/Legal",
        "super": "Government & Politics",
        "domains": 1143,
        "llms_txt": 53,
        "llms_editorial": 46,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 110,
        "blocks_gptbot": 70
      },
      {
        "sector": "Magazines",
        "super": "Entertainment",
        "domains": 1062,
        "llms_txt": 72,
        "llms_editorial": 49,
        "llms_boilerplate": 0,
        "agents_md": 2,
        "manifest": 0,
        "names_ai_bot": 406,
        "blocks_gptbot": 235
      },
      {
        "sector": "Information Security",
        "super": "Internet Communication",
        "domains": 1005,
        "llms_txt": 114,
        "llms_editorial": 86,
        "llms_boilerplate": 3,
        "agents_md": 5,
        "manifest": 5,
        "names_ai_bot": 98,
        "blocks_gptbot": 42
      },
      {
        "sector": "Questionable Activities",
        "super": "Questionable Content",
        "domains": 997,
        "llms_txt": 31,
        "llms_editorial": 22,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 271,
        "blocks_gptbot": 254
      },
      {
        "sector": "Food & Drink",
        "super": "Society & Lifestyle",
        "domains": 982,
        "llms_txt": 76,
        "llms_editorial": 46,
        "llms_boilerplate": 20,
        "agents_md": 24,
        "manifest": 0,
        "names_ai_bot": 169,
        "blocks_gptbot": 114
      },
      {
        "sector": "Television",
        "super": "Entertainment",
        "domains": 945,
        "llms_txt": 42,
        "llms_editorial": 26,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 1,
        "names_ai_bot": 165,
        "blocks_gptbot": 121
      },
      {
        "sector": "Reference",
        "super": "Education",
        "domains": 924,
        "llms_txt": 51,
        "llms_editorial": 43,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 1,
        "names_ai_bot": 224,
        "blocks_gptbot": 162
      },
      {
        "sector": "Hobbies & Interests",
        "super": "Society & Lifestyle",
        "domains": 884,
        "llms_txt": 99,
        "llms_editorial": 52,
        "llms_boilerplate": 23,
        "agents_md": 25,
        "manifest": 0,
        "names_ai_bot": 174,
        "blocks_gptbot": 115
      },
      {
        "sector": "Job Search & Careers",
        "super": "Job Search & Careers",
        "domains": 859,
        "llms_txt": 82,
        "llms_editorial": 59,
        "llms_boilerplate": 1,
        "agents_md": 2,
        "manifest": 2,
        "names_ai_bot": 134,
        "blocks_gptbot": 70
      },
      {
        "sector": "Music",
        "super": "Entertainment",
        "domains": 764,
        "llms_txt": 64,
        "llms_editorial": 37,
        "llms_boilerplate": 18,
        "agents_md": 18,
        "manifest": 0,
        "names_ai_bot": 180,
        "blocks_gptbot": 136
      },
      {
        "sector": "Shopping",
        "super": "Shopping & Auctions",
        "domains": 764,
        "llms_txt": 75,
        "llms_editorial": 44,
        "llms_boilerplate": 12,
        "agents_md": 13,
        "manifest": 2,
        "names_ai_bot": 179,
        "blocks_gptbot": 90
      },
      {
        "sector": "Real Estate",
        "super": "Real Estate",
        "domains": 761,
        "llms_txt": 69,
        "llms_editorial": 59,
        "llms_boilerplate": 3,
        "agents_md": 3,
        "manifest": 1,
        "names_ai_bot": 105,
        "blocks_gptbot": 40
      },
      {
        "sector": "Audio Streaming",
        "super": "Entertainment",
        "domains": 671,
        "llms_txt": 45,
        "llms_editorial": 31,
        "llms_boilerplate": 4,
        "agents_md": 5,
        "manifest": 1,
        "names_ai_bot": 133,
        "blocks_gptbot": 96
      },
      {
        "sector": "Auctions & Marketplaces",
        "super": "Shopping & Auctions",
        "domains": 647,
        "llms_txt": 55,
        "llms_editorial": 41,
        "llms_boilerplate": 1,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 198,
        "blocks_gptbot": 119
      },
      {
        "sector": "File Sharing",
        "super": "Technology",
        "domains": 562,
        "llms_txt": 35,
        "llms_editorial": 30,
        "llms_boilerplate": 1,
        "agents_md": 2,
        "manifest": 1,
        "names_ai_bot": 68,
        "blocks_gptbot": 43
      },
      {
        "sector": "Photography",
        "super": "Society & Lifestyle",
        "domains": 541,
        "llms_txt": 34,
        "llms_editorial": 21,
        "llms_boilerplate": 4,
        "agents_md": 5,
        "manifest": 0,
        "names_ai_bot": 114,
        "blocks_gptbot": 72
      },
      {
        "sector": "Charity and Non-profit",
        "super": "Government & Politics",
        "domains": 524,
        "llms_txt": 49,
        "llms_editorial": 36,
        "llms_boilerplate": 2,
        "agents_md": 4,
        "manifest": 1,
        "names_ai_bot": 54,
        "blocks_gptbot": 32
      },
      {
        "sector": "Artificial Intelligence",
        "super": "Technology",
        "domains": 478,
        "llms_txt": 175,
        "llms_editorial": 123,
        "llms_boilerplate": 2,
        "agents_md": 3,
        "manifest": 13,
        "names_ai_bot": 86,
        "blocks_gptbot": 23
      },
      {
        "sector": "Adult Themes",
        "super": "Adult Themes",
        "domains": 408,
        "llms_txt": 26,
        "llms_editorial": 14,
        "llms_boilerplate": 3,
        "agents_md": 3,
        "manifest": 2,
        "names_ai_bot": 60,
        "blocks_gptbot": 41
      },
      {
        "sector": "Brokerage & Investing",
        "super": "Business & Economy",
        "domains": 403,
        "llms_txt": 55,
        "llms_editorial": 46,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 1,
        "names_ai_bot": 62,
        "blocks_gptbot": 34
      },
      {
        "sector": "Dating & Relationships",
        "super": "Society & Lifestyle",
        "domains": 392,
        "llms_txt": 22,
        "llms_editorial": 15,
        "llms_boilerplate": 3,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 39,
        "blocks_gptbot": 24
      },
      {
        "sector": "Comic Books",
        "super": "Entertainment",
        "domains": 379,
        "llms_txt": 15,
        "llms_editorial": 10,
        "llms_boilerplate": 2,
        "agents_md": 3,
        "manifest": 1,
        "names_ai_bot": 97,
        "blocks_gptbot": 92
      },
      {
        "sector": "Cartoons & Anime",
        "super": "Entertainment",
        "domains": 378,
        "llms_txt": 12,
        "llms_editorial": 8,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 1,
        "names_ai_bot": 119,
        "blocks_gptbot": 107
      },
      {
        "sector": "Home & Garden",
        "super": "Society & Lifestyle",
        "domains": 359,
        "llms_txt": 57,
        "llms_editorial": 26,
        "llms_boilerplate": 19,
        "agents_md": 25,
        "manifest": 1,
        "names_ai_bot": 69,
        "blocks_gptbot": 22
      },
      {
        "sector": "Religion & Spirituality",
        "super": "Religion",
        "domains": 342,
        "llms_txt": 12,
        "llms_editorial": 11,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 62,
        "blocks_gptbot": 40
      },
      {
        "sector": "Advertisements",
        "super": "Ads",
        "domains": 336,
        "llms_txt": 16,
        "llms_editorial": 10,
        "llms_boilerplate": 1,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 14,
        "blocks_gptbot": 11
      },
      {
        "sector": "Chat",
        "super": "Internet Communication",
        "domains": 308,
        "llms_txt": 32,
        "llms_editorial": 20,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 66,
        "blocks_gptbot": 17
      },
      {
        "sector": "Coupons",
        "super": "Shopping & Auctions",
        "domains": 294,
        "llms_txt": 26,
        "llms_editorial": 21,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 56,
        "blocks_gptbot": 37
      },
      {
        "sector": "Science",
        "super": "Education",
        "domains": 273,
        "llms_txt": 19,
        "llms_editorial": 16,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 1,
        "names_ai_bot": 47,
        "blocks_gptbot": 32
      },
      {
        "sector": "Arts",
        "super": "Entertainment",
        "domains": 267,
        "llms_txt": 29,
        "llms_editorial": 15,
        "llms_boilerplate": 6,
        "agents_md": 7,
        "manifest": 0,
        "names_ai_bot": 65,
        "blocks_gptbot": 51
      },
      {
        "sector": "Social Networks",
        "super": "Society & Lifestyle",
        "domains": 266,
        "llms_txt": 7,
        "llms_editorial": 5,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 54,
        "blocks_gptbot": 43
      },
      {
        "sector": "No Content",
        "super": "Miscellaneous",
        "domains": 250,
        "llms_txt": 2,
        "llms_editorial": 2,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 3,
        "blocks_gptbot": 3
      },
      {
        "sector": "P2P",
        "super": "Internet Communication",
        "domains": 244,
        "llms_txt": 15,
        "llms_editorial": 12,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 38,
        "blocks_gptbot": 27
      },
      {
        "sector": "Weather",
        "super": "Weather",
        "domains": 237,
        "llms_txt": 16,
        "llms_editorial": 14,
        "llms_boilerplate": 0,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 39,
        "blocks_gptbot": 22
      },
      {
        "sector": "Pets",
        "super": "Society & Lifestyle",
        "domains": 213,
        "llms_txt": 22,
        "llms_editorial": 15,
        "llms_boilerplate": 3,
        "agents_md": 6,
        "manifest": 0,
        "names_ai_bot": 37,
        "blocks_gptbot": 23
      },
      {
        "sector": "Safe for Kids",
        "super": "Safe for Kids",
        "domains": 178,
        "llms_txt": 12,
        "llms_editorial": 5,
        "llms_boilerplate": 7,
        "agents_md": 10,
        "manifest": 0,
        "names_ai_bot": 32,
        "blocks_gptbot": 22
      },
      {
        "sector": "Lifestyle",
        "super": "Society & Lifestyle",
        "domains": 172,
        "llms_txt": 15,
        "llms_editorial": 11,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 26,
        "blocks_gptbot": 14
      },
      {
        "sector": "Webmail",
        "super": "Internet Communication",
        "domains": 170,
        "llms_txt": 13,
        "llms_editorial": 8,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 1,
        "names_ai_bot": 9,
        "blocks_gptbot": 3
      },
      {
        "sector": "Cryptocurrency",
        "super": "Business & Economy",
        "domains": 161,
        "llms_txt": 42,
        "llms_editorial": 29,
        "llms_boilerplate": 0,
        "agents_md": 4,
        "manifest": 2,
        "names_ai_bot": 17,
        "blocks_gptbot": 4
      },
      {
        "sector": "Parenting",
        "super": "Society & Lifestyle",
        "domains": 157,
        "llms_txt": 26,
        "llms_editorial": 9,
        "llms_boilerplate": 10,
        "agents_md": 11,
        "manifest": 0,
        "names_ai_bot": 43,
        "blocks_gptbot": 21
      },
      {
        "sector": "Internet Phone & VOIP",
        "super": "Internet Communication",
        "domains": 145,
        "llms_txt": 10,
        "llms_editorial": 5,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 4,
        "blocks_gptbot": 2
      },
      {
        "sector": "Military",
        "super": "Military & Weapons",
        "domains": 129,
        "llms_txt": 11,
        "llms_editorial": 9,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 10,
        "blocks_gptbot": 9
      },
      {
        "sector": "Instant Messengers",
        "super": "Internet Communication",
        "domains": 120,
        "llms_txt": 7,
        "llms_editorial": 4,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 5,
        "blocks_gptbot": 2
      },
      {
        "sector": "Alcohol",
        "super": "Society & Lifestyle",
        "domains": 98,
        "llms_txt": 7,
        "llms_editorial": 5,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 20,
        "blocks_gptbot": 10
      },
      {
        "sector": "URL Alias/Redirect",
        "super": "Miscellaneous",
        "domains": 97,
        "llms_txt": 5,
        "llms_editorial": 5,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 4,
        "blocks_gptbot": 3
      },
      {
        "sector": "Humor",
        "super": "Entertainment",
        "domains": 83,
        "llms_txt": 4,
        "llms_editorial": 2,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 15,
        "blocks_gptbot": 9
      },
      {
        "sector": "Astrology",
        "super": "Society & Lifestyle",
        "domains": 77,
        "llms_txt": 10,
        "llms_editorial": 9,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 12,
        "blocks_gptbot": 10
      },
      {
        "sector": "Translator",
        "super": "Technology",
        "domains": 76,
        "llms_txt": 8,
        "llms_editorial": 6,
        "llms_boilerplate": 0,
        "agents_md": 1,
        "manifest": 1,
        "names_ai_bot": 11,
        "blocks_gptbot": 9
      },
      {
        "sector": "Weapons",
        "super": "Military & Weapons",
        "domains": 75,
        "llms_txt": 8,
        "llms_editorial": 5,
        "llms_boilerplate": 2,
        "agents_md": 2,
        "manifest": 0,
        "names_ai_bot": 12,
        "blocks_gptbot": 8
      },
      {
        "sector": "Lingerie & Bikini",
        "super": "Society & Lifestyle",
        "domains": 69,
        "llms_txt": 18,
        "llms_editorial": 10,
        "llms_boilerplate": 7,
        "agents_md": 8,
        "manifest": 0,
        "names_ai_bot": 12,
        "blocks_gptbot": 5
      },
      {
        "sector": "Remote Access",
        "super": "Technology",
        "domains": 65,
        "llms_txt": 8,
        "llms_editorial": 6,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 3,
        "blocks_gptbot": 1
      },
      {
        "sector": "APIs",
        "super": "Technology",
        "domains": 64,
        "llms_txt": 10,
        "llms_editorial": 8,
        "llms_boilerplate": 0,
        "agents_md": 1,
        "manifest": 4,
        "names_ai_bot": 4,
        "blocks_gptbot": 3
      },
      {
        "sector": "Nudity",
        "super": "Adult Themes",
        "domains": 59,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 5,
        "blocks_gptbot": 4
      },
      {
        "sector": "Space & Astronomy",
        "super": "Education",
        "domains": 58,
        "llms_txt": 1,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 12,
        "blocks_gptbot": 10
      },
      {
        "sector": "Professional Networking",
        "super": "Society & Lifestyle",
        "domains": 46,
        "llms_txt": 3,
        "llms_editorial": 2,
        "llms_boilerplate": 0,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 13,
        "blocks_gptbot": 9
      },
      {
        "sector": "News, Portal & Search",
        "super": "Technology",
        "domains": 45,
        "llms_txt": 1,
        "llms_editorial": 1,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 8,
        "blocks_gptbot": 5
      },
      {
        "sector": "Fine Art",
        "super": "Entertainment",
        "domains": 38,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 14,
        "blocks_gptbot": 8
      },
      {
        "sector": "Miscellaneous",
        "super": "Miscellaneous",
        "domains": 36,
        "llms_txt": 1,
        "llms_editorial": 1,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 1,
        "blocks_gptbot": 0
      },
      {
        "sector": "Arts & Crafts",
        "super": "Society & Lifestyle",
        "domains": 27,
        "llms_txt": 5,
        "llms_editorial": 1,
        "llms_boilerplate": 3,
        "agents_md": 3,
        "manifest": 0,
        "names_ai_bot": 5,
        "blocks_gptbot": 2
      },
      {
        "sector": "Drugs",
        "super": "Questionable Content",
        "domains": 26,
        "llms_txt": 5,
        "llms_editorial": 2,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 6,
        "blocks_gptbot": 5
      },
      {
        "sector": "Digital Postcards",
        "super": "Society & Lifestyle",
        "domains": 18,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      },
      {
        "sector": "Violence",
        "super": "Violence",
        "domains": 18,
        "llms_txt": 1,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 2,
        "blocks_gptbot": 2
      },
      {
        "sector": "Messaging",
        "super": "Internet Communication",
        "domains": 16,
        "llms_txt": 5,
        "llms_editorial": 4,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 4,
        "blocks_gptbot": 2
      },
      {
        "sector": "School Cheating",
        "super": "Questionable Content",
        "domains": 15,
        "llms_txt": 5,
        "llms_editorial": 5,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 2,
        "blocks_gptbot": 1
      },
      {
        "sector": "Tobacco",
        "super": "Society & Lifestyle",
        "domains": 15,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 1,
        "blocks_gptbot": 0
      },
      {
        "sector": "Radio",
        "super": "Entertainment",
        "domains": 13,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 2,
        "blocks_gptbot": 1
      },
      {
        "sector": "Sex Education",
        "super": "Health",
        "domains": 12,
        "llms_txt": 1,
        "llms_editorial": 1,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 1,
        "blocks_gptbot": 1
      },
      {
        "sector": "Swimsuits",
        "super": "Society & Lifestyle",
        "domains": 11,
        "llms_txt": 5,
        "llms_editorial": 1,
        "llms_boilerplate": 4,
        "agents_md": 4,
        "manifest": 0,
        "names_ai_bot": 2,
        "blocks_gptbot": 0
      },
      {
        "sector": "Deceptive Ads",
        "super": "Questionable Content",
        "domains": 10,
        "llms_txt": 3,
        "llms_editorial": 1,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 4,
        "blocks_gptbot": 2
      },
      {
        "sector": "Home Video/DVD",
        "super": "Entertainment",
        "domains": 10,
        "llms_txt": 1,
        "llms_editorial": 0,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 5,
        "blocks_gptbot": 2
      },
      {
        "sector": "Hacking",
        "super": "Questionable Content",
        "domains": 9,
        "llms_txt": 1,
        "llms_editorial": 1,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      },
      {
        "sector": "Photo Sharing",
        "super": "Internet Communication",
        "domains": 7,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 1,
        "names_ai_bot": 1,
        "blocks_gptbot": 1
      },
      {
        "sector": "Militancy, Hate & Extremism",
        "super": "Questionable Content",
        "domains": 5,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      },
      {
        "sector": "Unreachable",
        "super": "Miscellaneous",
        "domains": 5,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      },
      {
        "sector": "Child Abuse",
        "super": "Blocked by Default",
        "domains": 4,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 1,
        "blocks_gptbot": 0
      },
      {
        "sector": "Paranormal",
        "super": "Entertainment",
        "domains": 4,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      },
      {
        "sector": "Unreliable Information",
        "super": "Questionable Content",
        "domains": 3,
        "llms_txt": 1,
        "llms_editorial": 0,
        "llms_boilerplate": 1,
        "agents_md": 1,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      },
      {
        "sector": "Sexuality",
        "super": "Society & Lifestyle",
        "domains": 2,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      },
      {
        "sector": "Body Art",
        "super": "Society & Lifestyle",
        "domains": 1,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 1,
        "blocks_gptbot": 1
      },
      {
        "sector": "Jewelry",
        "super": "Society & Lifestyle",
        "domains": 1,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 1,
        "blocks_gptbot": 0
      },
      {
        "sector": "Profanity",
        "super": "Questionable Content",
        "domains": 1,
        "llms_txt": 0,
        "llms_editorial": 0,
        "llms_boilerplate": 0,
        "agents_md": 0,
        "manifest": 0,
        "names_ai_bot": 0,
        "blocks_gptbot": 0
      }
    ]
  },
  "entryTypes": [
    {
      "family": "mcp",
      "label": "MCP server",
      "domains": 112,
      "domainsRecognized": 108,
      "domainsVariant": 6,
      "entries": 127,
      "entriesRecognized": 121,
      "variants": [
        {
          "type": "application/mcp-server+json",
          "entries": 5,
          "domains": 5
        },
        {
          "type": "application/vnd.mcp+json",
          "entries": 1,
          "domains": 1
        }
      ]
    },
    {
      "family": "agent_card",
      "label": "Agent card",
      "domains": 4,
      "domainsRecognized": 4,
      "domainsVariant": 0,
      "entries": 8,
      "entriesRecognized": 8,
      "variants": []
    },
    {
      "family": "skill",
      "label": "Agent skill",
      "domains": 42,
      "domainsRecognized": 23,
      "domainsVariant": 20,
      "entries": 116,
      "entriesRecognized": 58,
      "variants": [
        {
          "type": "application/ai-skill",
          "entries": 28,
          "domains": 5
        },
        {
          "type": "application/ai-skill+md",
          "entries": 19,
          "domains": 12
        },
        {
          "type": "application/ai-skill-archive+gzip",
          "entries": 8,
          "domains": 1
        },
        {
          "type": "application/agent-skills-index+json",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/json;profile=\"https://schemas.agentskills.io/discovery/0.2.0/schema.json\"",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/vnd.linken-sphere.agent-skills+json",
          "entries": 1,
          "domains": 1
        }
      ]
    },
    {
      "family": "plugin",
      "label": "Agent plugin",
      "domains": 1,
      "domainsRecognized": 0,
      "domainsVariant": 1,
      "entries": 11,
      "entriesRecognized": 0,
      "variants": [
        {
          "type": "application/agent-plugins+json",
          "entries": 4,
          "domains": 1
        },
        {
          "type": "application/vnd.anthropic.claude-plugin+json",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/vnd.cursor.cursor-plugin+json",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/vnd.github.copilot-plugin",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/vnd.moonshot.kimi-plugin+json",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/vnd.openai.codex-plugin+json",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/vnd.openclaw.plugin+json",
          "entries": 1,
          "domains": 1
        },
        {
          "type": "application/vnd.xai.grok-plugin+json",
          "entries": 1,
          "domains": 1
        }
      ]
    },
    {
      "family": "registry",
      "label": "Registry",
      "domains": 2,
      "domainsRecognized": 2,
      "domainsVariant": 0,
      "entries": 3,
      "entriesRecognized": 3,
      "variants": []
    },
    {
      "family": "catalog",
      "label": "Nested catalog",
      "domains": 3,
      "domainsRecognized": 3,
      "domainsVariant": 0,
      "entries": 3,
      "entriesRecognized": 3,
      "variants": []
    },
    {
      "family": "other",
      "label": "Other (not covered by §3.3)",
      "domains": 103,
      "domainsRecognized": 0,
      "domainsVariant": 0,
      "entries": 408,
      "entriesRecognized": 0,
      "variants": []
    }
  ],
  "aiBotDirectives": [
    {
      "bot": "GPTBot",
      "named": 10666,
      "fully_blocked": 8109
    },
    {
      "bot": "ClaudeBot",
      "named": 9765,
      "fully_blocked": 7387
    },
    {
      "bot": "CCBot",
      "named": 9193,
      "fully_blocked": 7903
    },
    {
      "bot": "Google-Extended",
      "named": 8748,
      "fully_blocked": 6688
    },
    {
      "bot": "Bytespider",
      "named": 8495,
      "fully_blocked": 7743
    },
    {
      "bot": "meta-externalagent",
      "named": 7823,
      "fully_blocked": 6702
    },
    {
      "bot": "Applebot-Extended",
      "named": 7595,
      "fully_blocked": 6519
    },
    {
      "bot": "anthropic-ai",
      "named": 7433,
      "fully_blocked": 6430
    },
    {
      "bot": "cohere-ai",
      "named": 6675,
      "fully_blocked": 6059
    },
    {
      "bot": "Claude-Web",
      "named": 6444,
      "fully_blocked": 5771
    },
    {
      "bot": "PerplexityBot",
      "named": 4909,
      "fully_blocked": 2378
    },
    {
      "bot": "ChatGPT-User",
      "named": 4822,
      "fully_blocked": 2422
    },
    {
      "bot": "OAI-SearchBot",
      "named": 3767,
      "fully_blocked": 1420
    },
    {
      "bot": "Perplexity-User",
      "named": 2223,
      "fully_blocked": 1120
    },
    {
      "bot": "Claude-User",
      "named": 2182,
      "fully_blocked": 1086
    },
    {
      "bot": "meta-externalfetcher",
      "named": 1650,
      "fully_blocked": 1275
    },
    {
      "bot": "DuckAssistBot",
      "named": 1498,
      "fully_blocked": 996
    },
    {
      "bot": "MistralAI-User",
      "named": 1272,
      "fully_blocked": 963
    },
    {
      "bot": "Google-NotebookLM",
      "named": 588,
      "fully_blocked": 513
    },
    {
      "bot": "Gemini-Deep-Research",
      "named": 494,
      "fully_blocked": 358
    },
    {
      "bot": "Google-Agent",
      "named": 468,
      "fully_blocked": 410
    },
    {
      "bot": "NovaAct",
      "named": 373,
      "fully_blocked": 342
    },
    {
      "bot": "Devin",
      "named": 363,
      "fully_blocked": 345
    },
    {
      "bot": "Kimi-User",
      "named": 320,
      "fully_blocked": 314
    },
    {
      "bot": "GoogleAgent-Mariner",
      "named": 315,
      "fully_blocked": 275
    },
    {
      "bot": "AmazonBuyForMe",
      "named": 280,
      "fully_blocked": 271
    },
    {
      "bot": "Manus-User",
      "named": 263,
      "fully_blocked": 247
    },
    {
      "bot": "TwinAgent",
      "named": 229,
      "fully_blocked": 224
    },
    {
      "bot": "Kagi-Fetcher",
      "named": 109,
      "fully_blocked": 104
    },
    {
      "bot": "Claude-Code",
      "named": 98,
      "fully_blocked": 78
    },
    {
      "bot": "OpenCode",
      "named": 65,
      "fully_blocked": 62
    },
    {
      "bot": "Trae",
      "named": 64,
      "fully_blocked": 62
    },
    {
      "bot": "GoogleAgent-URLContext",
      "named": 63,
      "fully_blocked": 55
    },
    {
      "bot": "Cursor",
      "named": 58,
      "fully_blocked": 55
    },
    {
      "bot": "YepBot",
      "named": 44,
      "fully_blocked": 44
    },
    {
      "bot": "Google-Gemini-CLI",
      "named": 40,
      "fully_blocked": 36
    },
    {
      "bot": "ChatGPT-Agent",
      "named": 10,
      "fully_blocked": 6
    },
    {
      "bot": "DesvelaBot",
      "named": 1,
      "fully_blocked": 1
    }
  ],
  "dnsAid": {
    "probed": 98756,
    "present": 42
  },
  "notable": {
    "ai_catalog": [
      {
        "host": "github.com",
        "rank": 34
      },
      {
        "host": "shop.app",
        "rank": 545
      },
      {
        "host": "huggingface.co",
        "rank": 1079
      },
      {
        "host": "oxylabs.io",
        "rank": 1333
      },
      {
        "host": "hostinger.com",
        "rank": 1832
      },
      {
        "host": "bitdefender.com",
        "rank": 1844
      },
      {
        "host": "padlet.com",
        "rank": 2720
      },
      {
        "host": "cookieyes.com",
        "rank": 2749
      },
      {
        "host": "rudderstack.com",
        "rank": 2854
      },
      {
        "host": "zapier.com",
        "rank": 3005
      },
      {
        "host": "clerk.com",
        "rank": 3430
      },
      {
        "host": "hootsuite.com",
        "rank": 3449
      },
      {
        "host": "netlify.com",
        "rank": 3988
      },
      {
        "host": "supabase.com",
        "rank": 4862
      },
      {
        "host": "beehiiv.com",
        "rank": 5770
      }
    ],
    "llms_txt": [
      {
        "host": "cloudflare.com",
        "rank": 2
      },
      {
        "host": "github.com",
        "rank": 34
      },
      {
        "host": "adobe.com",
        "rank": 61
      },
      {
        "host": "wordpress.org",
        "rank": 65
      },
      {
        "host": "opera.com",
        "rank": 76
      },
      {
        "host": "wordpress.com",
        "rank": 99
      },
      {
        "host": "intuit.com",
        "rank": 114
      },
      {
        "host": "samsung.com",
        "rank": 122
      },
      {
        "host": "dropbox.com",
        "rank": 136
      },
      {
        "host": "webex.com",
        "rank": 146
      },
      {
        "host": "gravatar.com",
        "rank": 167
      },
      {
        "host": "ubuntu.com",
        "rank": 168
      },
      {
        "host": "kaspersky.com",
        "rank": 171
      },
      {
        "host": "shopify.com",
        "rank": 172
      },
      {
        "host": "paypal.com",
        "rank": 175
      }
    ],
    "agents_md": [
      {
        "host": "shopify.com",
        "rank": 172
      },
      {
        "host": "lg.com",
        "rank": 916
      },
      {
        "host": "coinbase.com",
        "rank": 1477
      },
      {
        "host": "cookieyes.com",
        "rank": 2749
      },
      {
        "host": "crazyegg.com",
        "rank": 2988
      },
      {
        "host": "backblaze.com",
        "rank": 3334
      },
      {
        "host": "razorpay.com",
        "rank": 3369
      },
      {
        "host": "nazwa.pl",
        "rank": 3833
      },
      {
        "host": "nanit.com",
        "rank": 3942
      },
      {
        "host": "eufylife.com",
        "rank": 4848
      },
      {
        "host": "nownodes.io",
        "rank": 5585
      },
      {
        "host": "laravel-news.com",
        "rank": 5780
      },
      {
        "host": "wyze.com",
        "rank": 6027
      },
      {
        "host": "openweathermap.org",
        "rank": 6727
      },
      {
        "host": "anker.com",
        "rank": 8006
      }
    ],
    "ard_json": [
      {
        "host": "cm.com",
        "rank": 364
      },
      {
        "host": "bluehost.com",
        "rank": 467
      },
      {
        "host": "gameanalytics.com",
        "rank": 2439
      },
      {
        "host": "cookieyes.com",
        "rank": 2749
      },
      {
        "host": "hootsuite.com",
        "rank": 3449
      },
      {
        "host": "elementor.com",
        "rank": 3906
      },
      {
        "host": "openrouter.ai",
        "rank": 3932
      },
      {
        "host": "netlify.com",
        "rank": 3988
      },
      {
        "host": "supabase.com",
        "rank": 4862
      },
      {
        "host": "pixai.art",
        "rank": 5736
      },
      {
        "host": "activecampaign.com",
        "rank": 10820
      },
      {
        "host": "buildkite.com",
        "rank": 12353
      },
      {
        "host": "apify.com",
        "rank": 12404
      },
      {
        "host": "ati.su",
        "rank": 14022
      },
      {
        "host": "bestprice.gr",
        "rank": 15343
      }
    ]
  },
  "disclaimer": "Adoption and access signals do not demonstrate citations, recommendations or visibility inside any AI system."
}