{
  "methodology_version": "2026-08-10.1",
  "generated_utc": "2026-08-09T10:47:13.783Z",
  "coverage": {
    "tranco_top_n": 50000,
    "robots_parsed": 27067,
    "crawlers_tracked": 20
  },
  "evidence": {
    "version": "2026-08-10.1",
    "crawler_identity": "CrawlPriceIndexBot/1.0, self-identified, requests cryptographically signed (RFC 9421 / Web Bot Auth). Public key directory: https://crawlpriceindex.com/.well-known/http-message-signatures-directory",
    "signal_classes": {
      "robots_stance": {
        "what": "Per-crawler directive parsed from the site's own robots.txt",
        "method": "GET /robots.txt with our identified user agent, parsed for User-agent/Disallow/Allow groups",
        "evidence_type": "observed",
        "values": {
          "blocked": "an explicit Disallow rule applies to this crawler",
          "allowed": "an explicit Allow rule applies",
          "partial": "some paths disallowed, not the whole site",
          "unlisted": "robots.txt exists but names no rule for this crawler",
          "no_robots": "no robots.txt was served"
        },
        "caveat": "A declaration is not enforcement. See enforcement."
      },
      "observed_price": {
        "what": "A price a site quotes to an AI crawler",
        "method": "Recorded only when the site returns it in a machine-readable payment response (HTTP 402 with a crawler-price/payment header, or an equivalent x402 offer)",
        "evidence_type": "observed",
        "caveat": "We publish only prices we have actually been quoted. We do not estimate, model or extrapolate prices for domains that do not quote one."
      },
      "payment_signal": {
        "what": "Paywall / licensing / toll signals other than an explicit price",
        "method": "HTTP status and headers on identified-crawler requests to the homepage (402 responses, TollBit-style token walls, licensing redirects, payment:free declarations)",
        "evidence_type": "observed"
      },
      "block_rates": {
        "what": "Share of scanned domains blocking a given crawler",
        "method": "Aggregated from robots_stance across all parsed domains in the current sweep",
        "evidence_type": "derived",
        "caveat": "Derived arithmetic over observations. Denominator is robots_parsed, not tranco_top_n."
      },
      "country_editions": {
        "what": "Block rates segmented by country-code top-level domain",
        "method": "ccTLD of the domain, aggregated where at least 8 domains are present",
        "evidence_type": "derived",
        "caveat": "ccTLD is a proxy for country, not a measure of publisher nationality or audience."
      },
      "enforcement": {
        "what": "Whether a declared block is actually enforced when the crawler arrives",
        "method": "Compare robots.txt declaration against the HTTP status returned to a crawler-identified request, on the documented publisher panel only",
        "evidence_type": "observed (panel-limited)",
        "caveat": "Small n. Directional for the panel, not projectable to the whole web."
      },
      "trends": {
        "what": "Week-over-week movement of the above",
        "method": "Difference between consecutive weekly snapshots held in our own archive",
        "evidence_type": "derived",
        "caveat": "History cannot be backfilled: series begin the week we first observed them."
      },
      "suggested_price_band": {
        "what": "Guidance shown in the public checker for what a site might charge",
        "method": "Our judgement, anchored to observed quotes and the domain's Tranco rank",
        "evidence_type": "inferred",
        "caveat": "NOT a measurement. Explicitly labelled as judgement wherever it appears."
      }
    },
    "not_measured": [
      "Actual revenue earned by any site",
      "Traffic volumes of any site",
      "Prices for domains that do not publish one",
      "Private licensing deal values",
      "Whether an AI company honoured a payment request"
    ],
    "reproducibility": "Every figure is regenerated weekly from a full sweep. Method, crawler identity and signal definitions are published at https://crawlpriceindex.com/methodology.html and versioned above.",
    "corrections_policy": "Errors are corrected in the next weekly edition and listed in corrections[] with the date, what changed and why. We do not silently amend past editions."
  },
  "human_readable": "https://crawlpriceindex.com/methodology.html",
  "crawler_key_directory": "https://crawlpriceindex.com/.well-known/http-message-signatures-directory",
  "contact": "hello@crawlpriceindex.com"
}