From c53a916822ea34a4722c885173d6bebee6cb1ae9 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Mon, 6 Jul 2026 18:11:05 +0200 Subject: [PATCH 1/4] feat: add public_hash_list table definition Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .../output/performance/public_hash_list.js | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 definitions/output/performance/public_hash_list.js diff --git a/definitions/output/performance/public_hash_list.js b/definitions/output/performance/public_hash_list.js new file mode 100644 index 00000000..c458cdfc --- /dev/null +++ b/definitions/output/performance/public_hash_list.js @@ -0,0 +1,62 @@ +publish('public_hash_list', { + type: 'table', + schema: 'performance', + description: `Identifies web resources (scripts, CSS, fonts, WASM) whose SHA-256 body hash appears across >=100 independent origins in the HTTP Archive monthly crawl. +The >=100-origins threshold is the k-anonymity privacy gate: a resource that widespread cannot serve as a cross-site identifier. +Traffic-weighted score: each hash is scored by SUM(100000 / min_rank) across origins, where min_rank is the CrUX popularity bucket of the hosting page (1 000 = top 1K sites -> 100 pts; 1 000 000 = top 1M -> 0.1 pt). This lifts hashes carried by high-traffic pages to the top of the list. +Results are published as a world-readable Google Sheet: https://docs.google.com/spreadsheets/d/1Cw4wguQ0X4xMqZlTRYlo6OHQaUr7UVYlFZaIQkXK2Jw/edit?usp=sharing +The CSV export used by http-archive.js: https://docs.google.com/spreadsheets/d/e/2PACX-1vTOcTespiVHDRLIq16_3GsnnvJmut00x0fzWTLXSWBNya6Go_1kBrGoVJvxb8gEaP_L9FfKmXy3-kF-/pub?output=csv +Repo: https://github.com/tomayac/public-hash-list`, + columns: { + body_hash: 'SHA-256 body hash of the resource from WebPageTest payload', + type: 'Simplified type of the resource (script, css, font, wasm)', + num_origins: 'Number of independent origins hosting the resource (privacy-gated threshold of >= 100)', + traffic_weighted_score: 'Traffic-weighted popularity score (SUM(100000 / min_rank) across origins, where min_rank is the CrUX rank bucket of the hosting page)', + sample_url: 'A sample URL where this resource was detected' + }, + tags: ['crawl_complete'] +}).query(ctx => ` +WITH request_origins AS ( + SELECT + SAFE.STRING(payload._body_hash) AS body_hash, + type, + NET.HOST(url) AS origin, + MIN(rank) AS min_rank, + ANY_VALUE(url) AS sample_url + FROM ${ctx.ref('crawl', 'requests')} + WHERE + date = DATE_TRUNC(CURRENT_DATE(), MONTH) + AND SAFE.STRING(payload._body_hash) IS NOT NULL + AND type IN ('script', 'css', 'font', 'wasm') + AND SAFE.STRING(summary.method) = 'GET' + GROUP BY + body_hash, + type, + origin +), + +hash_popularity AS ( + SELECT + body_hash, + type, + COUNT(DISTINCT origin) AS num_origins, + SUM(100000.0 / min_rank) AS traffic_weighted_score, + ANY_VALUE(sample_url) AS sample_url + FROM request_origins + GROUP BY + body_hash, + type +) + +SELECT + body_hash, + type, + num_origins, + traffic_weighted_score, + sample_url +FROM hash_popularity +WHERE + num_origins >= 100 +ORDER BY + traffic_weighted_score DESC +`) From 853b7b7daf72417fc4a7c474d46abd355fe721b8 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:38:35 +0200 Subject: [PATCH 2/4] feat: export public hash list to cloud storage via automated job Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- definitions/output/performance/public_hash_list.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/definitions/output/performance/public_hash_list.js b/definitions/output/performance/public_hash_list.js index c458cdfc..3d4e1476 100644 --- a/definitions/output/performance/public_hash_list.js +++ b/definitions/output/performance/public_hash_list.js @@ -59,4 +59,18 @@ WHERE num_origins >= 100 ORDER BY traffic_weighted_score DESC +`).postOps(ctx => ` +SELECT + reports.run_export_job( + JSON '''{ + "destination": "cloud_storage", + "config": { + "bucket": "${constants.bucket}", + "name": "${constants.storagePath}public_hash_list.csv", + "format": "csv" + }, + "query": "SELECT * FROM ${ctx.self()}" + }''' + ); `) + From 5e73fc824a50dbe5d35447fae410263e6b97245b Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:39:17 +0200 Subject: [PATCH 3/4] lint Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- includes/reports.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/reports.js b/includes/reports.js index 5f03adff..80bf7316 100644 --- a/includes/reports.js +++ b/includes/reports.js @@ -3085,7 +3085,7 @@ const lensArrayExpression = `ARRAY[ IF('Drupal' IN UNNEST(technologies.technology), 'drupal', NULL), IF('Magento' IN UNNEST(technologies.technology), 'magento', NULL), IF('WordPress' IN UNNEST(technologies.technology), 'wordpress', NULL) -]`; +]` class HTTPArchiveReports { constructor() { From 010ecb938eba430b62ae27902e62f64fb7d5ac5f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:02:59 +0200 Subject: [PATCH 4/4] refactor: reformat public_hash_list query and postOps strings for improved readability Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .../output/performance/public_hash_list.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/definitions/output/performance/public_hash_list.js b/definitions/output/performance/public_hash_list.js index 3d4e1476..c7cb7dea 100644 --- a/definitions/output/performance/public_hash_list.js +++ b/definitions/output/performance/public_hash_list.js @@ -10,12 +10,16 @@ Repo: https://github.com/tomayac/public-hash-list`, columns: { body_hash: 'SHA-256 body hash of the resource from WebPageTest payload', type: 'Simplified type of the resource (script, css, font, wasm)', - num_origins: 'Number of independent origins hosting the resource (privacy-gated threshold of >= 100)', - traffic_weighted_score: 'Traffic-weighted popularity score (SUM(100000 / min_rank) across origins, where min_rank is the CrUX rank bucket of the hosting page)', + num_origins: + 'Number of independent origins hosting the resource (privacy-gated threshold of >= 100)', + traffic_weighted_score: + 'Traffic-weighted popularity score (SUM(100000 / min_rank) across origins, where min_rank is the CrUX rank bucket of the hosting page)', sample_url: 'A sample URL where this resource was detected' }, tags: ['crawl_complete'] -}).query(ctx => ` +}) + .query( + (ctx) => ` WITH request_origins AS ( SELECT SAFE.STRING(payload._body_hash) AS body_hash, @@ -59,7 +63,10 @@ WHERE num_origins >= 100 ORDER BY traffic_weighted_score DESC -`).postOps(ctx => ` +` + ) + .postOps( + (ctx) => ` SELECT reports.run_export_job( JSON '''{ @@ -72,5 +79,5 @@ SELECT "query": "SELECT * FROM ${ctx.self()}" }''' ); -`) - +` + )