From f7996294504b8053634079cc8c76bb84d3079ac7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Jul 2026 22:35:58 +0000 Subject: [PATCH 1/2] Support overriding the clickhouse binary with a PR build Add a `clickhouse_pr` env variable to the benchmark launcher. When set to a ClickHouse/ClickHouse pull-request number, the clickhouse* systems benchmark that PR's CI release build instead of the latest stable release; every other system ignores it. The value is set on the operator side (run-benchmark.sh), threaded to the VM through cloud-init, and consumed only by the clickhouse* install scripts via a new shared helper, lib/download-clickhouse. The helper resolves "the PR build" as the newest PR commit whose `Build (_release)` CI check finished successfully for the machine's architecture, then downloads the raw self-extracting binary that build uploaded to S3: https://clickhouse-builds.s3.amazonaws.com/PRs///build__release/clickhouse Walking commits newest-first (falling back past a tip whose build is still running, failed, or was skipped) implements "the last commit with a finished release build". With no clickhouse_pr set the helper keeps the historical `curl https://clickhouse.com/ | sh` behaviour. The chosen PR is recorded in the log as a `ClickHouse PR:` line and parsed into a new results.clickhouse_pr column (prepare-database.sql) so PR-build runs can be filtered apart from normal results later. Co-Authored-By: Claude Opus 4.8 (1M context) --- clickhouse-datalake-partitioned/install | 2 +- clickhouse-datalake/install | 2 +- clickhouse-parquet-partitioned/install | 2 +- clickhouse-parquet/install | 2 +- clickhouse-trash-schema/install | 2 +- clickhouse-web/install | 2 +- clickhouse/install | 2 +- cloud-init.sh.in | 11 +++ lib/download-clickhouse | 94 +++++++++++++++++++++++++ prepare-database.sql | 6 +- run-benchmark.sh | 9 ++- 11 files changed, 125 insertions(+), 9 deletions(-) create mode 100755 lib/download-clickhouse diff --git a/clickhouse-datalake-partitioned/install b/clickhouse-datalake-partitioned/install index ee46804c16..3f3e637339 100755 --- a/clickhouse-datalake-partitioned/install +++ b/clickhouse-datalake-partitioned/install @@ -2,7 +2,7 @@ set -e if [ ! -x ./clickhouse ]; then - curl https://clickhouse.com/ | sh + ../lib/download-clickhouse fi # Use a userspace page cache sized to ~80% of RAM for S3 object reads. diff --git a/clickhouse-datalake/install b/clickhouse-datalake/install index ee46804c16..3f3e637339 100755 --- a/clickhouse-datalake/install +++ b/clickhouse-datalake/install @@ -2,7 +2,7 @@ set -e if [ ! -x ./clickhouse ]; then - curl https://clickhouse.com/ | sh + ../lib/download-clickhouse fi # Use a userspace page cache sized to ~80% of RAM for S3 object reads. diff --git a/clickhouse-parquet-partitioned/install b/clickhouse-parquet-partitioned/install index 43a2ea1c30..861f2e6d25 100755 --- a/clickhouse-parquet-partitioned/install +++ b/clickhouse-parquet-partitioned/install @@ -2,5 +2,5 @@ set -e if [ ! -x ./clickhouse ]; then - curl https://clickhouse.com/ | sh + ../lib/download-clickhouse fi diff --git a/clickhouse-parquet/install b/clickhouse-parquet/install index 43a2ea1c30..861f2e6d25 100755 --- a/clickhouse-parquet/install +++ b/clickhouse-parquet/install @@ -2,5 +2,5 @@ set -e if [ ! -x ./clickhouse ]; then - curl https://clickhouse.com/ | sh + ../lib/download-clickhouse fi diff --git a/clickhouse-trash-schema/install b/clickhouse-trash-schema/install index 4b46afd44c..ac3ccec31f 100755 --- a/clickhouse-trash-schema/install +++ b/clickhouse-trash-schema/install @@ -2,7 +2,7 @@ set -e if [ ! -x /usr/bin/clickhouse ]; then - curl https://clickhouse.com/ | sh + ../lib/download-clickhouse sudo ./clickhouse install --noninteractive fi diff --git a/clickhouse-web/install b/clickhouse-web/install index 75715b33c6..10ab7419e9 100755 --- a/clickhouse-web/install +++ b/clickhouse-web/install @@ -6,7 +6,7 @@ set -e # over HTTP from a public ClickHouse-hosted dataset. if [ ! -x /usr/bin/clickhouse ]; then - curl https://clickhouse.com/ | sh + ../lib/download-clickhouse sudo ./clickhouse install --noninteractive fi diff --git a/clickhouse/install b/clickhouse/install index 98f91acdca..c46883ff1e 100755 --- a/clickhouse/install +++ b/clickhouse/install @@ -2,7 +2,7 @@ set -e if [ ! -x /usr/bin/clickhouse ]; then - curl https://clickhouse.com/ | sh + ../lib/download-clickhouse sudo ./clickhouse install --noninteractive fi diff --git a/cloud-init.sh.in b/cloud-init.sh.in index 9017df65ac..aab701efc5 100644 --- a/cloud-init.sh.in +++ b/cloud-init.sh.in @@ -9,6 +9,12 @@ # once here so every per-system install/start/load/query inherits it. export HOME="${HOME:-/root}" +# Optional ClickHouse PR override (see run-benchmark.sh / lib/download-clickhouse). +# Exported for every system but only consumed by the clickhouse* install +# scripts; empty for a normal run. Single-quoted; PR numbers are digits so +# there is nothing to escape. +export clickhouse_pr='@clickhouse_pr@' + # c6a.4xlarge has 32 GB RAM with no swap. Loading the 75 GB hits.tsv into # row-oriented databases (mysql, mariadb, postgres, mongodb, cratedb) and # in-process Python servers (pandas/polars/duckdb-dataframe) fills RAM @@ -43,6 +49,11 @@ jq -r '.tuned' template.json | tee -a log echo -n 'Tags: ' | tee -a log jq -c -r '.tags' template.json | tee -a log +# Record the ClickHouse PR override (empty on a normal run) so PR-build results +# can be filtered apart later. sink.parser extracts this into the +# results.clickhouse_pr column; see prepare-database.sql. +echo "ClickHouse PR: ${clickhouse_pr}" | tee -a log + echo -n 'Disk usage before: ' | tee -a log df -B1 / | tail -n1 | awk '{ print $3 }' | tee -a log diff --git a/lib/download-clickhouse b/lib/download-clickhouse new file mode 100755 index 0000000000..b3758d45fc --- /dev/null +++ b/lib/download-clickhouse @@ -0,0 +1,94 @@ +#!/bin/bash +# Put the `clickhouse` universal binary in the current directory. +# +# Default: download the latest stable build from clickhouse.com — exactly what +# every clickhouse* install script did inline (`curl https://clickhouse.com/ | +# sh`) before this helper existed. +# +# Override: when the `clickhouse_pr` environment variable names a +# ClickHouse/ClickHouse pull-request number, fetch that PR's CI release build +# instead of the stable one. This is how an operator benchmarks an unreleased +# binary straight from a PR: +# +# clickhouse_pr=81944 system=clickhouse ./run-benchmark.sh +# +# `clickhouse_pr` is set on the operator side, threaded to the VM through +# cloud-init, and consumed ONLY here — so only the clickhouse* systems react to +# it; every other system ignores it. +# +# "The binary from the PR" is resolved as: the newest commit in the PR whose +# release-build CI job for THIS machine's architecture finished successfully, +# then the raw self-extracting binary that job uploaded to S3. Walking from the +# newest commit backwards (rather than pinning the tip) means a tip commit whose +# build is still running, failed, or was skipped (e.g. a docs-only commit) falls +# back to the last commit that does have a finished build. +set -euo pipefail + +# No PR requested: the historical default path. +if [ -z "${clickhouse_pr:-}" ]; then + curl https://clickhouse.com/ | sh + exit 0 +fi + +pr="$clickhouse_pr" +api="https://api.github.com/repos/ClickHouse/ClickHouse" + +# ClickHouse names its release builds by short arch (amd / arm). Map this +# machine's architecture onto that; reject targets that have no release build. +case "$(uname -m)" in + x86_64 | amd64) arch=amd ;; + aarch64 | arm64) arch=arm ;; + *) + echo "clickhouse_pr: no release build for architecture '$(uname -m)'" >&2 + exit 1 + ;; +esac +check_name="Build (${arch}_release)" # CI check-run name for the release build +build_dir="build_${arch}_release" # S3 folder that build uploads into + +# GitHub's anonymous API limit is 60 requests/hour; if the operator exported a +# GITHUB_TOKEN, use it to lift the limit (helps when many machines launch at +# once and each makes a handful of calls). +gh() { + if [ -n "${GITHUB_TOKEN:-}" ]; then + curl -fsSL -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github+json" "$@" + else + curl -fsSL -H "Accept: application/vnd.github+json" "$@" + fi +} + +# Candidate commits, newest first: the PR head (the usual answer) followed by +# the rest of the PR's commits newest->oldest as a fallback. The PR-commits +# endpoint returns oldest->newest, so `tac` flips it; awk drops blanks and the +# duplicate head. (For a PR with >100 commits only the first page's commits act +# as fallback, but the head — checked first — covers the normal case.) +head_sha=$(gh "${api}/pulls/${pr}" | jq -r '.head.sha') +rest=$(gh "${api}/pulls/${pr}/commits?per_page=100" | jq -r '.[].sha' | tac) +candidates=$(printf '%s\n%s\n' "$head_sha" "$rest" | awk 'NF && !seen[$0]++') + +echo "clickhouse_pr=${pr}: looking for a successful '${check_name}' build..." >&2 +sha="" +while IFS= read -r commit; do + [ -z "$commit" ] && continue + # Filter server-side by check name so we don't have to page through the + # ~170 check-runs a ClickHouse commit accumulates. + if gh -G --data-urlencode "check_name=${check_name}" \ + "${api}/commits/${commit}/check-runs" \ + | jq -e 'any(.check_runs[]; .conclusion == "success")' >/dev/null; then + sha="$commit" + break + fi +done <<< "$candidates" + +if [ -z "$sha" ]; then + echo "clickhouse_pr=${pr}: found no commit with a successful '${check_name}'." >&2 + echo "clickhouse_pr=${pr}: is CI still running, or did the release build fail?" >&2 + exit 1 +fi + +url="https://clickhouse-builds.s3.amazonaws.com/PRs/${pr}/${sha}/${build_dir}/clickhouse" +echo "clickhouse_pr=${pr}: commit ${sha}" >&2 +echo "clickhouse_pr=${pr}: downloading ${url}" >&2 +wget --continue --progress=dot:giga "$url" -O clickhouse +chmod +x clickhouse diff --git a/prepare-database.sql b/prepare-database.sql index 9ea4e68c90..1e20e94bda 100644 --- a/prepare-database.sql +++ b/prepare-database.sql @@ -17,6 +17,9 @@ CREATE TABLE sink.results proprietary String, tuned String, tags String, + -- ClickHouse/ClickHouse PR number when the run benchmarked a PR build + -- (clickhouse_pr override, clickhouse* systems only); empty otherwise. + clickhouse_pr String, total_time UInt64, disk_space_diff UInt64, load_time Float64, @@ -43,6 +46,7 @@ WITH extract(content, 'Proprietary: ([^\n]+)') AS proprietary, extract(content, 'Tuned: ([^\n]+)') AS tuned, extract(content, 'Tags: ([^\n]+)') AS tags, + extract(content, 'ClickHouse PR: ([^\n]+)') AS clickhouse_pr, toUInt64OrZero(extract(content, 'Disk usage after: (\d+)')) - toUInt64OrZero(extract(content, 'Disk usage before: (\d+)')) AS disk_space_diff, toUInt64OrZero(extract(content, 'Total time: (\d+)')) AS total_time, @@ -65,7 +69,7 @@ WITH load_time IS NOT NULL AND length(runtimes) = 43 AND data_size >= 5000000000 AND arrayExists(x -> arrayExists(y -> toFloat64OrZero(y) > 0.1, x), runtimes) AS good -SELECT time, system, machine, system_name, proprietary, tuned, tags, total_time, disk_space_diff, load_time, data_size, length(runtimes) AS num_results, runtimes, runtimes_formatted, +SELECT time, system, machine, system_name, proprietary, tuned, tags, clickhouse_pr, total_time, disk_space_diff, load_time, data_size, length(runtimes) AS num_results, runtimes, runtimes_formatted, concurrent_qps, concurrent_error_ratio, '{ "system": "' || system_name || '", diff --git a/run-benchmark.sh b/run-benchmark.sh index bc6bc2e407..b3e40436d5 100755 --- a/run-benchmark.sh +++ b/run-benchmark.sh @@ -5,6 +5,12 @@ system=${system:=clickhouse} repo=${repo:=ClickHouse/ClickBench} branch=${branch:=main} +# Optional ClickHouse/ClickHouse PR number. When set, the clickhouse* systems +# benchmark that PR's CI release build instead of the latest stable one; every +# other system ignores it. Threaded to the VM below and resolved there by +# lib/download-clickhouse. +clickhouse_pr=${clickhouse_pr:-} + arch=$(aws ec2 describe-instance-types --instance-types $machine --query 'InstanceTypes[0].ProcessorInfo.SupportedArchitectures' --output text) ami=$(aws ec2 describe-images --owners amazon --filters "Name=name,Values=ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04*" "Name=architecture,Values=${arch}" "Name=state,Values=available" --query 'sort_by(Images, &CreationDate) | [-1].[ImageId]' --output text) @@ -12,12 +18,13 @@ ami=$(aws ec2 describe-images --owners amazon --filters "Name=name,Values=ubuntu # Default keeps the 10h cap that worked for the slowest OLTP systems. timeout="${timeout:-36000}" -awk -v sys="$system" -v repo="$repo" -v branch="$branch" -v t="$timeout" ' +awk -v sys="$system" -v repo="$repo" -v branch="$branch" -v t="$timeout" -v chpr="$clickhouse_pr" ' { gsub(/@system@/, sys) gsub(/@repo@/, repo) gsub(/@branch@/, branch) gsub(/@timeout@/, t) + gsub(/@clickhouse_pr@/, chpr) print }' cloud-init.sh.in > cloud-init.sh From ec036a93f064282ced12b23a72d51c1cbbfee446 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Jul 2026 01:12:37 +0200 Subject: [PATCH 2/2] Apply suggestion from @alexey-milovidov --- lib/download-clickhouse | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/download-clickhouse b/lib/download-clickhouse index b3758d45fc..fea19ae9e8 100755 --- a/lib/download-clickhouse +++ b/lib/download-clickhouse @@ -24,7 +24,7 @@ # back to the last commit that does have a finished build. set -euo pipefail -# No PR requested: the historical default path. +# No PR requested: the default path. if [ -z "${clickhouse_pr:-}" ]; then curl https://clickhouse.com/ | sh exit 0