Update ZigHouse

zighouse/README.md

@@ -1,12 +1,18 @@
 # ZigHouse
 
-ZigHouse is an experimental analytical database binary written in Zig.
+ZigHouse is an experimental, ClickHouse-compatible analytical database written
+in Zig. It ingests data into a MergeTree-compatible on-disk store and executes
+analytical SQL through a single native execution path (there is no
+ClickBench-specific query handling).
 
-This ClickBench entry uses the published Linux x86_64 benchmark binary from:
+This entry uses a **generic** published release binary — not a
+ClickBench-specific build:
 
-https://github.com/donge/zighouse/releases/tag/v0.1.0-clickbench
+https://github.com/donge/zighouse/releases/tag/v1.0.2
 
-The binary imports the ClickBench Parquet dataset into a local column-oriented store and runs the 43 ClickBench queries with its native engine.
+`./install` downloads the Linux x86_64 release, decompresses it, and verifies
+its SHA256 checksum. Override the version/checksum with the `ZIGHOUSE_VERSION`
+and `ZIGHOUSE_SHA256` environment variables.
 
 ## Running
 
@@ -16,8 +22,27 @@ From this directory inside the ClickBench repository:
 ./benchmark.sh
 ```
 
-The benchmark script downloads `hits.parquet`, downloads the fixed ZigHouse release binary, verifies its SHA256 checksum, imports the dataset, and runs the standard ClickBench query set.
+The shared driver (`../lib/benchmark-common.sh`) installs the binary, starts the
+server, loads the dataset, and runs the 43 ClickBench queries three times each.
+
+## How it works
+
+ZigHouse runs as a server that speaks the ClickHouse HTTP protocol:
+
+- `./start` / `./stop` — launch/stop `zighouse serve`. With `--port=<P>` the
+  native TCP interface listens on `<P>` and HTTP on `<P>+1` (default 28123 /
+  28124).
+- `./load` — creates the table (`create.sql`) and streams the dataset in over
+  HTTP. The server ingests JSONEachRow (it does not read Parquet or TSV
+  directly), so the ClickBench `hits.json` dataset is posted in ≤256 MiB,
+  line-aligned chunks, one `INSERT ... FORMAT JSONEachRow` per chunk. After the
+  load, `generic-smoke.sh` runs a few non-ClickBench statements as a sanity
+  check on the generic SQL engine.
+- `./query` — sends one SQL statement over HTTP and reports the wall-clock time.
+- `./data-size` — size on disk of the store directory.
 
 ## Notes
 
-The included AWS result was produced on `c6i.4xlarge` in AWS China. `c6a.4xlarge` was not available in the AWS China regions used for this run.
+The results directory holds per-machine result files produced by the shared
+driver; regenerate them by running `./benchmark.sh` on the target hardware and
+committing the resulting JSON under `results/YYYYMMDD/`.

zighouse/benchmark.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 set -e
 
-export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
-# Single-process engine: each query forks a fresh full-machine process with no
-# shared scheduler across connections, so the concurrent-QPS test only
-# oversubscribes RAM rather than measuring throughput. Skip it by default;
-# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946.
+# The dataset is fetched inside ./load: JSONEachRow is the only bulk format the
+# generic ZigHouse HTTP server can ingest, and there is no shared JSON download
+# helper, so there is no separate download step here.
+export BENCH_DOWNLOAD_SCRIPT=""
+# Skip the concurrent-QPS test by default (see issue #946); override
+# BENCH_CONCURRENT_DURATION to run it against the HTTP server.
 export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}"
 exec ../lib/benchmark-common.sh

zighouse/check

@@ -1,4 +1,7 @@
 #!/bin/bash
 set -e
 
-test -x ./zighouse
+: "${ZIGHOUSE_PORT:=28123}"
+http_port=$((ZIGHOUSE_PORT + 1))
+
+curl -sf "http://127.0.0.1:${http_port}/ping" >/dev/null

zighouse/create.sql

@@ -1,2 +1,110 @@
--- ZigHouse imports the ClickBench Parquet dataset directly with:
--- zighouse import-clickbench-parquet-hot hits.parquet <store>
+CREATE TABLE hits
+(
+    WatchID UInt64,
+    JavaEnable UInt8,
+    Title String,
+    GoodEvent Int16,
+    EventTime DateTime,
+    EventDate Date,
+    CounterID UInt32,
+    ClientIP UInt32,
+    RegionID UInt32,
+    UserID UInt64,
+    CounterClass Int8,
+    OS UInt8,
+    UserAgent UInt8,
+    URL String,
+    Referer String,
+    IsRefresh UInt8,
+    RefererCategoryID UInt16,
+    RefererRegionID UInt32,
+    URLCategoryID UInt16,
+    URLRegionID UInt32,
+    ResolutionWidth UInt16,
+    ResolutionHeight UInt16,
+    ResolutionDepth UInt8,
+    FlashMajor UInt8,
+    FlashMinor UInt8,
+    FlashMinor2 String,
+    NetMajor UInt8,
+    NetMinor UInt8,
+    UserAgentMajor UInt16,
+    UserAgentMinor String,
+    CookieEnable UInt8,
+    JavascriptEnable UInt8,
+    IsMobile UInt8,
+    MobilePhone UInt8,
+    MobilePhoneModel String,
+    Params String,
+    IPNetworkID UInt32,
+    TraficSourceID Int8,
+    SearchEngineID UInt16,
+    SearchPhrase String,
+    AdvEngineID UInt8,
+    IsArtifical UInt8,
+    WindowClientWidth UInt16,
+    WindowClientHeight UInt16,
+    ClientTimeZone Int16,
+    ClientEventTime DateTime,
+    SilverlightVersion1 UInt8,
+    SilverlightVersion2 UInt8,
+    SilverlightVersion3 UInt32,
+    SilverlightVersion4 UInt16,
+    PageCharset String,
+    CodeVersion UInt32,
+    IsLink UInt8,
+    IsDownload UInt8,
+    IsNotBounce UInt8,
+    FUniqID UInt64,
+    OriginalURL String,
+    HID UInt32,
+    IsOldCounter UInt8,
+    IsEvent UInt8,
+    IsParameter UInt8,
+    DontCountHits UInt8,
+    WithHash UInt8,
+    HitColor String,
+    LocalEventTime DateTime,
+    Age UInt8,
+    Sex UInt8,
+    Income UInt8,
+    Interests UInt16,
+    Robotness UInt8,
+    RemoteIP UInt32,
+    WindowName Int32,
+    OpenerName Int32,
+    HistoryLength Int16,
+    BrowserLanguage String,
+    BrowserCountry String,
+    SocialNetwork String,
+    SocialAction String,
+    HTTPError UInt16,
+    SendTiming Int32,
+    DNSTiming Int32,
+    ConnectTiming Int32,
+    ResponseStartTiming Int32,
+    ResponseEndTiming Int32,
+    FetchTiming Int32,
+    SocialSourceNetworkID UInt8,
+    SocialSourcePage String,
+    ParamPrice Int64,
+    ParamOrderID String,
+    ParamCurrency String,
+    ParamCurrencyID UInt16,
+    OpenstatServiceName String,
+    OpenstatCampaignID String,
+    OpenstatAdID String,
+    OpenstatSourceID String,
+    UTMSource String,
+    UTMMedium String,
+    UTMCampaign String,
+    UTMContent String,
+    UTMTerm String,
+    FromTag String,
+    HasGCLID UInt8,
+    RefererHash UInt64,
+    URLHash UInt64,
+    CLID UInt32
+)
+ENGINE = MergeTree
+ORDER BY (CounterID, EventDate, UserID, EventTime, WatchID)

zighouse/data-size

@@ -1,4 +1,8 @@
 #!/bin/bash
 set -e
 
-du -sb zighouse-store 2>/dev/null | awk '{ print $1 }' || du -sk zighouse-store | awk '{ print $1 * 1024 }'
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+: "${ZIGHOUSE_DATA_DIR:=${SCRIPT_DIR}/zighouse-store}"
+
+du -sb "$ZIGHOUSE_DATA_DIR" 2>/dev/null | awk '{ print $1 }' \
+    || du -sk "$ZIGHOUSE_DATA_DIR" | awk '{ print $1 * 1024 }'

zighouse/generic-smoke.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Smoke-test ZigHouse's SQL engine on a handful of non-ClickBench statements
+# through the ClickHouse-compatible HTTP interface. This runs automatically at
+# the end of ./load (so the capability frontier is exercised on every run) and
+# can also be invoked standalone from this directory once the server is up and
+# the dataset has been loaded.
+set -u
+
+: "${ZIGHOUSE_PORT:=28123}"
+http_port=$((ZIGHOUSE_PORT + 1))
+base="http://127.0.0.1:${http_port}"
+
+run() {
+    echo "== $1 =="
+    echo "SQL: $2"
+    curl -sS -G "${base}/" \
+        --data-urlencode "query=$2" \
+        --data-urlencode "default_format=TabSeparated" \
+        || echo "  -> error"
+    echo
+}
+
+run "count_all"       "SELECT COUNT(*) FROM hits"
+run "sum_with_filter" "SELECT SUM(AdvEngineID) FROM hits WHERE EventDate >= '2013-07-15'"
+run "min_max_date"    "SELECT MIN(EventDate), MAX(EventDate) FROM hits"
+run "count_distinct"  "SELECT COUNT(DISTINCT SearchEngineID) FROM hits"
+run "groupby_topk"    "SELECT RegionID, COUNT(*) AS c FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 5"
+
+exit 0

zighouse/install

@@ -1,10 +1,17 @@
 #!/bin/bash
 set -e
 
-: "${ZIGHOUSE_VERSION:=v0.1.0-clickbench}"
-: "${ZIGHOUSE_SHA256:=5a779eacf87082eeeb13b336d6f798a0399593c29b5a586ab67202104396dc83}"
+# Download a generic ZigHouse release binary (not a ClickBench-specific build)
+# from https://github.com/donge/zighouse/releases and verify its checksum.
+#
+# Release assets are gzip-compressed, so we decompress before checking the
+# SHA256 of the resulting executable.
+: "${ZIGHOUSE_VERSION:=v1.0.2}"
+# SHA256 of the *decompressed* linux-x86_64 binary.
+: "${ZIGHOUSE_SHA256:=ad6f0a6a98be1a6dac6a4bc99c4162a97e828ff273ee6417823c0ebf88f6470b}"
 
-url="https://github.com/donge/zighouse/releases/download/${ZIGHOUSE_VERSION}/zighouse-linux-x86_64"
-curl -L --fail -o zighouse "$url"
+url="https://github.com/donge/zighouse/releases/download/${ZIGHOUSE_VERSION}/zighouse-linux-x86_64.gz"
+curl -L --fail -o zighouse.gz "$url"
+gunzip -f zighouse.gz
 printf '%s  zighouse\n' "$ZIGHOUSE_SHA256" | sha256sum -c -
 chmod +x zighouse

zighouse/load

@@ -1,6 +1,47 @@
 #!/bin/bash
-set -e
+set -euo pipefail
+
+# ZigHouse's generic HTTP server ingests JSONEachRow (it cannot read Parquet or
+# TSV directly). We stream the ClickBench hits.json dataset to it in chunks:
+# the server buffers each request body whole and rejects anything over 256 MiB,
+# and `split -C` keeps every JSON line intact within a chunk.
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+: "${ZIGHOUSE_PORT:=28123}"
+: "${ZIGHOUSE_LOAD_CHUNK:=200M}"
+http_port=$((ZIGHOUSE_PORT + 1))
+base="http://127.0.0.1:${http_port}"
+
+# --- Fetch the dataset (JSONEachRow). Prefer the playground's pre-staged
+#     read-only copy; otherwise download and decompress. --------------------
+if [ ! -e hits.json ]; then
+    if [ -f /opt/clickbench/datasets_ro/hits.json ]; then
+        ln -sf /opt/clickbench/datasets_ro/hits.json hits.json
+    elif [ -f /opt/clickbench/datasets_ro/hits.json.gz ]; then
+        sudo apt-get install -y pigz
+        pigz -d -k -c /opt/clickbench/datasets_ro/hits.json.gz > hits.json
+    else
+        sudo apt-get install -y pigz
+        wget --continue --progress=dot:giga \
+            'https://datasets.clickhouse.com/hits_compatible/hits.json.gz'
+        pigz -d -f hits.json.gz
+    fi
+fi
+
+# --- Schema --------------------------------------------------------------
+curl -sS --fail-with-body -G "${base}/" \
+    --data-urlencode "query=$(cat "${SCRIPT_DIR}/create.sql")" >/dev/null
+
+# --- Bulk load: stream hits.json to the server in <=chunk-sized, line-aligned
+#     pieces, each posted as `INSERT INTO hits FORMAT JSONEachRow`. ----------
+export ZIGHOUSE_INSERT_URL="${base}/?query=INSERT%20INTO%20hits%20FORMAT%20JSONEachRow"
+split -C "${ZIGHOUSE_LOAD_CHUNK}" \
+    --filter='curl -sS --fail-with-body --data-binary @- "$ZIGHOUSE_INSERT_URL" >/dev/null' \
+    hits.json
 
-rm -rf zighouse-store
-ZIGHOUSE_CLICKBENCH_SUBMIT=1 ZIGHOUSE_IMPORT_TRACE=1 ./zighouse import-clickbench-parquet-hot hits.parquet zighouse-store
 sync
+
+# hits.json (a symlink or a downloaded copy) is only needed for the load.
+rm -f hits.json hits.json.gz
+
+# --- Post-load smoke test of the generic SQL engine (see generic-smoke.sh).
+"${SCRIPT_DIR}/generic-smoke.sh"

zighouse/query

@@ -1,5 +1,23 @@
 #!/bin/bash
-set -e
+# Reads a SQL query from stdin and runs it against the ZigHouse HTTP server.
+# Stdout: query result (TabSeparated).
+# Stderr: query runtime in fractional seconds on the last line.
+# Exit non-zero on error.
+set -euo pipefail
+
+: "${ZIGHOUSE_PORT:=28123}"
+http_port=$((ZIGHOUSE_PORT + 1))
 
 query=$(cat)
-ZIGHOUSE_CLICKBENCH_SUBMIT=1 ./zighouse --backend native query-timed zighouse-store "$query"
+
+# -G moves the url-encoded params into the ?query= URL parameter; the server
+# returns the result body, and curl's %{time_total} (whole-request wall time
+# in seconds) is appended on its own trailing line.
+out=$(curl -sS --fail-with-body -G "http://127.0.0.1:${http_port}/" \
+    --data-urlencode "query=${query}" \
+    --data-urlencode "default_format=TabSeparated" \
+    -w $'\n%{time_total}')
+
+# Everything but the last line is the result; the last line is the timing.
+printf '%s' "$out" | sed '$d'
+printf '%s\n' "$out" | tail -n1 >&2

zighouse/start

@@ -1,4 +1,22 @@
 #!/bin/bash
 set -e
 
-test -x ./zighouse
+# Start the ZigHouse server. It speaks the ClickHouse HTTP protocol; with
+# --port=<P> the native TCP interface listens on <P> and HTTP on <P>+1.
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+: "${ZIGHOUSE_PORT:=28123}"
+: "${ZIGHOUSE_DATA_DIR:=${SCRIPT_DIR}/zighouse-store}"
+http_port=$((ZIGHOUSE_PORT + 1))
+
+mkdir -p "$ZIGHOUSE_DATA_DIR"
+
+# Already up (e.g. ./start called twice)? Nothing to do.
+if curl -sf "http://127.0.0.1:${http_port}/ping" >/dev/null 2>&1; then
+    exit 0
+fi
+
+nohup "${SCRIPT_DIR}/zighouse" serve \
+    --data-dir="${ZIGHOUSE_DATA_DIR}" \
+    --port="${ZIGHOUSE_PORT}" \
+    >"${SCRIPT_DIR}/zighouse-server.log" 2>&1 &
+echo $! > "${SCRIPT_DIR}/zighouse-server.pid"

zighouse/stop

@@ -1,4 +1,13 @@
 #!/bin/bash
 set -e
 
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+pidfile="${SCRIPT_DIR}/zighouse-server.pid"
+
+if [ -f "$pidfile" ]; then
+    kill "$(cat "$pidfile")" 2>/dev/null || true
+    rm -f "$pidfile"
+fi
+# Fallback in case the pidfile is missing or stale.
+pkill -f "zighouse serve" 2>/dev/null || true
 exit 0

zighouse/template.json

@@ -2,12 +2,10 @@
   "system": "ZigHouse",
   "proprietary": "no",
   "hardware": "cpu",
-  "tuned": "yes",
+  "tuned": "no",
   "tags": [
     "Zig",
     "column-oriented",
-    "embedded",
-    "parquet",
-    "tuned"
+    "ClickHouse-compatible"
   ]
 }