diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 1e4c062f5a2..701c0f28a34 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -93,7 +93,7 @@ jobs: env: RUST_BACKTRACE: full run: | - target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json + bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json - name: Setup AWS CLI if: github.event.pull_request.head.repo.fork == false diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 852baa26cda..5f879b5e8e4 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -89,7 +89,7 @@ jobs: env: RUST_BACKTRACE: full run: | - target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json + bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json - name: Setup AWS CLI uses: aws-actions/configure-aws-credentials@v5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29d8b7c3590..bf47fe40356 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,9 +110,12 @@ jobs: uv run --all-packages pytest --benchmark-disable -n auto test/ working-directory: vortex-python/ + - name: Setup benchmark environment + run: sudo bash scripts/setup-benchmark.sh + - name: Pytest Benchmarks - Vortex run: | - uv run --all-packages pytest --benchmark-only benchmark/ + bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ working-directory: vortex-python/ - name: Doctest - PyVortex @@ -743,7 +746,7 @@ jobs: - name: Run benchmarks uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 with: - run: cargo codspeed run + run: bash scripts/bench-taskset.sh cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} mode: "simulation" diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index bb8233ae56e..92edb23f534 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -198,7 +198,7 @@ jobs: OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | - .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \ + bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \ ${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }} - name: Run ${{ matrix.name }} benchmark (remote) @@ -212,7 +212,7 @@ jobs: OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | - .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \ + bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \ --remote-storage "${{ matrix.remote_storage }}" \ --benchmark-id "${{ matrix.id }}" \ ${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }} diff --git a/scripts/bench-taskset.sh b/scripts/bench-taskset.sh new file mode 100644 index 00000000000..a86f6818ab3 --- /dev/null +++ b/scripts/bench-taskset.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +set -Eeu -o pipefail + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [args...]" >&2 + exit 1 +fi + +if [[ -f /tmp/vortex-benchmark.env ]]; then + # shellcheck disable=SC1091 + source /tmp/vortex-benchmark.env +fi + + + +if [[ -z "${BENCH_CPUS:-}" ]]; then + cpu_count="$(nproc)" + BENCH_CPUS="2-$((cpu_count - 1))" +fi + +if command -v numactl >/dev/null 2>&1; then + exec numactl --physcpubind="$BENCH_CPUS" --localalloc "$@" +fi + +exec taskset -c "$BENCH_CPUS" "$@" diff --git a/scripts/setup-benchmark.sh b/scripts/setup-benchmark.sh index d45dad2e7c5..48536092e17 100755 --- a/scripts/setup-benchmark.sh +++ b/scripts/setup-benchmark.sh @@ -10,17 +10,24 @@ if [ "$EUID" -ne 0 ]; then exit 0 fi -# Turn off frequency scaling -for gov in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do - echo performance > "$gov" 2>/dev/null || true -done +# Disable turbo boost so benchmark runs stay at a more stable clock rate. +[[ -w /sys/devices/system/cpu/intel_pstate/no_turbo ]] && echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo || true +[[ -w /sys/devices/system/cpu/cpufreq/boost ]] && echo 0 > /sys/devices/system/cpu/cpufreq/boost || true # Really discourage swapping to disk sysctl vm.swappiness=0 +swapoff -a || true + +# Might be worse if a single application uses the OS +# https://www.intel.com/content/www/us/en/developer/articles/technical/measuring-impact-of-numa-migrations-on-performance.html +sysctl -w kernel.numa_balancing=0 # Disable ASLR - https://docs.kernel.org/admin-guide/sysctl/kernel.html#randomize-va-space sysctl kernel.randomize_va_space=0 +# This is a desktop optimization, making sure its disabled +sysctl -w kernel.sched_autogroup_enabled=0 + # Reduce kernel logging to minimum dmesg -n 1 @@ -33,3 +40,37 @@ systemctl mask ModemManager # For apparmor specifically, also teardown loaded profiles aa-teardown + +# Reduce background activity (Ubuntu-specific) +for unit in \ + irqbalance \ + apt-daily.service \ + apt-daily-upgrade.service \ + apt-daily.timer \ + apt-daily-upgrade.timer \ + motd-news.service \ + motd-news.timer \ + apport +do + systemctl disable --now "$unit" 2>/dev/null +done +systemctl mask irqbalance 2>/dev/null + +CPU_COUNT="$(nproc)" +HOUSEKEEPING_CPUS="0-1" +BENCH_CPUS="2-$((CPU_COUNT - 1))" + +# Pin all IRQs to housekeeping CPUs +for f in /proc/irq/[0-9]*/smp_affinity_list; do + if [[ -w "$f" ]]; then + # Some IRQs are kernel-managed and reject writes with EPERM even as root. + echo "$HOUSEKEEPING_CPUS" > "$f" 2>/dev/null || true + fi +done + +# Persist CPU affinity ranges for non-root benchmark steps in CI. +cat > /tmp/vortex-benchmark.env <