diff --git a/.github/workflows/scripts/gh_runs_lib.sh b/.github/workflows/scripts/gh_runs_lib.sh new file mode 100755 index 00000000000..988438d5b89 --- /dev/null +++ b/.github/workflows/scripts/gh_runs_lib.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Shared helpers for the unit-test report scripts. The runs list paginates unstably on a busy +# repo (open-ended fetches duplicate and drop runs), so we fetch one bounded month at a time +# and dedupe client-side. Source this; do not execute. + +# enum_months START END -> YYYY-MM for each month in [START, END] inclusive. Endpoints are +# encoded as month indices (year*12 + month-1) so one counter spans years. 10# forces base +# 10, else "08"/"09" parse as invalid octal. +enum_months() { + local s=$(( ${1%-*} * 12 + 10#${1#*-} - 1 )) + local e=$(( ${2%-*} * 12 + 10#${2#*-} - 1 )) + local i + for (( i = s; i <= e; i++ )); do + printf '%04d-%02d\n' $(( i / 12 )) $(( i % 12 + 1 )) + done +} + +# last_day YYYY-MM -> last calendar day (28..31): first of next month minus one day. BSD +# then GNU date. +last_day() { + date -j -v+1m -v-1d -f %Y-%m-%d "${1}-01" +%d 2>/dev/null \ + || date -d "${1}-01 +1 month -1 day" +%d +} + +# run_ids_for_month WORKFLOW_ID MONTH -> deduped completed push run ids created in MONTH. The +# upper bound must be the real last day; an invalid date (2025-09-31) returns zero runs. +run_ids_for_month() { + local wf="$1" month="$2" + gh api --paginate \ + "repos/${REPO}/actions/workflows/${wf}/runs?event=push&created=${month}-01..${month}-$(last_day "$month")&per_page=100" \ + --jq '.workflow_runs[] | select(.status == "completed") | .id' \ + | sort -u +} diff --git a/.github/workflows/scripts/render_duration_readme.sh b/.github/workflows/scripts/render_duration_readme.sh new file mode 100644 index 00000000000..dd89705d336 --- /dev/null +++ b/.github/workflows/scripts/render_duration_readme.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Render README.md (intro + one Mermaid xychart-beta per DB engine + full table) from the +# duration CSV. Engines and versions are parsed from the job names (Test- ()), +# so new ones self-plot. Non-matrix jobs (e.g. Rubocop) appear in the table only. +# Usage: render_duration_readme.sh > README.md +# +set -euo pipefail + +CSV="${1:?usage: render_duration_readme.sh }" + +# Distinct months, ascending — the shared x-axis for every chart. +months_csv="$(awk -F, 'NR>1 {print $1}' "${CSV}" | sort -u | paste -sd, -)" +months_axis="$(printf '%s' "${months_csv}" | awk -F, '{for(i=1;i<=NF;i++) printf "%s\"%s\"", (i>1?", ":""), $i}')" +latest_month="$(awk -F, 'NR>1 {print $1}' "${CSV}" | sort -u | tail -1)" + +# Distinct engines (Test-), in first-seen-then-sorted order. +engines="$(awk -F, 'NR>1 && $2 ~ /^Test-[A-Za-z]+ \(/ { + e=$2; sub(/^Test-/,"",e); sub(/ \(.*/,"",e); print e }' "${CSV}" | sort -u)" + +cat < Auto-generated by \`.github/workflows/unit_test_duration_report.yml\`. Each run upserts the +> completed months still in the API into \`data/job_duration.csv\`, preserving history past +> GitHub's ~400-day run retention. Do not edit this branch by hand. +EOF + +# One chart per engine: a named line per version (image), sharing the month x-axis. +for engine in ${engines}; do + # Only versions present in the latest month are charted — a retired one would trail to a + # false 0; a new one self-appears once it reaches the latest month. + versions="$(awk -F, -v e="Test-${engine} (" -v last="${latest_month}" ' + NR>1 && $1==last && index($2, e)==1 { + v=$2; sub(/^.*\(/,"",v); sub(/\).*/,"",v); print v }' "${CSV}" | sort -u)" + + [ -n "${versions}" ] || continue + + printf '\n## %s\n\n```mermaid\nxychart-beta\n' "${engine}" + printf ' title "%s — median minutes on main"\n' "${engine}" + printf ' x-axis [%s]\n' "${months_axis}" + printf ' y-axis "Median minutes"\n' + + for v in ${versions}; do + # p50 per month aligned to the shared axis; months before this version ran get 0 + # (xychart-beta has no gap). The table shows true coverage. + series="$(awk -F, -v job="Test-${engine} (${v})" -v ms="${months_csv}" ' + NR>1 && $2==job { p[$1]=$4 } + END { + n=split(ms, m, ","); + for (i=1;i<=n;i++) printf "%s%s", (i>1?", ":""), (m[i] in p ? p[m[i]] : 0); + }' "${CSV}")" + printf ' line "%s" [%s]\n' "${v}" "${series}" + done + printf '```\n' +done + +cat <<'EOF' + +## Data + +| Month | Job | Runs | Median min | +|-------|-----|-----:|-----------:| +EOF + +awk -F, 'NR>1 {printf "| %s | %s | %s | %s |\n", $1, $2, $3, $4}' "${CSV}" diff --git a/.github/workflows/scripts/render_readme.sh b/.github/workflows/scripts/render_retrigger_readme.sh similarity index 84% rename from .github/workflows/scripts/render_readme.sh rename to .github/workflows/scripts/render_retrigger_readme.sh index 4c84d6a634c..c901b73e8a1 100755 --- a/.github/workflows/scripts/render_readme.sh +++ b/.github/workflows/scripts/render_retrigger_readme.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # # Render README.md (intro + Mermaid chart + table) from the CSV data file. -# Usage: render_readme.sh > README.md +# Usage: render_retrigger_readme.sh > README.md # set -euo pipefail -CSV="${1:?usage: render_readme.sh }" +CSV="${1:?usage: render_retrigger_readme.sh }" months="$(awk -F, 'NR>1 {printf "%s\"%s\"", sep, $1; sep=", "}' "${CSV}")" values="$(awk -F, 'NR>1 {printf "%s%s", sep, $4; sep=", "}' "${CSV}")" @@ -13,7 +13,7 @@ values="$(awk -F, 'NR>1 {printf "%s%s", sep, $4; sep=", "}' "${CSV}")" cat <= 2\` **and** it eventually succeeded — a proxy diff --git a/.github/workflows/scripts/unit_test_duration_metric.sh b/.github/workflows/scripts/unit_test_duration_metric.sh new file mode 100755 index 00000000000..6ffb32844e5 --- /dev/null +++ b/.github/workflows/scripts/unit_test_duration_metric.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Median (p50) duration of each "Unit Tests" job on main, per month. Green jobs only (a +# failed job's duration is meaningless), push events only, current month excluded. Series are +# discovered from the job name, so new DB versions self-register. Emits CSV to stdout: +# month,job,runs,p50_min +# +# Runs are fetched one bounded month at a time (see gh_runs_lib.sh) and each run's /jobs is +# fetched in turn. START defaults to 3 months back (self-heal margin); set START=YYYY-MM for +# a backfill. +# +# Usage: GH_TOKEN=$(gh auth token) unit_test_duration_metric.sh +# +set -euo pipefail + +REPO="${REPO:-cloudfoundry/cloud_controller_ng}" +WORKFLOW_NAME="${WORKFLOW_NAME:-Unit Tests}" +START="${START:-$(date -u -v-3m +%Y-%m 2>/dev/null || date -u -d '3 months ago' +%Y-%m)}" + +. "$(dirname "$0")/gh_runs_lib.sh" + +# jq picks the first match, avoiding a head(1) that would SIGPIPE under pipefail. +workflow_id="$(gh api --paginate "repos/${REPO}/actions/workflows" \ + --jq "[.workflows[] | select(.name == \"${WORKFLOW_NAME}\") | .id] | first")" + +if [ -z "${workflow_id}" ] || [ "${workflow_id}" = "null" ]; then + echo "error: workflow '${WORKFLOW_NAME}' not found in ${REPO}" >&2 + exit 1 +fi + +# Last completed month = the month before the current one. +end_month="$(date -u -v-1m +%Y-%m 2>/dev/null || date -u -d 'last month' +%Y-%m)" + +# Run ids per month, tagged with the month so jobs bucket by their run's month (not by job +# started_at, which can drift across a boundary). +run_pairs() { + local month + for month in $(enum_months "${START}" "${end_month}"); do + run_ids_for_month "${workflow_id}" "${month}" \ + | awk -v m="${month}" 'NF {print m "\t" $0}' + done +} + +pairs="$(run_pairs)" +run_total="$(printf '%s\n' "${pairs}" | grep -c .)" + +# Green jobs only, emitting jobseconds; awk prepends the month. A transient /jobs failure +# must not silently drop a run, so retry once then abort — a gap would corrupt the median. +JOBS_FILTER='.jobs[] + | select(.conclusion == "success" and .started_at != null and .completed_at != null) + | [ .name, + ((.completed_at | sub("\\.[0-9]+";"") | fromdateiso8601) + - (.started_at | sub("\\.[0-9]+";"") | fromdateiso8601)) ] + | @tsv' +failed=0 +durations() { + local month rid + while IFS=$'\t' read -r month rid; do + [ -n "${rid}" ] || continue + { gh api --paginate "repos/${REPO}/actions/runs/${rid}/jobs?per_page=100" --jq "${JOBS_FILTER}" \ + || gh api --paginate "repos/${REPO}/actions/runs/${rid}/jobs?per_page=100" --jq "${JOBS_FILTER}" \ + || { echo "warn: /jobs failed for run ${rid}" >&2; failed=$((failed + 1)); } + } | awk -v m="${month}" -F'\t' 'NF {print m "\t" $0}' + done <<< "${pairs}" + if [ "${failed}" -gt 0 ]; then + echo "error: ${failed}/${run_total} run(s) failed to fetch; aborting to avoid under-counting" >&2 + return 1 + fi +} + +{ + echo "month,job,runs,p50_min" + # Group by (month, job); p50 = element at index round((n-1)*0.5) of the sorted durations. + durations \ + | sort -t$'\t' -k1,1 -k2,2 -k3,3n \ + | awk -F'\t' ' + function flush( i) { + if (key == "") return; + i = int((cnt - 1) * 0.5 + 0.5); + printf "%s,%s,%d,%.1f\n", month, job, cnt, vals[i] / 60; + } + { + if ($1 SUBSEP $2 != key) { flush(); key = $1 SUBSEP $2; month = $1; job = $2; cnt = 0; delete vals; } + vals[cnt++] = $3; + } + END { flush() }' +} diff --git a/.github/workflows/scripts/unit_test_retrigger_metric.sh b/.github/workflows/scripts/unit_test_retrigger_metric.sh index 78e8c890390..57ba115615a 100755 --- a/.github/workflows/scripts/unit_test_retrigger_metric.sh +++ b/.github/workflows/scripts/unit_test_retrigger_metric.sh @@ -1,16 +1,22 @@ #!/usr/bin/env bash # -# Monthly re-trigger rate of the "Unit Tests" workflow on main. A run is flaky when its -# final run_attempt >= 2 and conclusion == success (re-run to green); still-failing -# re-runs are excluded. Push events only, completed only. Emits CSV to stdout: +# Monthly re-trigger rate of the "Unit Tests" workflow on main: share of runs that reached +# green only on a re-run (final run_attempt >= 2 and conclusion == success). Still-failing +# re-runs are excluded. Push events only, current month excluded. Emits CSV to stdout: # month,total_runs,retriggered,retrigger_pct # +# Months are fetched one bounded range at a time (see gh_runs_lib.sh). START defaults to the +# oldest month still retained; override for a shorter window. +# # Usage: GH_TOKEN=$(gh auth token) unit_test_retrigger_metric.sh # set -euo pipefail REPO="${REPO:-cloudfoundry/cloud_controller_ng}" WORKFLOW_NAME="${WORKFLOW_NAME:-Unit Tests}" +START="${START:-2025-08}" + +. "$(dirname "$0")/gh_runs_lib.sh" # jq picks the first match, avoiding a head(1) that would SIGPIPE under pipefail. workflow_id="$(gh api --paginate "repos/${REPO}/actions/workflows" \ @@ -21,22 +27,26 @@ if [ -z "${workflow_id}" ] || [ "${workflow_id}" = "null" ]; then exit 1 fi +# Last completed month = the month before the current one. +end_month="$(date -u -v-1m +%Y-%m 2>/dev/null || date -u -d 'last month' +%Y-%m)" + { echo "month,total_runs,retriggered,retrigger_pct" - gh api --paginate \ - "repos/${REPO}/actions/workflows/${workflow_id}/runs?event=push&per_page=100" \ - --jq '.workflow_runs[] - | select(.status == "completed") - | {month: .created_at[0:7], - retrig: (if .run_attempt >= 2 and .conclusion == "success" then 1 else 0 end)}' \ - | jq -rs ' - group_by(.month) - | map({month: .[0].month, total: length, retrig: (map(.retrig) | add)}) - | sort_by(.month) - | .[] - | [ .month, - .total, - .retrig, - ((.retrig * 1000 / .total | round) / 10) ] - | @csv' -} | sed 's/"//g' + for month in $(enum_months "${START}" "${end_month}"); do + gh api --paginate \ + "repos/${REPO}/actions/workflows/${workflow_id}/runs?event=push&created=${month}-01..${month}-$(last_day "$month")&per_page=100" \ + --jq '.workflow_runs[] + | select(.status == "completed") + | [ .id, + (if .run_attempt >= 2 and .conclusion == "success" then 1 else 0 end) ] + | @tsv' \ + | sort -u \ + | awk -F'\t' -v m="${month}" ' + { total++; retrig += $2 } + END { + if (total > 0) + printf "%s,%d,%d,%s\n", m, total, retrig, + (int(retrig * 1000 / total + 0.5) / 10); + }' + done +} diff --git a/.github/workflows/scripts/upsert_duration.sh b/.github/workflows/scripts/upsert_duration.sh new file mode 100644 index 00000000000..0705dc542f1 --- /dev/null +++ b/.github/workflows/scripts/upsert_duration.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Upsert fresh rows into the stored CSV, keyed by (month, job). Fresh wins only when its run +# count is >= stored's, so a partial refetch never clobbers a complete month; rows absent +# from fresh are preserved. Written back to , sorted by month then job. +# Usage: upsert_duration.sh +# +set -euo pipefail + +HEADER="month,job,runs,p50_min" +STORED="${1:?usage: upsert_duration.sh }" +FRESH="${2:?usage: upsert_duration.sh }" + +[ -f "${STORED}" ] || printf '%s\n' "${HEADER}" > "${STORED}" + +{ + printf '%s\n' "${HEADER}" + awk -F, ' + FNR==1 { next } + { k = $1 SUBSEP $2 } + NR==FNR { row[k]=$0; runs[k]=$3; next } + (k in row) && $3 < runs[k] { next } + { row[k]=$0 } + END { for (k in row) print row[k] } + ' "${STORED}" "${FRESH}" | sort -t, -k1,1 -k2,2 +} > "${STORED}.new" + +mv "${STORED}.new" "${STORED}" diff --git a/.github/workflows/scripts/upsert_metric.sh b/.github/workflows/scripts/upsert_metric.sh deleted file mode 100755 index d3c7bda3f11..00000000000 --- a/.github/workflows/scripts/upsert_metric.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Upsert fresh metric rows into the stored CSV, keyed by month. Fresh wins only when its -# run count is >= stored's, so a partially-aged-out month never overwrites a complete one; -# aged-out months (absent from fresh) are preserved. Result written back to . -# Usage: upsert_metric.sh -# -set -euo pipefail - -HEADER="month,total_runs,retriggered,retrigger_pct" -STORED="${1:?usage: upsert_metric.sh }" -FRESH="${2:?usage: upsert_metric.sh }" - -[ -f "${STORED}" ] || printf '%s\n' "${HEADER}" > "${STORED}" - -awk -F, -v header="${HEADER}" ' - FNR==1 { next } - NR==FNR { row[$1]=$0; total[$1]=$2; next } - ($1 in row) && $2 < total[$1] { next } - { row[$1]=$0 } - END { - print header; - n=0; for (m in row) keys[n++]=m; - for(i=0;i "${STORED}.new" - -mv "${STORED}.new" "${STORED}" diff --git a/.github/workflows/scripts/upsert_retrigger.sh b/.github/workflows/scripts/upsert_retrigger.sh new file mode 100755 index 00000000000..b0b15f8463a --- /dev/null +++ b/.github/workflows/scripts/upsert_retrigger.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# Upsert fresh rows into the stored CSV, keyed by month. Fresh wins only when its run count +# is >= stored's, so a partial refetch never clobbers a complete month; months absent from +# fresh are preserved. Written back to , sorted by month. +# Usage: upsert_retrigger.sh +# +set -euo pipefail + +HEADER="month,total_runs,retriggered,retrigger_pct" +STORED="${1:?usage: upsert_retrigger.sh }" +FRESH="${2:?usage: upsert_retrigger.sh }" + +[ -f "${STORED}" ] || printf '%s\n' "${HEADER}" > "${STORED}" + +{ + printf '%s\n' "${HEADER}" + awk -F, ' + FNR==1 { next } + NR==FNR { row[$1]=$0; total[$1]=$2; next } + ($1 in row) && $2 < total[$1] { next } + { row[$1]=$0 } + END { for (m in row) print row[m] } + ' "${STORED}" "${FRESH}" | sort -t, -k1,1 +} > "${STORED}.new" + +mv "${STORED}.new" "${STORED}" diff --git a/.github/workflows/unit_test_duration_report.yml b/.github/workflows/unit_test_duration_report.yml new file mode 100644 index 00000000000..c3f05f835ba --- /dev/null +++ b/.github/workflows/unit_test_duration_report.yml @@ -0,0 +1,66 @@ +name: Unit Test Duration Report + +# Monthly median duration of each "Unit Tests" job on main, upserted into a data file +# + README charts on the reports/unit-test-duration branch (survives ~400-day retention). + +on: + schedule: + - cron: "33 6 2 * *" # 2nd of each month, 06:33 UTC + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: unit-test-duration-report + cancel-in-progress: false + +env: + REPORT_BRANCH: reports/unit-test-duration + +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: hmarr/debug-action@v3 + + - uses: actions/checkout@v7 + with: + path: main-src + + - uses: actions/checkout@v7 + with: + ref: ${{ env.REPORT_BRANCH }} + path: report-out + + - name: Compute metric + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p report-out/data + bash main-src/.github/workflows/scripts/unit_test_duration_metric.sh > fresh.csv + cat fresh.csv + + - name: Upsert into data file + run: | + bash main-src/.github/workflows/scripts/upsert_duration.sh \ + report-out/data/job_duration.csv fresh.csv + + - name: Render README + run: | + bash main-src/.github/workflows/scripts/render_duration_readme.sh \ + report-out/data/job_duration.csv > report-out/README.md + cat report-out/README.md >> "$GITHUB_STEP_SUMMARY" + + - name: Commit & push + working-directory: report-out + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add data/job_duration.csv README.md + if git diff --cached --quiet; then + echo "No changes to commit." + else + git commit -m "Update unit-test job-duration metric ($(date -u +%Y-%m-%d))" + git push + fi diff --git a/.github/workflows/unit_test_flakiness_report.yml b/.github/workflows/unit_test_flakiness_report.yml index 4c15dfc134b..6df11480a99 100644 --- a/.github/workflows/unit_test_flakiness_report.yml +++ b/.github/workflows/unit_test_flakiness_report.yml @@ -28,7 +28,6 @@ jobs: with: path: main-src - # The reports/unit-test-flakiness branch must exist (created once, like gh-pages). - uses: actions/checkout@v7 with: ref: ${{ env.REPORT_BRANCH }} @@ -44,12 +43,12 @@ jobs: - name: Upsert into data file run: | - bash main-src/.github/workflows/scripts/upsert_metric.sh \ + bash main-src/.github/workflows/scripts/upsert_retrigger.sh \ report-out/data/retrigger_rate.csv fresh.csv - name: Render README run: | - bash main-src/.github/workflows/scripts/render_readme.sh \ + bash main-src/.github/workflows/scripts/render_retrigger_readme.sh \ report-out/data/retrigger_rate.csv > report-out/README.md cat report-out/README.md >> "$GITHUB_STEP_SUMMARY"