Add monthly unit-test job-duration report

.github/workflows/scripts/gh_runs_lib.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# Shared helpers for the unit-test report scripts. The runs list paginates unstably on a busy
+# repo (open-ended fetches duplicate and drop runs), so we fetch one bounded month at a time
+# and dedupe client-side. Source this; do not execute.
+
+# enum_months START END -> YYYY-MM for each month in [START, END] inclusive. Endpoints are
+# encoded as month indices (year*12 + month-1) so one counter spans years. 10# forces base
+# 10, else "08"/"09" parse as invalid octal.
+enum_months() {
+  local s=$(( ${1%-*} * 12 + 10#${1#*-} - 1 ))
+  local e=$(( ${2%-*} * 12 + 10#${2#*-} - 1 ))
+  local i
+  for (( i = s; i <= e; i++ )); do
+    printf '%04d-%02d\n' $(( i / 12 )) $(( i % 12 + 1 ))
+  done
+}
+
+# last_day YYYY-MM -> last calendar day (28..31): first of next month minus one day. BSD
+# then GNU date.
+last_day() {
+  date -j -v+1m -v-1d -f %Y-%m-%d "${1}-01" +%d 2>/dev/null \
+    || date -d "${1}-01 +1 month -1 day" +%d
+}
+
+# run_ids_for_month WORKFLOW_ID MONTH -> deduped completed push run ids created in MONTH. The
+# upper bound must be the real last day; an invalid date (2025-09-31) returns zero runs.
+run_ids_for_month() {
+  local wf="$1" month="$2"
+  gh api --paginate \
+    "repos/${REPO}/actions/workflows/${wf}/runs?event=push&created=${month}-01..${month}-$(last_day "$month")&per_page=100" \
+    --jq '.workflow_runs[] | select(.status == "completed") | .id' \
+  | sort -u
+}

.github/workflows/scripts/render_duration_readme.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Render README.md (intro + one Mermaid xychart-beta per DB engine + full table) from the
+# duration CSV. Engines and versions are parsed from the job names (Test-<Engine> (<image>)),
+# so new ones self-plot. Non-matrix jobs (e.g. Rubocop) appear in the table only.
+# Usage: render_duration_readme.sh <csv-path> > README.md
+#
+set -euo pipefail
+
+CSV="${1:?usage: render_duration_readme.sh <csv-path>}"
+
+# Distinct months, ascending — the shared x-axis for every chart.
+months_csv="$(awk -F, 'NR>1 {print $1}' "${CSV}" | sort -u | paste -sd, -)"
+months_axis="$(printf '%s' "${months_csv}" | awk -F, '{for(i=1;i<=NF;i++) printf "%s\"%s\"", (i>1?", ":""), $i}')"
+latest_month="$(awk -F, 'NR>1 {print $1}' "${CSV}" | sort -u | tail -1)"
+
+# Distinct engines (Test-<Engine>), in first-seen-then-sorted order.
+engines="$(awk -F, 'NR>1 && $2 ~ /^Test-[A-Za-z]+ \(/ {
+             e=$2; sub(/^Test-/,"",e); sub(/ \(.*/,"",e); print e }' "${CSV}" | sort -u)"
+
+cat <<EOF
+# Unit Tests — monthly median job duration (main)
+
+Median (p50) wall-clock duration of each [\`Unit Tests\`](../../blob/main/.github/workflows/unit_tests.yml)
+job on \`main\`, per month, split by database engine. Only jobs that **succeeded** count — a
+failed or timed-out job's duration is meaningless. Duration includes DB-service start-up but
+not queue time; it is comparable across months.
+
+> Auto-generated by \`.github/workflows/unit_test_duration_report.yml\`. Each run upserts the
+> completed months still in the API into \`data/job_duration.csv\`, preserving history past
+> GitHub's ~400-day run retention. Do not edit this branch by hand.
+EOF
+
+# One chart per engine: a named line per version (image), sharing the month x-axis.
+for engine in ${engines}; do
+  # Only versions present in the latest month are charted — a retired one would trail to a
+  # false 0; a new one self-appears once it reaches the latest month.
+  versions="$(awk -F, -v e="Test-${engine} (" -v last="${latest_month}" '
+                NR>1 && $1==last && index($2, e)==1 {
+                  v=$2; sub(/^.*\(/,"",v); sub(/\).*/,"",v); print v }' "${CSV}" | sort -u)"
+
+  [ -n "${versions}" ] || continue
+
+  printf '\n## %s\n\n```mermaid\nxychart-beta\n' "${engine}"
+  printf '    title "%s — median minutes on main"\n' "${engine}"
+  printf '    x-axis [%s]\n' "${months_axis}"
+  printf '    y-axis "Median minutes"\n'
+
+  for v in ${versions}; do
+    # p50 per month aligned to the shared axis; months before this version ran get 0
+    # (xychart-beta has no gap). The table shows true coverage.
+    series="$(awk -F, -v job="Test-${engine} (${v})" -v ms="${months_csv}" '
+      NR>1 && $2==job { p[$1]=$4 }
+      END {
+        n=split(ms, m, ",");
+        for (i=1;i<=n;i++) printf "%s%s", (i>1?", ":""), (m[i] in p ? p[m[i]] : 0);
+      }' "${CSV}")"
+    printf '    line "%s" [%s]\n' "${v}" "${series}"
+  done
+  printf '```\n'
+done
+
+cat <<'EOF'
+
+## Data
+
+| Month | Job | Runs | Median min |
+|-------|-----|-----:|-----------:|
+EOF
+
+awk -F, 'NR>1 {printf "| %s | %s | %s | %s |\n", $1, $2, $3, $4}' "${CSV}"

.github/workflows/scripts/render_readme.sh → .github/workflows/scripts/render_retrigger_readme.sh

@@ -1,19 +1,19 @@
 #!/usr/bin/env bash
 #
 # Render README.md (intro + Mermaid chart + table) from the CSV data file.
-# Usage: render_readme.sh <csv-path> > README.md
+# Usage: render_retrigger_readme.sh <csv-path> > README.md
 #
 set -euo pipefail
 
-CSV="${1:?usage: render_readme.sh <csv-path>}"
+CSV="${1:?usage: render_retrigger_readme.sh <csv-path>}"
 
 months="$(awk -F, 'NR>1 {printf "%s\"%s\"", sep, $1; sep=", "}' "${CSV}")"
 values="$(awk -F, 'NR>1 {printf "%s%s", sep, $4; sep=", "}' "${CSV}")"
 
 cat <<EOF
 # Unit Tests — monthly re-trigger rate (main)
 
-How often the [\`Unit Tests\`](../../actions/workflows/unit_tests.yml) workflow on \`main\`
+How often the [\`Unit Tests\`](../../blob/main/.github/workflows/unit_tests.yml) workflow on \`main\`
 was **re-triggered** to green, as a percentage of runs per month.
 
 A run counts when its final \`run_attempt >= 2\` **and** it eventually succeeded — a proxy

.github/workflows/scripts/unit_test_duration_metric.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+#
+# Median (p50) duration of each "Unit Tests" job on main, per month. Green jobs only (a
+# failed job's duration is meaningless), push events only, current month excluded. Series are
+# discovered from the job name, so new DB versions self-register. Emits CSV to stdout:
+# month,job,runs,p50_min
+#
+# Runs are fetched one bounded month at a time (see gh_runs_lib.sh) and each run's /jobs is
+# fetched in turn. START defaults to 3 months back (self-heal margin); set START=YYYY-MM for
+# a backfill.
+#
+# Usage: GH_TOKEN=$(gh auth token) unit_test_duration_metric.sh
+#
+set -euo pipefail
+
+REPO="${REPO:-cloudfoundry/cloud_controller_ng}"
+WORKFLOW_NAME="${WORKFLOW_NAME:-Unit Tests}"
+START="${START:-$(date -u -v-3m +%Y-%m 2>/dev/null || date -u -d '3 months ago' +%Y-%m)}"
+
+. "$(dirname "$0")/gh_runs_lib.sh"
+
+# jq picks the first match, avoiding a head(1) that would SIGPIPE under pipefail.
+workflow_id="$(gh api --paginate "repos/${REPO}/actions/workflows" \
+  --jq "[.workflows[] | select(.name == \"${WORKFLOW_NAME}\") | .id] | first")"
+
+if [ -z "${workflow_id}" ] || [ "${workflow_id}" = "null" ]; then
+  echo "error: workflow '${WORKFLOW_NAME}' not found in ${REPO}" >&2
+  exit 1
+fi
+
+# Last completed month = the month before the current one.
+end_month="$(date -u -v-1m +%Y-%m 2>/dev/null || date -u -d 'last month' +%Y-%m)"
+
+# Run ids per month, tagged with the month so jobs bucket by their run's month (not by job
+# started_at, which can drift across a boundary).
+run_pairs() {
+  local month
+  for month in $(enum_months "${START}" "${end_month}"); do
+    run_ids_for_month "${workflow_id}" "${month}" \
+      | awk -v m="${month}" 'NF {print m "\t" $0}'
+  done
+}
+
+pairs="$(run_pairs)"
+run_total="$(printf '%s\n' "${pairs}" | grep -c .)"
+
+# Green jobs only, emitting job<TAB>seconds; awk prepends the month. A transient /jobs failure
+# must not silently drop a run, so retry once then abort — a gap would corrupt the median.
+JOBS_FILTER='.jobs[]
+  | select(.conclusion == "success" and .started_at != null and .completed_at != null)
+  | [ .name,
+      ((.completed_at | sub("\\.[0-9]+";"") | fromdateiso8601)
+       - (.started_at | sub("\\.[0-9]+";"") | fromdateiso8601)) ]
+  | @tsv'
+failed=0
+durations() {
+  local month rid
+  while IFS=$'\t' read -r month rid; do
+    [ -n "${rid}" ] || continue
+    { gh api --paginate "repos/${REPO}/actions/runs/${rid}/jobs?per_page=100" --jq "${JOBS_FILTER}" \
+        || gh api --paginate "repos/${REPO}/actions/runs/${rid}/jobs?per_page=100" --jq "${JOBS_FILTER}" \
+        || { echo "warn: /jobs failed for run ${rid}" >&2; failed=$((failed + 1)); }
+    } | awk -v m="${month}" -F'\t' 'NF {print m "\t" $0}'
+  done <<< "${pairs}"
+  if [ "${failed}" -gt 0 ]; then
+    echo "error: ${failed}/${run_total} run(s) failed to fetch; aborting to avoid under-counting" >&2
+    return 1
+  fi
+}
+
+{
+  echo "month,job,runs,p50_min"
+  # Group by (month, job); p50 = element at index round((n-1)*0.5) of the sorted durations.
+  durations \
+  | sort -t$'\t' -k1,1 -k2,2 -k3,3n \
+  | awk -F'\t' '
+      function flush(  i) {
+        if (key == "") return;
+        i = int((cnt - 1) * 0.5 + 0.5);
+        printf "%s,%s,%d,%.1f\n", month, job, cnt, vals[i] / 60;
+      }
+      {
+        if ($1 SUBSEP $2 != key) { flush(); key = $1 SUBSEP $2; month = $1; job = $2; cnt = 0; delete vals; }
+        vals[cnt++] = $3;
+      }
+      END { flush() }'
+}

.github/workflows/scripts/unit_test_retrigger_metric.sh

@@ -1,16 +1,22 @@
 #!/usr/bin/env bash
 #
-# Monthly re-trigger rate of the "Unit Tests" workflow on main. A run is flaky when its
-# final run_attempt >= 2 and conclusion == success (re-run to green); still-failing
-# re-runs are excluded. Push events only, completed only. Emits CSV to stdout:
+# Monthly re-trigger rate of the "Unit Tests" workflow on main: share of runs that reached
+# green only on a re-run (final run_attempt >= 2 and conclusion == success). Still-failing
+# re-runs are excluded. Push events only, current month excluded. Emits CSV to stdout:
 # month,total_runs,retriggered,retrigger_pct
 #
+# Months are fetched one bounded range at a time (see gh_runs_lib.sh). START defaults to the
+# oldest month still retained; override for a shorter window.
+#
 # Usage: GH_TOKEN=$(gh auth token) unit_test_retrigger_metric.sh
 #
 set -euo pipefail
 
 REPO="${REPO:-cloudfoundry/cloud_controller_ng}"
 WORKFLOW_NAME="${WORKFLOW_NAME:-Unit Tests}"
+START="${START:-2025-08}"
+
+. "$(dirname "$0")/gh_runs_lib.sh"
 
 # jq picks the first match, avoiding a head(1) that would SIGPIPE under pipefail.
 workflow_id="$(gh api --paginate "repos/${REPO}/actions/workflows" \
@@ -21,22 +27,26 @@ if [ -z "${workflow_id}" ] || [ "${workflow_id}" = "null" ]; then
   exit 1
 fi
 
+# Last completed month = the month before the current one.
+end_month="$(date -u -v-1m +%Y-%m 2>/dev/null || date -u -d 'last month' +%Y-%m)"
+
 {
   echo "month,total_runs,retriggered,retrigger_pct"
-  gh api --paginate \
-    "repos/${REPO}/actions/workflows/${workflow_id}/runs?event=push&per_page=100" \
-    --jq '.workflow_runs[]
-          | select(.status == "completed")
-          | {month: .created_at[0:7],
-             retrig: (if .run_attempt >= 2 and .conclusion == "success" then 1 else 0 end)}' \
-  | jq -rs '
-      group_by(.month)
-      | map({month: .[0].month, total: length, retrig: (map(.retrig) | add)})
-      | sort_by(.month)
-      | .[]
-      | [ .month,
-          .total,
-          .retrig,
-          ((.retrig * 1000 / .total | round) / 10) ]
-      | @csv'
-} | sed 's/"//g'
+  for month in $(enum_months "${START}" "${end_month}"); do
+    gh api --paginate \
+      "repos/${REPO}/actions/workflows/${workflow_id}/runs?event=push&created=${month}-01..${month}-$(last_day "$month")&per_page=100" \
+      --jq '.workflow_runs[]
+            | select(.status == "completed")
+            | [ .id,
+                (if .run_attempt >= 2 and .conclusion == "success" then 1 else 0 end) ]
+            | @tsv' \
+    | sort -u \
+    | awk -F'\t' -v m="${month}" '
+        { total++; retrig += $2 }
+        END {
+          if (total > 0)
+            printf "%s,%d,%d,%s\n", m, total, retrig,
+                   (int(retrig * 1000 / total + 0.5) / 10);
+        }'
+  done
+}

.github/workflows/scripts/upsert_duration.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#
+# Upsert fresh rows into the stored CSV, keyed by (month, job). Fresh wins only when its run
+# count is >= stored's, so a partial refetch never clobbers a complete month; rows absent
+# from fresh are preserved. Written back to <stored>, sorted by month then job.
+# Usage: upsert_duration.sh <stored-csv> <fresh-csv>
+#
+set -euo pipefail
+
+HEADER="month,job,runs,p50_min"
+STORED="${1:?usage: upsert_duration.sh <stored-csv> <fresh-csv>}"
+FRESH="${2:?usage: upsert_duration.sh <stored-csv> <fresh-csv>}"
+
+[ -f "${STORED}" ] || printf '%s\n' "${HEADER}" > "${STORED}"
+
+{
+  printf '%s\n' "${HEADER}"
+  awk -F, '
+    FNR==1 { next }
+    { k = $1 SUBSEP $2 }
+    NR==FNR { row[k]=$0; runs[k]=$3; next }
+    (k in row) && $3 < runs[k] { next }
+    { row[k]=$0 }
+    END { for (k in row) print row[k] }
+  ' "${STORED}" "${FRESH}" | sort -t, -k1,1 -k2,2
+} > "${STORED}.new"
+
+mv "${STORED}.new" "${STORED}"

.github/workflows/scripts/upsert_retrigger.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Upsert fresh rows into the stored CSV, keyed by month. Fresh wins only when its run count
+# is >= stored's, so a partial refetch never clobbers a complete month; months absent from
+# fresh are preserved. Written back to <stored>, sorted by month.
+# Usage: upsert_retrigger.sh <stored-csv> <fresh-csv>
+#
+set -euo pipefail
+
+HEADER="month,total_runs,retriggered,retrigger_pct"
+STORED="${1:?usage: upsert_retrigger.sh <stored-csv> <fresh-csv>}"
+FRESH="${2:?usage: upsert_retrigger.sh <stored-csv> <fresh-csv>}"
+
+[ -f "${STORED}" ] || printf '%s\n' "${HEADER}" > "${STORED}"
+
+{
+  printf '%s\n' "${HEADER}"
+  awk -F, '
+    FNR==1 { next }
+    NR==FNR { row[$1]=$0; total[$1]=$2; next }
+    ($1 in row) && $2 < total[$1] { next }
+    { row[$1]=$0 }
+    END { for (m in row) print row[m] }
+  ' "${STORED}" "${FRESH}" | sort -t, -k1,1
+} > "${STORED}.new"
+
+mv "${STORED}.new" "${STORED}"