Add deferred Orion summary report step for consolidated regression reporting

ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml

@@ -30,7 +30,7 @@ tests:
       EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s --service-latency
       IGNORE_JOB_ITERATIONS: "true"
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.21"
       ZONES_COUNT: "3"
@@ -46,7 +46,7 @@ tests:
     env:
       ES_TYPE: qe
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.21"
     test:
     - chain: openshift-qe-orion-consolidated
@@ -55,7 +55,7 @@ tests:
   steps:
     cluster_profile: aws-perfscale-qe
     env:
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.19"
     test:
     - chain: openshift-qe-orion-consolidated-virt

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml

@@ -45,7 +45,7 @@ tests:
         --churn-cycles=2
       ND_EXTRA_FLAGS: --churn-mode=objects --churn-delay=1m --churn-percent=50 --churn-cycles=2
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.21"
       ZONES_COUNT: "3"
@@ -185,6 +185,7 @@ tests:
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
       KB_FLAGS: --local-indexing
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.21"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml

@@ -45,7 +45,7 @@ tests:
         --churn-cycles=2
       ND_EXTRA_FLAGS: --churn-mode=objects --churn-delay=1m --churn-percent=50 --churn-cycles=2
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.22"
       ZONES_COUNT: "3"
@@ -296,6 +296,7 @@ tests:
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
       KB_FLAGS: --local-indexing
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.22"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml

@@ -46,7 +46,7 @@ tests:
         --churn-cycles=2
       ND_EXTRA_FLAGS: --churn-mode=objects --churn-delay=1m --churn-percent=50 --churn-cycles=2
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "5.0"
       ZONES_COUNT: "3"
@@ -172,6 +172,7 @@ tests:
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
       KB_FLAGS: --local-indexing
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "5.0"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml

@@ -38,7 +38,7 @@ tests:
       CONFIG: config/standard-scalelab.yml
       INFRA: "true"
       PUBLIC_VLAN: "false"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.19"
       VM: "true"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml

@@ -281,7 +281,7 @@ tests:
   steps:
     cluster_profile: metal-perfscale-cpt
     env:
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.19"
     test:
     - chain: openshift-qe-orion-consolidated-virt

ci-operator/step-registry/openshift-qe/control-plane/openshift-qe-control-plane-chain.yaml

@@ -9,3 +9,4 @@ chain:
   - chain: openshift-qe-udn-density-pods
   documentation: |-
     This chain executes cluster density v2, node density-cni, crd-scale, node-density, and udn density workloads.
+

ci-operator/step-registry/openshift-qe/orion/consolidated-virt/openshift-qe-orion-consolidated-virt-chain.yaml

@@ -5,5 +5,6 @@ chain:
   - ref: openshift-qe-orion-virt-data-path
   - ref: openshift-qe-orion-virt-density
   - ref: openshift-qe-orion-virt-udn-density
+  - ref: openshift-qe-orion-report
   documentation: |-
-    Chain of all orion virtualization workloads. 
+    Chain of all orion virt workloads with aggregated summary reporting.

ci-operator/step-registry/openshift-qe/orion/consolidated/openshift-qe-orion-consolidated-chain.yaml

@@ -5,5 +5,6 @@ chain:
   - ref: openshift-qe-orion-node-density
   - ref: openshift-qe-orion-node-density-cni
   - ref: openshift-qe-orion-crd-scale
+  - ref: openshift-qe-orion-report
   documentation: |-
-    Chain of all orion workloads. 
+    Chain of all orion workloads with aggregated summary reporting.

ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-commands.sh

@@ -305,4 +305,9 @@ if [ $orion_exit_status -eq 3 ]; then
   exit 0
 fi
 
+if [ "${RUN_ORION}" == "deferred" ]; then
+  echo "RUN_ORION=deferred. Exit status $orion_exit_status deferred to report step."
+  exit 0
+fi
+
 exit $orion_exit_status

ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-ref.yaml

@@ -80,6 +80,7 @@ ref:
     default: ""
     documentation:
       Comma separated list of repositories to monitor before and after change points.
+
   commands: openshift-qe-orion-commands.sh
   timeout: 6h
   credentials:

ci-operator/step-registry/openshift-qe/orion/report/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- perfscale-ocp-approvers
+reviewers:
+- perfscale-ocp-reviewers

ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+set -x
+set -euo pipefail
+
+if [ "${RUN_ORION}" == "false" ]; then
+  echo "RUN_ORION is false, skipping report."
+  exit 0
+fi
+
+# Fetch deferred JSON results from previous steps' GCS artifacts
+GCS_BUCKET="gs://test-platform-results"
+GCS_BASE=""
+
+case "${JOB_TYPE:-}" in
+    presubmit)
+        if [[ -n "${REPO_OWNER:-}" && -n "${REPO_NAME:-}" && -n "${PULL_NUMBER:-}" && -n "${JOB_NAME:-}" && -n "${BUILD_ID:-}" ]]; then
+            GCS_BASE="${GCS_BUCKET}/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}/artifacts"
+        fi
+        ;;
+    periodic)
+        if [[ -n "${JOB_NAME:-}" && -n "${BUILD_ID:-}" ]]; then
+            GCS_BASE="${GCS_BUCKET}/logs/${JOB_NAME}/${BUILD_ID}/artifacts"
+        fi
+        ;;
+esac
+
+if [[ -z "$GCS_BASE" ]]; then
+    echo "Could not determine GCS artifacts path. Skipping report."
+    exit 0
+fi
+
+echo "Fetching deferred orion JSONs from GCS: ${GCS_BASE}"
+mkdir -p /tmp/orion-jsons
+gsutil -m cp "${GCS_BASE}/*/openshift-qe-orion-*/artifacts/junit_*.json" /tmp/orion-jsons/ 2>/dev/null || true
+
+shopt -s nullglob
+json_files=(/tmp/orion-jsons/junit*.json)
+shopt -u nullglob
+
+if [ ${#json_files[@]} -eq 0 ]; then
+  echo "No deferred orion JSON results found in GCS."
+  exit 0
+fi
+
+# Copy JSONs to ARTIFACT_DIR for archival
+cp "${json_files[@]}" "${ARTIFACT_DIR}/" 2>/dev/null || true
+
+# Set up Python environment and install orion
+python --version
+pushd /tmp
+python -m virtualenv ./venv_report
+source ./venv_report/bin/activate
+
+if [[ $TAG == "latest" ]]; then
+    LATEST_TAG=$(curl -s "https://api.github.com/repos/cloud-bulldozer/orion/releases/latest" | jq -r '.tag_name')
+else
+    LATEST_TAG=$TAG
+fi
+git clone --branch "$LATEST_TAG" "$ORION_REPO" --depth 1
+pushd orion
+pip install -r requirements.txt
+pip install .
+popd && popd
+
+# Build comma-separated file list for orion --report
+json_file_list=""
+for f in "${json_files[@]}"; do
+  json_file_list="${json_file_list:+${json_file_list},}${f}"
+done
+
+# Run orion report on all deferred JSONs
+# orion --report exits 2 if regressions found, 0 otherwise
+set +e
+orion --report "$json_file_list" | tee "${ARTIFACT_DIR}/orion-report-summary.txt"
+report_exit=$?
+set -e
+
+if [ "$report_exit" -eq 2 ]; then
+  echo "Orion report detected regressions."
+  exit 1
+fi
+
+exit "$report_exit"

ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.metadata.json

@@ -0,0 +1,11 @@
+{
+	"path": "openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml",
+	"owners": {
+		"approvers": [
+			"perfscale-ocp-approvers"
+		],
+		"reviewers": [
+			"perfscale-ocp-reviewers"
+		]
+	}
+}

ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml

@@ -0,0 +1,29 @@
+ref:
+  as: openshift-qe-orion-report
+  from_image:
+    namespace: ci
+    name: ocp-qe-perfscale-ci
+    tag: latest
+  env:
+  - name: ORION_REPO
+    default: "https://github.com/cloud-bulldozer/orion.git"
+    documentation:
+      Orion github repository to clone for report generation.
+  - name: TAG
+    default: "latest"
+    documentation:
+      Orion version/tag to clone.
+  - name: RUN_ORION
+    default: "false"
+    documentation:
+      When false, the report step is skipped. Set to deferred at test level to enable deferred failure reporting.
+  commands: openshift-qe-orion-report-commands.sh
+  timeout: 30m
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  documentation: >-
+    Aggregates deferred orion regression results from SHARED_DIR using
+    orion --report to produce a consolidated regression report. Must be
+    used as the last step in an orion chain when RUN_ORION is set to deferred.

ci-operator/step-registry/openshift-qe/udn-density-pods/openshift-qe-udn-density-pods-workflow.yaml

@@ -4,5 +4,6 @@ workflow:
   steps:
     test:
       - chain: openshift-qe-udn-density-pods
+      - ref: openshift-qe-orion-report
   documentation: |-
-    This workflow executes udn-density-pods workload.
+    This workflow executes udn-density-pods workload and Orion change detection with summary reporting.