python all the things

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

scripts/compare-benchmark-jsons.py

@@ -63,9 +63,31 @@ def extract_dataset_key(df):
 improvement_threshold = 1.0 - (threshold_pct / 100.0)  # e.g., 0.7 for 30%, 0.9 for 10%
 regression_threshold = 1.0 + (threshold_pct / 100.0)  # e.g., 1.3 for 30%, 1.1 for 10%
 
-# After merge with suffixes, z-score columns become abs_z_score_base and abs_z_score_pr
-has_z_base = "abs_z_score_base" in df3.columns
-has_z_pr = "abs_z_score_pr" in df3.columns
+def compute_abs_z_score(runtimes):
+    """Compute |median - mean| / stddev from a list of runtimes."""
+    if not isinstance(runtimes, list) or len(runtimes) < 2:
+        return float("nan")
+    n = len(runtimes)
+    mean = sum(runtimes) / n
+    variance = sum((x - mean) ** 2 for x in runtimes) / (n - 1)
+    stddev = math.sqrt(variance)
+    if stddev == 0:
+        return 0.0
+    sorted_rt = sorted(runtimes)
+    if n % 2 == 1:
+        median = sorted_rt[n // 2]
+    else:
+        median = (sorted_rt[n // 2 - 1] + sorted_rt[n // 2]) / 2
+    return abs((median - mean) / stddev)
+
+
+# Compute |z-score| from all_runtimes when available
+has_z_pr = "all_runtimes_pr" in df3.columns
+has_z_base = "all_runtimes_base" in df3.columns
+if has_z_pr:
+    df3["abs_z_score_pr"] = df3["all_runtimes_pr"].apply(compute_abs_z_score)
+if has_z_base:
+    df3["abs_z_score_base"] = df3["all_runtimes_base"].apply(compute_abs_z_score)
 
 # Generate summary statistics
 df3["ratio"] = df3["value_pr"] / df3["value_base"]

vortex-bench/src/measurements.rs

@@ -272,27 +272,6 @@ impl QueryMeasurement {
             )
         }
     }
-
-    /// Compute |z-score| = |median - mean| / stddev for the runs.
-    /// Returns `None` if fewer than 2 runs (stddev is undefined).
-    pub fn abs_z_score(&self) -> Option<f64> {
-        let n = self.runs.len();
-        if n < 2 {
-            return None;
-        }
-
-        let nanos: Vec<f64> = self.runs.iter().map(|d| d.as_nanos() as f64).collect();
-        let mean = nanos.iter().sum::<f64>() / n as f64;
-        let variance = nanos.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / (n - 1) as f64;
-        let stddev = variance.sqrt();
-
-        if stddev == 0.0 {
-            return Some(0.0);
-        }
-
-        let median = self.median_run().as_nanos() as f64;
-        Some(((median - mean) / stddev).abs())
-    }
 }
 
 #[derive(Serialize, Deserialize)]
@@ -303,10 +282,6 @@ pub struct QueryMeasurementJson {
     pub unit: String,
     pub value: u128,
     pub all_runtimes: Vec<u128>,
-    /// Absolute z-score of the median relative to the mean: |median - mean| / stddev.
-    /// Indicates how representative the reported median is. `None` when fewer than 2 runs.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub abs_z_score: Option<f64>,
     pub target: Target,
     pub commit_id: String,
     pub env_triple: TripleJson,
@@ -338,7 +313,6 @@ impl ToJson for QueryMeasurement {
             unit: "ns".to_string(),
             value: self.median_run().as_nanos(),
             all_runtimes: self.runs.iter().map(|r| r.as_nanos()).collect_vec(),
-            abs_z_score: self.abs_z_score(),
             commit_id: GIT_COMMIT_ID.to_string(),
             target: self.target,
             env_triple: TripleJson {