sui-eval 0.1.198

//! Head-to-head performance comparison: sui vs CppNix on the oracle
//! corpus.
//!
//! # Scope
//!
//! For every non-skipped `(defnix …)` in `tests/oracle_corpus/`, run
//! the same Nix source through:
//!
//! 1. `sui_eval::eval` — in-process, no IPC
//! 2. `nix-instantiate --eval --json --strict -E "$src"` — out-of-
//!    process, includes a `fork+exec` overhead of ~30 ms per call
//!
//! Each is iterated a few times and timed with `std::time::Instant`;
//! we report the median. A markdown report lands at
//! `target/sui-vs-cppnix.md` alongside `target/oracle-perf.md`.
//!
//! # Honesty
//!
//! The CppNix timing includes process-spawn overhead that a
//! persistent daemon or REPL wouldn't pay. That's the user-perspective
//! wall cost of running `nix-instantiate` from a shell; it is NOT a
//! pure eval-engine-to-eval-engine comparison. A 30× advantage to sui
//! on tiny inputs is mostly the spawn cost, not a meaningful engine
//! delta. The report surfaces two columns so you can judge:
//!
//! - `cppnix_µs` — the full wall time per call
//! - `cppnix_eval_µs` — estimated eval time (cppnix_µs minus the
//!   median spawn floor measured from a trivial `eval 1`)
//!
//! and the `ratio` column uses `cppnix_eval_µs / sui_µs` so it
//! reflects engine delta, not spawn cost.
//!
//! # Opt-in
//!
//! Requires `SUI_TEST_ONLINE=1` (same gate as the differential oracle).
//! Also skips silently when `nix-instantiate` isn't on PATH. Not a CI
//! gate — human review of the diff is the feedback loop.

mod common;

use common::load_corpus;
use std::fmt::Write as _;
use std::process::Command;
use std::time::{Duration, Instant};

const ITERATIONS: usize = 5;

/// Eval `src` via `nix-instantiate --eval --json --strict -E src` and
/// return wall time on success, `None` on any failure (parse error,
/// non-zero exit, unparseable output). We don't want a single broken
/// program to zero the report.
fn time_cppnix(src: &str) -> Option<Duration> {
    let wrapped;
    let passed: &str = if src.trim_start().starts_with('-') {
        wrapped = format!("({src})");
        &wrapped
    } else {
        src
    };
    let start = Instant::now();
    let output = Command::new("nix-instantiate")
        .args(["--eval", "--json", "--strict", "-E", passed])
        .output()
        .ok()?;
    let elapsed = start.elapsed();
    if !output.status.success() {
        return None;
    }
    // Parse stdout as JSON to confirm we got a real answer, not a
    // spurious exit-0 with garbage.
    let stdout = String::from_utf8_lossy(&output.stdout);
    let _parsed: serde_json::Value = serde_json::from_str(stdout.trim()).ok()?;
    Some(elapsed)
}

/// Eval `src` via `sui_eval::eval` and return wall time on success.
fn time_sui(src: &str) -> Option<Duration> {
    let start = Instant::now();
    let out = sui_eval::eval(src);
    let elapsed = start.elapsed();
    out.ok().map(|_| elapsed)
}

/// Median of a small sample (we assume len > 0). For an even count
/// we return the lower mid rather than averaging — cheaper, and
/// with 5 samples the difference is noise.
fn median(mut xs: Vec<Duration>) -> Duration {
    xs.sort();
    xs[xs.len() / 2]
}

/// Measure the irreducible `nix-instantiate` spawn floor from evaluating
/// a trivial expression many times. This is subtracted from each
/// cppnix measurement to get an estimated pure-eval cost.
fn measure_cppnix_spawn_floor() -> Duration {
    let samples: Vec<Duration> = (0..5).filter_map(|_| time_cppnix("1")).collect();
    if samples.is_empty() {
        return Duration::ZERO;
    }
    median(samples)
}

#[derive(Debug, Clone)]
struct Row {
    name: String,
    source_short: String,
    sui_us: u128,
    cppnix_us: u128,
    cppnix_eval_us: u128,
    engine_ratio: f64, // cppnix_eval_us / sui_us; >1.0 means sui is faster
}

/// Programs whose cppnix wall time differs from the spawn floor by
/// less than this threshold are NOT engine-comparable — both runs
/// spent ~all their time in process setup, and subtracting them
/// leaves measurement noise. We still report them, but we exclude
/// them from the engine-ratio geomean.
const MEANINGFUL_CPPNIX_EVAL_US: u128 = 500;

fn render_report(rows: Vec<Row>, spawn_floor_us: u128) -> String {
    let mut out = String::new();
    writeln!(out, "# sui vs CppNix — head-to-head on the oracle corpus").unwrap();
    writeln!(out).unwrap();
    writeln!(
        out,
        "> Regenerated by `SUI_TEST_ONLINE=1 cargo test -p sui-eval --test vs_cppnix`."
    )
    .unwrap();
    writeln!(out).unwrap();
    writeln!(out, "## Method").unwrap();
    writeln!(out).unwrap();
    writeln!(
        out,
        "Each program runs {ITERATIONS}× on each engine; we report the \
         median per-call wall time in microseconds."
    )
    .unwrap();
    writeln!(out).unwrap();
    writeln!(
        out,
        "**The CppNix spawn floor on this machine is {spawn_floor_us} µs** \
         (median of `nix-instantiate --eval -E 1`). That's the fork+exec+ \
         builtin-init cost every invocation pays. Most programs in this \
         corpus evaluate in <1 ms inside CppNix, so the spawn cost \
         dominates the wall clock. Two consequences:"
    )
    .unwrap();
    writeln!(out).unwrap();
    writeln!(
        out,
        "1. **User perspective** — running `nix-instantiate` from a \
            shell means paying ~{}× sui's total eval cost on typical \
            short expressions. `cppnix_µs` captures this.",
        (spawn_floor_us / 100).max(1)
    )
    .unwrap();
    writeln!(
        out,
        "2. **Engine perspective** — `cppnix_eval_µs` = `cppnix_µs − spawn_floor` \
            tries to isolate pure eval cost. This works reliably only for \
            programs where `cppnix_eval_µs > {MEANINGFUL_CPPNIX_EVAL_US}` \
            (the threshold above measurement noise). Programs below that \
            threshold are marked `-` in the engine table and excluded from \
            the engine geomean."
    )
    .unwrap();
    writeln!(out).unwrap();

    // Summary block ------------------------------------------------
    let n = rows.len();
    let meaningful: Vec<&Row> = rows
        .iter()
        .filter(|r| r.cppnix_eval_us >= MEANINGFUL_CPPNIX_EVAL_US)
        .collect();
    let sui_faster_engine = meaningful.iter().filter(|r| r.engine_ratio > 1.0).count();
    let cppnix_faster_engine = meaningful.iter().filter(|r| r.engine_ratio < 1.0).count();

    let engine_geomean: f64 = if meaningful.is_empty() {
        0.0
    } else {
        let ratios: Vec<f64> = meaningful.iter().map(|r| r.engine_ratio.max(1e-6)).collect();
        let log_sum: f64 = ratios.iter().map(|r| r.ln()).sum();
        (log_sum / ratios.len() as f64).exp()
    };

    // Wall-clock geomean — always meaningful because we're comparing
    // directly-measured wall times, not subtracted deltas.
    let wall_geomean: f64 = {
        let ratios: Vec<f64> = rows
            .iter()
            .map(|r| {
                let sui = r.sui_us.max(1) as f64;
                let cpp = r.cppnix_us.max(1) as f64;
                cpp / sui
            })
            .collect();
        let log_sum: f64 = ratios.iter().map(|r| r.ln()).sum();
        (log_sum / ratios.len() as f64).exp()
    };

    let mean_sui: f64 = rows.iter().map(|r| r.sui_us as f64).sum::<f64>() / n as f64;
    let mean_cppnix_wall: f64 = rows.iter().map(|r| r.cppnix_us as f64).sum::<f64>() / n as f64;

    writeln!(out, "## Summary").unwrap();
    writeln!(out).unwrap();
    writeln!(out, "| metric | value |").unwrap();
    writeln!(out, "|--------|------:|").unwrap();
    writeln!(out, "| programs | {n} |").unwrap();
    writeln!(out, "| programs with cppnix_eval > {MEANINGFUL_CPPNIX_EVAL_US} µs (engine-comparable) | {} |", meaningful.len()).unwrap();
    writeln!(
        out,
        "| **wall-clock geomean (sui vs `nix-instantiate`)** | **{wall_geomean:.0}×** |"
    )
    .unwrap();
    writeln!(
        out,
        "| **engine geomean (on comparable programs)** | **{engine_geomean:.1}×** |"
    )
    .unwrap();
    writeln!(
        out,
        "| sui faster engine / CppNix faster engine | {sui_faster_engine} / {cppnix_faster_engine} |"
    )
    .unwrap();
    writeln!(out, "| mean sui time | {mean_sui:.0} µs |").unwrap();
    writeln!(
        out,
        "| mean CppNix wall | {mean_cppnix_wall:.0} µs |"
    )
    .unwrap();
    writeln!(
        out,
        "| CppNix spawn floor | {spawn_floor_us} µs |"
    )
    .unwrap();
    writeln!(out).unwrap();

    // Only engine-comparable programs produce meaningful ratios.
    writeln!(out, "## Programs where CppNix beats sui on pure eval").unwrap();
    writeln!(out).unwrap();
    writeln!(
        out,
        "Filtered to `cppnix_eval_µs ≥ {MEANINGFUL_CPPNIX_EVAL_US}` so \
         only real engine differences show. Programs below the threshold \
         are spawn-noise — both engines are too fast to differentiate."
    )
    .unwrap();
    writeln!(out).unwrap();
    let losses: Vec<&Row> = meaningful
        .iter()
        .copied()
        .filter(|r| r.engine_ratio < 1.0)
        .collect();
    if losses.is_empty() {
        writeln!(
            out,
            "*(none — sui beats CppNix on every engine-comparable program.)*"
        )
        .unwrap();
    } else {
        let mut sorted = losses.clone();
        sorted.sort_by(|a, b| a.engine_ratio.partial_cmp(&b.engine_ratio).unwrap());
        writeln!(
            out,
            "| program | sui µs | cppnix_eval µs | ratio (sui×) | source |"
        )
        .unwrap();
        writeln!(out, "|---------|------:|---------------:|-------------:|--------|").unwrap();
        for r in sorted.iter().take(10) {
            writeln!(
                out,
                "| `{}` | {} | {} | {:.2}× | `{}` |",
                r.name, r.sui_us, r.cppnix_eval_us, r.engine_ratio, r.source_short,
            )
            .unwrap();
        }
    }
    writeln!(out).unwrap();

    writeln!(out, "## Top 10 biggest sui wins (engine-comparable only)").unwrap();
    writeln!(out).unwrap();
    let mut wins: Vec<&Row> = meaningful.iter().copied().collect();
    wins.sort_by(|a, b| b.engine_ratio.partial_cmp(&a.engine_ratio).unwrap());
    writeln!(
        out,
        "| program | sui µs | cppnix_eval µs | ratio (sui×) | source |"
    )
    .unwrap();
    writeln!(out, "|---------|------:|---------------:|-------------:|--------|").unwrap();
    for r in wins.iter().take(10) {
        writeln!(
            out,
            "| `{}` | {} | {} | {:.2}× | `{}` |",
            r.name, r.sui_us, r.cppnix_eval_us, r.engine_ratio, r.source_short,
        )
        .unwrap();
    }
    writeln!(out).unwrap();

    // Full alphabetical table
    writeln!(out, "## Full table").unwrap();
    writeln!(out).unwrap();
    writeln!(out, "Sorted alphabetically for stable diffs.").unwrap();
    writeln!(out).unwrap();
    let mut full = rows;
    full.sort_by(|a, b| a.name.cmp(&b.name));
    writeln!(
        out,
        "| program | sui µs | cppnix µs | cppnix_eval µs | ratio |"
    )
    .unwrap();
    writeln!(
        out,
        "|---------|------:|----------:|---------------:|------:|"
    )
    .unwrap();
    for r in full {
        writeln!(
            out,
            "| `{}` | {} | {} | {} | {:.2}× |",
            r.name, r.sui_us, r.cppnix_us, r.cppnix_eval_us, r.engine_ratio,
        )
        .unwrap();
    }
    out
}

fn write_report(body: &str) -> std::path::PathBuf {
    let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .expect("workspace root exists")
        .join("target");
    std::fs::create_dir_all(&target).ok();
    let path = target.join("sui-vs-cppnix.md");
    std::fs::write(&path, body).expect("write sui-vs-cppnix.md");
    path
}

#[test]
fn compare_sui_vs_cppnix() {
    if common::skip_if_offline("compare_sui_vs_cppnix") {
        return;
    }

    eprintln!("measuring CppNix spawn floor…");
    let spawn_floor = measure_cppnix_spawn_floor();
    let spawn_floor_us = spawn_floor.as_micros();
    eprintln!("CppNix spawn floor: {spawn_floor_us} µs");

    let cases = load_corpus();
    let mut rows: Vec<Row> = Vec::with_capacity(cases.len());

    for case in cases {
        if case.spec.skip {
            continue;
        }

        // Quick compat gate — if sui or cppnix can't handle this
        // program, skip it rather than poisoning the comparison. The
        // differential oracle is where "we disagree" gets surfaced.
        let mut sui_samples: Vec<Duration> = Vec::with_capacity(ITERATIONS);
        for _ in 0..ITERATIONS {
            if let Some(d) = time_sui(&case.spec.source) {
                sui_samples.push(d);
            }
        }
        if sui_samples.is_empty() {
            continue;
        }

        let mut cpp_samples: Vec<Duration> = Vec::with_capacity(ITERATIONS);
        for _ in 0..ITERATIONS {
            if let Some(d) = time_cppnix(&case.spec.source) {
                cpp_samples.push(d);
            }
        }
        if cpp_samples.is_empty() {
            continue;
        }

        let sui_us = median(sui_samples).as_micros();
        let cppnix_us = median(cpp_samples).as_micros();
        let cppnix_eval_us = cppnix_us.saturating_sub(spawn_floor_us);
        let ratio = if sui_us == 0 {
            f64::INFINITY
        } else {
            // guard against cppnix_eval_us == 0 on ultra-fast cases
            #[allow(clippy::cast_precision_loss)]
            let r = cppnix_eval_us.max(1) as f64 / sui_us as f64;
            r
        };

        let source_short: String = case
            .spec
            .source
            .chars()
            .take(60)
            .collect::<String>()
            .replace('\n', " ")
            .replace('|', "\\|");

        rows.push(Row {
            name: case.name.clone(),
            source_short,
            sui_us,
            cppnix_us,
            cppnix_eval_us,
            engine_ratio: ratio,
        });
    }

    assert!(!rows.is_empty(), "no rows produced — both engines failed");

    let body = render_report(rows, spawn_floor_us);
    let path = write_report(&body);
    eprintln!("\nwrote head-to-head report to {}", path.display());
}