inferencelayer 0.2.13

//! The quiet-machine guard + pins ledger shared by every perf harness (`scoreboard`,
//! `encoder_scoreboard`, `enc-kernel-lab`).
//!
//! Factored out of `src/bin/scoreboard.rs` so the encoder harness cannot drift from the decoder
//! harness on refusal semantics: a benchmark number is only comparable to another benchmark
//! number if both were captured under the same contention policy. The 2026-07-13 GLiNER
//! benchmarking session is the cautionary tale — the same input measured 29.6 ms and 82.9 ms
//! minutes apart on a box at load average 46–209, and two optimization decisions were made on
//! numbers that later turned out to be noise.
//!
//! Two independent contention axes, because one does not imply the other:
//!   * CPU — ambient load (compile jobs, browsers) stealing the submitting/worker threads;
//!   * GPU — a foreign compute process pinning the SMs while `uptime` reads near-idle
//!     (observed on the shared V100 box 2026-07-12: load 3.53 with three foreign 9–18 GiB
//!     compute processes resident).
//!
//! `OSFKB_SCOREBOARD_IGNORE_LOAD=1` downgrades refusal to a warning for exploratory runs; the
//! harnesses then refuse `--pin`, so a contended number can never become the reference every
//! later run is judged against.

use anyhow::{Context, Result, bail};

/// 1-minute load average, portable via `uptime` (macOS + Linux). `None` = unavailable.
pub fn load_average_1m() -> Option<f64> {
    let out = std::process::Command::new("uptime").output().ok()?;
    let text = String::from_utf8(out.stdout).ok()?;
    let tail = text.split("load average").nth(1)?;
    tail.split(|c: char| !c.is_ascii_digit() && c != '.')
        .find(|s| !s.is_empty())
        .and_then(|s| s.parse().ok())
}

/// Cumulative CPU seconds this process has burned (user + sys, summed across threads).
///
/// Needed because the 1-minute load average COUNTS US. A 16-thread benchmark drives the load average
/// up by most of a core count all on its own, so "the load rose while we were measuring" is a
/// statement about our own benchmark unless we subtract our own contribution first. Without this,
/// the end-of-sweep guard fires on every run and means nothing.
pub fn self_cpu_seconds() -> Option<f64> {
    let out = std::process::Command::new("ps")
        .args(["-o", "cputime=", "-p", &std::process::id().to_string()])
        .output()
        .ok()?;
    let t = String::from_utf8(out.stdout).ok()?;
    // `[[HH:]MM:]SS[.ss]`
    let mut secs = 0.0f64;
    for part in t.trim().split(':') {
        secs = secs * 60.0 + part.parse::<f64>().ok()?;
    }
    Some(secs)
}

/// The load average with our own CPU usage removed — i.e. the contention we did NOT cause.
///
/// `self_cores` is our average core occupancy over the window (CPU seconds / wall seconds).
pub fn foreign_load(self_cores: f64) -> Option<f64> {
    load_average_1m().map(|l| (l - self_cores).max(0.0))
}

/// Compute processes occupying the GPUs, excluding our own. `None` when there is no nvidia-smi
/// (Metal exposes no per-process compute query — absence of the tool is NOT evidence of a quiet
/// GPU, so callers must not read `None` as "clear").
pub fn foreign_gpu_procs() -> Option<Vec<String>> {
    let out = std::process::Command::new("nvidia-smi")
        .args([
            "--query-compute-apps=pid,used_memory,process_name",
            "--format=csv,noheader",
        ])
        .output()
        .ok()?;
    if !out.status.success() {
        return None;
    }
    let text = String::from_utf8_lossy(&out.stdout);
    Some(parse_compute_apps(&text, std::process::id()))
}

/// Rows of `nvidia-smi --query-compute-apps` that are not us. Split from the process spawn so the
/// filtering is testable without a GPU.
pub fn parse_compute_apps(text: &str, me: u32) -> Vec<String> {
    let me = me.to_string();
    text.lines()
        .map(str::trim)
        .filter(|l| !l.is_empty())
        .filter(|l| l.split(',').next().map(str::trim) != Some(me.as_str()))
        .map(str::to_owned)
        .collect()
}

/// Refuse to measure on a busy machine; returns whether the box was genuinely quiet. A `false`
/// return is not fatal (exploratory runs under contention are useful) but callers must refuse
/// `--pin` on it — `run` in both scoreboards enforces that.
pub fn ensure_quiet_machine() -> Result<bool> {
    let overridden = std::env::var("OSFKB_SCOREBOARD_IGNORE_LOAD").as_deref() == Ok("1");
    let mut quiet = true;

    match foreign_gpu_procs() {
        Some(procs) if !procs.is_empty() => {
            quiet = false;
            eprintln!(
                "quiet-machine guard: {} foreign GPU process(es):",
                procs.len()
            );
            for p in &procs {
                eprintln!("  {p}");
            }
            if !overridden {
                bail!(
                    "the GPUs are not idle: {} foreign compute process(es) are resident, so \
                     absolutes would be distorted (CPU load average cannot see this). Wait for a \
                     clear GPU or set OSFKB_SCOREBOARD_IGNORE_LOAD=1 for a non-pinnable \
                     exploratory run.",
                    procs.len()
                )
            }
        }
        Some(_) => eprintln!("quiet-machine guard: GPUs idle (no foreign compute procs) — ok"),
        None => eprintln!("quiet-machine guard: no nvidia-smi; GPU contention UNCHECKED"),
    }

    let cores = std::thread::available_parallelism()
        .map(std::num::NonZero::get)
        .unwrap_or(8) as f64;
    match load_average_1m() {
        None => eprintln!("quiet-machine guard: load average unavailable; CPU load UNCHECKED"),
        Some(load) => {
            let limit = cores / 2.0;
            if load < limit {
                eprintln!(
                    "quiet-machine guard: load {load:.2} < {limit:.1} ({cores:.0} cores) — ok"
                );
            } else {
                quiet = false;
                if !overridden {
                    bail!(
                        "machine is busy (1-min load {load:.2} ≥ {limit:.1} on {cores:.0} cores): \
                         absolutes would be distorted. Wait for quiet or set \
                         OSFKB_SCOREBOARD_IGNORE_LOAD=1 for a non-pinnable exploratory run."
                    )
                }
                eprintln!("quiet-machine guard: load {load:.2} ≥ {limit:.1} ({cores:.0} cores)");
            }
        }
    }

    if !quiet {
        eprintln!("quiet-machine guard OVERRIDDEN — these numbers are exploratory, not pinnable");
    }
    Ok(quiet)
}

/// Print NVIDIA clock + throttle state when nvidia-smi is present (informational; pinning
/// absolute numbers on NVIDIA requires locked clocks — `nvidia-smi -lgc <mhz>`).
pub fn nvidia_clock_note() {
    let Ok(out) = std::process::Command::new("nvidia-smi")
        .args([
            "--query-gpu=index,clocks.sm,clocks_throttle_reasons.active",
            "--format=csv,noheader",
        ])
        .output()
    else {
        return;
    };
    if out.status.success() {
        let text = String::from_utf8_lossy(&out.stdout);
        for line in text.lines() {
            eprintln!("nvidia clocks: {line}");
        }
    }
}

/// Read a pins ledger; a missing file is an empty ledger (first run on a branch).
pub fn read_pins(path: &str) -> Result<serde_json::Value> {
    match std::fs::read(path) {
        Ok(bytes) => serde_json::from_slice(&bytes).with_context(|| format!("parse {path}")),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(serde_json::json!({"rows": {}})),
        Err(e) => Err(e).with_context(|| format!("read {path}")),
    }
}

pub fn write_pins(path: &str, pins: &serde_json::Value) -> Result<()> {
    if let Some(dir) = std::path::Path::new(path).parent() {
        std::fs::create_dir_all(dir).ok();
    }
    std::fs::write(path, serde_json::to_vec_pretty(pins)?).with_context(|| format!("write {path}"))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parse_compute_apps_excludes_self_and_blank_lines() {
        let text = "1234, 512 MiB, python\n\n  5678, 9 GiB, train.py \n";
        let got = parse_compute_apps(text, 1234);
        assert_eq!(got, vec!["5678, 9 GiB, train.py".to_string()]);
        assert!(parse_compute_apps("", 1).is_empty());
    }
}