launchbound_report/
build.rs

1//! Build a report.v1 from a run directory: verdicts.json (the gate record)
2//! + plan.json + results.json (measurement, if the box has run).
3
4use crate::{
5    CandidateReport, ChosenInfo, DeviceInfo, RejectedFaster, Report, ReportError, RuleRef, Totals,
6};
7use launchbound_bench::{BenchPlan, Results, indistinguishable};
8use serde_json::Value;
9use std::path::{Path, PathBuf};
10
11/// The documents a run directory holds, loaded into memory.
12///
13/// `verdicts.json` is required — a run that never gated is not a run this
14/// tool reports on. The other two are optional because `prune` alone
15/// produces a directory worth reporting, with no timings in it.
16pub struct RunDir {
17    /// `verdicts.json`, kept as raw JSON: it is the gate's output and this
18    /// crate reads rather than owns its shape.
19    pub verdicts: Value,
20    /// `plan.json`, when the run got as far as planning a benchmark.
21    pub plan: Option<BenchPlan>,
22    /// `results.json`, when the run got as far as measuring.
23    pub results: Option<Results>,
24}
25
26impl RunDir {
27    /// Read a run directory. Fails if `verdicts.json` is missing or does
28    /// not declare `verdicts.v1`; tolerates the absence of the rest.
29    pub fn load(dir: &Path) -> Result<Self, ReportError> {
30        let verdicts_path = dir.join("verdicts.json");
31        let text = std::fs::read_to_string(&verdicts_path).map_err(|e| {
32            ReportError::RunDir(format!(
33                "{}: {e} (stage writes it)",
34                verdicts_path.display()
35            ))
36        })?;
37        let verdicts: Value =
38            serde_json::from_str(&text).map_err(|e| ReportError::RunDir(e.to_string()))?;
39        if verdicts["schema"] != "verdicts.v1" {
40            return Err(ReportError::RunDir(format!(
41                "unsupported verdicts schema {}",
42                verdicts["schema"]
43            )));
44        }
45        let plan = BenchPlan::load(&dir.join("plan.json")).ok();
46        // A results.json that exists but cannot be read is an error, not
47        // "the box has not run yet". The two call for opposite actions —
48        // wait, or go and look — and rendering both as `unmeasured` at
49        // exit 0 told them apart for nobody. Same treatment `verdicts.v1`
50        // gets fifteen lines above.
51        let results = Results::load(&dir.join("results.json")).map_err(ReportError::RunDir)?;
52        Ok(RunDir {
53            verdicts,
54            plan,
55            results,
56        })
57    }
58
59    /// A run directory path from a CLI argument.
60    pub fn path_of(dir: &str) -> PathBuf {
61        PathBuf::from(dir)
62    }
63}
64
65fn rules_of(candidate: &Value) -> Vec<RuleRef> {
66    let mut rules = Vec::new();
67    for key in ["records", "caveats"] {
68        if let Some(list) = candidate.get(key).and_then(Value::as_array) {
69            for r in list {
70                rules.push(RuleRef {
71                    rule: r["rule"].as_str().unwrap_or("?").to_string(),
72                    span: r["span"].as_str().map(String::from),
73                    reason: r
74                        .get("reason")
75                        .or_else(|| r.get("message"))
76                        .and_then(Value::as_str)
77                        .unwrap_or("")
78                        .to_string(),
79                });
80            }
81        }
82    }
83    rules
84}
85
86/// Assemble a `report.v1` document from a loaded run directory.
87///
88/// Joins the gate's verdicts to the measurements by candidate ID, picks the
89/// chosen configuration, and fills the two sections a reader acts on:
90/// candidates indistinguishable from the chosen one (overlapping intervals,
91/// reported rather than ranked) and refused candidates that measurably beat
92/// it.
93///
94/// Float ordering throughout uses [`f64::total_cmp`], so a NaN median
95/// produces a report rather than a panic — see `tests/nan_ranking.rs`.
96pub fn build_report(run: &RunDir) -> Result<Report, ReportError> {
97    let kernel = run.verdicts["kernel"].as_str().unwrap_or("?").to_string();
98    let gate_cc = run.verdicts["cc"].as_str().unwrap_or("?").to_string();
99    let convergence_gate = run.verdicts["gate"].as_str().unwrap_or("full").to_string();
100    let empty = Vec::new();
101    let verdict_candidates = run.verdicts["candidates"].as_array().unwrap_or(&empty);
102
103    let mut candidates = Vec::new();
104    let mut admitted = 0usize;
105    let mut refused = 0usize;
106    let mut measured_ok = 0usize;
107    let mut gpu_seconds = 0.0f64;
108
109    for vc in verdict_candidates {
110        let id = vc["id"].as_str().unwrap_or("?").to_string();
111        let verdict = vc["verdict"].as_str().unwrap_or("?").to_string();
112        match verdict.as_str() {
113            "clean" | "admitted_with_caveats" | "ungated" => admitted += 1,
114            "disqualified" => refused += 1,
115            _ => {}
116        }
117        let measurement = run
118            .results
119            .as_ref()
120            .and_then(|r| r.candidates.iter().find(|c| c.id == id));
121        let (status, summary, error, secs) = match measurement {
122            Some(m) => {
123                if m.status == "ok" {
124                    measured_ok += 1;
125                }
126                gpu_seconds += m.gpu_seconds;
127                (
128                    m.status.clone(),
129                    m.summary.clone(),
130                    m.error.clone(),
131                    m.gpu_seconds,
132                )
133            }
134            None => ("unmeasured".to_string(), None, None, 0.0),
135        };
136        candidates.push(CandidateReport {
137            id,
138            config: vc["config"].as_str().unwrap_or("?").to_string(),
139            verdict,
140            rules: rules_of(vc),
141            measurement_status: status,
142            summary,
143            measurement_error: error,
144            gpu_seconds: secs,
145        });
146    }
147
148    // The chosen configuration: best median among measured, admitted, ok.
149    let chosen = candidates
150        .iter()
151        .filter(|c| {
152            matches!(
153                c.verdict.as_str(),
154                "clean" | "admitted_with_caveats" | "ungated"
155            ) && c.measurement_status == "ok"
156        })
157        // Carry the summary, not just its median: the `filter_map` above is
158        // what proves it exists, and re-reaching for `c.summary` afterwards
159        // put the proof and the use in different expressions. `s` is the
160        // guarantee, held in the type.
161        .filter_map(|c| c.summary.as_ref().map(|s| (c, s)))
162        .min_by(|a, b| a.1.median_ms.total_cmp(&b.1.median_ms))
163        .map(|(c, s)| ChosenInfo {
164            id: c.id.clone(),
165            config: c.config.clone(),
166            summary: s.clone(),
167        });
168
169    let mut indistinguishable_from_chosen = Vec::new();
170    let mut rejected_faster = Vec::new();
171    if let Some(chosen) = &chosen {
172        for c in &candidates {
173            let Some(summary) = &c.summary else { continue };
174            if c.id == chosen.id || c.measurement_status != "ok" {
175                continue;
176            }
177            let is_admitted = matches!(
178                c.verdict.as_str(),
179                "clean" | "admitted_with_caveats" | "ungated"
180            );
181            if is_admitted && indistinguishable(summary, &chosen.summary) {
182                indistinguishable_from_chosen.push(c.id.clone());
183            }
184            // Measurably faster: the whole interval sits below the chosen's.
185            if c.verdict == "disqualified" && summary.ci95_hi_ms < chosen.summary.ci95_lo_ms {
186                rejected_faster.push(RejectedFaster {
187                    id: c.id.clone(),
188                    config: c.config.clone(),
189                    summary: summary.clone(),
190                    speedup_vs_chosen: chosen.summary.median_ms / summary.median_ms,
191                    rules: c.rules.clone(),
192                });
193            }
194        }
195        rejected_faster.sort_by(|a, b| b.speedup_vs_chosen.total_cmp(&a.speedup_vs_chosen));
196    }
197
198    let total = candidates.len();
199    Ok(Report {
200        schema: "report.v1".into(),
201        kernel,
202        gate_cc,
203        // A report with no measurements must say so — labelling it
204        // `measured` would be the exact dishonesty §1.9 forbids.
205        measurement_kind: if run.results.is_some() {
206            "measured".into()
207        } else {
208            "unmeasured".into()
209        },
210        convergence_gate,
211        device: run.results.as_ref().map(|r| DeviceInfo {
212            name: r.device_name.clone(),
213            cc: r.device_cc.clone(),
214            driver_version: r.driver_version.clone(),
215        }),
216        allow_unsafe_reason: run
217            .plan
218            .as_ref()
219            .and_then(|p| p.allow_unsafe_reason.clone()),
220        chosen,
221        indistinguishable_from_chosen,
222        rejected_faster,
223        candidates,
224        totals: Totals {
225            candidates: total,
226            admitted,
227            refused,
228            measured_ok,
229            gpu_seconds,
230        },
231    })
232}
launchbound_report/build.rs

launchbound_report/
build.rs
