Skip to main content

firstpass_proxy/
cli.rs

1//! CLI surfaces for the `firstpass` binary (SPEC §7.3/§7.4): `doctor` validates a setup before
2//! you route real traffic through it, and `trace` reads recent audit records from the store. Both
3//! are kept here (not in the binary) so the judgment is unit-tested.
4
5use crate::config::ProxyConfig;
6use firstpass_core::{Mode, Trace};
7
8/// A single doctor check outcome.
9#[derive(Debug, PartialEq, Eq)]
10pub enum CheckStatus {
11    /// Healthy.
12    Ok,
13    /// Works, but worth knowing (e.g. no key in env — observe still runs).
14    Warn,
15    /// Broken; `firstpass doctor` exits non-zero.
16    Fail,
17}
18
19/// One line of the doctor report.
20#[derive(Debug)]
21pub struct Check {
22    /// Short check name.
23    pub name: String,
24    /// Outcome.
25    pub status: CheckStatus,
26    /// Human-readable detail.
27    pub detail: String,
28}
29
30impl Check {
31    fn new(name: impl Into<String>, status: CheckStatus, detail: impl Into<String>) -> Self {
32        Self {
33            name: name.into(),
34            status,
35            detail: detail.into(),
36        }
37    }
38}
39
40/// The result of `firstpass doctor`.
41#[derive(Debug)]
42pub struct DoctorReport {
43    /// One entry per check, in report order.
44    pub checks: Vec<Check>,
45}
46
47impl DoctorReport {
48    /// Healthy iff no check failed (warnings are fine).
49    #[must_use]
50    pub fn healthy(&self) -> bool {
51        self.checks.iter().all(|c| c.status != CheckStatus::Fail)
52    }
53
54    /// Render the report as human-readable lines (a rendering of the structured checks).
55    #[must_use]
56    pub fn render(&self) -> String {
57        let mut out = String::new();
58        for c in &self.checks {
59            let mark = match c.status {
60                CheckStatus::Ok => "✓",
61                CheckStatus::Warn => "!",
62                CheckStatus::Fail => "✗",
63            };
64            out.push_str(&format!("{mark} {}: {}\n", c.name, c.detail));
65        }
66        out.push_str(if self.healthy() {
67            "\nhealthy — ready to route.\n"
68        } else {
69            "\nnot healthy — fix the ✗ items above.\n"
70        });
71        out
72    }
73}
74
75/// Validate a loaded config against the environment: routing sanity, provider key presence, and
76/// that every configured gate's command actually exists. `env` looks up environment variables
77/// (injected so this is testable).
78#[must_use]
79pub fn doctor(config: &ProxyConfig, env: impl Fn(&str) -> Option<String>) -> DoctorReport {
80    let mut checks = Vec::new();
81
82    // Config parsed (we hold a ProxyConfig), report the shape.
83    let route_count = config.routing.as_ref().map_or(0, |c| c.routes.len());
84    checks.push(Check::new(
85        "config",
86        CheckStatus::Ok,
87        format!("default mode {:?}, {route_count} route(s)", config.mode),
88    ));
89
90    // Enforce is only meaningful with an enforce route to activate the engine.
91    let enforce_routes = config.routing.as_ref().map_or(0, |c| {
92        c.routes.iter().filter(|r| r.mode == Mode::Enforce).count()
93    });
94    if config.mode == Mode::Enforce && enforce_routes == 0 {
95        checks.push(Check::new(
96            "routing",
97            CheckStatus::Warn,
98            "mode is enforce but no enforce route is defined — all traffic will observe",
99        ));
100    } else {
101        checks.push(Check::new(
102            "routing",
103            CheckStatus::Ok,
104            format!("{enforce_routes} enforce route(s)"),
105        ));
106    }
107
108    // A provider key in the environment. Observe uses the caller's key, so absence is a warning.
109    if env("ANTHROPIC_API_KEY").is_some_and(|k| !k.is_empty()) {
110        checks.push(Check::new("anthropic-key", CheckStatus::Ok, "present"));
111    } else {
112        checks.push(Check::new(
113            "anthropic-key",
114            CheckStatus::Warn,
115            "ANTHROPIC_API_KEY not set — observe uses the caller's key; enforce needs one reachable",
116        ));
117    }
118
119    // A ladder rung on a built-in provider whose key is absent fails at the first REQUEST, not at
120    // startup — the least useful moment to find out. Checked per rung actually configured rather
121    // than for all eight built-ins, because warning about providers nobody is using is noise that
122    // trains an operator to skip the report.
123    let mut missing_keys: Vec<(String, &str)> = Vec::new();
124    if let Some(routing) = config.routing.as_ref() {
125        let mut seen = std::collections::HashSet::new();
126        for rung in routing.routes.iter().flat_map(|r| r.ladder.iter()) {
127            let Some((prov, _)) = rung.split_once('/') else {
128                continue;
129            };
130            let Some(key_env) = crate::provider::builtin_key_env(prov) else {
131                continue; // anthropic/openai are covered above; a [[provider]] block owns its own
132            };
133            if seen.insert(prov.to_owned()) && env(key_env).is_none_or(|k| k.is_empty()) {
134                missing_keys.push((prov.to_owned(), key_env));
135            }
136        }
137    }
138    for (prov, key_env) in missing_keys {
139        checks.push(Check::new(
140            format!("provider:{prov}"),
141            CheckStatus::Fail,
142            format!("{key_env} not set — a rung on `{prov}` will fail at the first request"),
143        ));
144    }
145
146    // Every configured gate command must resolve, or that gate silently abstains at runtime.
147    let path = env("PATH");
148    let gate_defs = config.routing.as_ref().map_or(&[][..], |c| &c.gate_defs);
149    for def in gate_defs {
150        match def.cmd.first() {
151            Some(program) if command_on_path(program, path.as_deref()) => checks.push(Check::new(
152                format!("gate:{}", def.id),
153                CheckStatus::Ok,
154                format!("`{program}` found"),
155            )),
156            Some(program) => checks.push(Check::new(
157                format!("gate:{}", def.id),
158                CheckStatus::Fail,
159                format!("`{program}` not found on PATH"),
160            )),
161            None => checks.push(Check::new(
162                format!("gate:{}", def.id),
163                CheckStatus::Fail,
164                "empty command",
165            )),
166        }
167    }
168
169    // The trace store must be writable, or we'd trade the audit trail (or availability) for it.
170    if can_write_db(&config.db_path) {
171        checks.push(Check::new(
172            "trace-store",
173            CheckStatus::Ok,
174            format!("{} is writable", config.db_path),
175        ));
176    } else {
177        checks.push(Check::new(
178            "trace-store",
179            CheckStatus::Fail,
180            format!("cannot write near {}", config.db_path),
181        ));
182    }
183
184    DoctorReport { checks }
185}
186
187/// Whether `program` is runnable: an explicit path (contains a separator) that exists, or a bare
188/// name found in one of `PATH`'s directories.
189#[must_use]
190pub fn command_on_path(program: &str, path_var: Option<&str>) -> bool {
191    if program.contains('/') || program.contains('\\') {
192        return std::path::Path::new(program).is_file();
193    }
194    let Some(path) = path_var else { return false };
195    std::env::split_paths(path).any(|dir| dir.join(program).is_file())
196}
197
198/// How this binary got onto the machine. Upgrading is channel-specific — running the wrong
199/// updater either does nothing or fights the package manager that owns the file — so
200/// `firstpass upgrade` detects the channel instead of guessing one.
201#[derive(Debug, Clone, Copy, PartialEq, Eq)]
202pub enum Channel {
203    /// The shell/powershell installer, which ships its own `firstpass-update` (dist-workspace.toml
204    /// sets `install-updater = true`). This is the only channel we can upgrade in place.
205    Dist,
206    /// `brew install dshakes/tap/firstpass-proxy`.
207    Homebrew,
208    /// `cargo install --git`.
209    Cargo,
210    /// pip / uv / uvx wheel.
211    Python,
212    /// pipx-managed install (its own venv, symlinked onto PATH).
213    Pipx,
214    /// The npm installer package.
215    Npm,
216    /// Running inside the container image.
217    Docker,
218    /// Built from source, vendored, or something we do not recognise.
219    Unknown,
220}
221
222impl Channel {
223    /// Human label used in the report.
224    #[must_use]
225    pub const fn label(self) -> &'static str {
226        match self {
227            Self::Dist => "installer script (self-updating)",
228            Self::Homebrew => "Homebrew",
229            Self::Cargo => "cargo install",
230            Self::Python => "Python wheel (pip / uv / uvx)",
231            Self::Pipx => "pipx",
232            Self::Npm => "npm",
233            Self::Docker => "Docker image",
234            Self::Unknown => "unrecognised",
235        }
236    }
237
238    /// The command that upgrades this channel. `None` when we cannot tell.
239    #[must_use]
240    pub const fn upgrade_command(self) -> Option<&'static str> {
241        match self {
242            Self::Dist => Some("firstpass-update"),
243            Self::Homebrew => Some("brew upgrade firstpass-proxy"),
244            Self::Cargo => Some(
245                "cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force",
246            ),
247            Self::Python => Some("uv tool upgrade firstpass    # or: pip install -U firstpass"),
248            Self::Pipx => Some("pipx upgrade firstpass"),
249            Self::Npm => Some("npm install -g @dshakesnotbot/firstpass-proxy@latest"),
250            Self::Docker => Some("docker pull ghcr.io/dshakes/firstpass:latest"),
251            Self::Unknown => None,
252        }
253    }
254}
255
256/// Work out how this binary was installed from the path it runs as. Pure: the executable path,
257/// the updater probe and the container check are all injected, so every branch is unit-tested
258/// without installing anything.
259///
260/// Package-manager path markers are checked BEFORE the updater probe: a Homebrew or wheel install
261/// can still carry the dist updater alongside it, and in that case the package manager owns the
262/// file — running the self-updater would leave the manager's metadata pointing at a binary it no
263/// longer controls.
264#[must_use]
265pub fn detect_channel(exe: &std::path::Path, updater_present: bool, in_container: bool) -> Channel {
266    let p = exe.to_string_lossy().to_lowercase().replace('\\', "/");
267    let has = |needle: &str| p.contains(needle);
268    if in_container {
269        return Channel::Docker;
270    }
271    if has("/.cargo/bin/") || has("/.cargo/registry/") {
272        return Channel::Cargo;
273    }
274    if has("/cellar/") || has("/homebrew/") || has("/linuxbrew/") {
275        return Channel::Homebrew;
276    }
277    if has("/pipx/") {
278        return Channel::Pipx; // before the generic markers: a pipx venv also has site-packages
279    }
280    if has("/site-packages/")
281        || has("/dist-packages/")
282        || has("/.venv/")
283        || has("/uv/")
284        || has("/uv-cache/")
285    {
286        return Channel::Python;
287    }
288    if has("/node_modules/") {
289        return Channel::Npm;
290    }
291    if updater_present {
292        return Channel::Dist;
293    }
294    Channel::Unknown
295}
296
297/// The text `firstpass upgrade` prints. Separated from the doing so the wording is testable.
298#[must_use]
299pub fn upgrade_report(channel: Channel, current_version: &str) -> String {
300    let mut out = format!(
301        "firstpass {current_version} · installed via {}\n",
302        channel.label()
303    );
304    match channel.upgrade_command() {
305        Some(cmd) if channel == Channel::Dist => {
306            out.push_str(&format!("\nupgrading in place — running `{cmd}`\n"));
307        }
308        Some(cmd) => {
309            out.push_str(&format!(
310                "\nthis channel is owned by its package manager, so upgrade with:\n    {cmd}\n"
311            ));
312        }
313        None => {
314            out.push_str(
315                "\ncould not tell how this binary was installed. Pick the line that matches:\n\
316                 \x20   firstpass-update                              # installer script\n\
317                 \x20   brew upgrade firstpass-proxy                  # Homebrew\n\
318                 \x20   uv tool upgrade firstpass                     # uv / pip: pip install -U firstpass\n\
319                 \x20   pipx upgrade firstpass                        # pipx\n\
320                 \x20   npm install -g @dshakesnotbot/firstpass-proxy@latest\n\
321                 \x20   docker pull ghcr.io/dshakes/firstpass:latest\n\
322                 \x20   cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force\n",
323            );
324        }
325    }
326    out
327}
328
329/// Probe whether the trace DB's directory is writable, without creating the DB itself.
330fn can_write_db(db_path: &str) -> bool {
331    let path = std::path::Path::new(db_path);
332    let dir = path
333        .parent()
334        .filter(|d| !d.as_os_str().is_empty())
335        .map_or_else(
336            || std::path::PathBuf::from("."),
337            std::path::Path::to_path_buf,
338        );
339    let probe = dir.join(format!(".firstpass-doctor-probe-{}", std::process::id()));
340    match std::fs::File::create(&probe) {
341        Ok(_) => {
342            let _ = std::fs::remove_file(&probe);
343            true
344        }
345        Err(_) => false,
346    }
347}
348
349/// Per-gate and per-rung evaluation summary computed from receipts — the operator's live
350/// eval suite: how each gate is verdict-ing, how often routing escalates, where serves land.
351#[derive(Debug, serde::Serialize)]
352pub struct EvalsSummary {
353    /// Traces aggregated (enforce mode only — observe records no gate decisions on-path).
354    pub enforce_traces: usize,
355    /// Total escalations across those traces.
356    pub escalations: u64,
357    /// gate_id → (pass, fail, abstain) counts across every attempt.
358    pub gates: std::collections::BTreeMap<String, (u64, u64, u64)>,
359    /// rung index → times an attempt at that rung was the served one.
360    pub served_by_rung: std::collections::BTreeMap<u32, u64>,
361}
362
363/// Aggregate gate verdicts + routing behavior over `traces` (enforce only).
364#[must_use]
365/// Prequential evaluation of the per-query gate-pass predictor (ADR 0008 Phase 2) — the honest
366/// "is it any good yet" check before the predictor is ever allowed to *act*. Replays the receipts
367/// in order through a fresh predictor: for each attempt with a clear Pass/Fail verdict, **predict
368/// before update** (so every prediction is on data the model has not trained on), then update.
369/// Reports AUC (predicted -> passed) and Brier score over those pairs.
370#[derive(Debug, serde::Serialize)]
371pub struct PredictorEval {
372    /// Labeled attempts evaluated.
373    pub n: usize,
374    /// AUC of predicted P(pass) ranking actual passes above fails. 0.5 = no signal; 1.0 = perfect.
375    /// `None` when one class is absent (all pass or all fail — AUC undefined).
376    pub auc: Option<f64>,
377    /// Mean squared error of the probabilistic prediction, in `[0, 1]`. Lower is better; 0.25 is
378    /// the always-0.5 baseline.
379    pub brier: f64,
380}
381
382/// AUC via the Mann-Whitney rank-sum identity (ties = 0.5). `None` if either class is empty.
383fn predictor_auc(pairs: &[(f64, bool)]) -> Option<f64> {
384    let pos: Vec<f64> = pairs.iter().filter(|(_, y)| *y).map(|(s, _)| *s).collect();
385    let neg: Vec<f64> = pairs.iter().filter(|(_, y)| !*y).map(|(s, _)| *s).collect();
386    if pos.is_empty() || neg.is_empty() {
387        return None;
388    }
389    let mut wins = 0.0_f64;
390    for &p in &pos {
391        for &n in &neg {
392            wins += if p > n {
393                1.0
394            } else if (p - n).abs() < f64::EPSILON {
395                0.5
396            } else {
397                0.0
398            };
399        }
400    }
401    Some(wins / (pos.len() as f64 * neg.len() as f64))
402}
403
404/// Run the prequential predictor evaluation over `traces` with the given `lr`/`l2`.
405pub fn evaluate_predictor(traces: &[Trace], lr: f64, l2: f64) -> PredictorEval {
406    use firstpass_core::{PassPredictor, Verdict};
407    let mut model = PassPredictor::new(lr, l2);
408    let mut pairs: Vec<(f64, bool)> = Vec::new();
409    let mut se = 0.0_f64;
410    for t in traces {
411        for a in &t.attempts {
412            let y = match a.verdict {
413                Verdict::Pass => true,
414                Verdict::Fail => false,
415                Verdict::Abstain => continue,
416            };
417            let p = model.predict(&t.request.features, a.rung);
418            let target = f64::from(u8::from(y));
419            se += (p - target) * (p - target);
420            pairs.push((p, y));
421            model.update(&t.request.features, a.rung, y);
422        }
423    }
424    let n = pairs.len();
425    PredictorEval {
426        n,
427        auc: predictor_auc(&pairs),
428        brier: if n == 0 { 0.0 } else { se / n as f64 },
429    }
430}
431
432pub fn summarize_evals(traces: &[Trace]) -> EvalsSummary {
433    let mut s = EvalsSummary {
434        enforce_traces: 0,
435        escalations: 0,
436        gates: std::collections::BTreeMap::new(),
437        served_by_rung: std::collections::BTreeMap::new(),
438    };
439    for t in traces {
440        if t.mode != Mode::Enforce {
441            continue;
442        }
443        s.enforce_traces += 1;
444        s.escalations += u64::from(t.final_.escalations);
445        if let Some(rung) = t.final_.served_rung {
446            *s.served_by_rung.entry(rung).or_insert(0) += 1;
447        }
448        for attempt in &t.attempts {
449            for g in &attempt.gates {
450                let e = s.gates.entry(g.gate_id.clone()).or_insert((0, 0, 0));
451                match g.verdict {
452                    firstpass_core::Verdict::Pass => e.0 += 1,
453                    firstpass_core::Verdict::Fail => e.1 += 1,
454                    firstpass_core::Verdict::Abstain => e.2 += 1,
455                }
456            }
457        }
458    }
459    s
460}
461
462/// Render an [`EvalsSummary`] for humans (`--json` callers serialize the struct instead).
463#[must_use]
464pub fn format_evals(s: &EvalsSummary) -> String {
465    if s.enforce_traces == 0 {
466        return "no enforce traces yet — route some traffic in enforce mode first".to_owned();
467    }
468    let mut out = format!(
469        "enforce traces: {} · escalations: {}\n",
470        s.enforce_traces, s.escalations
471    );
472    out.push_str("gates (pass / fail / abstain):\n");
473    for (id, (p, f, a)) in &s.gates {
474        out.push_str(&format!("  {id:<24} {p:>6} / {f:>6} / {a:>6}\n"));
475    }
476    out.push_str("served by rung:\n");
477    for (rung, n) in &s.served_by_rung {
478        out.push_str(&format!("  rung {rung:<2} {n:>6}\n"));
479    }
480    out.trim_end().to_owned()
481}
482
483/// A structured, agent-readable explanation of a single routing decision — why the ladder
484/// went where it did, built purely from a sealed [`Trace`].
485#[derive(Debug, serde::Serialize)]
486pub struct RouteExplanation {
487    pub trace_id: String,
488    pub mode: String,
489    pub policy: String,
490    /// The model actually served, if any.
491    pub served_model: Option<String>,
492    pub served_rung: Option<u32>,
493    pub escalations: u32,
494    pub total_cost_usd: f64,
495    pub baseline_usd: f64,
496    pub savings_usd: f64,
497    /// Per attempt, in ladder order: what was tried and how it was judged.
498    pub attempts: Vec<AttemptExplanation>,
499    /// One-line human summary.
500    pub summary: String,
501}
502
503/// Per-attempt slice of a [`RouteExplanation`].
504#[derive(Debug, serde::Serialize)]
505pub struct AttemptExplanation {
506    pub rung: u32,
507    pub model: String,
508    pub verdict: String,
509    /// gate_id → verdict, for every gate that ran on this attempt.
510    pub gates: Vec<(String, String)>,
511}
512
513fn verdict_str(v: firstpass_core::Verdict) -> &'static str {
514    match v {
515        firstpass_core::Verdict::Pass => "pass",
516        firstpass_core::Verdict::Fail => "fail",
517        firstpass_core::Verdict::Abstain => "abstain",
518    }
519}
520
521/// Explain one routing decision from its sealed receipt — the "why this model" answer, as
522/// structured data an agent can act on (and the backing for the MCP `explain_route` tool).
523#[must_use]
524pub fn explain_trace(t: &Trace) -> RouteExplanation {
525    let attempts: Vec<AttemptExplanation> = t
526        .attempts
527        .iter()
528        .map(|a| AttemptExplanation {
529            rung: a.rung,
530            model: a.model.clone(),
531            verdict: verdict_str(a.verdict).to_owned(),
532            gates: a
533                .gates
534                .iter()
535                .map(|g| (g.gate_id.clone(), verdict_str(g.verdict).to_owned()))
536                .collect(),
537        })
538        .collect();
539    let served_model = t
540        .final_
541        .served_rung
542        .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
543        .map(|a| a.model.clone());
544    let summary = match &served_model {
545        Some(m) => format!(
546            "served {m} at rung {} after {} escalation(s); spent ${:.4} vs ${:.4} always-top (saved ${:.4})",
547            t.final_.served_rung.unwrap_or(0),
548            t.final_.escalations,
549            t.final_.total_cost_usd,
550            t.final_.counterfactual_baseline_usd,
551            t.final_.savings_usd
552        ),
553        None => "no rung served (all attempts failed the gate or errored)".to_owned(),
554    };
555    RouteExplanation {
556        trace_id: t.trace_id.to_string(),
557        mode: match t.mode {
558            Mode::Enforce => "enforce",
559            Mode::Observe => "observe",
560        }
561        .to_owned(),
562        policy: t.policy.id.clone(),
563        served_model,
564        served_rung: t.final_.served_rung,
565        escalations: t.final_.escalations,
566        total_cost_usd: t.final_.total_cost_usd,
567        baseline_usd: t.final_.counterfactual_baseline_usd,
568        savings_usd: t.final_.savings_usd,
569        attempts,
570        summary,
571    }
572}
573
574/// Outcome of an independent receipt-chain verification — the compliance artifact.
575#[derive(Debug, serde::Serialize)]
576pub struct VerifyReport {
577    /// Receipts checked.
578    pub receipts: usize,
579    /// `true` iff the hash chain re-derives cleanly from genesis (tamper-evident proof).
580    pub valid: bool,
581    /// On failure, the 0-based index of the first broken link and why.
582    #[serde(skip_serializing_if = "Option::is_none")]
583    pub broken_at: Option<usize>,
584    #[serde(skip_serializing_if = "Option::is_none")]
585    pub detail: Option<String>,
586}
587
588/// Independently re-derive the hash chain over `traces` from genesis — the same computation an
589/// external auditor runs, with no trust in the proxy or the database. A single altered or
590/// reordered receipt breaks the chain at its index.
591#[must_use]
592pub fn verify_receipts(traces: &[Trace]) -> VerifyReport {
593    match firstpass_core::verify_chain(traces, firstpass_core::GENESIS_HASH) {
594        Ok(()) => VerifyReport {
595            receipts: traces.len(),
596            valid: true,
597            broken_at: None,
598            detail: None,
599        },
600        Err(firstpass_core::Error::ChainBroken { index, detail }) => VerifyReport {
601            receipts: traces.len(),
602            valid: false,
603            broken_at: Some(index),
604            detail: Some(detail),
605        },
606        Err(e) => VerifyReport {
607            receipts: traces.len(),
608            valid: false,
609            broken_at: None,
610            detail: Some(e.to_string()),
611        },
612    }
613}
614
615/// Parse a JSONL receipt export (one [`Trace`] per line) back into traces, for offline
616/// verification. Blank lines are skipped; a malformed line fails loudly with its line number.
617///
618/// # Errors
619/// Returns the 1-based line number and parse error of the first unparseable line.
620pub fn parse_receipt_jsonl(text: &str) -> Result<Vec<Trace>, String> {
621    let mut traces = Vec::new();
622    for (i, line) in text.lines().enumerate() {
623        if line.trim().is_empty() {
624            continue;
625        }
626        let t: Trace = serde_json::from_str(line).map_err(|e| format!("line {}: {e}", i + 1))?;
627        traces.push(t);
628    }
629    Ok(traces)
630}
631
632/// Render `traces` as a JSONL receipt export — one sealed receipt per line, in chain order.
633/// This is the artifact an operator hands an auditor; the deferred-verdict side table is never
634/// included (it is not part of the hashed body).
635#[must_use]
636/// One decision, reshaped as a training example: context, action, reward, propensity.
637///
638/// A receipt is an audit record — nested, provenance-heavy, and shaped for a human or an auditor.
639/// A learner wants a flat row. The conversion is trivial except for one thing that is not, and is
640/// the reason this is worth shipping rather than leaving to a downstream script:
641///
642/// **The propensity is already logged.** Firstpass records the probability its logging policy
643/// assigned to the rung it chose, because it needs that denominator for IPS/SNIPS/DR off-policy
644/// evaluation. That is exactly the field offline RL needs and exactly the field a routing log
645/// normally lacks — without it, a learner trained on this data inherits the logging policy's bias
646/// with no way to correct for it. Rows where it is absent are marked, not silently defaulted to 1.
647#[derive(Debug, Clone, serde::Serialize)]
648pub struct TrainingRow {
649    /// Decision id, so a row can be traced back to the receipt it came from.
650    pub trace_id: String,
651    /// Context the decision was made in.
652    pub features: firstpass_core::Features,
653    /// The action taken: which rung the ladder started on.
654    pub start_rung: u32,
655    /// Which rung actually served, if any.
656    pub served_rung: Option<u32>,
657    /// Whether the served output passed its gate — the reward signal, and the honest one, because
658    /// it is a verified outcome rather than a predicted score.
659    pub served_passed: bool,
660    /// Total USD spent on the decision. Cost-aware objectives need this alongside the reward.
661    pub cost_usd: f64,
662    /// Probability the logging policy gave the chosen action. `None` for a deterministic policy —
663    /// an un-corrected row, which a learner must handle rather than assume away.
664    pub propensity: Option<f64>,
665    /// Whether this was a deliberate exploration draw.
666    pub explore: bool,
667    /// Logging policy identity, so rows from different policies can be separated.
668    pub policy_id: String,
669}
670
671impl TrainingRow {
672    /// Reshape one receipt.
673    pub fn from_trace(t: &Trace) -> Self {
674        let served_rung = t.final_.served_rung;
675        Self {
676            trace_id: t.trace_id.to_string(),
677            features: t.request.features.clone(),
678            // The first attempt is where the ladder started; absent attempts means nothing ran.
679            start_rung: t.attempts.first().map_or(0, |a| a.rung),
680            served_rung,
681            served_passed: served_rung.is_some_and(|r| {
682                t.attempts
683                    .iter()
684                    .any(|a| a.rung == r && a.verdict == firstpass_core::Verdict::Pass)
685            }),
686            cost_usd: t.final_.total_cost_usd,
687            propensity: t.policy.propensity,
688            explore: t.policy.explore,
689            policy_id: t.policy.id.clone(),
690        }
691    }
692}
693
694/// Firstpass's own added latency, summarized from receipts.
695///
696/// Percentiles rather than a mean: overhead is right-skewed (a cold gate process, a slow
697/// subprocess spawn), and a mean hides exactly the tail a caller feels.
698#[derive(Debug, Clone, serde::Serialize)]
699pub struct OverheadSummary {
700    /// Receipts that yielded an honest figure.
701    pub n: usize,
702    /// Receipts skipped because concurrency made the subtraction meaningless (speculation).
703    ///
704    /// Reported, never silently dropped: if most of a deployment's traffic is speculative, the
705    /// percentiles below describe a minority of it and the reader needs to know that.
706    pub skipped_concurrent: usize,
707    pub p50_ms: u64,
708    pub p95_ms: u64,
709    pub p99_ms: u64,
710    pub max_ms: u64,
711}
712
713/// Summarize proxy overhead across `traces`.
714#[must_use]
715pub fn summarize_overhead(traces: &[Trace]) -> OverheadSummary {
716    let mut v: Vec<u64> = Vec::with_capacity(traces.len());
717    let mut skipped = 0usize;
718    for t in traces {
719        match t.overhead_ms() {
720            Some(ms) => v.push(ms),
721            None => skipped += 1,
722        }
723    }
724    v.sort_unstable();
725    // Nearest-rank percentile: with a handful of receipts an interpolated one invents a value
726    // between two real measurements, and every number here should be one we actually observed.
727    let pct = |p: f64| -> u64 {
728        if v.is_empty() {
729            return 0;
730        }
731        let rank = (p * v.len() as f64).ceil() as usize;
732        v[rank.clamp(1, v.len()) - 1]
733    };
734    OverheadSummary {
735        n: v.len(),
736        skipped_concurrent: skipped,
737        p50_ms: pct(0.50),
738        p95_ms: pct(0.95),
739        p99_ms: pct(0.99),
740        max_ms: v.last().copied().unwrap_or(0),
741    }
742}
743
744/// Render [`summarize_overhead`] for a terminal.
745#[must_use]
746pub fn format_overhead(s: &OverheadSummary) -> String {
747    if s.n == 0 {
748        return format!(
749            "no receipts with a measurable overhead ({} skipped as concurrent).\n\
750             Route some traffic first, or run with speculation off to measure it.\n",
751            s.skipped_concurrent
752        );
753    }
754    // Built line by line rather than as one continued literal: a `\`-continuation silently keeps
755    // the next line's indentation, which is how the first version of this shipped a wrapped
756    // sentence indented fifteen spaces into the middle of a report.
757    let mut lines = vec![
758        "Firstpass overhead — time added beyond the provider calls and gates it ran".to_owned(),
759        String::new(),
760        format!("  p50  {:>6} ms", s.p50_ms),
761        format!("  p95  {:>6} ms", s.p95_ms),
762        format!("  p99  {:>6} ms", s.p99_ms),
763        format!("  max  {:>6} ms", s.max_ms),
764        String::new(),
765        format!("  over {} receipt(s)", s.n),
766    ];
767    // A p95 of 0 is a real result for a Rust proxy, but printed bare it reads like the number is
768    // not being measured at all. Receipt timings are whole milliseconds, so say what 0 means.
769    if s.p95_ms == 0 {
770        lines.push(
771            "  0 ms is the floor, not a missing measurement: receipts record whole".to_owned(),
772        );
773        lines.push(
774            "  milliseconds, so the proxy's own work stayed under 1 ms per request.".to_owned(),
775        );
776    }
777    if s.skipped_concurrent > 0 {
778        lines.push(format!(
779            "  {} skipped: speculative rungs run concurrently, so subtracting dispatched",
780            s.skipped_concurrent
781        ));
782        lines.push(
783            "  work from wall-clock is meaningless there — excluded, not floored to 0.".to_owned(),
784        );
785    }
786    lines.push(String::new());
787    lines.join("\n")
788}
789
790/// Reshape receipts into JSONL training rows (`firstpass export --format rl`).
791#[must_use]
792pub fn export_training_jsonl(traces: &[Trace]) -> String {
793    let mut out = String::new();
794    for t in traces {
795        if let Ok(line) = serde_json::to_string(&TrainingRow::from_trace(t)) {
796            out.push_str(&line);
797            out.push('\n');
798        }
799    }
800    out
801}
802
803pub fn export_receipts_jsonl(traces: &[Trace]) -> String {
804    let mut out = String::new();
805    for t in traces {
806        if let Ok(line) = serde_json::to_string(t) {
807            out.push_str(&line);
808            out.push('\n');
809        }
810    }
811    out
812}
813
814/// Aggregated spend/savings over a set of traces — the number the operator screenshots.
815/// Pure so it's unit-testable; `firstpass savings` feeds it the trace store.
816#[derive(Debug, serde::Serialize)]
817pub struct SavingsSummary {
818    /// Traces aggregated.
819    pub traces: usize,
820    /// Enforce-mode traces (the ones where routing actually decided).
821    pub enforce_traces: usize,
822    /// USD actually spent (model + gate calls), summed over all traces.
823    pub spent_usd: f64,
824    /// USD spent on gates alone (the price of proof).
825    pub gate_usd: f64,
826    /// What always calling the top rung would have cost (§9.1 counterfactual), summed.
827    pub baseline_usd: f64,
828    /// `baseline - spent` — the savings the receipts prove.
829    pub savings_usd: f64,
830    /// Savings as a fraction of baseline, `0.0` when there is no baseline.
831    pub savings_pct: f64,
832}
833
834/// Aggregate spend vs the always-top counterfactual over `traces`.
835#[must_use]
836pub fn summarize_savings(traces: &[Trace]) -> SavingsSummary {
837    let mut s = SavingsSummary {
838        traces: traces.len(),
839        enforce_traces: 0,
840        spent_usd: 0.0,
841        gate_usd: 0.0,
842        baseline_usd: 0.0,
843        savings_usd: 0.0,
844        savings_pct: 0.0,
845    };
846    for t in traces {
847        if t.mode == Mode::Enforce {
848            s.enforce_traces += 1;
849        }
850        s.spent_usd += t.final_.total_cost_usd;
851        s.gate_usd += t.final_.gate_cost_usd;
852        s.baseline_usd += t.final_.counterfactual_baseline_usd;
853    }
854    s.savings_usd = s.baseline_usd - s.spent_usd;
855    if s.baseline_usd > 0.0 {
856        s.savings_pct = s.savings_usd / s.baseline_usd;
857    }
858    s
859}
860
861/// Render a [`SavingsSummary`] for humans (`--json` callers serialize the struct instead).
862#[must_use]
863pub fn format_savings(s: &SavingsSummary) -> String {
864    if s.traces == 0 {
865        return "no traces recorded yet — route some traffic through the proxy first".to_owned();
866    }
867    format!(
868        "traces: {} ({} enforce)\nspent:    ${:.4}  (gates ${:.4})\nbaseline: ${:.4}  (always top rung)\nsavings:  ${:.4}  ({:.1}%)",
869        s.traces,
870        s.enforce_traces,
871        s.spent_usd,
872        s.gate_usd,
873        s.baseline_usd,
874        s.savings_usd,
875        s.savings_pct * 100.0
876    )
877}
878
879/// Render the most recent `limit` traces as JSON lines — machine-first (SPEC §0.2): each line is a
880/// full [`Trace`], newest last.
881#[must_use]
882pub fn format_traces(traces: &[Trace], limit: usize) -> String {
883    let mut lines: Vec<String> = traces
884        .iter()
885        .rev()
886        .take(limit)
887        .filter_map(|t| serde_json::to_string(t).ok())
888        .collect();
889    lines.reverse();
890    if lines.is_empty() {
891        "no traces recorded yet".to_owned()
892    } else {
893        lines.join("\n")
894    }
895}
896
897/// What `firstpass experiment` prints when there is nothing to report: the SPECIFIC reason and the
898/// next step, instead of an empty table. `routing` is the loaded config, `db_exists` whether the
899/// receipt store file is on disk.
900#[must_use]
901pub fn experiment_next_step(
902    routing: Option<&firstpass_core::Config>,
903    db_path: &str,
904    db_exists: bool,
905) -> String {
906    const SNIPPET: &str = "  [[route]]\n  mode   = \"enforce\"\n  ladder = [\"anthropic/claude-opus-4-5\"]   # what you run today\n\n  [route.challenger]\n  ladder  = [\"openrouter/minimax-m2.5\"]    # what you would like to switch to\n  percent = 10                              # % of sessions served by it\n";
907    let challengers = routing.map_or(0, |c| {
908        c.routes.iter().filter(|r| r.challenger.is_some()).count()
909    });
910    if challengers == 0 {
911        let why = if routing.is_none() {
912            "no routing config is loaded (set FIRSTPASS_CONFIG=./firstpass.toml, or run \
913             `firstpass onboard`)"
914        } else {
915            "no route in the loaded config has a [route.challenger]"
916        };
917        return format!(
918            "No experiment to report: {why}.\n\nAdd a challenger to the route you want to \
919             re-evaluate, then restart the proxy:\n\n{SNIPPET}\nThen post task outcomes to \
920             POST /v1/feedback as you already do, and run `firstpass experiment` again.\n\
921             Guide: docs/guide/experiments.html\n"
922        );
923    }
924    let store = if db_exists {
925        format!("The receipt store ({db_path}) has no experiment traffic for this tenant yet.")
926    } else {
927        format!("No receipt store at {db_path} yet: the proxy has not served a request.")
928    };
929    format!(
930        "{challengers} route(s) have a [route.challenger], but there is nothing to report yet.\n\
931         {store}\n\nNext: send requests through the proxy (each session is randomized to an arm), \
932         then post one task outcome per session:\n\n  curl -s localhost:8080/v1/feedback -d \
933         '{{\"trace_id\":\"<id from the receipt>\",\"gate_id\":\"tests\",\"verdict\":\"pass\",\
934         \"reporter\":\"ci\"}}'\n\nThe panel at /panel shows the same evidence live. Nothing is \
935         adopted for you.\n"
936    )
937}
938
939#[cfg(test)]
940mod tests {
941    use super::*;
942
943    #[test]
944    fn experiment_next_step_names_the_missing_piece() {
945        // No config at all: say how to load one and show the snippet.
946        let none = experiment_next_step(None, "x.db", false);
947        assert!(none.contains("no routing config is loaded"));
948        assert!(none.contains("[route.challenger]"));
949
950        // Config without a challenger: a different reason, same snippet.
951        let plain = config_with(
952            Some(
953                "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
954            ),
955            "x.db",
956        );
957        let r = experiment_next_step(plain.routing.as_ref(), "x.db", false);
958        assert!(r.contains("no route in the loaded config has a [route.challenger]"));
959
960        // Challenger configured: the next step is traffic + feedback, not config.
961        let with = config_with(
962            Some(
963                "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n\
964                 [route.challenger]\nladder = [\"anthropic/claude-haiku-4-5\"]\npercent = 10.0\n",
965            ),
966            "x.db",
967        );
968        let r = experiment_next_step(with.routing.as_ref(), "x.db", false);
969        assert!(r.contains("1 route(s) have a [route.challenger]"));
970        assert!(r.contains("has not served a request"));
971        assert!(r.contains("/v1/feedback"));
972        let r = experiment_next_step(with.routing.as_ref(), "x.db", true);
973        assert!(r.contains("no experiment traffic"));
974    }
975
976    fn config_with(toml: Option<&str>, db_path: &str) -> ProxyConfig {
977        ProxyConfig::from_lookup(|k| match k {
978            "FIRSTPASS_MODE" if toml.is_some() => Some("enforce".to_owned()),
979            "FIRSTPASS_CONFIG_TOML" => toml.map(str::to_owned),
980            "FIRSTPASS_DB" => Some(db_path.to_owned()),
981            _ => None,
982        })
983        .unwrap()
984    }
985
986    #[test]
987    fn doctor_flags_a_ladder_rung_whose_provider_key_is_missing() {
988        // Without this the failure surfaces on the first real request, in production, as an
989        // upstream error — the least useful moment to learn the key was never set.
990        let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"groq/llama-3.3-70b-versatile\"]\n\
991                    [[price]]\nmodel = \"groq/llama-3.3-70b-versatile\"\n\
992                    input_per_mtok = 0.59\noutput_per_mtok = 0.79\n";
993        let config = config_with(Some(toml), "/tmp/doctor-groq.db");
994
995        let report = doctor(&config, |k| match k {
996            "PATH" => Some("/usr/bin".to_owned()),
997            _ => None, // no GROQ_API_KEY
998        });
999
1000        let groq = report
1001            .checks
1002            .iter()
1003            .find(|c| c.name == "provider:groq")
1004            .expect("groq rung should be checked");
1005        assert_eq!(groq.status, CheckStatus::Fail);
1006        assert!(groq.detail.contains("GROQ_API_KEY"), "{}", groq.detail);
1007        assert!(
1008            !report.healthy(),
1009            "a rung that cannot authenticate is not healthy"
1010        );
1011    }
1012
1013    #[test]
1014    fn doctor_stays_quiet_about_providers_the_ladder_does_not_use() {
1015        // Warning about all eight built-ins would be noise, and noise trains an operator to skip
1016        // the report — which is how the one real warning gets missed.
1017        let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\n";
1018        let config = config_with(Some(toml), "/tmp/doctor-quiet.db");
1019
1020        let report = doctor(&config, |k| match k {
1021            "PATH" => Some("/usr/bin".to_owned()),
1022            "ANTHROPIC_API_KEY" => Some("sk-test".to_owned()),
1023            _ => None,
1024        });
1025
1026        assert!(
1027            !report
1028                .checks
1029                .iter()
1030                .any(|c| c.name.starts_with("provider:")),
1031            "unused providers must not be reported: {:?}",
1032            report.checks.iter().map(|c| &c.name).collect::<Vec<_>>()
1033        );
1034    }
1035
1036    #[test]
1037    fn command_on_path_finds_real_and_rejects_fake() {
1038        let path = std::env::var("PATH").ok();
1039        assert!(command_on_path("sh", path.as_deref()), "sh is on PATH");
1040        assert!(!command_on_path(
1041            "firstpass-definitely-not-a-real-binary",
1042            path.as_deref()
1043        ));
1044        assert!(!command_on_path("/nonexistent/abs/path", None));
1045    }
1046
1047    #[test]
1048    fn doctor_flags_a_missing_gate_binary() {
1049        let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"good\", \"bad\"]\n\
1050                    [[gate]]\nid = \"good\"\ncmd = [\"sh\"]\n\
1051                    [[gate]]\nid = \"bad\"\ncmd = [\"firstpass-nope-not-real\"]\n";
1052        let db = std::env::temp_dir().join("firstpass-doctor-test.db");
1053        let config = config_with(Some(toml), db.to_str().unwrap());
1054
1055        // Real PATH so `sh` resolves; no ANTHROPIC_API_KEY -> a warning, not a failure.
1056        let report = doctor(&config, |k| match k {
1057            "PATH" => std::env::var("PATH").ok(),
1058            _ => None,
1059        });
1060
1061        assert!(
1062            !report.healthy(),
1063            "a missing gate binary must fail the report"
1064        );
1065        let bad = report.checks.iter().find(|c| c.name == "gate:bad").unwrap();
1066        assert_eq!(bad.status, CheckStatus::Fail);
1067        let good = report
1068            .checks
1069            .iter()
1070            .find(|c| c.name == "gate:good")
1071            .unwrap();
1072        assert_eq!(good.status, CheckStatus::Ok);
1073        let key = report
1074            .checks
1075            .iter()
1076            .find(|c| c.name == "anthropic-key")
1077            .unwrap();
1078        assert_eq!(key.status, CheckStatus::Warn);
1079    }
1080
1081    #[test]
1082    fn doctor_is_healthy_for_a_plain_observe_setup() {
1083        let db = std::env::temp_dir().join("firstpass-doctor-ok.db");
1084        let config = config_with(None, db.to_str().unwrap());
1085        let report = doctor(&config, |k| {
1086            (k == "ANTHROPIC_API_KEY").then(|| "sk-test".to_owned())
1087        });
1088        assert!(report.healthy(), "{}", report.render());
1089    }
1090
1091    #[test]
1092    fn format_traces_handles_empty() {
1093        assert_eq!(format_traces(&[], 10), "no traces recorded yet");
1094    }
1095
1096    #[test]
1097    fn savings_summary_empty_and_nonempty() {
1098        let empty = summarize_savings(&[]);
1099        assert_eq!(empty.traces, 0);
1100        assert!(format_savings(&empty).contains("no traces"));
1101    }
1102
1103    #[test]
1104    fn evals_summary_empty_zero_state() {
1105        let s = summarize_evals(&[]);
1106        assert_eq!(s.enforce_traces, 0);
1107        assert!(format_evals(&s).contains("no enforce traces"));
1108    }
1109
1110    #[test]
1111    fn verify_and_export_round_trip_detects_tampering() {
1112        use firstpass_core::{
1113            Attempt, Features, FinalOutcome, GENESIS_HASH, PolicyRef, RequestInfo, ServedFrom,
1114            TaskKind, Verdict,
1115        };
1116
1117        // Build a valid 3-link chain: each prev_hash = the previous record's hash.
1118        let mk = |prev: &str, session: &str| Trace {
1119            trace_id: uuid::Uuid::now_v7(),
1120            prev_hash: prev.to_owned(),
1121            tenant_id: "default".to_owned(),
1122            session_id: session.to_owned(),
1123            ts: jiff::Timestamp::UNIX_EPOCH,
1124            mode: Mode::Observe,
1125            policy: PolicyRef {
1126                id: "observe-passthrough@v0".to_owned(),
1127                explore: false,
1128                propensity: None,
1129                mode_profile: None,
1130            },
1131            request: RequestInfo {
1132                api: "anthropic.messages".to_owned(),
1133                prompt_hash: "deadbeef".to_owned(),
1134                features: Features::new(TaskKind::Other),
1135            },
1136            attempts: vec![Attempt {
1137                rung: 0,
1138                model: "anthropic/claude-haiku-4-5".to_owned(),
1139                provider: "anthropic".to_owned(),
1140                in_tokens: 10,
1141                cache_write_tokens: 0,
1142                cache_read_tokens: 0,
1143                out_tokens: 5,
1144                cost_usd: 0.001,
1145                latency_ms: 12,
1146                gates: vec![],
1147                verdict: Verdict::Pass,
1148                truncated: false,
1149            }],
1150            deferred: Vec::new(),
1151            final_: FinalOutcome {
1152                served_rung: Some(0),
1153                served_from: ServedFrom::Attempt,
1154                total_cost_usd: 0.001,
1155                gate_cost_usd: 0.0,
1156                total_latency_ms: 12,
1157                escalations: 0,
1158                counterfactual_baseline_usd: 0.001,
1159                savings_usd: 0.0,
1160                cache_source: None,
1161            },
1162            probe: None,
1163            rollout: None,
1164            shadow: None,
1165            route_ix: None,
1166            predicted_pass: None,
1167            decision_prior: None,
1168            elastic: None,
1169            risk: None,
1170            experiment: None,
1171            upstream_provider: String::new(),
1172        };
1173        let t0 = mk(GENESIS_HASH, "s0");
1174        let t1 = mk(&t0.hash().unwrap(), "s1");
1175        let t2 = mk(&t1.hash().unwrap(), "s2");
1176        let chain = vec![t0, t1, t2];
1177
1178        // Clean chain verifies.
1179        let report = verify_receipts(&chain);
1180        assert!(report.valid, "intact chain must verify: {report:?}");
1181        assert_eq!(report.receipts, 3);
1182
1183        // Export → parse round-trips and still verifies (the auditor's offline path).
1184        let jsonl = export_receipts_jsonl(&chain);
1185        assert_eq!(jsonl.lines().count(), 3);
1186        let reparsed = parse_receipt_jsonl(&jsonl).expect("export must re-parse");
1187        assert!(
1188            verify_receipts(&reparsed).valid,
1189            "round-tripped chain must verify"
1190        );
1191
1192        // Tamper with a middle receipt's body → verification catches it at that index.
1193        let mut tampered = reparsed;
1194        tampered[1].final_.total_cost_usd = 999.0; // alter a sealed field
1195        let bad = verify_receipts(&tampered);
1196        assert!(!bad.valid, "a mutated receipt must break the chain");
1197        assert_eq!(
1198            bad.broken_at,
1199            Some(2),
1200            "the break surfaces at the next link"
1201        );
1202
1203        // Reordering is also caught.
1204        let mut reordered = parse_receipt_jsonl(&jsonl).unwrap();
1205        reordered.swap(0, 1);
1206        assert!(
1207            !verify_receipts(&reordered).valid,
1208            "reordering breaks the chain"
1209        );
1210    }
1211
1212    #[test]
1213    fn parse_receipt_jsonl_reports_bad_line() {
1214        let err = parse_receipt_jsonl("not json\n").unwrap_err();
1215        assert!(err.starts_with("line 1:"), "must name the bad line: {err}");
1216        assert!(
1217            parse_receipt_jsonl("\n\n").unwrap().is_empty(),
1218            "blank lines skipped"
1219        );
1220    }
1221
1222    fn trace_with_timing(total_ms: u64, attempts: &[(u64, u64)]) -> Trace {
1223        use firstpass_core::{
1224            Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1225            ServedFrom, TaskKind, Verdict,
1226        };
1227        Trace {
1228            trace_id: uuid::Uuid::now_v7(),
1229            prev_hash: GENESIS_HASH.to_owned(),
1230            tenant_id: "default".to_owned(),
1231            session_id: "s".to_owned(),
1232            ts: jiff::Timestamp::UNIX_EPOCH,
1233            mode: Mode::Enforce,
1234            policy: PolicyRef {
1235                id: "static-ladder@v0".to_owned(),
1236                explore: false,
1237                propensity: None,
1238                mode_profile: None,
1239            },
1240            request: RequestInfo {
1241                api: "anthropic.messages".to_owned(),
1242                prompt_hash: "x".to_owned(),
1243                features: Features::new(TaskKind::CodeEdit),
1244            },
1245            attempts: attempts
1246                .iter()
1247                .enumerate()
1248                .map(|(i, (model_ms, gate_ms))| Attempt {
1249                    rung: i as u32,
1250                    model: "anthropic/claude-haiku-4-5".to_owned(),
1251                    provider: "anthropic".to_owned(),
1252                    in_tokens: 1,
1253                    cache_write_tokens: 0,
1254                    cache_read_tokens: 0,
1255                    out_tokens: 1,
1256                    cost_usd: 0.0,
1257                    latency_ms: *model_ms,
1258                    gates: vec![GateResult::deterministic(
1259                        "non-empty",
1260                        Verdict::Pass,
1261                        *gate_ms,
1262                    )],
1263                    verdict: Verdict::Pass,
1264                    truncated: false,
1265                })
1266                .collect(),
1267            final_: FinalOutcome {
1268                served_rung: Some(0),
1269                served_from: ServedFrom::Attempt,
1270                total_cost_usd: 0.0,
1271                gate_cost_usd: 0.0,
1272                total_latency_ms: total_ms,
1273                escalations: 0,
1274                counterfactual_baseline_usd: 0.0,
1275                savings_usd: 0.0,
1276                cache_source: None,
1277            },
1278            deferred: vec![],
1279            predicted_pass: None,
1280            decision_prior: None,
1281            probe: None,
1282            elastic: None,
1283            risk: None,
1284            rollout: None,
1285            shadow: None,
1286            route_ix: None,
1287            experiment: None,
1288            upstream_provider: String::new(),
1289        }
1290    }
1291
1292    #[test]
1293    fn overhead_is_wall_clock_minus_the_work_actually_dispatched() {
1294        // 500ms total, 400ms in the provider, 60ms in the gate → 40ms is ours.
1295        let t = trace_with_timing(500, &[(400, 60)]);
1296        assert_eq!(t.overhead_ms(), Some(40));
1297
1298        // Escalation: two provider calls and two gate runs all count against wall-clock.
1299        let t = trace_with_timing(1000, &[(400, 50), (500, 20)]);
1300        assert_eq!(t.overhead_ms(), Some(30));
1301    }
1302
1303    #[test]
1304    fn a_concurrent_receipt_is_excluded_rather_than_floored_to_zero() {
1305        // Speculation runs rungs in parallel, so dispatched work legitimately exceeds wall-clock.
1306        // Flooring that to 0 would be a fabricated measurement that drags a p95 down — exactly the
1307        // kind of quietly-wrong number this codebase keeps having to remove.
1308        let t = trace_with_timing(600, &[(400, 50), (500, 50)]);
1309        assert_eq!(t.overhead_ms(), None, "must not report a made-up 0");
1310
1311        let s = summarize_overhead(&[t, trace_with_timing(500, &[(400, 60)])]);
1312        assert_eq!(s.n, 1, "only the measurable one counts");
1313        assert_eq!(
1314            s.skipped_concurrent, 1,
1315            "and the skip is reported, not hidden"
1316        );
1317        assert_eq!(s.p50_ms, 40);
1318    }
1319
1320    #[test]
1321    fn percentiles_are_observed_values_not_interpolated_ones() {
1322        // Nearest-rank: every number printed is one we actually measured. Interpolation would
1323        // invent a value between two real observations and report it as fact.
1324        let traces: Vec<Trace> = [10u64, 20, 30, 40, 100]
1325            .iter()
1326            .map(|ms| trace_with_timing(100 + ms, &[(100, 0)]))
1327            .collect();
1328
1329        let s = summarize_overhead(&traces);
1330
1331        assert_eq!(s.n, 5);
1332        assert_eq!(s.p50_ms, 30);
1333        assert_eq!(s.p95_ms, 100);
1334        assert_eq!(s.p99_ms, 100);
1335        assert_eq!(s.max_ms, 100);
1336        for v in [s.p50_ms, s.p95_ms, s.p99_ms] {
1337            assert!([10, 20, 30, 40, 100].contains(&v), "{v} was never observed");
1338        }
1339    }
1340
1341    #[test]
1342    fn no_receipts_reports_that_rather_than_zeros() {
1343        let s = summarize_overhead(&[]);
1344        assert_eq!(s.n, 0);
1345        assert!(
1346            format_overhead(&s).contains("no receipts"),
1347            "{}",
1348            format_overhead(&s)
1349        );
1350    }
1351
1352    #[test]
1353    fn training_rows_carry_the_logged_propensity() {
1354        use firstpass_core::{
1355            Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1356            ServedFrom, TaskKind, Verdict,
1357        };
1358        let mk = |propensity: Option<f64>, explore: bool| Trace {
1359            trace_id: uuid::Uuid::now_v7(),
1360            prev_hash: GENESIS_HASH.to_owned(),
1361            tenant_id: "default".to_owned(),
1362            session_id: "s".to_owned(),
1363            ts: jiff::Timestamp::UNIX_EPOCH,
1364            mode: Mode::Enforce,
1365            policy: PolicyRef {
1366                id: "bandit@v1+eps".to_owned(),
1367                explore,
1368                propensity,
1369                mode_profile: None,
1370            },
1371            request: RequestInfo {
1372                api: "anthropic.messages".to_owned(),
1373                prompt_hash: "x".to_owned(),
1374                features: Features::new(TaskKind::CodeEdit),
1375            },
1376            attempts: vec![Attempt {
1377                rung: 1,
1378                model: "anthropic/claude-sonnet-5".to_owned(),
1379                provider: "anthropic".to_owned(),
1380                in_tokens: 10,
1381                cache_write_tokens: 0,
1382                cache_read_tokens: 0,
1383                out_tokens: 5,
1384                cost_usd: 0.004,
1385                latency_ms: 7,
1386                gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
1387                verdict: Verdict::Pass,
1388                truncated: false,
1389            }],
1390            final_: FinalOutcome {
1391                served_rung: Some(1),
1392                served_from: ServedFrom::Attempt,
1393                total_cost_usd: 0.004,
1394                gate_cost_usd: 0.0,
1395                total_latency_ms: 7,
1396                escalations: 0,
1397                counterfactual_baseline_usd: 0.02,
1398                savings_usd: 0.016,
1399                cache_source: None,
1400            },
1401            deferred: vec![],
1402            predicted_pass: None,
1403            decision_prior: None,
1404            probe: None,
1405            elastic: None,
1406            risk: None,
1407            rollout: None,
1408            shadow: None,
1409            route_ix: None,
1410            experiment: None,
1411            upstream_provider: String::new(),
1412        };
1413
1414        let rows = export_training_jsonl(&[mk(Some(0.25), true), mk(None, false)]);
1415        let lines: Vec<&str> = rows.lines().collect();
1416        assert_eq!(lines.len(), 2);
1417
1418        let a: serde_json::Value = serde_json::from_str(lines[0]).expect("row is json");
1419        // The whole point: the propensity survives into the training row. Without it a learner
1420        // inherits the logging policy's bias with no way to correct for it.
1421        assert_eq!(a["propensity"], 0.25);
1422        assert_eq!(a["explore"], true);
1423        assert_eq!(a["start_rung"], 1);
1424        assert_eq!(a["served_rung"], 1);
1425        assert_eq!(a["served_passed"], true);
1426        assert_eq!(a["policy_id"], "bandit@v1+eps");
1427
1428        // A deterministic decision must be marked un-corrected, NOT silently defaulted to 1.0 —
1429        // that would look like a uniformly-sampled row and quietly bias any estimate built on it.
1430        let b: serde_json::Value = serde_json::from_str(lines[1]).expect("row is json");
1431        assert!(b["propensity"].is_null(), "got {}", b["propensity"]);
1432    }
1433
1434    #[test]
1435    fn explain_trace_summarizes_served_and_escalation() {
1436        use firstpass_core::{
1437            Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1438            ServedFrom, TaskKind, Verdict,
1439        };
1440        let t = Trace {
1441            trace_id: uuid::Uuid::now_v7(),
1442            prev_hash: GENESIS_HASH.to_owned(),
1443            tenant_id: "default".to_owned(),
1444            session_id: "s".to_owned(),
1445            ts: jiff::Timestamp::UNIX_EPOCH,
1446            mode: Mode::Enforce,
1447            policy: PolicyRef {
1448                id: "static-ladder@v0".to_owned(),
1449                explore: false,
1450                propensity: None,
1451                mode_profile: None,
1452            },
1453            request: RequestInfo {
1454                api: "anthropic.messages".to_owned(),
1455                prompt_hash: "x".to_owned(),
1456                features: Features::new(TaskKind::Other),
1457            },
1458            attempts: vec![
1459                Attempt {
1460                    rung: 0,
1461                    model: "anthropic/claude-haiku-4-5".to_owned(),
1462                    provider: "anthropic".to_owned(),
1463                    in_tokens: 1,
1464                    cache_write_tokens: 0,
1465                    cache_read_tokens: 0,
1466                    out_tokens: 1,
1467                    cost_usd: 0.001,
1468                    latency_ms: 5,
1469                    gates: vec![GateResult::deterministic("non-empty", Verdict::Fail, 0)],
1470                    verdict: Verdict::Fail,
1471                    truncated: false,
1472                },
1473                Attempt {
1474                    rung: 1,
1475                    model: "anthropic/claude-sonnet-5".to_owned(),
1476                    provider: "anthropic".to_owned(),
1477                    in_tokens: 1,
1478                    cache_write_tokens: 0,
1479                    cache_read_tokens: 0,
1480                    out_tokens: 1,
1481                    cost_usd: 0.01,
1482                    latency_ms: 9,
1483                    gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
1484                    verdict: Verdict::Pass,
1485                    truncated: false,
1486                },
1487            ],
1488            deferred: Vec::new(),
1489            final_: FinalOutcome {
1490                served_rung: Some(1),
1491                served_from: ServedFrom::Attempt,
1492                total_cost_usd: 0.011,
1493                gate_cost_usd: 0.0,
1494                total_latency_ms: 14,
1495                escalations: 1,
1496                counterfactual_baseline_usd: 0.02,
1497                savings_usd: 0.009,
1498                cache_source: None,
1499            },
1500            probe: None,
1501            rollout: None,
1502            shadow: None,
1503            route_ix: None,
1504            predicted_pass: None,
1505            decision_prior: None,
1506            elastic: None,
1507            risk: None,
1508            experiment: None,
1509            upstream_provider: String::new(),
1510        };
1511        let ex = explain_trace(&t);
1512        assert_eq!(
1513            ex.served_model.as_deref(),
1514            Some("anthropic/claude-sonnet-5")
1515        );
1516        assert_eq!(ex.escalations, 1);
1517        assert_eq!(ex.attempts.len(), 2);
1518        assert_eq!(ex.attempts[0].verdict, "fail");
1519        assert_eq!(
1520            ex.attempts[1].gates[0],
1521            ("non-empty".to_owned(), "pass".to_owned())
1522        );
1523        assert!(ex.summary.contains("served anthropic/claude-sonnet-5"));
1524    }
1525
1526    #[test]
1527    fn evaluate_predictor_empty_and_signal() {
1528        // empty store => n=0, no crash
1529        let e = evaluate_predictor(&[], 0.05, 1e-4);
1530        assert_eq!(e.n, 0);
1531        assert!(e.auc.is_none());
1532    }
1533
1534    /// Each channel must be recognised from a realistic install path. Getting this wrong is not
1535    /// cosmetic: it would print an upgrade command that silently does nothing.
1536    #[test]
1537    fn detect_channel_reads_real_install_paths() {
1538        use std::path::Path;
1539        let cases = [
1540            ("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
1541            (
1542                "/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
1543                Channel::Homebrew,
1544            ),
1545            (
1546                "/home/linuxbrew/.linuxbrew/bin/firstpass",
1547                Channel::Homebrew,
1548            ),
1549            (
1550                "/Users/u/.venv/lib/python3.12/site-packages/firstpass/firstpass",
1551                Channel::Python,
1552            ),
1553            // Both observed on a real machine rather than invented: uvx runs from an ephemeral
1554            // env under archive-v0, and `uv tool install` lands under the uv tools dir.
1555            (
1556                "/Users/u/.cache/uv/archive-v0/DQ4usH_XuqNyywgHKB8qB/bin/firstpass",
1557                Channel::Python,
1558            ),
1559            (
1560                "/Users/u/.local/share/uv/tools/firstpass/bin/firstpass",
1561                Channel::Python,
1562            ),
1563            (
1564                "/usr/local/lib/node_modules/@dshakesnotbot/firstpass-proxy/bin/firstpass",
1565                Channel::Npm,
1566            ),
1567            (r"C:\Users\u\.cargo\bin\firstpass.exe", Channel::Cargo),
1568        ];
1569        for (path, want) in cases {
1570            assert_eq!(
1571                detect_channel(Path::new(path), false, false),
1572                want,
1573                "misread {path}"
1574            );
1575        }
1576        // The installer channel is identified by its updater, not by its path.
1577        assert_eq!(
1578            detect_channel(Path::new("/Users/u/.local/bin/firstpass"), true, false),
1579            Channel::Dist
1580        );
1581        assert_eq!(
1582            detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
1583            Channel::Unknown
1584        );
1585        // A container wins outright — nothing on the host owns that file.
1586        assert_eq!(
1587            detect_channel(Path::new("/usr/local/bin/firstpass"), true, true),
1588            Channel::Docker
1589        );
1590    }
1591
1592    /// A package manager owning the file must beat the updater probe: a brew or wheel install can
1593    /// carry the updater too, and self-updating underneath the manager desynchronises its metadata.
1594    #[test]
1595    fn package_manager_paths_win_over_the_self_updater() {
1596        use std::path::Path;
1597        for (path, want) in [
1598            (
1599                "/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
1600                Channel::Homebrew,
1601            ),
1602            ("/Users/u/.venv/bin/firstpass", Channel::Python),
1603            ("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
1604        ] {
1605            assert_eq!(detect_channel(Path::new(path), true, false), want, "{path}");
1606        }
1607    }
1608
1609    /// Only the channel we own may be upgraded in place; the rest must be advice, and the
1610    /// unknown case must still leave the user with something runnable.
1611    #[test]
1612    fn upgrade_report_runs_only_our_own_channel() {
1613        assert!(upgrade_report(Channel::Dist, "0.2.2").contains("upgrading in place"));
1614        for c in [
1615            Channel::Homebrew,
1616            Channel::Python,
1617            Channel::Npm,
1618            Channel::Docker,
1619            Channel::Cargo,
1620        ] {
1621            let r = upgrade_report(c, "0.2.2");
1622            assert!(
1623                !r.contains("upgrading in place"),
1624                "{c:?} must not self-update"
1625            );
1626            assert!(
1627                r.contains("owned by its package manager"),
1628                "{c:?} must explain why"
1629            );
1630            assert!(
1631                r.contains(c.upgrade_command().expect("has a command")),
1632                "{c:?} must print its exact command"
1633            );
1634        }
1635        let unknown = upgrade_report(Channel::Unknown, "0.2.2");
1636        assert!(unknown.contains("brew upgrade") && unknown.contains("docker pull"));
1637        assert!(
1638            unknown.contains("0.2.2"),
1639            "always states the running version"
1640        );
1641    }
1642
1643    /// Regression: reported from a real machine. pipx installs into its own venv and puts a
1644    /// symlink on PATH, so the launch path is `~/.local/bin/firstpass` and carries no marker at
1645    /// all. Detection runs on the CANONICAL path; these are the two forms of that install.
1646    #[test]
1647    fn pipx_install_is_recognised_from_its_real_path() {
1648        use std::path::Path;
1649        let real = "/Users/u/.local/pipx/venvs/firstpass/bin/firstpass";
1650        assert_eq!(detect_channel(Path::new(real), false, false), Channel::Pipx);
1651        assert_eq!(
1652            Channel::Pipx.upgrade_command(),
1653            Some("pipx upgrade firstpass")
1654        );
1655
1656        // The launch path alone is genuinely unidentifiable — which is why the caller must
1657        // canonicalize rather than this function pretending it can tell.
1658        assert_eq!(
1659            detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
1660            Channel::Unknown
1661        );
1662
1663        // A pipx venv also contains site-packages; pipx must win so the command is right.
1664        assert_eq!(
1665            detect_channel(
1666                Path::new(
1667                    "/Users/u/.local/pipx/venvs/firstpass/lib/python3.14/site-packages/x/firstpass"
1668                ),
1669                false,
1670                false
1671            ),
1672            Channel::Pipx
1673        );
1674    }
1675}