Skip to main content

firstpass_proxy/
cli.rs

1//! CLI surfaces for the `firstpass` binary (SPEC §7.3/§7.4): `doctor` validates a setup before
2//! you route real traffic through it, and `trace` reads recent audit records from the store. Both
3//! are kept here (not in the binary) so the judgment is unit-tested.
4
5use crate::config::ProxyConfig;
6use firstpass_core::{Mode, Trace};
7
8/// A single doctor check outcome.
9#[derive(Debug, PartialEq, Eq)]
10pub enum CheckStatus {
11    /// Healthy.
12    Ok,
13    /// Works, but worth knowing (e.g. no key in env — observe still runs).
14    Warn,
15    /// Broken; `firstpass doctor` exits non-zero.
16    Fail,
17}
18
19/// One line of the doctor report.
20#[derive(Debug)]
21pub struct Check {
22    /// Short check name.
23    pub name: String,
24    /// Outcome.
25    pub status: CheckStatus,
26    /// Human-readable detail.
27    pub detail: String,
28}
29
30impl Check {
31    fn new(name: impl Into<String>, status: CheckStatus, detail: impl Into<String>) -> Self {
32        Self {
33            name: name.into(),
34            status,
35            detail: detail.into(),
36        }
37    }
38}
39
40/// The result of `firstpass doctor`.
41#[derive(Debug)]
42pub struct DoctorReport {
43    /// One entry per check, in report order.
44    pub checks: Vec<Check>,
45}
46
47impl DoctorReport {
48    /// Healthy iff no check failed (warnings are fine).
49    #[must_use]
50    pub fn healthy(&self) -> bool {
51        self.checks.iter().all(|c| c.status != CheckStatus::Fail)
52    }
53
54    /// Render the report as human-readable lines (a rendering of the structured checks).
55    #[must_use]
56    pub fn render(&self) -> String {
57        let mut out = String::new();
58        for c in &self.checks {
59            let mark = match c.status {
60                CheckStatus::Ok => "✓",
61                CheckStatus::Warn => "!",
62                CheckStatus::Fail => "✗",
63            };
64            out.push_str(&format!("{mark} {}: {}\n", c.name, c.detail));
65        }
66        out.push_str(if self.healthy() {
67            "\nhealthy — ready to route.\n"
68        } else {
69            "\nnot healthy — fix the ✗ items above.\n"
70        });
71        out
72    }
73}
74
75/// Validate a loaded config against the environment: routing sanity, provider key presence, and
76/// that every configured gate's command actually exists. `env` looks up environment variables
77/// (injected so this is testable).
78#[must_use]
79pub fn doctor(config: &ProxyConfig, env: impl Fn(&str) -> Option<String>) -> DoctorReport {
80    let mut checks = Vec::new();
81
82    // Config parsed (we hold a ProxyConfig), report the shape.
83    let route_count = config.routing.as_ref().map_or(0, |c| c.routes.len());
84    checks.push(Check::new(
85        "config",
86        CheckStatus::Ok,
87        format!("default mode {:?}, {route_count} route(s)", config.mode),
88    ));
89
90    // Enforce is only meaningful with an enforce route to activate the engine.
91    let enforce_routes = config.routing.as_ref().map_or(0, |c| {
92        c.routes.iter().filter(|r| r.mode == Mode::Enforce).count()
93    });
94    if config.mode == Mode::Enforce && enforce_routes == 0 {
95        checks.push(Check::new(
96            "routing",
97            CheckStatus::Warn,
98            "mode is enforce but no enforce route is defined — all traffic will observe",
99        ));
100    } else {
101        checks.push(Check::new(
102            "routing",
103            CheckStatus::Ok,
104            format!("{enforce_routes} enforce route(s)"),
105        ));
106    }
107
108    // A provider key in the environment. Observe uses the caller's key, so absence is a warning.
109    if env("ANTHROPIC_API_KEY").is_some_and(|k| !k.is_empty()) {
110        checks.push(Check::new("anthropic-key", CheckStatus::Ok, "present"));
111    } else {
112        checks.push(Check::new(
113            "anthropic-key",
114            CheckStatus::Warn,
115            "ANTHROPIC_API_KEY not set — observe uses the caller's key; enforce needs one reachable",
116        ));
117    }
118
119    // A ladder rung on a built-in provider whose key is absent fails at the first REQUEST, not at
120    // startup — the least useful moment to find out. Checked per rung actually configured rather
121    // than for all eight built-ins, because warning about providers nobody is using is noise that
122    // trains an operator to skip the report.
123    let mut missing_keys: Vec<(String, &str)> = Vec::new();
124    if let Some(routing) = config.routing.as_ref() {
125        let mut seen = std::collections::HashSet::new();
126        for rung in routing.routes.iter().flat_map(|r| r.ladder.iter()) {
127            let Some((prov, _)) = rung.split_once('/') else {
128                continue;
129            };
130            let Some(key_env) = crate::provider::builtin_key_env(prov) else {
131                continue; // anthropic/openai are covered above; a [[provider]] block owns its own
132            };
133            if seen.insert(prov.to_owned()) && env(key_env).is_none_or(|k| k.is_empty()) {
134                missing_keys.push((prov.to_owned(), key_env));
135            }
136        }
137    }
138    for (prov, key_env) in missing_keys {
139        checks.push(Check::new(
140            format!("provider:{prov}"),
141            CheckStatus::Fail,
142            format!("{key_env} not set — a rung on `{prov}` will fail at the first request"),
143        ));
144    }
145
146    // Every configured gate command must resolve, or that gate silently abstains at runtime.
147    let path = env("PATH");
148    let gate_defs = config.routing.as_ref().map_or(&[][..], |c| &c.gate_defs);
149    for def in gate_defs {
150        match def.cmd.first() {
151            Some(program) if command_on_path(program, path.as_deref()) => checks.push(Check::new(
152                format!("gate:{}", def.id),
153                CheckStatus::Ok,
154                format!("`{program}` found"),
155            )),
156            Some(program) => checks.push(Check::new(
157                format!("gate:{}", def.id),
158                CheckStatus::Fail,
159                format!("`{program}` not found on PATH"),
160            )),
161            None => checks.push(Check::new(
162                format!("gate:{}", def.id),
163                CheckStatus::Fail,
164                "empty command",
165            )),
166        }
167    }
168
169    // The trace store must be writable, or we'd trade the audit trail (or availability) for it.
170    if can_write_db(&config.db_path) {
171        checks.push(Check::new(
172            "trace-store",
173            CheckStatus::Ok,
174            format!("{} is writable", config.db_path),
175        ));
176    } else {
177        checks.push(Check::new(
178            "trace-store",
179            CheckStatus::Fail,
180            format!("cannot write near {}", config.db_path),
181        ));
182    }
183
184    DoctorReport { checks }
185}
186
187/// Whether `program` is runnable: an explicit path (contains a separator) that exists, or a bare
188/// name found in one of `PATH`'s directories.
189#[must_use]
190pub fn command_on_path(program: &str, path_var: Option<&str>) -> bool {
191    if program.contains('/') || program.contains('\\') {
192        return std::path::Path::new(program).is_file();
193    }
194    let Some(path) = path_var else { return false };
195    std::env::split_paths(path).any(|dir| dir.join(program).is_file())
196}
197
198/// How this binary got onto the machine. Upgrading is channel-specific — running the wrong
199/// updater either does nothing or fights the package manager that owns the file — so
200/// `firstpass upgrade` detects the channel instead of guessing one.
201#[derive(Debug, Clone, Copy, PartialEq, Eq)]
202pub enum Channel {
203    /// The shell/powershell installer, which ships its own `firstpass-update` (dist-workspace.toml
204    /// sets `install-updater = true`). This is the only channel we can upgrade in place.
205    Dist,
206    /// `brew install dshakes/tap/firstpass-proxy`.
207    Homebrew,
208    /// `cargo install --git`.
209    Cargo,
210    /// pip / uv / uvx wheel.
211    Python,
212    /// pipx-managed install (its own venv, symlinked onto PATH).
213    Pipx,
214    /// The npm installer package.
215    Npm,
216    /// Running inside the container image.
217    Docker,
218    /// Built from source, vendored, or something we do not recognise.
219    Unknown,
220}
221
222impl Channel {
223    /// Human label used in the report.
224    #[must_use]
225    pub const fn label(self) -> &'static str {
226        match self {
227            Self::Dist => "installer script (self-updating)",
228            Self::Homebrew => "Homebrew",
229            Self::Cargo => "cargo install",
230            Self::Python => "Python wheel (pip / uv / uvx)",
231            Self::Pipx => "pipx",
232            Self::Npm => "npm",
233            Self::Docker => "Docker image",
234            Self::Unknown => "unrecognised",
235        }
236    }
237
238    /// The command that upgrades this channel. `None` when we cannot tell.
239    #[must_use]
240    pub const fn upgrade_command(self) -> Option<&'static str> {
241        match self {
242            Self::Dist => Some("firstpass-update"),
243            Self::Homebrew => Some("brew upgrade firstpass-proxy"),
244            Self::Cargo => Some(
245                "cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force",
246            ),
247            Self::Python => Some("uv tool upgrade firstpass    # or: pip install -U firstpass"),
248            Self::Pipx => Some("pipx upgrade firstpass"),
249            Self::Npm => Some("npm install -g @dshakesnotbot/firstpass-proxy@latest"),
250            Self::Docker => Some("docker pull ghcr.io/dshakes/firstpass:latest"),
251            Self::Unknown => None,
252        }
253    }
254}
255
256/// Work out how this binary was installed from the path it runs as. Pure: the executable path,
257/// the updater probe and the container check are all injected, so every branch is unit-tested
258/// without installing anything.
259///
260/// Package-manager path markers are checked BEFORE the updater probe: a Homebrew or wheel install
261/// can still carry the dist updater alongside it, and in that case the package manager owns the
262/// file — running the self-updater would leave the manager's metadata pointing at a binary it no
263/// longer controls.
264#[must_use]
265pub fn detect_channel(exe: &std::path::Path, updater_present: bool, in_container: bool) -> Channel {
266    let p = exe.to_string_lossy().to_lowercase().replace('\\', "/");
267    let has = |needle: &str| p.contains(needle);
268    if in_container {
269        return Channel::Docker;
270    }
271    if has("/.cargo/bin/") || has("/.cargo/registry/") {
272        return Channel::Cargo;
273    }
274    if has("/cellar/") || has("/homebrew/") || has("/linuxbrew/") {
275        return Channel::Homebrew;
276    }
277    if has("/pipx/") {
278        return Channel::Pipx; // before the generic markers: a pipx venv also has site-packages
279    }
280    if has("/site-packages/")
281        || has("/dist-packages/")
282        || has("/.venv/")
283        || has("/uv/")
284        || has("/uv-cache/")
285    {
286        return Channel::Python;
287    }
288    if has("/node_modules/") {
289        return Channel::Npm;
290    }
291    if updater_present {
292        return Channel::Dist;
293    }
294    Channel::Unknown
295}
296
297/// The text `firstpass upgrade` prints. Separated from the doing so the wording is testable.
298#[must_use]
299pub fn upgrade_report(channel: Channel, current_version: &str) -> String {
300    let mut out = format!(
301        "firstpass {current_version} · installed via {}\n",
302        channel.label()
303    );
304    match channel.upgrade_command() {
305        Some(cmd) if channel == Channel::Dist => {
306            out.push_str(&format!("\nupgrading in place — running `{cmd}`\n"));
307        }
308        Some(cmd) => {
309            out.push_str(&format!(
310                "\nthis channel is owned by its package manager, so upgrade with:\n    {cmd}\n"
311            ));
312        }
313        None => {
314            out.push_str(
315                "\ncould not tell how this binary was installed. Pick the line that matches:\n\
316                 \x20   firstpass-update                              # installer script\n\
317                 \x20   brew upgrade firstpass-proxy                  # Homebrew\n\
318                 \x20   uv tool upgrade firstpass                     # uv / pip: pip install -U firstpass\n\
319                 \x20   pipx upgrade firstpass                        # pipx\n\
320                 \x20   npm install -g @dshakesnotbot/firstpass-proxy@latest\n\
321                 \x20   docker pull ghcr.io/dshakes/firstpass:latest\n\
322                 \x20   cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force\n",
323            );
324        }
325    }
326    out
327}
328
329/// Probe whether the trace DB's directory is writable, without creating the DB itself.
330fn can_write_db(db_path: &str) -> bool {
331    let path = std::path::Path::new(db_path);
332    let dir = path
333        .parent()
334        .filter(|d| !d.as_os_str().is_empty())
335        .map_or_else(
336            || std::path::PathBuf::from("."),
337            std::path::Path::to_path_buf,
338        );
339    let probe = dir.join(format!(".firstpass-doctor-probe-{}", std::process::id()));
340    match std::fs::File::create(&probe) {
341        Ok(_) => {
342            let _ = std::fs::remove_file(&probe);
343            true
344        }
345        Err(_) => false,
346    }
347}
348
349/// Per-gate and per-rung evaluation summary computed from receipts — the operator's live
350/// eval suite: how each gate is verdict-ing, how often routing escalates, where serves land.
351#[derive(Debug, serde::Serialize)]
352pub struct EvalsSummary {
353    /// Traces aggregated (enforce mode only — observe records no gate decisions on-path).
354    pub enforce_traces: usize,
355    /// Total escalations across those traces.
356    pub escalations: u64,
357    /// gate_id → (pass, fail, abstain) counts across every attempt.
358    pub gates: std::collections::BTreeMap<String, (u64, u64, u64)>,
359    /// rung index → times an attempt at that rung was the served one.
360    pub served_by_rung: std::collections::BTreeMap<u32, u64>,
361}
362
363/// Aggregate gate verdicts + routing behavior over `traces` (enforce only).
364#[must_use]
365/// Prequential evaluation of the per-query gate-pass predictor (ADR 0008 Phase 2) — the honest
366/// "is it any good yet" check before the predictor is ever allowed to *act*. Replays the receipts
367/// in order through a fresh predictor: for each attempt with a clear Pass/Fail verdict, **predict
368/// before update** (so every prediction is on data the model has not trained on), then update.
369/// Reports AUC (predicted -> passed) and Brier score over those pairs.
370#[derive(Debug, serde::Serialize)]
371pub struct PredictorEval {
372    /// Labeled attempts evaluated.
373    pub n: usize,
374    /// AUC of predicted P(pass) ranking actual passes above fails. 0.5 = no signal; 1.0 = perfect.
375    /// `None` when one class is absent (all pass or all fail — AUC undefined).
376    pub auc: Option<f64>,
377    /// Mean squared error of the probabilistic prediction, in `[0, 1]`. Lower is better; 0.25 is
378    /// the always-0.5 baseline.
379    pub brier: f64,
380}
381
382/// AUC via the Mann-Whitney rank-sum identity (ties = 0.5). `None` if either class is empty.
383fn predictor_auc(pairs: &[(f64, bool)]) -> Option<f64> {
384    let pos: Vec<f64> = pairs.iter().filter(|(_, y)| *y).map(|(s, _)| *s).collect();
385    let neg: Vec<f64> = pairs.iter().filter(|(_, y)| !*y).map(|(s, _)| *s).collect();
386    if pos.is_empty() || neg.is_empty() {
387        return None;
388    }
389    let mut wins = 0.0_f64;
390    for &p in &pos {
391        for &n in &neg {
392            wins += if p > n {
393                1.0
394            } else if (p - n).abs() < f64::EPSILON {
395                0.5
396            } else {
397                0.0
398            };
399        }
400    }
401    Some(wins / (pos.len() as f64 * neg.len() as f64))
402}
403
404/// Run the prequential predictor evaluation over `traces` with the given `lr`/`l2`.
405pub fn evaluate_predictor(traces: &[Trace], lr: f64, l2: f64) -> PredictorEval {
406    use firstpass_core::{PassPredictor, Verdict};
407    let mut model = PassPredictor::new(lr, l2);
408    let mut pairs: Vec<(f64, bool)> = Vec::new();
409    let mut se = 0.0_f64;
410    for t in traces {
411        for a in &t.attempts {
412            let y = match a.verdict {
413                Verdict::Pass => true,
414                Verdict::Fail => false,
415                Verdict::Abstain => continue,
416            };
417            let p = model.predict(&t.request.features, a.rung);
418            let target = f64::from(u8::from(y));
419            se += (p - target) * (p - target);
420            pairs.push((p, y));
421            model.update(&t.request.features, a.rung, y);
422        }
423    }
424    let n = pairs.len();
425    PredictorEval {
426        n,
427        auc: predictor_auc(&pairs),
428        brier: if n == 0 { 0.0 } else { se / n as f64 },
429    }
430}
431
432pub fn summarize_evals(traces: &[Trace]) -> EvalsSummary {
433    let mut s = EvalsSummary {
434        enforce_traces: 0,
435        escalations: 0,
436        gates: std::collections::BTreeMap::new(),
437        served_by_rung: std::collections::BTreeMap::new(),
438    };
439    for t in traces {
440        if t.mode != Mode::Enforce {
441            continue;
442        }
443        s.enforce_traces += 1;
444        s.escalations += u64::from(t.final_.escalations);
445        if let Some(rung) = t.final_.served_rung {
446            *s.served_by_rung.entry(rung).or_insert(0) += 1;
447        }
448        for attempt in &t.attempts {
449            for g in &attempt.gates {
450                let e = s.gates.entry(g.gate_id.clone()).or_insert((0, 0, 0));
451                match g.verdict {
452                    firstpass_core::Verdict::Pass => e.0 += 1,
453                    firstpass_core::Verdict::Fail => e.1 += 1,
454                    firstpass_core::Verdict::Abstain => e.2 += 1,
455                }
456            }
457        }
458    }
459    s
460}
461
462/// Render an [`EvalsSummary`] for humans (`--json` callers serialize the struct instead).
463#[must_use]
464pub fn format_evals(s: &EvalsSummary) -> String {
465    if s.enforce_traces == 0 {
466        return "no enforce traces yet — route some traffic in enforce mode first".to_owned();
467    }
468    let mut out = format!(
469        "enforce traces: {} · escalations: {}\n",
470        s.enforce_traces, s.escalations
471    );
472    out.push_str("gates (pass / fail / abstain):\n");
473    for (id, (p, f, a)) in &s.gates {
474        out.push_str(&format!("  {id:<24} {p:>6} / {f:>6} / {a:>6}\n"));
475    }
476    out.push_str("served by rung:\n");
477    for (rung, n) in &s.served_by_rung {
478        out.push_str(&format!("  rung {rung:<2} {n:>6}\n"));
479    }
480    out.trim_end().to_owned()
481}
482
483/// A structured, agent-readable explanation of a single routing decision — why the ladder
484/// went where it did, built purely from a sealed [`Trace`].
485#[derive(Debug, serde::Serialize)]
486pub struct RouteExplanation {
487    pub trace_id: String,
488    pub mode: String,
489    pub policy: String,
490    /// The model actually served, if any.
491    pub served_model: Option<String>,
492    pub served_rung: Option<u32>,
493    pub escalations: u32,
494    pub total_cost_usd: f64,
495    pub baseline_usd: f64,
496    pub savings_usd: f64,
497    /// Per attempt, in ladder order: what was tried and how it was judged.
498    pub attempts: Vec<AttemptExplanation>,
499    /// One-line human summary.
500    pub summary: String,
501}
502
503/// Per-attempt slice of a [`RouteExplanation`].
504#[derive(Debug, serde::Serialize)]
505pub struct AttemptExplanation {
506    pub rung: u32,
507    pub model: String,
508    pub verdict: String,
509    /// gate_id → verdict, for every gate that ran on this attempt.
510    pub gates: Vec<(String, String)>,
511}
512
513fn verdict_str(v: firstpass_core::Verdict) -> &'static str {
514    match v {
515        firstpass_core::Verdict::Pass => "pass",
516        firstpass_core::Verdict::Fail => "fail",
517        firstpass_core::Verdict::Abstain => "abstain",
518    }
519}
520
521/// Explain one routing decision from its sealed receipt — the "why this model" answer, as
522/// structured data an agent can act on (and the backing for the MCP `explain_route` tool).
523#[must_use]
524pub fn explain_trace(t: &Trace) -> RouteExplanation {
525    let attempts: Vec<AttemptExplanation> = t
526        .attempts
527        .iter()
528        .map(|a| AttemptExplanation {
529            rung: a.rung,
530            model: a.model.clone(),
531            verdict: verdict_str(a.verdict).to_owned(),
532            gates: a
533                .gates
534                .iter()
535                .map(|g| (g.gate_id.clone(), verdict_str(g.verdict).to_owned()))
536                .collect(),
537        })
538        .collect();
539    let served_model = t
540        .final_
541        .served_rung
542        .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
543        .map(|a| a.model.clone());
544    let summary = match &served_model {
545        Some(m) => format!(
546            "served {m} at rung {} after {} escalation(s); spent ${:.4} vs ${:.4} always-top (saved ${:.4})",
547            t.final_.served_rung.unwrap_or(0),
548            t.final_.escalations,
549            t.final_.total_cost_usd,
550            t.final_.counterfactual_baseline_usd,
551            t.final_.savings_usd
552        ),
553        None => "no rung served (all attempts failed the gate or errored)".to_owned(),
554    };
555    RouteExplanation {
556        trace_id: t.trace_id.to_string(),
557        mode: match t.mode {
558            Mode::Enforce => "enforce",
559            Mode::Observe => "observe",
560        }
561        .to_owned(),
562        policy: t.policy.id.clone(),
563        served_model,
564        served_rung: t.final_.served_rung,
565        escalations: t.final_.escalations,
566        total_cost_usd: t.final_.total_cost_usd,
567        baseline_usd: t.final_.counterfactual_baseline_usd,
568        savings_usd: t.final_.savings_usd,
569        attempts,
570        summary,
571    }
572}
573
574/// Outcome of an independent receipt-chain verification — the compliance artifact.
575#[derive(Debug, serde::Serialize)]
576pub struct VerifyReport {
577    /// Receipts checked.
578    pub receipts: usize,
579    /// `true` iff the hash chain re-derives cleanly from genesis (tamper-evident proof).
580    pub valid: bool,
581    /// On failure, the 0-based index of the first broken link and why.
582    #[serde(skip_serializing_if = "Option::is_none")]
583    pub broken_at: Option<usize>,
584    #[serde(skip_serializing_if = "Option::is_none")]
585    pub detail: Option<String>,
586}
587
588/// Independently re-derive the hash chain over `traces` from genesis — the same computation an
589/// external auditor runs, with no trust in the proxy or the database. A single altered or
590/// reordered receipt breaks the chain at its index.
591#[must_use]
592pub fn verify_receipts(traces: &[Trace]) -> VerifyReport {
593    match firstpass_core::verify_chain(traces, firstpass_core::GENESIS_HASH) {
594        Ok(()) => VerifyReport {
595            receipts: traces.len(),
596            valid: true,
597            broken_at: None,
598            detail: None,
599        },
600        Err(firstpass_core::Error::ChainBroken { index, detail }) => VerifyReport {
601            receipts: traces.len(),
602            valid: false,
603            broken_at: Some(index),
604            detail: Some(detail),
605        },
606        Err(e) => VerifyReport {
607            receipts: traces.len(),
608            valid: false,
609            broken_at: None,
610            detail: Some(e.to_string()),
611        },
612    }
613}
614
615/// Parse a JSONL receipt export (one [`Trace`] per line) back into traces, for offline
616/// verification. Blank lines are skipped; a malformed line fails loudly with its line number.
617///
618/// # Errors
619/// Returns the 1-based line number and parse error of the first unparseable line.
620pub fn parse_receipt_jsonl(text: &str) -> Result<Vec<Trace>, String> {
621    let mut traces = Vec::new();
622    for (i, line) in text.lines().enumerate() {
623        if line.trim().is_empty() {
624            continue;
625        }
626        let t: Trace = serde_json::from_str(line).map_err(|e| format!("line {}: {e}", i + 1))?;
627        traces.push(t);
628    }
629    Ok(traces)
630}
631
632/// Render `traces` as a JSONL receipt export — one sealed receipt per line, in chain order.
633/// This is the artifact an operator hands an auditor; the deferred-verdict side table is never
634/// included (it is not part of the hashed body).
635#[must_use]
636/// One decision, reshaped as a training example: context, action, reward, propensity.
637///
638/// A receipt is an audit record — nested, provenance-heavy, and shaped for a human or an auditor.
639/// A learner wants a flat row. The conversion is trivial except for one thing that is not, and is
640/// the reason this is worth shipping rather than leaving to a downstream script:
641///
642/// **The propensity is already logged.** Firstpass records the probability its logging policy
643/// assigned to the rung it chose, because it needs that denominator for IPS/SNIPS/DR off-policy
644/// evaluation. That is exactly the field offline RL needs and exactly the field a routing log
645/// normally lacks — without it, a learner trained on this data inherits the logging policy's bias
646/// with no way to correct for it. Rows where it is absent are marked, not silently defaulted to 1.
647#[derive(Debug, Clone, serde::Serialize)]
648pub struct TrainingRow {
649    /// Decision id, so a row can be traced back to the receipt it came from.
650    pub trace_id: String,
651    /// Context the decision was made in.
652    pub features: firstpass_core::Features,
653    /// The action taken: which rung the ladder started on.
654    pub start_rung: u32,
655    /// Which rung actually served, if any.
656    pub served_rung: Option<u32>,
657    /// Whether the served output passed its gate — the reward signal, and the honest one, because
658    /// it is a verified outcome rather than a predicted score.
659    pub served_passed: bool,
660    /// Total USD spent on the decision. Cost-aware objectives need this alongside the reward.
661    pub cost_usd: f64,
662    /// Probability the logging policy gave the chosen action. `None` for a deterministic policy —
663    /// an un-corrected row, which a learner must handle rather than assume away.
664    pub propensity: Option<f64>,
665    /// Whether this was a deliberate exploration draw.
666    pub explore: bool,
667    /// Logging policy identity, so rows from different policies can be separated.
668    pub policy_id: String,
669}
670
671impl TrainingRow {
672    /// Reshape one receipt.
673    pub fn from_trace(t: &Trace) -> Self {
674        let served_rung = t.final_.served_rung;
675        Self {
676            trace_id: t.trace_id.to_string(),
677            features: t.request.features.clone(),
678            // The first attempt is where the ladder started; absent attempts means nothing ran.
679            start_rung: t.attempts.first().map_or(0, |a| a.rung),
680            served_rung,
681            served_passed: served_rung.is_some_and(|r| {
682                t.attempts
683                    .iter()
684                    .any(|a| a.rung == r && a.verdict == firstpass_core::Verdict::Pass)
685            }),
686            cost_usd: t.final_.total_cost_usd,
687            propensity: t.policy.propensity,
688            explore: t.policy.explore,
689            policy_id: t.policy.id.clone(),
690        }
691    }
692}
693
694/// Firstpass's own added latency, summarized from receipts.
695///
696/// Percentiles rather than a mean: overhead is right-skewed (a cold gate process, a slow
697/// subprocess spawn), and a mean hides exactly the tail a caller feels.
698#[derive(Debug, Clone, serde::Serialize)]
699pub struct OverheadSummary {
700    /// Receipts that yielded an honest figure.
701    pub n: usize,
702    /// Receipts skipped because concurrency made the subtraction meaningless (speculation).
703    ///
704    /// Reported, never silently dropped: if most of a deployment's traffic is speculative, the
705    /// percentiles below describe a minority of it and the reader needs to know that.
706    pub skipped_concurrent: usize,
707    pub p50_ms: u64,
708    pub p95_ms: u64,
709    pub p99_ms: u64,
710    pub max_ms: u64,
711}
712
713/// Summarize proxy overhead across `traces`.
714#[must_use]
715pub fn summarize_overhead(traces: &[Trace]) -> OverheadSummary {
716    let mut v: Vec<u64> = Vec::with_capacity(traces.len());
717    let mut skipped = 0usize;
718    for t in traces {
719        match t.overhead_ms() {
720            Some(ms) => v.push(ms),
721            None => skipped += 1,
722        }
723    }
724    v.sort_unstable();
725    // Nearest-rank percentile: with a handful of receipts an interpolated one invents a value
726    // between two real measurements, and every number here should be one we actually observed.
727    let pct = |p: f64| -> u64 {
728        if v.is_empty() {
729            return 0;
730        }
731        let rank = (p * v.len() as f64).ceil() as usize;
732        v[rank.clamp(1, v.len()) - 1]
733    };
734    OverheadSummary {
735        n: v.len(),
736        skipped_concurrent: skipped,
737        p50_ms: pct(0.50),
738        p95_ms: pct(0.95),
739        p99_ms: pct(0.99),
740        max_ms: v.last().copied().unwrap_or(0),
741    }
742}
743
744/// Render [`summarize_overhead`] for a terminal.
745#[must_use]
746pub fn format_overhead(s: &OverheadSummary) -> String {
747    if s.n == 0 {
748        return format!(
749            "no receipts with a measurable overhead ({} skipped as concurrent).\n\
750             Route some traffic first, or run with speculation off to measure it.\n",
751            s.skipped_concurrent
752        );
753    }
754    // Built line by line rather than as one continued literal: a `\`-continuation silently keeps
755    // the next line's indentation, which is how the first version of this shipped a wrapped
756    // sentence indented fifteen spaces into the middle of a report.
757    let mut lines = vec![
758        "Firstpass overhead — time added beyond the provider calls and gates it ran".to_owned(),
759        String::new(),
760        format!("  p50  {:>6} ms", s.p50_ms),
761        format!("  p95  {:>6} ms", s.p95_ms),
762        format!("  p99  {:>6} ms", s.p99_ms),
763        format!("  max  {:>6} ms", s.max_ms),
764        String::new(),
765        format!("  over {} receipt(s)", s.n),
766    ];
767    // A p95 of 0 is a real result for a Rust proxy, but printed bare it reads like the number is
768    // not being measured at all. Receipt timings are whole milliseconds, so say what 0 means.
769    if s.p95_ms == 0 {
770        lines.push(
771            "  0 ms is the floor, not a missing measurement: receipts record whole".to_owned(),
772        );
773        lines.push(
774            "  milliseconds, so the proxy's own work stayed under 1 ms per request.".to_owned(),
775        );
776    }
777    if s.skipped_concurrent > 0 {
778        lines.push(format!(
779            "  {} skipped: speculative rungs run concurrently, so subtracting dispatched",
780            s.skipped_concurrent
781        ));
782        lines.push(
783            "  work from wall-clock is meaningless there — excluded, not floored to 0.".to_owned(),
784        );
785    }
786    lines.push(String::new());
787    lines.join("\n")
788}
789
790/// Reshape receipts into JSONL training rows (`firstpass export --format rl`).
791#[must_use]
792pub fn export_training_jsonl(traces: &[Trace]) -> String {
793    let mut out = String::new();
794    for t in traces {
795        if let Ok(line) = serde_json::to_string(&TrainingRow::from_trace(t)) {
796            out.push_str(&line);
797            out.push('\n');
798        }
799    }
800    out
801}
802
803pub fn export_receipts_jsonl(traces: &[Trace]) -> String {
804    let mut out = String::new();
805    for t in traces {
806        if let Ok(line) = serde_json::to_string(t) {
807            out.push_str(&line);
808            out.push('\n');
809        }
810    }
811    out
812}
813
814/// Aggregated spend/savings over a set of traces — the number the operator screenshots.
815/// Pure so it's unit-testable; `firstpass savings` feeds it the trace store.
816#[derive(Debug, serde::Serialize)]
817pub struct SavingsSummary {
818    /// Traces aggregated.
819    pub traces: usize,
820    /// Enforce-mode traces (the ones where routing actually decided).
821    pub enforce_traces: usize,
822    /// USD actually spent (model + gate calls), summed over all traces.
823    pub spent_usd: f64,
824    /// USD spent on gates alone (the price of proof).
825    pub gate_usd: f64,
826    /// What always calling the top rung would have cost (§9.1 counterfactual), summed.
827    pub baseline_usd: f64,
828    /// `baseline - spent` — the savings the receipts prove.
829    pub savings_usd: f64,
830    /// Savings as a fraction of baseline, `0.0` when there is no baseline.
831    pub savings_pct: f64,
832}
833
834/// Aggregate spend vs the always-top counterfactual over `traces`.
835#[must_use]
836pub fn summarize_savings(traces: &[Trace]) -> SavingsSummary {
837    let mut s = SavingsSummary {
838        traces: traces.len(),
839        enforce_traces: 0,
840        spent_usd: 0.0,
841        gate_usd: 0.0,
842        baseline_usd: 0.0,
843        savings_usd: 0.0,
844        savings_pct: 0.0,
845    };
846    for t in traces {
847        if t.mode == Mode::Enforce {
848            s.enforce_traces += 1;
849        }
850        s.spent_usd += t.final_.total_cost_usd;
851        s.gate_usd += t.final_.gate_cost_usd;
852        s.baseline_usd += t.final_.counterfactual_baseline_usd;
853    }
854    s.savings_usd = s.baseline_usd - s.spent_usd;
855    if s.baseline_usd > 0.0 {
856        s.savings_pct = s.savings_usd / s.baseline_usd;
857    }
858    s
859}
860
861/// Render a [`SavingsSummary`] for humans (`--json` callers serialize the struct instead).
862#[must_use]
863pub fn format_savings(s: &SavingsSummary) -> String {
864    if s.traces == 0 {
865        return "no traces recorded yet — route some traffic through the proxy first".to_owned();
866    }
867    format!(
868        "traces: {} ({} enforce)\nspent:    ${:.4}  (gates ${:.4})\nbaseline: ${:.4}  (always top rung)\nsavings:  ${:.4}  ({:.1}%)",
869        s.traces,
870        s.enforce_traces,
871        s.spent_usd,
872        s.gate_usd,
873        s.baseline_usd,
874        s.savings_usd,
875        s.savings_pct * 100.0
876    )
877}
878
879/// Render the most recent `limit` traces as JSON lines — machine-first (SPEC §0.2): each line is a
880/// full [`Trace`], newest last.
881#[must_use]
882pub fn format_traces(traces: &[Trace], limit: usize) -> String {
883    let mut lines: Vec<String> = traces
884        .iter()
885        .rev()
886        .take(limit)
887        .filter_map(|t| serde_json::to_string(t).ok())
888        .collect();
889    lines.reverse();
890    if lines.is_empty() {
891        "no traces recorded yet".to_owned()
892    } else {
893        lines.join("\n")
894    }
895}
896
897#[cfg(test)]
898mod tests {
899    use super::*;
900
901    fn config_with(toml: Option<&str>, db_path: &str) -> ProxyConfig {
902        ProxyConfig::from_lookup(|k| match k {
903            "FIRSTPASS_MODE" if toml.is_some() => Some("enforce".to_owned()),
904            "FIRSTPASS_CONFIG_TOML" => toml.map(str::to_owned),
905            "FIRSTPASS_DB" => Some(db_path.to_owned()),
906            _ => None,
907        })
908        .unwrap()
909    }
910
911    #[test]
912    fn doctor_flags_a_ladder_rung_whose_provider_key_is_missing() {
913        // Without this the failure surfaces on the first real request, in production, as an
914        // upstream error — the least useful moment to learn the key was never set.
915        let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"groq/llama-3.3-70b-versatile\"]\n\
916                    [[price]]\nmodel = \"groq/llama-3.3-70b-versatile\"\n\
917                    input_per_mtok = 0.59\noutput_per_mtok = 0.79\n";
918        let config = config_with(Some(toml), "/tmp/doctor-groq.db");
919
920        let report = doctor(&config, |k| match k {
921            "PATH" => Some("/usr/bin".to_owned()),
922            _ => None, // no GROQ_API_KEY
923        });
924
925        let groq = report
926            .checks
927            .iter()
928            .find(|c| c.name == "provider:groq")
929            .expect("groq rung should be checked");
930        assert_eq!(groq.status, CheckStatus::Fail);
931        assert!(groq.detail.contains("GROQ_API_KEY"), "{}", groq.detail);
932        assert!(
933            !report.healthy(),
934            "a rung that cannot authenticate is not healthy"
935        );
936    }
937
938    #[test]
939    fn doctor_stays_quiet_about_providers_the_ladder_does_not_use() {
940        // Warning about all eight built-ins would be noise, and noise trains an operator to skip
941        // the report — which is how the one real warning gets missed.
942        let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\n";
943        let config = config_with(Some(toml), "/tmp/doctor-quiet.db");
944
945        let report = doctor(&config, |k| match k {
946            "PATH" => Some("/usr/bin".to_owned()),
947            "ANTHROPIC_API_KEY" => Some("sk-test".to_owned()),
948            _ => None,
949        });
950
951        assert!(
952            !report
953                .checks
954                .iter()
955                .any(|c| c.name.starts_with("provider:")),
956            "unused providers must not be reported: {:?}",
957            report.checks.iter().map(|c| &c.name).collect::<Vec<_>>()
958        );
959    }
960
961    #[test]
962    fn command_on_path_finds_real_and_rejects_fake() {
963        let path = std::env::var("PATH").ok();
964        assert!(command_on_path("sh", path.as_deref()), "sh is on PATH");
965        assert!(!command_on_path(
966            "firstpass-definitely-not-a-real-binary",
967            path.as_deref()
968        ));
969        assert!(!command_on_path("/nonexistent/abs/path", None));
970    }
971
972    #[test]
973    fn doctor_flags_a_missing_gate_binary() {
974        let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"good\", \"bad\"]\n\
975                    [[gate]]\nid = \"good\"\ncmd = [\"sh\"]\n\
976                    [[gate]]\nid = \"bad\"\ncmd = [\"firstpass-nope-not-real\"]\n";
977        let db = std::env::temp_dir().join("firstpass-doctor-test.db");
978        let config = config_with(Some(toml), db.to_str().unwrap());
979
980        // Real PATH so `sh` resolves; no ANTHROPIC_API_KEY -> a warning, not a failure.
981        let report = doctor(&config, |k| match k {
982            "PATH" => std::env::var("PATH").ok(),
983            _ => None,
984        });
985
986        assert!(
987            !report.healthy(),
988            "a missing gate binary must fail the report"
989        );
990        let bad = report.checks.iter().find(|c| c.name == "gate:bad").unwrap();
991        assert_eq!(bad.status, CheckStatus::Fail);
992        let good = report
993            .checks
994            .iter()
995            .find(|c| c.name == "gate:good")
996            .unwrap();
997        assert_eq!(good.status, CheckStatus::Ok);
998        let key = report
999            .checks
1000            .iter()
1001            .find(|c| c.name == "anthropic-key")
1002            .unwrap();
1003        assert_eq!(key.status, CheckStatus::Warn);
1004    }
1005
1006    #[test]
1007    fn doctor_is_healthy_for_a_plain_observe_setup() {
1008        let db = std::env::temp_dir().join("firstpass-doctor-ok.db");
1009        let config = config_with(None, db.to_str().unwrap());
1010        let report = doctor(&config, |k| {
1011            (k == "ANTHROPIC_API_KEY").then(|| "sk-test".to_owned())
1012        });
1013        assert!(report.healthy(), "{}", report.render());
1014    }
1015
1016    #[test]
1017    fn format_traces_handles_empty() {
1018        assert_eq!(format_traces(&[], 10), "no traces recorded yet");
1019    }
1020
1021    #[test]
1022    fn savings_summary_empty_and_nonempty() {
1023        let empty = summarize_savings(&[]);
1024        assert_eq!(empty.traces, 0);
1025        assert!(format_savings(&empty).contains("no traces"));
1026    }
1027
1028    #[test]
1029    fn evals_summary_empty_zero_state() {
1030        let s = summarize_evals(&[]);
1031        assert_eq!(s.enforce_traces, 0);
1032        assert!(format_evals(&s).contains("no enforce traces"));
1033    }
1034
1035    #[test]
1036    fn verify_and_export_round_trip_detects_tampering() {
1037        use firstpass_core::{
1038            Attempt, Features, FinalOutcome, GENESIS_HASH, PolicyRef, RequestInfo, ServedFrom,
1039            TaskKind, Verdict,
1040        };
1041
1042        // Build a valid 3-link chain: each prev_hash = the previous record's hash.
1043        let mk = |prev: &str, session: &str| Trace {
1044            trace_id: uuid::Uuid::now_v7(),
1045            prev_hash: prev.to_owned(),
1046            tenant_id: "default".to_owned(),
1047            session_id: session.to_owned(),
1048            ts: jiff::Timestamp::UNIX_EPOCH,
1049            mode: Mode::Observe,
1050            policy: PolicyRef {
1051                id: "observe-passthrough@v0".to_owned(),
1052                explore: false,
1053                propensity: None,
1054                mode_profile: None,
1055            },
1056            request: RequestInfo {
1057                api: "anthropic.messages".to_owned(),
1058                prompt_hash: "deadbeef".to_owned(),
1059                features: Features::new(TaskKind::Other),
1060            },
1061            attempts: vec![Attempt {
1062                rung: 0,
1063                model: "anthropic/claude-haiku-4-5".to_owned(),
1064                provider: "anthropic".to_owned(),
1065                in_tokens: 10,
1066                cache_write_tokens: 0,
1067                cache_read_tokens: 0,
1068                out_tokens: 5,
1069                cost_usd: 0.001,
1070                latency_ms: 12,
1071                gates: vec![],
1072                verdict: Verdict::Pass,
1073            }],
1074            deferred: Vec::new(),
1075            final_: FinalOutcome {
1076                served_rung: Some(0),
1077                served_from: ServedFrom::Attempt,
1078                total_cost_usd: 0.001,
1079                gate_cost_usd: 0.0,
1080                total_latency_ms: 12,
1081                escalations: 0,
1082                counterfactual_baseline_usd: 0.001,
1083                savings_usd: 0.0,
1084                cache_source: None,
1085            },
1086            probe: None,
1087            rollout: None,
1088            shadow: None,
1089            route_ix: None,
1090            predicted_pass: None,
1091            elastic: None,
1092        };
1093        let t0 = mk(GENESIS_HASH, "s0");
1094        let t1 = mk(&t0.hash().unwrap(), "s1");
1095        let t2 = mk(&t1.hash().unwrap(), "s2");
1096        let chain = vec![t0, t1, t2];
1097
1098        // Clean chain verifies.
1099        let report = verify_receipts(&chain);
1100        assert!(report.valid, "intact chain must verify: {report:?}");
1101        assert_eq!(report.receipts, 3);
1102
1103        // Export → parse round-trips and still verifies (the auditor's offline path).
1104        let jsonl = export_receipts_jsonl(&chain);
1105        assert_eq!(jsonl.lines().count(), 3);
1106        let reparsed = parse_receipt_jsonl(&jsonl).expect("export must re-parse");
1107        assert!(
1108            verify_receipts(&reparsed).valid,
1109            "round-tripped chain must verify"
1110        );
1111
1112        // Tamper with a middle receipt's body → verification catches it at that index.
1113        let mut tampered = reparsed;
1114        tampered[1].final_.total_cost_usd = 999.0; // alter a sealed field
1115        let bad = verify_receipts(&tampered);
1116        assert!(!bad.valid, "a mutated receipt must break the chain");
1117        assert_eq!(
1118            bad.broken_at,
1119            Some(2),
1120            "the break surfaces at the next link"
1121        );
1122
1123        // Reordering is also caught.
1124        let mut reordered = parse_receipt_jsonl(&jsonl).unwrap();
1125        reordered.swap(0, 1);
1126        assert!(
1127            !verify_receipts(&reordered).valid,
1128            "reordering breaks the chain"
1129        );
1130    }
1131
1132    #[test]
1133    fn parse_receipt_jsonl_reports_bad_line() {
1134        let err = parse_receipt_jsonl("not json\n").unwrap_err();
1135        assert!(err.starts_with("line 1:"), "must name the bad line: {err}");
1136        assert!(
1137            parse_receipt_jsonl("\n\n").unwrap().is_empty(),
1138            "blank lines skipped"
1139        );
1140    }
1141
1142    fn trace_with_timing(total_ms: u64, attempts: &[(u64, u64)]) -> Trace {
1143        use firstpass_core::{
1144            Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1145            ServedFrom, TaskKind, Verdict,
1146        };
1147        Trace {
1148            trace_id: uuid::Uuid::now_v7(),
1149            prev_hash: GENESIS_HASH.to_owned(),
1150            tenant_id: "default".to_owned(),
1151            session_id: "s".to_owned(),
1152            ts: jiff::Timestamp::UNIX_EPOCH,
1153            mode: Mode::Enforce,
1154            policy: PolicyRef {
1155                id: "static-ladder@v0".to_owned(),
1156                explore: false,
1157                propensity: None,
1158                mode_profile: None,
1159            },
1160            request: RequestInfo {
1161                api: "anthropic.messages".to_owned(),
1162                prompt_hash: "x".to_owned(),
1163                features: Features::new(TaskKind::CodeEdit),
1164            },
1165            attempts: attempts
1166                .iter()
1167                .enumerate()
1168                .map(|(i, (model_ms, gate_ms))| Attempt {
1169                    rung: i as u32,
1170                    model: "anthropic/claude-haiku-4-5".to_owned(),
1171                    provider: "anthropic".to_owned(),
1172                    in_tokens: 1,
1173                    cache_write_tokens: 0,
1174                    cache_read_tokens: 0,
1175                    out_tokens: 1,
1176                    cost_usd: 0.0,
1177                    latency_ms: *model_ms,
1178                    gates: vec![GateResult::deterministic(
1179                        "non-empty",
1180                        Verdict::Pass,
1181                        *gate_ms,
1182                    )],
1183                    verdict: Verdict::Pass,
1184                })
1185                .collect(),
1186            final_: FinalOutcome {
1187                served_rung: Some(0),
1188                served_from: ServedFrom::Attempt,
1189                total_cost_usd: 0.0,
1190                gate_cost_usd: 0.0,
1191                total_latency_ms: total_ms,
1192                escalations: 0,
1193                counterfactual_baseline_usd: 0.0,
1194                savings_usd: 0.0,
1195                cache_source: None,
1196            },
1197            deferred: vec![],
1198            predicted_pass: None,
1199            probe: None,
1200            elastic: None,
1201            rollout: None,
1202            shadow: None,
1203            route_ix: None,
1204        }
1205    }
1206
1207    #[test]
1208    fn overhead_is_wall_clock_minus_the_work_actually_dispatched() {
1209        // 500ms total, 400ms in the provider, 60ms in the gate → 40ms is ours.
1210        let t = trace_with_timing(500, &[(400, 60)]);
1211        assert_eq!(t.overhead_ms(), Some(40));
1212
1213        // Escalation: two provider calls and two gate runs all count against wall-clock.
1214        let t = trace_with_timing(1000, &[(400, 50), (500, 20)]);
1215        assert_eq!(t.overhead_ms(), Some(30));
1216    }
1217
1218    #[test]
1219    fn a_concurrent_receipt_is_excluded_rather_than_floored_to_zero() {
1220        // Speculation runs rungs in parallel, so dispatched work legitimately exceeds wall-clock.
1221        // Flooring that to 0 would be a fabricated measurement that drags a p95 down — exactly the
1222        // kind of quietly-wrong number this codebase keeps having to remove.
1223        let t = trace_with_timing(600, &[(400, 50), (500, 50)]);
1224        assert_eq!(t.overhead_ms(), None, "must not report a made-up 0");
1225
1226        let s = summarize_overhead(&[t, trace_with_timing(500, &[(400, 60)])]);
1227        assert_eq!(s.n, 1, "only the measurable one counts");
1228        assert_eq!(
1229            s.skipped_concurrent, 1,
1230            "and the skip is reported, not hidden"
1231        );
1232        assert_eq!(s.p50_ms, 40);
1233    }
1234
1235    #[test]
1236    fn percentiles_are_observed_values_not_interpolated_ones() {
1237        // Nearest-rank: every number printed is one we actually measured. Interpolation would
1238        // invent a value between two real observations and report it as fact.
1239        let traces: Vec<Trace> = [10u64, 20, 30, 40, 100]
1240            .iter()
1241            .map(|ms| trace_with_timing(100 + ms, &[(100, 0)]))
1242            .collect();
1243
1244        let s = summarize_overhead(&traces);
1245
1246        assert_eq!(s.n, 5);
1247        assert_eq!(s.p50_ms, 30);
1248        assert_eq!(s.p95_ms, 100);
1249        assert_eq!(s.p99_ms, 100);
1250        assert_eq!(s.max_ms, 100);
1251        for v in [s.p50_ms, s.p95_ms, s.p99_ms] {
1252            assert!([10, 20, 30, 40, 100].contains(&v), "{v} was never observed");
1253        }
1254    }
1255
1256    #[test]
1257    fn no_receipts_reports_that_rather_than_zeros() {
1258        let s = summarize_overhead(&[]);
1259        assert_eq!(s.n, 0);
1260        assert!(
1261            format_overhead(&s).contains("no receipts"),
1262            "{}",
1263            format_overhead(&s)
1264        );
1265    }
1266
1267    #[test]
1268    fn training_rows_carry_the_logged_propensity() {
1269        use firstpass_core::{
1270            Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1271            ServedFrom, TaskKind, Verdict,
1272        };
1273        let mk = |propensity: Option<f64>, explore: bool| Trace {
1274            trace_id: uuid::Uuid::now_v7(),
1275            prev_hash: GENESIS_HASH.to_owned(),
1276            tenant_id: "default".to_owned(),
1277            session_id: "s".to_owned(),
1278            ts: jiff::Timestamp::UNIX_EPOCH,
1279            mode: Mode::Enforce,
1280            policy: PolicyRef {
1281                id: "bandit@v1+eps".to_owned(),
1282                explore,
1283                propensity,
1284                mode_profile: None,
1285            },
1286            request: RequestInfo {
1287                api: "anthropic.messages".to_owned(),
1288                prompt_hash: "x".to_owned(),
1289                features: Features::new(TaskKind::CodeEdit),
1290            },
1291            attempts: vec![Attempt {
1292                rung: 1,
1293                model: "anthropic/claude-sonnet-5".to_owned(),
1294                provider: "anthropic".to_owned(),
1295                in_tokens: 10,
1296                cache_write_tokens: 0,
1297                cache_read_tokens: 0,
1298                out_tokens: 5,
1299                cost_usd: 0.004,
1300                latency_ms: 7,
1301                gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
1302                verdict: Verdict::Pass,
1303            }],
1304            final_: FinalOutcome {
1305                served_rung: Some(1),
1306                served_from: ServedFrom::Attempt,
1307                total_cost_usd: 0.004,
1308                gate_cost_usd: 0.0,
1309                total_latency_ms: 7,
1310                escalations: 0,
1311                counterfactual_baseline_usd: 0.02,
1312                savings_usd: 0.016,
1313                cache_source: None,
1314            },
1315            deferred: vec![],
1316            predicted_pass: None,
1317            probe: None,
1318            elastic: None,
1319            rollout: None,
1320            shadow: None,
1321            route_ix: None,
1322        };
1323
1324        let rows = export_training_jsonl(&[mk(Some(0.25), true), mk(None, false)]);
1325        let lines: Vec<&str> = rows.lines().collect();
1326        assert_eq!(lines.len(), 2);
1327
1328        let a: serde_json::Value = serde_json::from_str(lines[0]).expect("row is json");
1329        // The whole point: the propensity survives into the training row. Without it a learner
1330        // inherits the logging policy's bias with no way to correct for it.
1331        assert_eq!(a["propensity"], 0.25);
1332        assert_eq!(a["explore"], true);
1333        assert_eq!(a["start_rung"], 1);
1334        assert_eq!(a["served_rung"], 1);
1335        assert_eq!(a["served_passed"], true);
1336        assert_eq!(a["policy_id"], "bandit@v1+eps");
1337
1338        // A deterministic decision must be marked un-corrected, NOT silently defaulted to 1.0 —
1339        // that would look like a uniformly-sampled row and quietly bias any estimate built on it.
1340        let b: serde_json::Value = serde_json::from_str(lines[1]).expect("row is json");
1341        assert!(b["propensity"].is_null(), "got {}", b["propensity"]);
1342    }
1343
1344    #[test]
1345    fn explain_trace_summarizes_served_and_escalation() {
1346        use firstpass_core::{
1347            Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1348            ServedFrom, TaskKind, Verdict,
1349        };
1350        let t = Trace {
1351            trace_id: uuid::Uuid::now_v7(),
1352            prev_hash: GENESIS_HASH.to_owned(),
1353            tenant_id: "default".to_owned(),
1354            session_id: "s".to_owned(),
1355            ts: jiff::Timestamp::UNIX_EPOCH,
1356            mode: Mode::Enforce,
1357            policy: PolicyRef {
1358                id: "static-ladder@v0".to_owned(),
1359                explore: false,
1360                propensity: None,
1361                mode_profile: None,
1362            },
1363            request: RequestInfo {
1364                api: "anthropic.messages".to_owned(),
1365                prompt_hash: "x".to_owned(),
1366                features: Features::new(TaskKind::Other),
1367            },
1368            attempts: vec![
1369                Attempt {
1370                    rung: 0,
1371                    model: "anthropic/claude-haiku-4-5".to_owned(),
1372                    provider: "anthropic".to_owned(),
1373                    in_tokens: 1,
1374                    cache_write_tokens: 0,
1375                    cache_read_tokens: 0,
1376                    out_tokens: 1,
1377                    cost_usd: 0.001,
1378                    latency_ms: 5,
1379                    gates: vec![GateResult::deterministic("non-empty", Verdict::Fail, 0)],
1380                    verdict: Verdict::Fail,
1381                },
1382                Attempt {
1383                    rung: 1,
1384                    model: "anthropic/claude-sonnet-5".to_owned(),
1385                    provider: "anthropic".to_owned(),
1386                    in_tokens: 1,
1387                    cache_write_tokens: 0,
1388                    cache_read_tokens: 0,
1389                    out_tokens: 1,
1390                    cost_usd: 0.01,
1391                    latency_ms: 9,
1392                    gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
1393                    verdict: Verdict::Pass,
1394                },
1395            ],
1396            deferred: Vec::new(),
1397            final_: FinalOutcome {
1398                served_rung: Some(1),
1399                served_from: ServedFrom::Attempt,
1400                total_cost_usd: 0.011,
1401                gate_cost_usd: 0.0,
1402                total_latency_ms: 14,
1403                escalations: 1,
1404                counterfactual_baseline_usd: 0.02,
1405                savings_usd: 0.009,
1406                cache_source: None,
1407            },
1408            probe: None,
1409            rollout: None,
1410            shadow: None,
1411            route_ix: None,
1412            predicted_pass: None,
1413            elastic: None,
1414        };
1415        let ex = explain_trace(&t);
1416        assert_eq!(
1417            ex.served_model.as_deref(),
1418            Some("anthropic/claude-sonnet-5")
1419        );
1420        assert_eq!(ex.escalations, 1);
1421        assert_eq!(ex.attempts.len(), 2);
1422        assert_eq!(ex.attempts[0].verdict, "fail");
1423        assert_eq!(
1424            ex.attempts[1].gates[0],
1425            ("non-empty".to_owned(), "pass".to_owned())
1426        );
1427        assert!(ex.summary.contains("served anthropic/claude-sonnet-5"));
1428    }
1429
1430    #[test]
1431    fn evaluate_predictor_empty_and_signal() {
1432        // empty store => n=0, no crash
1433        let e = evaluate_predictor(&[], 0.05, 1e-4);
1434        assert_eq!(e.n, 0);
1435        assert!(e.auc.is_none());
1436    }
1437
1438    /// Each channel must be recognised from a realistic install path. Getting this wrong is not
1439    /// cosmetic: it would print an upgrade command that silently does nothing.
1440    #[test]
1441    fn detect_channel_reads_real_install_paths() {
1442        use std::path::Path;
1443        let cases = [
1444            ("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
1445            (
1446                "/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
1447                Channel::Homebrew,
1448            ),
1449            (
1450                "/home/linuxbrew/.linuxbrew/bin/firstpass",
1451                Channel::Homebrew,
1452            ),
1453            (
1454                "/Users/u/.venv/lib/python3.12/site-packages/firstpass/firstpass",
1455                Channel::Python,
1456            ),
1457            // Both observed on a real machine rather than invented: uvx runs from an ephemeral
1458            // env under archive-v0, and `uv tool install` lands under the uv tools dir.
1459            (
1460                "/Users/u/.cache/uv/archive-v0/DQ4usH_XuqNyywgHKB8qB/bin/firstpass",
1461                Channel::Python,
1462            ),
1463            (
1464                "/Users/u/.local/share/uv/tools/firstpass/bin/firstpass",
1465                Channel::Python,
1466            ),
1467            (
1468                "/usr/local/lib/node_modules/@dshakesnotbot/firstpass-proxy/bin/firstpass",
1469                Channel::Npm,
1470            ),
1471            (r"C:\Users\u\.cargo\bin\firstpass.exe", Channel::Cargo),
1472        ];
1473        for (path, want) in cases {
1474            assert_eq!(
1475                detect_channel(Path::new(path), false, false),
1476                want,
1477                "misread {path}"
1478            );
1479        }
1480        // The installer channel is identified by its updater, not by its path.
1481        assert_eq!(
1482            detect_channel(Path::new("/Users/u/.local/bin/firstpass"), true, false),
1483            Channel::Dist
1484        );
1485        assert_eq!(
1486            detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
1487            Channel::Unknown
1488        );
1489        // A container wins outright — nothing on the host owns that file.
1490        assert_eq!(
1491            detect_channel(Path::new("/usr/local/bin/firstpass"), true, true),
1492            Channel::Docker
1493        );
1494    }
1495
1496    /// A package manager owning the file must beat the updater probe: a brew or wheel install can
1497    /// carry the updater too, and self-updating underneath the manager desynchronises its metadata.
1498    #[test]
1499    fn package_manager_paths_win_over_the_self_updater() {
1500        use std::path::Path;
1501        for (path, want) in [
1502            (
1503                "/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
1504                Channel::Homebrew,
1505            ),
1506            ("/Users/u/.venv/bin/firstpass", Channel::Python),
1507            ("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
1508        ] {
1509            assert_eq!(detect_channel(Path::new(path), true, false), want, "{path}");
1510        }
1511    }
1512
1513    /// Only the channel we own may be upgraded in place; the rest must be advice, and the
1514    /// unknown case must still leave the user with something runnable.
1515    #[test]
1516    fn upgrade_report_runs_only_our_own_channel() {
1517        assert!(upgrade_report(Channel::Dist, "0.2.2").contains("upgrading in place"));
1518        for c in [
1519            Channel::Homebrew,
1520            Channel::Python,
1521            Channel::Npm,
1522            Channel::Docker,
1523            Channel::Cargo,
1524        ] {
1525            let r = upgrade_report(c, "0.2.2");
1526            assert!(
1527                !r.contains("upgrading in place"),
1528                "{c:?} must not self-update"
1529            );
1530            assert!(
1531                r.contains("owned by its package manager"),
1532                "{c:?} must explain why"
1533            );
1534            assert!(
1535                r.contains(c.upgrade_command().expect("has a command")),
1536                "{c:?} must print its exact command"
1537            );
1538        }
1539        let unknown = upgrade_report(Channel::Unknown, "0.2.2");
1540        assert!(unknown.contains("brew upgrade") && unknown.contains("docker pull"));
1541        assert!(
1542            unknown.contains("0.2.2"),
1543            "always states the running version"
1544        );
1545    }
1546
1547    /// Regression: reported from a real machine. pipx installs into its own venv and puts a
1548    /// symlink on PATH, so the launch path is `~/.local/bin/firstpass` and carries no marker at
1549    /// all. Detection runs on the CANONICAL path; these are the two forms of that install.
1550    #[test]
1551    fn pipx_install_is_recognised_from_its_real_path() {
1552        use std::path::Path;
1553        let real = "/Users/u/.local/pipx/venvs/firstpass/bin/firstpass";
1554        assert_eq!(detect_channel(Path::new(real), false, false), Channel::Pipx);
1555        assert_eq!(
1556            Channel::Pipx.upgrade_command(),
1557            Some("pipx upgrade firstpass")
1558        );
1559
1560        // The launch path alone is genuinely unidentifiable — which is why the caller must
1561        // canonicalize rather than this function pretending it can tell.
1562        assert_eq!(
1563            detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
1564            Channel::Unknown
1565        );
1566
1567        // A pipx venv also contains site-packages; pipx must win so the command is right.
1568        assert_eq!(
1569            detect_channel(
1570                Path::new(
1571                    "/Users/u/.local/pipx/venvs/firstpass/lib/python3.14/site-packages/x/firstpass"
1572                ),
1573                false,
1574                false
1575            ),
1576            Channel::Pipx
1577        );
1578    }
1579}