firstpass_proxy/
cli.rs

1//! CLI surfaces for the `firstpass` binary (SPEC §7.3/§7.4): `doctor` validates a setup before
2//! you route real traffic through it, and `trace` reads recent audit records from the store. Both
3//! are kept here (not in the binary) so the judgment is unit-tested.
4
5use crate::config::ProxyConfig;
6use firstpass_core::{Mode, Trace};
7
8/// A single doctor check outcome.
9#[derive(Debug, PartialEq, Eq)]
10pub enum CheckStatus {
11    /// Healthy.
12    Ok,
13    /// Works, but worth knowing (e.g. no key in env — observe still runs).
14    Warn,
15    /// Broken; `firstpass doctor` exits non-zero.
16    Fail,
17}
18
19/// One line of the doctor report.
20#[derive(Debug)]
21pub struct Check {
22    /// Short check name.
23    pub name: String,
24    /// Outcome.
25    pub status: CheckStatus,
26    /// Human-readable detail.
27    pub detail: String,
28}
29
30impl Check {
31    fn new(name: impl Into<String>, status: CheckStatus, detail: impl Into<String>) -> Self {
32        Self {
33            name: name.into(),
34            status,
35            detail: detail.into(),
36        }
37    }
38}
39
40/// The result of `firstpass doctor`.
41#[derive(Debug)]
42pub struct DoctorReport {
43    /// One entry per check, in report order.
44    pub checks: Vec<Check>,
45}
46
47impl DoctorReport {
48    /// Healthy iff no check failed (warnings are fine).
49    #[must_use]
50    pub fn healthy(&self) -> bool {
51        self.checks.iter().all(|c| c.status != CheckStatus::Fail)
52    }
53
54    /// Render the report as human-readable lines (a rendering of the structured checks).
55    #[must_use]
56    pub fn render(&self) -> String {
57        let mut out = String::new();
58        for c in &self.checks {
59            let mark = match c.status {
60                CheckStatus::Ok => "✓",
61                CheckStatus::Warn => "!",
62                CheckStatus::Fail => "✗",
63            };
64            out.push_str(&format!("{mark} {}: {}\n", c.name, c.detail));
65        }
66        out.push_str(if self.healthy() {
67            "\nhealthy — ready to route.\n"
68        } else {
69            "\nnot healthy — fix the ✗ items above.\n"
70        });
71        out
72    }
73}
74
75/// Validate a loaded config against the environment: routing sanity, provider key presence, and
76/// that every configured gate's command actually exists. `env` looks up environment variables
77/// (injected so this is testable).
78#[must_use]
79pub fn doctor(config: &ProxyConfig, env: impl Fn(&str) -> Option<String>) -> DoctorReport {
80    let mut checks = Vec::new();
81
82    // Config parsed (we hold a ProxyConfig), report the shape.
83    let route_count = config.routing.as_ref().map_or(0, |c| c.routes.len());
84    checks.push(Check::new(
85        "config",
86        CheckStatus::Ok,
87        format!("default mode {:?}, {route_count} route(s)", config.mode),
88    ));
89
90    // Enforce is only meaningful with an enforce route to activate the engine.
91    let enforce_routes = config.routing.as_ref().map_or(0, |c| {
92        c.routes.iter().filter(|r| r.mode == Mode::Enforce).count()
93    });
94    if config.mode == Mode::Enforce && enforce_routes == 0 {
95        checks.push(Check::new(
96            "routing",
97            CheckStatus::Warn,
98            "mode is enforce but no enforce route is defined — all traffic will observe",
99        ));
100    } else {
101        checks.push(Check::new(
102            "routing",
103            CheckStatus::Ok,
104            format!("{enforce_routes} enforce route(s)"),
105        ));
106    }
107
108    // A provider key in the environment. Observe uses the caller's key, so absence is a warning.
109    if env("ANTHROPIC_API_KEY").is_some_and(|k| !k.is_empty()) {
110        checks.push(Check::new("anthropic-key", CheckStatus::Ok, "present"));
111    } else {
112        checks.push(Check::new(
113            "anthropic-key",
114            CheckStatus::Warn,
115            "ANTHROPIC_API_KEY not set — observe uses the caller's key; enforce needs one reachable",
116        ));
117    }
118
119    // Every configured gate command must resolve, or that gate silently abstains at runtime.
120    let path = env("PATH");
121    let gate_defs = config.routing.as_ref().map_or(&[][..], |c| &c.gate_defs);
122    for def in gate_defs {
123        match def.cmd.first() {
124            Some(program) if command_on_path(program, path.as_deref()) => checks.push(Check::new(
125                format!("gate:{}", def.id),
126                CheckStatus::Ok,
127                format!("`{program}` found"),
128            )),
129            Some(program) => checks.push(Check::new(
130                format!("gate:{}", def.id),
131                CheckStatus::Fail,
132                format!("`{program}` not found on PATH"),
133            )),
134            None => checks.push(Check::new(
135                format!("gate:{}", def.id),
136                CheckStatus::Fail,
137                "empty command",
138            )),
139        }
140    }
141
142    // The trace store must be writable, or we'd trade the audit trail (or availability) for it.
143    if can_write_db(&config.db_path) {
144        checks.push(Check::new(
145            "trace-store",
146            CheckStatus::Ok,
147            format!("{} is writable", config.db_path),
148        ));
149    } else {
150        checks.push(Check::new(
151            "trace-store",
152            CheckStatus::Fail,
153            format!("cannot write near {}", config.db_path),
154        ));
155    }
156
157    DoctorReport { checks }
158}
159
160/// Whether `program` is runnable: an explicit path (contains a separator) that exists, or a bare
161/// name found in one of `PATH`'s directories.
162#[must_use]
163pub fn command_on_path(program: &str, path_var: Option<&str>) -> bool {
164    if program.contains('/') || program.contains('\\') {
165        return std::path::Path::new(program).is_file();
166    }
167    let Some(path) = path_var else { return false };
168    std::env::split_paths(path).any(|dir| dir.join(program).is_file())
169}
170
171/// Probe whether the trace DB's directory is writable, without creating the DB itself.
172fn can_write_db(db_path: &str) -> bool {
173    let path = std::path::Path::new(db_path);
174    let dir = path
175        .parent()
176        .filter(|d| !d.as_os_str().is_empty())
177        .map_or_else(
178            || std::path::PathBuf::from("."),
179            std::path::Path::to_path_buf,
180        );
181    let probe = dir.join(format!(".firstpass-doctor-probe-{}", std::process::id()));
182    match std::fs::File::create(&probe) {
183        Ok(_) => {
184            let _ = std::fs::remove_file(&probe);
185            true
186        }
187        Err(_) => false,
188    }
189}
190
191/// Per-gate and per-rung evaluation summary computed from receipts — the operator's live
192/// eval suite: how each gate is verdict-ing, how often routing escalates, where serves land.
193#[derive(Debug, serde::Serialize)]
194pub struct EvalsSummary {
195    /// Traces aggregated (enforce mode only — observe records no gate decisions on-path).
196    pub enforce_traces: usize,
197    /// Total escalations across those traces.
198    pub escalations: u64,
199    /// gate_id → (pass, fail, abstain) counts across every attempt.
200    pub gates: std::collections::BTreeMap<String, (u64, u64, u64)>,
201    /// rung index → times an attempt at that rung was the served one.
202    pub served_by_rung: std::collections::BTreeMap<u32, u64>,
203}
204
205/// Aggregate gate verdicts + routing behavior over `traces` (enforce only).
206#[must_use]
207/// Prequential evaluation of the per-query gate-pass predictor (ADR 0008 Phase 2) — the honest
208/// "is it any good yet" check before the predictor is ever allowed to *act*. Replays the receipts
209/// in order through a fresh predictor: for each attempt with a clear Pass/Fail verdict, **predict
210/// before update** (so every prediction is on data the model has not trained on), then update.
211/// Reports AUC (predicted -> passed) and Brier score over those pairs.
212#[derive(Debug, serde::Serialize)]
213pub struct PredictorEval {
214    /// Labeled attempts evaluated.
215    pub n: usize,
216    /// AUC of predicted P(pass) ranking actual passes above fails. 0.5 = no signal; 1.0 = perfect.
217    /// `None` when one class is absent (all pass or all fail — AUC undefined).
218    pub auc: Option<f64>,
219    /// Mean squared error of the probabilistic prediction, in `[0, 1]`. Lower is better; 0.25 is
220    /// the always-0.5 baseline.
221    pub brier: f64,
222}
223
224/// AUC via the Mann-Whitney rank-sum identity (ties = 0.5). `None` if either class is empty.
225fn predictor_auc(pairs: &[(f64, bool)]) -> Option<f64> {
226    let pos: Vec<f64> = pairs.iter().filter(|(_, y)| *y).map(|(s, _)| *s).collect();
227    let neg: Vec<f64> = pairs.iter().filter(|(_, y)| !*y).map(|(s, _)| *s).collect();
228    if pos.is_empty() || neg.is_empty() {
229        return None;
230    }
231    let mut wins = 0.0_f64;
232    for &p in &pos {
233        for &n in &neg {
234            wins += if p > n {
235                1.0
236            } else if (p - n).abs() < f64::EPSILON {
237                0.5
238            } else {
239                0.0
240            };
241        }
242    }
243    Some(wins / (pos.len() as f64 * neg.len() as f64))
244}
245
246/// Run the prequential predictor evaluation over `traces` with the given `lr`/`l2`.
247pub fn evaluate_predictor(traces: &[Trace], lr: f64, l2: f64) -> PredictorEval {
248    use firstpass_core::{PassPredictor, Verdict};
249    let mut model = PassPredictor::new(lr, l2);
250    let mut pairs: Vec<(f64, bool)> = Vec::new();
251    let mut se = 0.0_f64;
252    for t in traces {
253        for a in &t.attempts {
254            let y = match a.verdict {
255                Verdict::Pass => true,
256                Verdict::Fail => false,
257                Verdict::Abstain => continue,
258            };
259            let p = model.predict(&t.request.features, a.rung);
260            let target = f64::from(u8::from(y));
261            se += (p - target) * (p - target);
262            pairs.push((p, y));
263            model.update(&t.request.features, a.rung, y);
264        }
265    }
266    let n = pairs.len();
267    PredictorEval {
268        n,
269        auc: predictor_auc(&pairs),
270        brier: if n == 0 { 0.0 } else { se / n as f64 },
271    }
272}
273
274pub fn summarize_evals(traces: &[Trace]) -> EvalsSummary {
275    let mut s = EvalsSummary {
276        enforce_traces: 0,
277        escalations: 0,
278        gates: std::collections::BTreeMap::new(),
279        served_by_rung: std::collections::BTreeMap::new(),
280    };
281    for t in traces {
282        if t.mode != Mode::Enforce {
283            continue;
284        }
285        s.enforce_traces += 1;
286        s.escalations += u64::from(t.final_.escalations);
287        if let Some(rung) = t.final_.served_rung {
288            *s.served_by_rung.entry(rung).or_insert(0) += 1;
289        }
290        for attempt in &t.attempts {
291            for g in &attempt.gates {
292                let e = s.gates.entry(g.gate_id.clone()).or_insert((0, 0, 0));
293                match g.verdict {
294                    firstpass_core::Verdict::Pass => e.0 += 1,
295                    firstpass_core::Verdict::Fail => e.1 += 1,
296                    firstpass_core::Verdict::Abstain => e.2 += 1,
297                }
298            }
299        }
300    }
301    s
302}
303
304/// Render an [`EvalsSummary`] for humans (`--json` callers serialize the struct instead).
305#[must_use]
306pub fn format_evals(s: &EvalsSummary) -> String {
307    if s.enforce_traces == 0 {
308        return "no enforce traces yet — route some traffic in enforce mode first".to_owned();
309    }
310    let mut out = format!(
311        "enforce traces: {} · escalations: {}\n",
312        s.enforce_traces, s.escalations
313    );
314    out.push_str("gates (pass / fail / abstain):\n");
315    for (id, (p, f, a)) in &s.gates {
316        out.push_str(&format!("  {id:<24} {p:>6} / {f:>6} / {a:>6}\n"));
317    }
318    out.push_str("served by rung:\n");
319    for (rung, n) in &s.served_by_rung {
320        out.push_str(&format!("  rung {rung:<2} {n:>6}\n"));
321    }
322    out.trim_end().to_owned()
323}
324
325/// A structured, agent-readable explanation of a single routing decision — why the ladder
326/// went where it did, built purely from a sealed [`Trace`].
327#[derive(Debug, serde::Serialize)]
328pub struct RouteExplanation {
329    pub trace_id: String,
330    pub mode: String,
331    pub policy: String,
332    /// The model actually served, if any.
333    pub served_model: Option<String>,
334    pub served_rung: Option<u32>,
335    pub escalations: u32,
336    pub total_cost_usd: f64,
337    pub baseline_usd: f64,
338    pub savings_usd: f64,
339    /// Per attempt, in ladder order: what was tried and how it was judged.
340    pub attempts: Vec<AttemptExplanation>,
341    /// One-line human summary.
342    pub summary: String,
343}
344
345/// Per-attempt slice of a [`RouteExplanation`].
346#[derive(Debug, serde::Serialize)]
347pub struct AttemptExplanation {
348    pub rung: u32,
349    pub model: String,
350    pub verdict: String,
351    /// gate_id → verdict, for every gate that ran on this attempt.
352    pub gates: Vec<(String, String)>,
353}
354
355fn verdict_str(v: firstpass_core::Verdict) -> &'static str {
356    match v {
357        firstpass_core::Verdict::Pass => "pass",
358        firstpass_core::Verdict::Fail => "fail",
359        firstpass_core::Verdict::Abstain => "abstain",
360    }
361}
362
363/// Explain one routing decision from its sealed receipt — the "why this model" answer, as
364/// structured data an agent can act on (and the backing for the MCP `explain_route` tool).
365#[must_use]
366pub fn explain_trace(t: &Trace) -> RouteExplanation {
367    let attempts: Vec<AttemptExplanation> = t
368        .attempts
369        .iter()
370        .map(|a| AttemptExplanation {
371            rung: a.rung,
372            model: a.model.clone(),
373            verdict: verdict_str(a.verdict).to_owned(),
374            gates: a
375                .gates
376                .iter()
377                .map(|g| (g.gate_id.clone(), verdict_str(g.verdict).to_owned()))
378                .collect(),
379        })
380        .collect();
381    let served_model = t
382        .final_
383        .served_rung
384        .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
385        .map(|a| a.model.clone());
386    let summary = match &served_model {
387        Some(m) => format!(
388            "served {m} at rung {} after {} escalation(s); spent ${:.4} vs ${:.4} always-top (saved ${:.4})",
389            t.final_.served_rung.unwrap_or(0),
390            t.final_.escalations,
391            t.final_.total_cost_usd,
392            t.final_.counterfactual_baseline_usd,
393            t.final_.savings_usd
394        ),
395        None => "no rung served (all attempts failed the gate or errored)".to_owned(),
396    };
397    RouteExplanation {
398        trace_id: t.trace_id.to_string(),
399        mode: match t.mode {
400            Mode::Enforce => "enforce",
401            Mode::Observe => "observe",
402        }
403        .to_owned(),
404        policy: t.policy.id.clone(),
405        served_model,
406        served_rung: t.final_.served_rung,
407        escalations: t.final_.escalations,
408        total_cost_usd: t.final_.total_cost_usd,
409        baseline_usd: t.final_.counterfactual_baseline_usd,
410        savings_usd: t.final_.savings_usd,
411        attempts,
412        summary,
413    }
414}
415
416/// Outcome of an independent receipt-chain verification — the compliance artifact.
417#[derive(Debug, serde::Serialize)]
418pub struct VerifyReport {
419    /// Receipts checked.
420    pub receipts: usize,
421    /// `true` iff the hash chain re-derives cleanly from genesis (tamper-evident proof).
422    pub valid: bool,
423    /// On failure, the 0-based index of the first broken link and why.
424    #[serde(skip_serializing_if = "Option::is_none")]
425    pub broken_at: Option<usize>,
426    #[serde(skip_serializing_if = "Option::is_none")]
427    pub detail: Option<String>,
428}
429
430/// Independently re-derive the hash chain over `traces` from genesis — the same computation an
431/// external auditor runs, with no trust in the proxy or the database. A single altered or
432/// reordered receipt breaks the chain at its index.
433#[must_use]
434pub fn verify_receipts(traces: &[Trace]) -> VerifyReport {
435    match firstpass_core::verify_chain(traces, firstpass_core::GENESIS_HASH) {
436        Ok(()) => VerifyReport {
437            receipts: traces.len(),
438            valid: true,
439            broken_at: None,
440            detail: None,
441        },
442        Err(firstpass_core::Error::ChainBroken { index, detail }) => VerifyReport {
443            receipts: traces.len(),
444            valid: false,
445            broken_at: Some(index),
446            detail: Some(detail),
447        },
448        Err(e) => VerifyReport {
449            receipts: traces.len(),
450            valid: false,
451            broken_at: None,
452            detail: Some(e.to_string()),
453        },
454    }
455}
456
457/// Parse a JSONL receipt export (one [`Trace`] per line) back into traces, for offline
458/// verification. Blank lines are skipped; a malformed line fails loudly with its line number.
459///
460/// # Errors
461/// Returns the 1-based line number and parse error of the first unparseable line.
462pub fn parse_receipt_jsonl(text: &str) -> Result<Vec<Trace>, String> {
463    let mut traces = Vec::new();
464    for (i, line) in text.lines().enumerate() {
465        if line.trim().is_empty() {
466            continue;
467        }
468        let t: Trace = serde_json::from_str(line).map_err(|e| format!("line {}: {e}", i + 1))?;
469        traces.push(t);
470    }
471    Ok(traces)
472}
473
474/// Render `traces` as a JSONL receipt export — one sealed receipt per line, in chain order.
475/// This is the artifact an operator hands an auditor; the deferred-verdict side table is never
476/// included (it is not part of the hashed body).
477#[must_use]
478pub fn export_receipts_jsonl(traces: &[Trace]) -> String {
479    let mut out = String::new();
480    for t in traces {
481        if let Ok(line) = serde_json::to_string(t) {
482            out.push_str(&line);
483            out.push('\n');
484        }
485    }
486    out
487}
488
489/// Aggregated spend/savings over a set of traces — the number the operator screenshots.
490/// Pure so it's unit-testable; `firstpass savings` feeds it the trace store.
491#[derive(Debug, serde::Serialize)]
492pub struct SavingsSummary {
493    /// Traces aggregated.
494    pub traces: usize,
495    /// Enforce-mode traces (the ones where routing actually decided).
496    pub enforce_traces: usize,
497    /// USD actually spent (model + gate calls), summed over all traces.
498    pub spent_usd: f64,
499    /// USD spent on gates alone (the price of proof).
500    pub gate_usd: f64,
501    /// What always calling the top rung would have cost (§9.1 counterfactual), summed.
502    pub baseline_usd: f64,
503    /// `baseline - spent` — the savings the receipts prove.
504    pub savings_usd: f64,
505    /// Savings as a fraction of baseline, `0.0` when there is no baseline.
506    pub savings_pct: f64,
507}
508
509/// Aggregate spend vs the always-top counterfactual over `traces`.
510#[must_use]
511pub fn summarize_savings(traces: &[Trace]) -> SavingsSummary {
512    let mut s = SavingsSummary {
513        traces: traces.len(),
514        enforce_traces: 0,
515        spent_usd: 0.0,
516        gate_usd: 0.0,
517        baseline_usd: 0.0,
518        savings_usd: 0.0,
519        savings_pct: 0.0,
520    };
521    for t in traces {
522        if t.mode == Mode::Enforce {
523            s.enforce_traces += 1;
524        }
525        s.spent_usd += t.final_.total_cost_usd;
526        s.gate_usd += t.final_.gate_cost_usd;
527        s.baseline_usd += t.final_.counterfactual_baseline_usd;
528    }
529    s.savings_usd = s.baseline_usd - s.spent_usd;
530    if s.baseline_usd > 0.0 {
531        s.savings_pct = s.savings_usd / s.baseline_usd;
532    }
533    s
534}
535
536/// Render a [`SavingsSummary`] for humans (`--json` callers serialize the struct instead).
537#[must_use]
538pub fn format_savings(s: &SavingsSummary) -> String {
539    if s.traces == 0 {
540        return "no traces recorded yet — route some traffic through the proxy first".to_owned();
541    }
542    format!(
543        "traces: {} ({} enforce)\nspent:    ${:.4}  (gates ${:.4})\nbaseline: ${:.4}  (always top rung)\nsavings:  ${:.4}  ({:.1}%)",
544        s.traces,
545        s.enforce_traces,
546        s.spent_usd,
547        s.gate_usd,
548        s.baseline_usd,
549        s.savings_usd,
550        s.savings_pct * 100.0
551    )
552}
553
554/// Render the most recent `limit` traces as JSON lines — machine-first (SPEC §0.2): each line is a
555/// full [`Trace`], newest last.
556#[must_use]
557pub fn format_traces(traces: &[Trace], limit: usize) -> String {
558    let mut lines: Vec<String> = traces
559        .iter()
560        .rev()
561        .take(limit)
562        .filter_map(|t| serde_json::to_string(t).ok())
563        .collect();
564    lines.reverse();
565    if lines.is_empty() {
566        "no traces recorded yet".to_owned()
567    } else {
568        lines.join("\n")
569    }
570}
571
572#[cfg(test)]
573mod tests {
574    use super::*;
575
576    fn config_with(toml: Option<&str>, db_path: &str) -> ProxyConfig {
577        ProxyConfig::from_lookup(|k| match k {
578            "FIRSTPASS_MODE" if toml.is_some() => Some("enforce".to_owned()),
579            "FIRSTPASS_CONFIG_TOML" => toml.map(str::to_owned),
580            "FIRSTPASS_DB" => Some(db_path.to_owned()),
581            _ => None,
582        })
583        .unwrap()
584    }
585
586    #[test]
587    fn command_on_path_finds_real_and_rejects_fake() {
588        let path = std::env::var("PATH").ok();
589        assert!(command_on_path("sh", path.as_deref()), "sh is on PATH");
590        assert!(!command_on_path(
591            "firstpass-definitely-not-a-real-binary",
592            path.as_deref()
593        ));
594        assert!(!command_on_path("/nonexistent/abs/path", None));
595    }
596
597    #[test]
598    fn doctor_flags_a_missing_gate_binary() {
599        let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"good\", \"bad\"]\n\
600                    [[gate]]\nid = \"good\"\ncmd = [\"sh\"]\n\
601                    [[gate]]\nid = \"bad\"\ncmd = [\"firstpass-nope-not-real\"]\n";
602        let db = std::env::temp_dir().join("firstpass-doctor-test.db");
603        let config = config_with(Some(toml), db.to_str().unwrap());
604
605        // Real PATH so `sh` resolves; no ANTHROPIC_API_KEY -> a warning, not a failure.
606        let report = doctor(&config, |k| match k {
607            "PATH" => std::env::var("PATH").ok(),
608            _ => None,
609        });
610
611        assert!(
612            !report.healthy(),
613            "a missing gate binary must fail the report"
614        );
615        let bad = report.checks.iter().find(|c| c.name == "gate:bad").unwrap();
616        assert_eq!(bad.status, CheckStatus::Fail);
617        let good = report
618            .checks
619            .iter()
620            .find(|c| c.name == "gate:good")
621            .unwrap();
622        assert_eq!(good.status, CheckStatus::Ok);
623        let key = report
624            .checks
625            .iter()
626            .find(|c| c.name == "anthropic-key")
627            .unwrap();
628        assert_eq!(key.status, CheckStatus::Warn);
629    }
630
631    #[test]
632    fn doctor_is_healthy_for_a_plain_observe_setup() {
633        let db = std::env::temp_dir().join("firstpass-doctor-ok.db");
634        let config = config_with(None, db.to_str().unwrap());
635        let report = doctor(&config, |k| {
636            (k == "ANTHROPIC_API_KEY").then(|| "sk-test".to_owned())
637        });
638        assert!(report.healthy(), "{}", report.render());
639    }
640
641    #[test]
642    fn format_traces_handles_empty() {
643        assert_eq!(format_traces(&[], 10), "no traces recorded yet");
644    }
645
646    #[test]
647    fn savings_summary_empty_and_nonempty() {
648        let empty = summarize_savings(&[]);
649        assert_eq!(empty.traces, 0);
650        assert!(format_savings(&empty).contains("no traces"));
651    }
652
653    #[test]
654    fn evals_summary_empty_zero_state() {
655        let s = summarize_evals(&[]);
656        assert_eq!(s.enforce_traces, 0);
657        assert!(format_evals(&s).contains("no enforce traces"));
658    }
659
660    #[test]
661    fn verify_and_export_round_trip_detects_tampering() {
662        use firstpass_core::{
663            Attempt, Features, FinalOutcome, GENESIS_HASH, PolicyRef, RequestInfo, ServedFrom,
664            TaskKind, Verdict,
665        };
666
667        // Build a valid 3-link chain: each prev_hash = the previous record's hash.
668        let mk = |prev: &str, session: &str| Trace {
669            trace_id: uuid::Uuid::now_v7(),
670            prev_hash: prev.to_owned(),
671            tenant_id: "default".to_owned(),
672            session_id: session.to_owned(),
673            ts: jiff::Timestamp::UNIX_EPOCH,
674            mode: Mode::Observe,
675            policy: PolicyRef {
676                id: "observe-passthrough@v0".to_owned(),
677                explore: false,
678                propensity: None,
679                mode_profile: None,
680            },
681            request: RequestInfo {
682                api: "anthropic.messages".to_owned(),
683                prompt_hash: "deadbeef".to_owned(),
684                features: Features::new(TaskKind::Other),
685            },
686            attempts: vec![Attempt {
687                rung: 0,
688                model: "anthropic/claude-haiku-4-5".to_owned(),
689                provider: "anthropic".to_owned(),
690                in_tokens: 10,
691                out_tokens: 5,
692                cost_usd: 0.001,
693                latency_ms: 12,
694                gates: vec![],
695                verdict: Verdict::Pass,
696            }],
697            deferred: Vec::new(),
698            final_: FinalOutcome {
699                served_rung: Some(0),
700                served_from: ServedFrom::Attempt,
701                total_cost_usd: 0.001,
702                gate_cost_usd: 0.0,
703                total_latency_ms: 12,
704                escalations: 0,
705                counterfactual_baseline_usd: 0.001,
706                savings_usd: 0.0,
707            },
708            probe: None,
709            predicted_pass: None,
710            elastic: None,
711        };
712        let t0 = mk(GENESIS_HASH, "s0");
713        let t1 = mk(&t0.hash().unwrap(), "s1");
714        let t2 = mk(&t1.hash().unwrap(), "s2");
715        let chain = vec![t0, t1, t2];
716
717        // Clean chain verifies.
718        let report = verify_receipts(&chain);
719        assert!(report.valid, "intact chain must verify: {report:?}");
720        assert_eq!(report.receipts, 3);
721
722        // Export → parse round-trips and still verifies (the auditor's offline path).
723        let jsonl = export_receipts_jsonl(&chain);
724        assert_eq!(jsonl.lines().count(), 3);
725        let reparsed = parse_receipt_jsonl(&jsonl).expect("export must re-parse");
726        assert!(
727            verify_receipts(&reparsed).valid,
728            "round-tripped chain must verify"
729        );
730
731        // Tamper with a middle receipt's body → verification catches it at that index.
732        let mut tampered = reparsed;
733        tampered[1].final_.total_cost_usd = 999.0; // alter a sealed field
734        let bad = verify_receipts(&tampered);
735        assert!(!bad.valid, "a mutated receipt must break the chain");
736        assert_eq!(
737            bad.broken_at,
738            Some(2),
739            "the break surfaces at the next link"
740        );
741
742        // Reordering is also caught.
743        let mut reordered = parse_receipt_jsonl(&jsonl).unwrap();
744        reordered.swap(0, 1);
745        assert!(
746            !verify_receipts(&reordered).valid,
747            "reordering breaks the chain"
748        );
749    }
750
751    #[test]
752    fn parse_receipt_jsonl_reports_bad_line() {
753        let err = parse_receipt_jsonl("not json\n").unwrap_err();
754        assert!(err.starts_with("line 1:"), "must name the bad line: {err}");
755        assert!(
756            parse_receipt_jsonl("\n\n").unwrap().is_empty(),
757            "blank lines skipped"
758        );
759    }
760
761    #[test]
762    fn explain_trace_summarizes_served_and_escalation() {
763        use firstpass_core::{
764            Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
765            ServedFrom, TaskKind, Verdict,
766        };
767        let t = Trace {
768            trace_id: uuid::Uuid::now_v7(),
769            prev_hash: GENESIS_HASH.to_owned(),
770            tenant_id: "default".to_owned(),
771            session_id: "s".to_owned(),
772            ts: jiff::Timestamp::UNIX_EPOCH,
773            mode: Mode::Enforce,
774            policy: PolicyRef {
775                id: "static-ladder@v0".to_owned(),
776                explore: false,
777                propensity: None,
778                mode_profile: None,
779            },
780            request: RequestInfo {
781                api: "anthropic.messages".to_owned(),
782                prompt_hash: "x".to_owned(),
783                features: Features::new(TaskKind::Other),
784            },
785            attempts: vec![
786                Attempt {
787                    rung: 0,
788                    model: "anthropic/claude-haiku-4-5".to_owned(),
789                    provider: "anthropic".to_owned(),
790                    in_tokens: 1,
791                    out_tokens: 1,
792                    cost_usd: 0.001,
793                    latency_ms: 5,
794                    gates: vec![GateResult::deterministic("non-empty", Verdict::Fail, 0)],
795                    verdict: Verdict::Fail,
796                },
797                Attempt {
798                    rung: 1,
799                    model: "anthropic/claude-sonnet-5".to_owned(),
800                    provider: "anthropic".to_owned(),
801                    in_tokens: 1,
802                    out_tokens: 1,
803                    cost_usd: 0.01,
804                    latency_ms: 9,
805                    gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
806                    verdict: Verdict::Pass,
807                },
808            ],
809            deferred: Vec::new(),
810            final_: FinalOutcome {
811                served_rung: Some(1),
812                served_from: ServedFrom::Attempt,
813                total_cost_usd: 0.011,
814                gate_cost_usd: 0.0,
815                total_latency_ms: 14,
816                escalations: 1,
817                counterfactual_baseline_usd: 0.02,
818                savings_usd: 0.009,
819            },
820            probe: None,
821            predicted_pass: None,
822            elastic: None,
823        };
824        let ex = explain_trace(&t);
825        assert_eq!(
826            ex.served_model.as_deref(),
827            Some("anthropic/claude-sonnet-5")
828        );
829        assert_eq!(ex.escalations, 1);
830        assert_eq!(ex.attempts.len(), 2);
831        assert_eq!(ex.attempts[0].verdict, "fail");
832        assert_eq!(
833            ex.attempts[1].gates[0],
834            ("non-empty".to_owned(), "pass".to_owned())
835        );
836        assert!(ex.summary.contains("served anthropic/claude-sonnet-5"));
837    }
838
839    #[test]
840    fn evaluate_predictor_empty_and_signal() {
841        // empty store => n=0, no crash
842        let e = evaluate_predictor(&[], 0.05, 1e-4);
843        assert_eq!(e.n, 0);
844        assert!(e.auc.is_none());
845    }
846}
firstpass_proxy/cli.rs

firstpass_proxy/
cli.rs
