1use crate::config::ProxyConfig;
6use firstpass_core::{Mode, Trace};
7
8#[derive(Debug, PartialEq, Eq)]
10pub enum CheckStatus {
11 Ok,
13 Warn,
15 Fail,
17}
18
19#[derive(Debug)]
21pub struct Check {
22 pub name: String,
24 pub status: CheckStatus,
26 pub detail: String,
28}
29
30impl Check {
31 fn new(name: impl Into<String>, status: CheckStatus, detail: impl Into<String>) -> Self {
32 Self {
33 name: name.into(),
34 status,
35 detail: detail.into(),
36 }
37 }
38}
39
40#[derive(Debug)]
42pub struct DoctorReport {
43 pub checks: Vec<Check>,
45}
46
47impl DoctorReport {
48 #[must_use]
50 pub fn healthy(&self) -> bool {
51 self.checks.iter().all(|c| c.status != CheckStatus::Fail)
52 }
53
54 #[must_use]
56 pub fn render(&self) -> String {
57 let mut out = String::new();
58 for c in &self.checks {
59 let mark = match c.status {
60 CheckStatus::Ok => "✓",
61 CheckStatus::Warn => "!",
62 CheckStatus::Fail => "✗",
63 };
64 out.push_str(&format!("{mark} {}: {}\n", c.name, c.detail));
65 }
66 out.push_str(if self.healthy() {
67 "\nhealthy — ready to route.\n"
68 } else {
69 "\nnot healthy — fix the ✗ items above.\n"
70 });
71 out
72 }
73}
74
75#[must_use]
79pub fn doctor(config: &ProxyConfig, env: impl Fn(&str) -> Option<String>) -> DoctorReport {
80 let mut checks = Vec::new();
81
82 let route_count = config.routing.as_ref().map_or(0, |c| c.routes.len());
84 checks.push(Check::new(
85 "config",
86 CheckStatus::Ok,
87 format!("default mode {:?}, {route_count} route(s)", config.mode),
88 ));
89
90 let enforce_routes = config.routing.as_ref().map_or(0, |c| {
92 c.routes.iter().filter(|r| r.mode == Mode::Enforce).count()
93 });
94 if config.mode == Mode::Enforce && enforce_routes == 0 {
95 checks.push(Check::new(
96 "routing",
97 CheckStatus::Warn,
98 "mode is enforce but no enforce route is defined — all traffic will observe",
99 ));
100 } else {
101 checks.push(Check::new(
102 "routing",
103 CheckStatus::Ok,
104 format!("{enforce_routes} enforce route(s)"),
105 ));
106 }
107
108 if env("ANTHROPIC_API_KEY").is_some_and(|k| !k.is_empty()) {
110 checks.push(Check::new("anthropic-key", CheckStatus::Ok, "present"));
111 } else {
112 checks.push(Check::new(
113 "anthropic-key",
114 CheckStatus::Warn,
115 "ANTHROPIC_API_KEY not set — observe uses the caller's key; enforce needs one reachable",
116 ));
117 }
118
119 let mut missing_keys: Vec<(String, &str)> = Vec::new();
124 if let Some(routing) = config.routing.as_ref() {
125 let mut seen = std::collections::HashSet::new();
126 for rung in routing.routes.iter().flat_map(|r| r.ladder.iter()) {
127 let Some((prov, _)) = rung.split_once('/') else {
128 continue;
129 };
130 let Some(key_env) = crate::provider::builtin_key_env(prov) else {
131 continue; };
133 if seen.insert(prov.to_owned()) && env(key_env).is_none_or(|k| k.is_empty()) {
134 missing_keys.push((prov.to_owned(), key_env));
135 }
136 }
137 }
138 for (prov, key_env) in missing_keys {
139 checks.push(Check::new(
140 format!("provider:{prov}"),
141 CheckStatus::Fail,
142 format!("{key_env} not set — a rung on `{prov}` will fail at the first request"),
143 ));
144 }
145
146 let path = env("PATH");
148 let gate_defs = config.routing.as_ref().map_or(&[][..], |c| &c.gate_defs);
149 for def in gate_defs {
150 match def.cmd.first() {
151 Some(program) if command_on_path(program, path.as_deref()) => checks.push(Check::new(
152 format!("gate:{}", def.id),
153 CheckStatus::Ok,
154 format!("`{program}` found"),
155 )),
156 Some(program) => checks.push(Check::new(
157 format!("gate:{}", def.id),
158 CheckStatus::Fail,
159 format!("`{program}` not found on PATH"),
160 )),
161 None => checks.push(Check::new(
162 format!("gate:{}", def.id),
163 CheckStatus::Fail,
164 "empty command",
165 )),
166 }
167 }
168
169 if can_write_db(&config.db_path) {
171 checks.push(Check::new(
172 "trace-store",
173 CheckStatus::Ok,
174 format!("{} is writable", config.db_path),
175 ));
176 } else {
177 checks.push(Check::new(
178 "trace-store",
179 CheckStatus::Fail,
180 format!("cannot write near {}", config.db_path),
181 ));
182 }
183
184 DoctorReport { checks }
185}
186
187#[must_use]
190pub fn command_on_path(program: &str, path_var: Option<&str>) -> bool {
191 if program.contains('/') || program.contains('\\') {
192 return std::path::Path::new(program).is_file();
193 }
194 let Some(path) = path_var else { return false };
195 std::env::split_paths(path).any(|dir| dir.join(program).is_file())
196}
197
198#[derive(Debug, Clone, Copy, PartialEq, Eq)]
202pub enum Channel {
203 Dist,
206 Homebrew,
208 Cargo,
210 Python,
212 Pipx,
214 Npm,
216 Docker,
218 Unknown,
220}
221
222impl Channel {
223 #[must_use]
225 pub const fn label(self) -> &'static str {
226 match self {
227 Self::Dist => "installer script (self-updating)",
228 Self::Homebrew => "Homebrew",
229 Self::Cargo => "cargo install",
230 Self::Python => "Python wheel (pip / uv / uvx)",
231 Self::Pipx => "pipx",
232 Self::Npm => "npm",
233 Self::Docker => "Docker image",
234 Self::Unknown => "unrecognised",
235 }
236 }
237
238 #[must_use]
240 pub const fn upgrade_command(self) -> Option<&'static str> {
241 match self {
242 Self::Dist => Some("firstpass-update"),
243 Self::Homebrew => Some("brew upgrade firstpass-proxy"),
244 Self::Cargo => Some(
245 "cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force",
246 ),
247 Self::Python => Some("uv tool upgrade firstpass # or: pip install -U firstpass"),
248 Self::Pipx => Some("pipx upgrade firstpass"),
249 Self::Npm => Some("npm install -g @dshakesnotbot/firstpass-proxy@latest"),
250 Self::Docker => Some("docker pull ghcr.io/dshakes/firstpass:latest"),
251 Self::Unknown => None,
252 }
253 }
254}
255
256#[must_use]
265pub fn detect_channel(exe: &std::path::Path, updater_present: bool, in_container: bool) -> Channel {
266 let p = exe.to_string_lossy().to_lowercase().replace('\\', "/");
267 let has = |needle: &str| p.contains(needle);
268 if in_container {
269 return Channel::Docker;
270 }
271 if has("/.cargo/bin/") || has("/.cargo/registry/") {
272 return Channel::Cargo;
273 }
274 if has("/cellar/") || has("/homebrew/") || has("/linuxbrew/") {
275 return Channel::Homebrew;
276 }
277 if has("/pipx/") {
278 return Channel::Pipx; }
280 if has("/site-packages/")
281 || has("/dist-packages/")
282 || has("/.venv/")
283 || has("/uv/")
284 || has("/uv-cache/")
285 {
286 return Channel::Python;
287 }
288 if has("/node_modules/") {
289 return Channel::Npm;
290 }
291 if updater_present {
292 return Channel::Dist;
293 }
294 Channel::Unknown
295}
296
297#[must_use]
299pub fn upgrade_report(channel: Channel, current_version: &str) -> String {
300 let mut out = format!(
301 "firstpass {current_version} · installed via {}\n",
302 channel.label()
303 );
304 match channel.upgrade_command() {
305 Some(cmd) if channel == Channel::Dist => {
306 out.push_str(&format!("\nupgrading in place — running `{cmd}`\n"));
307 }
308 Some(cmd) => {
309 out.push_str(&format!(
310 "\nthis channel is owned by its package manager, so upgrade with:\n {cmd}\n"
311 ));
312 }
313 None => {
314 out.push_str(
315 "\ncould not tell how this binary was installed. Pick the line that matches:\n\
316 \x20 firstpass-update # installer script\n\
317 \x20 brew upgrade firstpass-proxy # Homebrew\n\
318 \x20 uv tool upgrade firstpass # uv / pip: pip install -U firstpass\n\
319 \x20 pipx upgrade firstpass # pipx\n\
320 \x20 npm install -g @dshakesnotbot/firstpass-proxy@latest\n\
321 \x20 docker pull ghcr.io/dshakes/firstpass:latest\n\
322 \x20 cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force\n",
323 );
324 }
325 }
326 out
327}
328
329fn can_write_db(db_path: &str) -> bool {
331 let path = std::path::Path::new(db_path);
332 let dir = path
333 .parent()
334 .filter(|d| !d.as_os_str().is_empty())
335 .map_or_else(
336 || std::path::PathBuf::from("."),
337 std::path::Path::to_path_buf,
338 );
339 let probe = dir.join(format!(".firstpass-doctor-probe-{}", std::process::id()));
340 match std::fs::File::create(&probe) {
341 Ok(_) => {
342 let _ = std::fs::remove_file(&probe);
343 true
344 }
345 Err(_) => false,
346 }
347}
348
349#[derive(Debug, serde::Serialize)]
352pub struct EvalsSummary {
353 pub enforce_traces: usize,
355 pub escalations: u64,
357 pub gates: std::collections::BTreeMap<String, (u64, u64, u64)>,
359 pub served_by_rung: std::collections::BTreeMap<u32, u64>,
361}
362
363#[must_use]
365#[derive(Debug, serde::Serialize)]
371pub struct PredictorEval {
372 pub n: usize,
374 pub auc: Option<f64>,
377 pub brier: f64,
380}
381
382fn predictor_auc(pairs: &[(f64, bool)]) -> Option<f64> {
384 let pos: Vec<f64> = pairs.iter().filter(|(_, y)| *y).map(|(s, _)| *s).collect();
385 let neg: Vec<f64> = pairs.iter().filter(|(_, y)| !*y).map(|(s, _)| *s).collect();
386 if pos.is_empty() || neg.is_empty() {
387 return None;
388 }
389 let mut wins = 0.0_f64;
390 for &p in &pos {
391 for &n in &neg {
392 wins += if p > n {
393 1.0
394 } else if (p - n).abs() < f64::EPSILON {
395 0.5
396 } else {
397 0.0
398 };
399 }
400 }
401 Some(wins / (pos.len() as f64 * neg.len() as f64))
402}
403
404pub fn evaluate_predictor(traces: &[Trace], lr: f64, l2: f64) -> PredictorEval {
406 use firstpass_core::{PassPredictor, Verdict};
407 let mut model = PassPredictor::new(lr, l2);
408 let mut pairs: Vec<(f64, bool)> = Vec::new();
409 let mut se = 0.0_f64;
410 for t in traces {
411 for a in &t.attempts {
412 let y = match a.verdict {
413 Verdict::Pass => true,
414 Verdict::Fail => false,
415 Verdict::Abstain => continue,
416 };
417 let p = model.predict(&t.request.features, a.rung);
418 let target = f64::from(u8::from(y));
419 se += (p - target) * (p - target);
420 pairs.push((p, y));
421 model.update(&t.request.features, a.rung, y);
422 }
423 }
424 let n = pairs.len();
425 PredictorEval {
426 n,
427 auc: predictor_auc(&pairs),
428 brier: if n == 0 { 0.0 } else { se / n as f64 },
429 }
430}
431
432pub fn summarize_evals(traces: &[Trace]) -> EvalsSummary {
433 let mut s = EvalsSummary {
434 enforce_traces: 0,
435 escalations: 0,
436 gates: std::collections::BTreeMap::new(),
437 served_by_rung: std::collections::BTreeMap::new(),
438 };
439 for t in traces {
440 if t.mode != Mode::Enforce {
441 continue;
442 }
443 s.enforce_traces += 1;
444 s.escalations += u64::from(t.final_.escalations);
445 if let Some(rung) = t.final_.served_rung {
446 *s.served_by_rung.entry(rung).or_insert(0) += 1;
447 }
448 for attempt in &t.attempts {
449 for g in &attempt.gates {
450 let e = s.gates.entry(g.gate_id.clone()).or_insert((0, 0, 0));
451 match g.verdict {
452 firstpass_core::Verdict::Pass => e.0 += 1,
453 firstpass_core::Verdict::Fail => e.1 += 1,
454 firstpass_core::Verdict::Abstain => e.2 += 1,
455 }
456 }
457 }
458 }
459 s
460}
461
462#[must_use]
464pub fn format_evals(s: &EvalsSummary) -> String {
465 if s.enforce_traces == 0 {
466 return "no enforce traces yet — route some traffic in enforce mode first".to_owned();
467 }
468 let mut out = format!(
469 "enforce traces: {} · escalations: {}\n",
470 s.enforce_traces, s.escalations
471 );
472 out.push_str("gates (pass / fail / abstain):\n");
473 for (id, (p, f, a)) in &s.gates {
474 out.push_str(&format!(" {id:<24} {p:>6} / {f:>6} / {a:>6}\n"));
475 }
476 out.push_str("served by rung:\n");
477 for (rung, n) in &s.served_by_rung {
478 out.push_str(&format!(" rung {rung:<2} {n:>6}\n"));
479 }
480 out.trim_end().to_owned()
481}
482
483#[derive(Debug, serde::Serialize)]
486pub struct RouteExplanation {
487 pub trace_id: String,
488 pub mode: String,
489 pub policy: String,
490 pub served_model: Option<String>,
492 pub served_rung: Option<u32>,
493 pub escalations: u32,
494 pub total_cost_usd: f64,
495 pub baseline_usd: f64,
496 pub savings_usd: f64,
497 pub attempts: Vec<AttemptExplanation>,
499 pub summary: String,
501}
502
503#[derive(Debug, serde::Serialize)]
505pub struct AttemptExplanation {
506 pub rung: u32,
507 pub model: String,
508 pub verdict: String,
509 pub gates: Vec<(String, String)>,
511}
512
513fn verdict_str(v: firstpass_core::Verdict) -> &'static str {
514 match v {
515 firstpass_core::Verdict::Pass => "pass",
516 firstpass_core::Verdict::Fail => "fail",
517 firstpass_core::Verdict::Abstain => "abstain",
518 }
519}
520
521#[must_use]
524pub fn explain_trace(t: &Trace) -> RouteExplanation {
525 let attempts: Vec<AttemptExplanation> = t
526 .attempts
527 .iter()
528 .map(|a| AttemptExplanation {
529 rung: a.rung,
530 model: a.model.clone(),
531 verdict: verdict_str(a.verdict).to_owned(),
532 gates: a
533 .gates
534 .iter()
535 .map(|g| (g.gate_id.clone(), verdict_str(g.verdict).to_owned()))
536 .collect(),
537 })
538 .collect();
539 let served_model = t
540 .final_
541 .served_rung
542 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
543 .map(|a| a.model.clone());
544 let summary = match &served_model {
545 Some(m) => format!(
546 "served {m} at rung {} after {} escalation(s); spent ${:.4} vs ${:.4} always-top (saved ${:.4})",
547 t.final_.served_rung.unwrap_or(0),
548 t.final_.escalations,
549 t.final_.total_cost_usd,
550 t.final_.counterfactual_baseline_usd,
551 t.final_.savings_usd
552 ),
553 None => "no rung served (all attempts failed the gate or errored)".to_owned(),
554 };
555 RouteExplanation {
556 trace_id: t.trace_id.to_string(),
557 mode: match t.mode {
558 Mode::Enforce => "enforce",
559 Mode::Observe => "observe",
560 }
561 .to_owned(),
562 policy: t.policy.id.clone(),
563 served_model,
564 served_rung: t.final_.served_rung,
565 escalations: t.final_.escalations,
566 total_cost_usd: t.final_.total_cost_usd,
567 baseline_usd: t.final_.counterfactual_baseline_usd,
568 savings_usd: t.final_.savings_usd,
569 attempts,
570 summary,
571 }
572}
573
574#[derive(Debug, serde::Serialize)]
576pub struct VerifyReport {
577 pub receipts: usize,
579 pub valid: bool,
581 #[serde(skip_serializing_if = "Option::is_none")]
583 pub broken_at: Option<usize>,
584 #[serde(skip_serializing_if = "Option::is_none")]
585 pub detail: Option<String>,
586}
587
588#[must_use]
592pub fn verify_receipts(traces: &[Trace]) -> VerifyReport {
593 match firstpass_core::verify_chain(traces, firstpass_core::GENESIS_HASH) {
594 Ok(()) => VerifyReport {
595 receipts: traces.len(),
596 valid: true,
597 broken_at: None,
598 detail: None,
599 },
600 Err(firstpass_core::Error::ChainBroken { index, detail }) => VerifyReport {
601 receipts: traces.len(),
602 valid: false,
603 broken_at: Some(index),
604 detail: Some(detail),
605 },
606 Err(e) => VerifyReport {
607 receipts: traces.len(),
608 valid: false,
609 broken_at: None,
610 detail: Some(e.to_string()),
611 },
612 }
613}
614
615pub fn parse_receipt_jsonl(text: &str) -> Result<Vec<Trace>, String> {
621 let mut traces = Vec::new();
622 for (i, line) in text.lines().enumerate() {
623 if line.trim().is_empty() {
624 continue;
625 }
626 let t: Trace = serde_json::from_str(line).map_err(|e| format!("line {}: {e}", i + 1))?;
627 traces.push(t);
628 }
629 Ok(traces)
630}
631
632#[must_use]
636#[derive(Debug, Clone, serde::Serialize)]
648pub struct TrainingRow {
649 pub trace_id: String,
651 pub features: firstpass_core::Features,
653 pub start_rung: u32,
655 pub served_rung: Option<u32>,
657 pub served_passed: bool,
660 pub cost_usd: f64,
662 pub propensity: Option<f64>,
665 pub explore: bool,
667 pub policy_id: String,
669}
670
671impl TrainingRow {
672 pub fn from_trace(t: &Trace) -> Self {
674 let served_rung = t.final_.served_rung;
675 Self {
676 trace_id: t.trace_id.to_string(),
677 features: t.request.features.clone(),
678 start_rung: t.attempts.first().map_or(0, |a| a.rung),
680 served_rung,
681 served_passed: served_rung.is_some_and(|r| {
682 t.attempts
683 .iter()
684 .any(|a| a.rung == r && a.verdict == firstpass_core::Verdict::Pass)
685 }),
686 cost_usd: t.final_.total_cost_usd,
687 propensity: t.policy.propensity,
688 explore: t.policy.explore,
689 policy_id: t.policy.id.clone(),
690 }
691 }
692}
693
694#[derive(Debug, Clone, serde::Serialize)]
699pub struct OverheadSummary {
700 pub n: usize,
702 pub skipped_concurrent: usize,
707 pub p50_ms: u64,
708 pub p95_ms: u64,
709 pub p99_ms: u64,
710 pub max_ms: u64,
711}
712
713#[must_use]
715pub fn summarize_overhead(traces: &[Trace]) -> OverheadSummary {
716 let mut v: Vec<u64> = Vec::with_capacity(traces.len());
717 let mut skipped = 0usize;
718 for t in traces {
719 match t.overhead_ms() {
720 Some(ms) => v.push(ms),
721 None => skipped += 1,
722 }
723 }
724 v.sort_unstable();
725 let pct = |p: f64| -> u64 {
728 if v.is_empty() {
729 return 0;
730 }
731 let rank = (p * v.len() as f64).ceil() as usize;
732 v[rank.clamp(1, v.len()) - 1]
733 };
734 OverheadSummary {
735 n: v.len(),
736 skipped_concurrent: skipped,
737 p50_ms: pct(0.50),
738 p95_ms: pct(0.95),
739 p99_ms: pct(0.99),
740 max_ms: v.last().copied().unwrap_or(0),
741 }
742}
743
744#[must_use]
746pub fn format_overhead(s: &OverheadSummary) -> String {
747 if s.n == 0 {
748 return format!(
749 "no receipts with a measurable overhead ({} skipped as concurrent).\n\
750 Route some traffic first, or run with speculation off to measure it.\n",
751 s.skipped_concurrent
752 );
753 }
754 let mut lines = vec![
758 "Firstpass overhead — time added beyond the provider calls and gates it ran".to_owned(),
759 String::new(),
760 format!(" p50 {:>6} ms", s.p50_ms),
761 format!(" p95 {:>6} ms", s.p95_ms),
762 format!(" p99 {:>6} ms", s.p99_ms),
763 format!(" max {:>6} ms", s.max_ms),
764 String::new(),
765 format!(" over {} receipt(s)", s.n),
766 ];
767 if s.p95_ms == 0 {
770 lines.push(
771 " 0 ms is the floor, not a missing measurement: receipts record whole".to_owned(),
772 );
773 lines.push(
774 " milliseconds, so the proxy's own work stayed under 1 ms per request.".to_owned(),
775 );
776 }
777 if s.skipped_concurrent > 0 {
778 lines.push(format!(
779 " {} skipped: speculative rungs run concurrently, so subtracting dispatched",
780 s.skipped_concurrent
781 ));
782 lines.push(
783 " work from wall-clock is meaningless there — excluded, not floored to 0.".to_owned(),
784 );
785 }
786 lines.push(String::new());
787 lines.join("\n")
788}
789
790#[must_use]
792pub fn export_training_jsonl(traces: &[Trace]) -> String {
793 let mut out = String::new();
794 for t in traces {
795 if let Ok(line) = serde_json::to_string(&TrainingRow::from_trace(t)) {
796 out.push_str(&line);
797 out.push('\n');
798 }
799 }
800 out
801}
802
803pub fn export_receipts_jsonl(traces: &[Trace]) -> String {
804 let mut out = String::new();
805 for t in traces {
806 if let Ok(line) = serde_json::to_string(t) {
807 out.push_str(&line);
808 out.push('\n');
809 }
810 }
811 out
812}
813
814#[derive(Debug, serde::Serialize)]
817pub struct SavingsSummary {
818 pub traces: usize,
820 pub enforce_traces: usize,
822 pub spent_usd: f64,
824 pub gate_usd: f64,
826 pub baseline_usd: f64,
828 pub savings_usd: f64,
830 pub savings_pct: f64,
832}
833
834#[must_use]
836pub fn summarize_savings(traces: &[Trace]) -> SavingsSummary {
837 let mut s = SavingsSummary {
838 traces: traces.len(),
839 enforce_traces: 0,
840 spent_usd: 0.0,
841 gate_usd: 0.0,
842 baseline_usd: 0.0,
843 savings_usd: 0.0,
844 savings_pct: 0.0,
845 };
846 for t in traces {
847 if t.mode == Mode::Enforce {
848 s.enforce_traces += 1;
849 }
850 s.spent_usd += t.final_.total_cost_usd;
851 s.gate_usd += t.final_.gate_cost_usd;
852 s.baseline_usd += t.final_.counterfactual_baseline_usd;
853 }
854 s.savings_usd = s.baseline_usd - s.spent_usd;
855 if s.baseline_usd > 0.0 {
856 s.savings_pct = s.savings_usd / s.baseline_usd;
857 }
858 s
859}
860
861#[must_use]
863pub fn format_savings(s: &SavingsSummary) -> String {
864 if s.traces == 0 {
865 return "no traces recorded yet — route some traffic through the proxy first".to_owned();
866 }
867 format!(
868 "traces: {} ({} enforce)\nspent: ${:.4} (gates ${:.4})\nbaseline: ${:.4} (always top rung)\nsavings: ${:.4} ({:.1}%)",
869 s.traces,
870 s.enforce_traces,
871 s.spent_usd,
872 s.gate_usd,
873 s.baseline_usd,
874 s.savings_usd,
875 s.savings_pct * 100.0
876 )
877}
878
879#[must_use]
882pub fn format_traces(traces: &[Trace], limit: usize) -> String {
883 let mut lines: Vec<String> = traces
884 .iter()
885 .rev()
886 .take(limit)
887 .filter_map(|t| serde_json::to_string(t).ok())
888 .collect();
889 lines.reverse();
890 if lines.is_empty() {
891 "no traces recorded yet".to_owned()
892 } else {
893 lines.join("\n")
894 }
895}
896
897#[cfg(test)]
898mod tests {
899 use super::*;
900
901 fn config_with(toml: Option<&str>, db_path: &str) -> ProxyConfig {
902 ProxyConfig::from_lookup(|k| match k {
903 "FIRSTPASS_MODE" if toml.is_some() => Some("enforce".to_owned()),
904 "FIRSTPASS_CONFIG_TOML" => toml.map(str::to_owned),
905 "FIRSTPASS_DB" => Some(db_path.to_owned()),
906 _ => None,
907 })
908 .unwrap()
909 }
910
911 #[test]
912 fn doctor_flags_a_ladder_rung_whose_provider_key_is_missing() {
913 let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"groq/llama-3.3-70b-versatile\"]\n\
916 [[price]]\nmodel = \"groq/llama-3.3-70b-versatile\"\n\
917 input_per_mtok = 0.59\noutput_per_mtok = 0.79\n";
918 let config = config_with(Some(toml), "/tmp/doctor-groq.db");
919
920 let report = doctor(&config, |k| match k {
921 "PATH" => Some("/usr/bin".to_owned()),
922 _ => None, });
924
925 let groq = report
926 .checks
927 .iter()
928 .find(|c| c.name == "provider:groq")
929 .expect("groq rung should be checked");
930 assert_eq!(groq.status, CheckStatus::Fail);
931 assert!(groq.detail.contains("GROQ_API_KEY"), "{}", groq.detail);
932 assert!(
933 !report.healthy(),
934 "a rung that cannot authenticate is not healthy"
935 );
936 }
937
938 #[test]
939 fn doctor_stays_quiet_about_providers_the_ladder_does_not_use() {
940 let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\n";
943 let config = config_with(Some(toml), "/tmp/doctor-quiet.db");
944
945 let report = doctor(&config, |k| match k {
946 "PATH" => Some("/usr/bin".to_owned()),
947 "ANTHROPIC_API_KEY" => Some("sk-test".to_owned()),
948 _ => None,
949 });
950
951 assert!(
952 !report
953 .checks
954 .iter()
955 .any(|c| c.name.starts_with("provider:")),
956 "unused providers must not be reported: {:?}",
957 report.checks.iter().map(|c| &c.name).collect::<Vec<_>>()
958 );
959 }
960
961 #[test]
962 fn command_on_path_finds_real_and_rejects_fake() {
963 let path = std::env::var("PATH").ok();
964 assert!(command_on_path("sh", path.as_deref()), "sh is on PATH");
965 assert!(!command_on_path(
966 "firstpass-definitely-not-a-real-binary",
967 path.as_deref()
968 ));
969 assert!(!command_on_path("/nonexistent/abs/path", None));
970 }
971
972 #[test]
973 fn doctor_flags_a_missing_gate_binary() {
974 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"good\", \"bad\"]\n\
975 [[gate]]\nid = \"good\"\ncmd = [\"sh\"]\n\
976 [[gate]]\nid = \"bad\"\ncmd = [\"firstpass-nope-not-real\"]\n";
977 let db = std::env::temp_dir().join("firstpass-doctor-test.db");
978 let config = config_with(Some(toml), db.to_str().unwrap());
979
980 let report = doctor(&config, |k| match k {
982 "PATH" => std::env::var("PATH").ok(),
983 _ => None,
984 });
985
986 assert!(
987 !report.healthy(),
988 "a missing gate binary must fail the report"
989 );
990 let bad = report.checks.iter().find(|c| c.name == "gate:bad").unwrap();
991 assert_eq!(bad.status, CheckStatus::Fail);
992 let good = report
993 .checks
994 .iter()
995 .find(|c| c.name == "gate:good")
996 .unwrap();
997 assert_eq!(good.status, CheckStatus::Ok);
998 let key = report
999 .checks
1000 .iter()
1001 .find(|c| c.name == "anthropic-key")
1002 .unwrap();
1003 assert_eq!(key.status, CheckStatus::Warn);
1004 }
1005
1006 #[test]
1007 fn doctor_is_healthy_for_a_plain_observe_setup() {
1008 let db = std::env::temp_dir().join("firstpass-doctor-ok.db");
1009 let config = config_with(None, db.to_str().unwrap());
1010 let report = doctor(&config, |k| {
1011 (k == "ANTHROPIC_API_KEY").then(|| "sk-test".to_owned())
1012 });
1013 assert!(report.healthy(), "{}", report.render());
1014 }
1015
1016 #[test]
1017 fn format_traces_handles_empty() {
1018 assert_eq!(format_traces(&[], 10), "no traces recorded yet");
1019 }
1020
1021 #[test]
1022 fn savings_summary_empty_and_nonempty() {
1023 let empty = summarize_savings(&[]);
1024 assert_eq!(empty.traces, 0);
1025 assert!(format_savings(&empty).contains("no traces"));
1026 }
1027
1028 #[test]
1029 fn evals_summary_empty_zero_state() {
1030 let s = summarize_evals(&[]);
1031 assert_eq!(s.enforce_traces, 0);
1032 assert!(format_evals(&s).contains("no enforce traces"));
1033 }
1034
1035 #[test]
1036 fn verify_and_export_round_trip_detects_tampering() {
1037 use firstpass_core::{
1038 Attempt, Features, FinalOutcome, GENESIS_HASH, PolicyRef, RequestInfo, ServedFrom,
1039 TaskKind, Verdict,
1040 };
1041
1042 let mk = |prev: &str, session: &str| Trace {
1044 trace_id: uuid::Uuid::now_v7(),
1045 prev_hash: prev.to_owned(),
1046 tenant_id: "default".to_owned(),
1047 session_id: session.to_owned(),
1048 ts: jiff::Timestamp::UNIX_EPOCH,
1049 mode: Mode::Observe,
1050 policy: PolicyRef {
1051 id: "observe-passthrough@v0".to_owned(),
1052 explore: false,
1053 propensity: None,
1054 mode_profile: None,
1055 },
1056 request: RequestInfo {
1057 api: "anthropic.messages".to_owned(),
1058 prompt_hash: "deadbeef".to_owned(),
1059 features: Features::new(TaskKind::Other),
1060 },
1061 attempts: vec![Attempt {
1062 rung: 0,
1063 model: "anthropic/claude-haiku-4-5".to_owned(),
1064 provider: "anthropic".to_owned(),
1065 in_tokens: 10,
1066 cache_write_tokens: 0,
1067 cache_read_tokens: 0,
1068 out_tokens: 5,
1069 cost_usd: 0.001,
1070 latency_ms: 12,
1071 gates: vec![],
1072 verdict: Verdict::Pass,
1073 }],
1074 deferred: Vec::new(),
1075 final_: FinalOutcome {
1076 served_rung: Some(0),
1077 served_from: ServedFrom::Attempt,
1078 total_cost_usd: 0.001,
1079 gate_cost_usd: 0.0,
1080 total_latency_ms: 12,
1081 escalations: 0,
1082 counterfactual_baseline_usd: 0.001,
1083 savings_usd: 0.0,
1084 cache_source: None,
1085 },
1086 probe: None,
1087 rollout: None,
1088 shadow: None,
1089 route_ix: None,
1090 predicted_pass: None,
1091 elastic: None,
1092 };
1093 let t0 = mk(GENESIS_HASH, "s0");
1094 let t1 = mk(&t0.hash().unwrap(), "s1");
1095 let t2 = mk(&t1.hash().unwrap(), "s2");
1096 let chain = vec![t0, t1, t2];
1097
1098 let report = verify_receipts(&chain);
1100 assert!(report.valid, "intact chain must verify: {report:?}");
1101 assert_eq!(report.receipts, 3);
1102
1103 let jsonl = export_receipts_jsonl(&chain);
1105 assert_eq!(jsonl.lines().count(), 3);
1106 let reparsed = parse_receipt_jsonl(&jsonl).expect("export must re-parse");
1107 assert!(
1108 verify_receipts(&reparsed).valid,
1109 "round-tripped chain must verify"
1110 );
1111
1112 let mut tampered = reparsed;
1114 tampered[1].final_.total_cost_usd = 999.0; let bad = verify_receipts(&tampered);
1116 assert!(!bad.valid, "a mutated receipt must break the chain");
1117 assert_eq!(
1118 bad.broken_at,
1119 Some(2),
1120 "the break surfaces at the next link"
1121 );
1122
1123 let mut reordered = parse_receipt_jsonl(&jsonl).unwrap();
1125 reordered.swap(0, 1);
1126 assert!(
1127 !verify_receipts(&reordered).valid,
1128 "reordering breaks the chain"
1129 );
1130 }
1131
1132 #[test]
1133 fn parse_receipt_jsonl_reports_bad_line() {
1134 let err = parse_receipt_jsonl("not json\n").unwrap_err();
1135 assert!(err.starts_with("line 1:"), "must name the bad line: {err}");
1136 assert!(
1137 parse_receipt_jsonl("\n\n").unwrap().is_empty(),
1138 "blank lines skipped"
1139 );
1140 }
1141
1142 fn trace_with_timing(total_ms: u64, attempts: &[(u64, u64)]) -> Trace {
1143 use firstpass_core::{
1144 Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1145 ServedFrom, TaskKind, Verdict,
1146 };
1147 Trace {
1148 trace_id: uuid::Uuid::now_v7(),
1149 prev_hash: GENESIS_HASH.to_owned(),
1150 tenant_id: "default".to_owned(),
1151 session_id: "s".to_owned(),
1152 ts: jiff::Timestamp::UNIX_EPOCH,
1153 mode: Mode::Enforce,
1154 policy: PolicyRef {
1155 id: "static-ladder@v0".to_owned(),
1156 explore: false,
1157 propensity: None,
1158 mode_profile: None,
1159 },
1160 request: RequestInfo {
1161 api: "anthropic.messages".to_owned(),
1162 prompt_hash: "x".to_owned(),
1163 features: Features::new(TaskKind::CodeEdit),
1164 },
1165 attempts: attempts
1166 .iter()
1167 .enumerate()
1168 .map(|(i, (model_ms, gate_ms))| Attempt {
1169 rung: i as u32,
1170 model: "anthropic/claude-haiku-4-5".to_owned(),
1171 provider: "anthropic".to_owned(),
1172 in_tokens: 1,
1173 cache_write_tokens: 0,
1174 cache_read_tokens: 0,
1175 out_tokens: 1,
1176 cost_usd: 0.0,
1177 latency_ms: *model_ms,
1178 gates: vec![GateResult::deterministic(
1179 "non-empty",
1180 Verdict::Pass,
1181 *gate_ms,
1182 )],
1183 verdict: Verdict::Pass,
1184 })
1185 .collect(),
1186 final_: FinalOutcome {
1187 served_rung: Some(0),
1188 served_from: ServedFrom::Attempt,
1189 total_cost_usd: 0.0,
1190 gate_cost_usd: 0.0,
1191 total_latency_ms: total_ms,
1192 escalations: 0,
1193 counterfactual_baseline_usd: 0.0,
1194 savings_usd: 0.0,
1195 cache_source: None,
1196 },
1197 deferred: vec![],
1198 predicted_pass: None,
1199 probe: None,
1200 elastic: None,
1201 rollout: None,
1202 shadow: None,
1203 route_ix: None,
1204 }
1205 }
1206
1207 #[test]
1208 fn overhead_is_wall_clock_minus_the_work_actually_dispatched() {
1209 let t = trace_with_timing(500, &[(400, 60)]);
1211 assert_eq!(t.overhead_ms(), Some(40));
1212
1213 let t = trace_with_timing(1000, &[(400, 50), (500, 20)]);
1215 assert_eq!(t.overhead_ms(), Some(30));
1216 }
1217
1218 #[test]
1219 fn a_concurrent_receipt_is_excluded_rather_than_floored_to_zero() {
1220 let t = trace_with_timing(600, &[(400, 50), (500, 50)]);
1224 assert_eq!(t.overhead_ms(), None, "must not report a made-up 0");
1225
1226 let s = summarize_overhead(&[t, trace_with_timing(500, &[(400, 60)])]);
1227 assert_eq!(s.n, 1, "only the measurable one counts");
1228 assert_eq!(
1229 s.skipped_concurrent, 1,
1230 "and the skip is reported, not hidden"
1231 );
1232 assert_eq!(s.p50_ms, 40);
1233 }
1234
1235 #[test]
1236 fn percentiles_are_observed_values_not_interpolated_ones() {
1237 let traces: Vec<Trace> = [10u64, 20, 30, 40, 100]
1240 .iter()
1241 .map(|ms| trace_with_timing(100 + ms, &[(100, 0)]))
1242 .collect();
1243
1244 let s = summarize_overhead(&traces);
1245
1246 assert_eq!(s.n, 5);
1247 assert_eq!(s.p50_ms, 30);
1248 assert_eq!(s.p95_ms, 100);
1249 assert_eq!(s.p99_ms, 100);
1250 assert_eq!(s.max_ms, 100);
1251 for v in [s.p50_ms, s.p95_ms, s.p99_ms] {
1252 assert!([10, 20, 30, 40, 100].contains(&v), "{v} was never observed");
1253 }
1254 }
1255
1256 #[test]
1257 fn no_receipts_reports_that_rather_than_zeros() {
1258 let s = summarize_overhead(&[]);
1259 assert_eq!(s.n, 0);
1260 assert!(
1261 format_overhead(&s).contains("no receipts"),
1262 "{}",
1263 format_overhead(&s)
1264 );
1265 }
1266
1267 #[test]
1268 fn training_rows_carry_the_logged_propensity() {
1269 use firstpass_core::{
1270 Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1271 ServedFrom, TaskKind, Verdict,
1272 };
1273 let mk = |propensity: Option<f64>, explore: bool| Trace {
1274 trace_id: uuid::Uuid::now_v7(),
1275 prev_hash: GENESIS_HASH.to_owned(),
1276 tenant_id: "default".to_owned(),
1277 session_id: "s".to_owned(),
1278 ts: jiff::Timestamp::UNIX_EPOCH,
1279 mode: Mode::Enforce,
1280 policy: PolicyRef {
1281 id: "bandit@v1+eps".to_owned(),
1282 explore,
1283 propensity,
1284 mode_profile: None,
1285 },
1286 request: RequestInfo {
1287 api: "anthropic.messages".to_owned(),
1288 prompt_hash: "x".to_owned(),
1289 features: Features::new(TaskKind::CodeEdit),
1290 },
1291 attempts: vec![Attempt {
1292 rung: 1,
1293 model: "anthropic/claude-sonnet-5".to_owned(),
1294 provider: "anthropic".to_owned(),
1295 in_tokens: 10,
1296 cache_write_tokens: 0,
1297 cache_read_tokens: 0,
1298 out_tokens: 5,
1299 cost_usd: 0.004,
1300 latency_ms: 7,
1301 gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
1302 verdict: Verdict::Pass,
1303 }],
1304 final_: FinalOutcome {
1305 served_rung: Some(1),
1306 served_from: ServedFrom::Attempt,
1307 total_cost_usd: 0.004,
1308 gate_cost_usd: 0.0,
1309 total_latency_ms: 7,
1310 escalations: 0,
1311 counterfactual_baseline_usd: 0.02,
1312 savings_usd: 0.016,
1313 cache_source: None,
1314 },
1315 deferred: vec![],
1316 predicted_pass: None,
1317 probe: None,
1318 elastic: None,
1319 rollout: None,
1320 shadow: None,
1321 route_ix: None,
1322 };
1323
1324 let rows = export_training_jsonl(&[mk(Some(0.25), true), mk(None, false)]);
1325 let lines: Vec<&str> = rows.lines().collect();
1326 assert_eq!(lines.len(), 2);
1327
1328 let a: serde_json::Value = serde_json::from_str(lines[0]).expect("row is json");
1329 assert_eq!(a["propensity"], 0.25);
1332 assert_eq!(a["explore"], true);
1333 assert_eq!(a["start_rung"], 1);
1334 assert_eq!(a["served_rung"], 1);
1335 assert_eq!(a["served_passed"], true);
1336 assert_eq!(a["policy_id"], "bandit@v1+eps");
1337
1338 let b: serde_json::Value = serde_json::from_str(lines[1]).expect("row is json");
1341 assert!(b["propensity"].is_null(), "got {}", b["propensity"]);
1342 }
1343
1344 #[test]
1345 fn explain_trace_summarizes_served_and_escalation() {
1346 use firstpass_core::{
1347 Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
1348 ServedFrom, TaskKind, Verdict,
1349 };
1350 let t = Trace {
1351 trace_id: uuid::Uuid::now_v7(),
1352 prev_hash: GENESIS_HASH.to_owned(),
1353 tenant_id: "default".to_owned(),
1354 session_id: "s".to_owned(),
1355 ts: jiff::Timestamp::UNIX_EPOCH,
1356 mode: Mode::Enforce,
1357 policy: PolicyRef {
1358 id: "static-ladder@v0".to_owned(),
1359 explore: false,
1360 propensity: None,
1361 mode_profile: None,
1362 },
1363 request: RequestInfo {
1364 api: "anthropic.messages".to_owned(),
1365 prompt_hash: "x".to_owned(),
1366 features: Features::new(TaskKind::Other),
1367 },
1368 attempts: vec![
1369 Attempt {
1370 rung: 0,
1371 model: "anthropic/claude-haiku-4-5".to_owned(),
1372 provider: "anthropic".to_owned(),
1373 in_tokens: 1,
1374 cache_write_tokens: 0,
1375 cache_read_tokens: 0,
1376 out_tokens: 1,
1377 cost_usd: 0.001,
1378 latency_ms: 5,
1379 gates: vec![GateResult::deterministic("non-empty", Verdict::Fail, 0)],
1380 verdict: Verdict::Fail,
1381 },
1382 Attempt {
1383 rung: 1,
1384 model: "anthropic/claude-sonnet-5".to_owned(),
1385 provider: "anthropic".to_owned(),
1386 in_tokens: 1,
1387 cache_write_tokens: 0,
1388 cache_read_tokens: 0,
1389 out_tokens: 1,
1390 cost_usd: 0.01,
1391 latency_ms: 9,
1392 gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
1393 verdict: Verdict::Pass,
1394 },
1395 ],
1396 deferred: Vec::new(),
1397 final_: FinalOutcome {
1398 served_rung: Some(1),
1399 served_from: ServedFrom::Attempt,
1400 total_cost_usd: 0.011,
1401 gate_cost_usd: 0.0,
1402 total_latency_ms: 14,
1403 escalations: 1,
1404 counterfactual_baseline_usd: 0.02,
1405 savings_usd: 0.009,
1406 cache_source: None,
1407 },
1408 probe: None,
1409 rollout: None,
1410 shadow: None,
1411 route_ix: None,
1412 predicted_pass: None,
1413 elastic: None,
1414 };
1415 let ex = explain_trace(&t);
1416 assert_eq!(
1417 ex.served_model.as_deref(),
1418 Some("anthropic/claude-sonnet-5")
1419 );
1420 assert_eq!(ex.escalations, 1);
1421 assert_eq!(ex.attempts.len(), 2);
1422 assert_eq!(ex.attempts[0].verdict, "fail");
1423 assert_eq!(
1424 ex.attempts[1].gates[0],
1425 ("non-empty".to_owned(), "pass".to_owned())
1426 );
1427 assert!(ex.summary.contains("served anthropic/claude-sonnet-5"));
1428 }
1429
1430 #[test]
1431 fn evaluate_predictor_empty_and_signal() {
1432 let e = evaluate_predictor(&[], 0.05, 1e-4);
1434 assert_eq!(e.n, 0);
1435 assert!(e.auc.is_none());
1436 }
1437
1438 #[test]
1441 fn detect_channel_reads_real_install_paths() {
1442 use std::path::Path;
1443 let cases = [
1444 ("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
1445 (
1446 "/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
1447 Channel::Homebrew,
1448 ),
1449 (
1450 "/home/linuxbrew/.linuxbrew/bin/firstpass",
1451 Channel::Homebrew,
1452 ),
1453 (
1454 "/Users/u/.venv/lib/python3.12/site-packages/firstpass/firstpass",
1455 Channel::Python,
1456 ),
1457 (
1460 "/Users/u/.cache/uv/archive-v0/DQ4usH_XuqNyywgHKB8qB/bin/firstpass",
1461 Channel::Python,
1462 ),
1463 (
1464 "/Users/u/.local/share/uv/tools/firstpass/bin/firstpass",
1465 Channel::Python,
1466 ),
1467 (
1468 "/usr/local/lib/node_modules/@dshakesnotbot/firstpass-proxy/bin/firstpass",
1469 Channel::Npm,
1470 ),
1471 (r"C:\Users\u\.cargo\bin\firstpass.exe", Channel::Cargo),
1472 ];
1473 for (path, want) in cases {
1474 assert_eq!(
1475 detect_channel(Path::new(path), false, false),
1476 want,
1477 "misread {path}"
1478 );
1479 }
1480 assert_eq!(
1482 detect_channel(Path::new("/Users/u/.local/bin/firstpass"), true, false),
1483 Channel::Dist
1484 );
1485 assert_eq!(
1486 detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
1487 Channel::Unknown
1488 );
1489 assert_eq!(
1491 detect_channel(Path::new("/usr/local/bin/firstpass"), true, true),
1492 Channel::Docker
1493 );
1494 }
1495
1496 #[test]
1499 fn package_manager_paths_win_over_the_self_updater() {
1500 use std::path::Path;
1501 for (path, want) in [
1502 (
1503 "/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
1504 Channel::Homebrew,
1505 ),
1506 ("/Users/u/.venv/bin/firstpass", Channel::Python),
1507 ("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
1508 ] {
1509 assert_eq!(detect_channel(Path::new(path), true, false), want, "{path}");
1510 }
1511 }
1512
1513 #[test]
1516 fn upgrade_report_runs_only_our_own_channel() {
1517 assert!(upgrade_report(Channel::Dist, "0.2.2").contains("upgrading in place"));
1518 for c in [
1519 Channel::Homebrew,
1520 Channel::Python,
1521 Channel::Npm,
1522 Channel::Docker,
1523 Channel::Cargo,
1524 ] {
1525 let r = upgrade_report(c, "0.2.2");
1526 assert!(
1527 !r.contains("upgrading in place"),
1528 "{c:?} must not self-update"
1529 );
1530 assert!(
1531 r.contains("owned by its package manager"),
1532 "{c:?} must explain why"
1533 );
1534 assert!(
1535 r.contains(c.upgrade_command().expect("has a command")),
1536 "{c:?} must print its exact command"
1537 );
1538 }
1539 let unknown = upgrade_report(Channel::Unknown, "0.2.2");
1540 assert!(unknown.contains("brew upgrade") && unknown.contains("docker pull"));
1541 assert!(
1542 unknown.contains("0.2.2"),
1543 "always states the running version"
1544 );
1545 }
1546
1547 #[test]
1551 fn pipx_install_is_recognised_from_its_real_path() {
1552 use std::path::Path;
1553 let real = "/Users/u/.local/pipx/venvs/firstpass/bin/firstpass";
1554 assert_eq!(detect_channel(Path::new(real), false, false), Channel::Pipx);
1555 assert_eq!(
1556 Channel::Pipx.upgrade_command(),
1557 Some("pipx upgrade firstpass")
1558 );
1559
1560 assert_eq!(
1563 detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
1564 Channel::Unknown
1565 );
1566
1567 assert_eq!(
1569 detect_channel(
1570 Path::new(
1571 "/Users/u/.local/pipx/venvs/firstpass/lib/python3.14/site-packages/x/firstpass"
1572 ),
1573 false,
1574 false
1575 ),
1576 Channel::Pipx
1577 );
1578 }
1579}