use crate::config::ProxyConfig;
use firstpass_core::{Mode, Trace};
#[derive(Debug, PartialEq, Eq)]
pub enum CheckStatus {
Ok,
Warn,
Fail,
}
#[derive(Debug)]
pub struct Check {
pub name: String,
pub status: CheckStatus,
pub detail: String,
}
impl Check {
fn new(name: impl Into<String>, status: CheckStatus, detail: impl Into<String>) -> Self {
Self {
name: name.into(),
status,
detail: detail.into(),
}
}
}
#[derive(Debug)]
pub struct DoctorReport {
pub checks: Vec<Check>,
}
impl DoctorReport {
#[must_use]
pub fn healthy(&self) -> bool {
self.checks.iter().all(|c| c.status != CheckStatus::Fail)
}
#[must_use]
pub fn render(&self) -> String {
let mut out = String::new();
for c in &self.checks {
let mark = match c.status {
CheckStatus::Ok => "✓",
CheckStatus::Warn => "!",
CheckStatus::Fail => "✗",
};
out.push_str(&format!("{mark} {}: {}\n", c.name, c.detail));
}
out.push_str(if self.healthy() {
"\nhealthy — ready to route.\n"
} else {
"\nnot healthy — fix the ✗ items above.\n"
});
out
}
}
#[must_use]
pub fn doctor(config: &ProxyConfig, env: impl Fn(&str) -> Option<String>) -> DoctorReport {
let mut checks = Vec::new();
let route_count = config.routing.as_ref().map_or(0, |c| c.routes.len());
checks.push(Check::new(
"config",
CheckStatus::Ok,
format!("default mode {:?}, {route_count} route(s)", config.mode),
));
let enforce_routes = config.routing.as_ref().map_or(0, |c| {
c.routes.iter().filter(|r| r.mode == Mode::Enforce).count()
});
if config.mode == Mode::Enforce && enforce_routes == 0 {
checks.push(Check::new(
"routing",
CheckStatus::Warn,
"mode is enforce but no enforce route is defined — all traffic will observe",
));
} else {
checks.push(Check::new(
"routing",
CheckStatus::Ok,
format!("{enforce_routes} enforce route(s)"),
));
}
if env("ANTHROPIC_API_KEY").is_some_and(|k| !k.is_empty()) {
checks.push(Check::new("anthropic-key", CheckStatus::Ok, "present"));
} else {
checks.push(Check::new(
"anthropic-key",
CheckStatus::Warn,
"ANTHROPIC_API_KEY not set — observe uses the caller's key; enforce needs one reachable",
));
}
let mut missing_keys: Vec<(String, &str)> = Vec::new();
if let Some(routing) = config.routing.as_ref() {
let mut seen = std::collections::HashSet::new();
for rung in routing.routes.iter().flat_map(|r| r.ladder.iter()) {
let Some((prov, _)) = rung.split_once('/') else {
continue;
};
let Some(key_env) = crate::provider::builtin_key_env(prov) else {
continue; };
if seen.insert(prov.to_owned()) && env(key_env).is_none_or(|k| k.is_empty()) {
missing_keys.push((prov.to_owned(), key_env));
}
}
}
for (prov, key_env) in missing_keys {
checks.push(Check::new(
format!("provider:{prov}"),
CheckStatus::Fail,
format!("{key_env} not set — a rung on `{prov}` will fail at the first request"),
));
}
let path = env("PATH");
let gate_defs = config.routing.as_ref().map_or(&[][..], |c| &c.gate_defs);
for def in gate_defs {
match def.cmd.first() {
Some(program) if command_on_path(program, path.as_deref()) => checks.push(Check::new(
format!("gate:{}", def.id),
CheckStatus::Ok,
format!("`{program}` found"),
)),
Some(program) => checks.push(Check::new(
format!("gate:{}", def.id),
CheckStatus::Fail,
format!("`{program}` not found on PATH"),
)),
None => checks.push(Check::new(
format!("gate:{}", def.id),
CheckStatus::Fail,
"empty command",
)),
}
}
if can_write_db(&config.db_path) {
checks.push(Check::new(
"trace-store",
CheckStatus::Ok,
format!("{} is writable", config.db_path),
));
} else {
checks.push(Check::new(
"trace-store",
CheckStatus::Fail,
format!("cannot write near {}", config.db_path),
));
}
DoctorReport { checks }
}
#[must_use]
pub fn command_on_path(program: &str, path_var: Option<&str>) -> bool {
if program.contains('/') || program.contains('\\') {
return std::path::Path::new(program).is_file();
}
let Some(path) = path_var else { return false };
std::env::split_paths(path).any(|dir| dir.join(program).is_file())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Channel {
Dist,
Homebrew,
Cargo,
Python,
Pipx,
Npm,
Docker,
Unknown,
}
impl Channel {
#[must_use]
pub const fn label(self) -> &'static str {
match self {
Self::Dist => "installer script (self-updating)",
Self::Homebrew => "Homebrew",
Self::Cargo => "cargo install",
Self::Python => "Python wheel (pip / uv / uvx)",
Self::Pipx => "pipx",
Self::Npm => "npm",
Self::Docker => "Docker image",
Self::Unknown => "unrecognised",
}
}
#[must_use]
pub const fn upgrade_command(self) -> Option<&'static str> {
match self {
Self::Dist => Some("firstpass-update"),
Self::Homebrew => Some("brew upgrade firstpass-proxy"),
Self::Cargo => Some(
"cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force",
),
Self::Python => Some("uv tool upgrade firstpass # or: pip install -U firstpass"),
Self::Pipx => Some("pipx upgrade firstpass"),
Self::Npm => Some("npm install -g @dshakesnotbot/firstpass-proxy@latest"),
Self::Docker => Some("docker pull ghcr.io/dshakes/firstpass:latest"),
Self::Unknown => None,
}
}
}
#[must_use]
pub fn detect_channel(exe: &std::path::Path, updater_present: bool, in_container: bool) -> Channel {
let p = exe.to_string_lossy().to_lowercase().replace('\\', "/");
let has = |needle: &str| p.contains(needle);
if in_container {
return Channel::Docker;
}
if has("/.cargo/bin/") || has("/.cargo/registry/") {
return Channel::Cargo;
}
if has("/cellar/") || has("/homebrew/") || has("/linuxbrew/") {
return Channel::Homebrew;
}
if has("/pipx/") {
return Channel::Pipx; }
if has("/site-packages/")
|| has("/dist-packages/")
|| has("/.venv/")
|| has("/uv/")
|| has("/uv-cache/")
{
return Channel::Python;
}
if has("/node_modules/") {
return Channel::Npm;
}
if updater_present {
return Channel::Dist;
}
Channel::Unknown
}
#[must_use]
pub fn upgrade_report(channel: Channel, current_version: &str) -> String {
let mut out = format!(
"firstpass {current_version} · installed via {}\n",
channel.label()
);
match channel.upgrade_command() {
Some(cmd) if channel == Channel::Dist => {
out.push_str(&format!("\nupgrading in place — running `{cmd}`\n"));
}
Some(cmd) => {
out.push_str(&format!(
"\nthis channel is owned by its package manager, so upgrade with:\n {cmd}\n"
));
}
None => {
out.push_str(
"\ncould not tell how this binary was installed. Pick the line that matches:\n\
\x20 firstpass-update # installer script\n\
\x20 brew upgrade firstpass-proxy # Homebrew\n\
\x20 uv tool upgrade firstpass # uv / pip: pip install -U firstpass\n\
\x20 pipx upgrade firstpass # pipx\n\
\x20 npm install -g @dshakesnotbot/firstpass-proxy@latest\n\
\x20 docker pull ghcr.io/dshakes/firstpass:latest\n\
\x20 cargo install --git https://github.com/dshakes/firstpass firstpass-proxy --force\n",
);
}
}
out
}
fn can_write_db(db_path: &str) -> bool {
let path = std::path::Path::new(db_path);
let dir = path
.parent()
.filter(|d| !d.as_os_str().is_empty())
.map_or_else(
|| std::path::PathBuf::from("."),
std::path::Path::to_path_buf,
);
let probe = dir.join(format!(".firstpass-doctor-probe-{}", std::process::id()));
match std::fs::File::create(&probe) {
Ok(_) => {
let _ = std::fs::remove_file(&probe);
true
}
Err(_) => false,
}
}
#[derive(Debug, serde::Serialize)]
pub struct EvalsSummary {
pub enforce_traces: usize,
pub escalations: u64,
pub gates: std::collections::BTreeMap<String, (u64, u64, u64)>,
pub served_by_rung: std::collections::BTreeMap<u32, u64>,
}
#[must_use]
#[derive(Debug, serde::Serialize)]
pub struct PredictorEval {
pub n: usize,
pub auc: Option<f64>,
pub brier: f64,
}
fn predictor_auc(pairs: &[(f64, bool)]) -> Option<f64> {
let pos: Vec<f64> = pairs.iter().filter(|(_, y)| *y).map(|(s, _)| *s).collect();
let neg: Vec<f64> = pairs.iter().filter(|(_, y)| !*y).map(|(s, _)| *s).collect();
if pos.is_empty() || neg.is_empty() {
return None;
}
let mut wins = 0.0_f64;
for &p in &pos {
for &n in &neg {
wins += if p > n {
1.0
} else if (p - n).abs() < f64::EPSILON {
0.5
} else {
0.0
};
}
}
Some(wins / (pos.len() as f64 * neg.len() as f64))
}
pub fn evaluate_predictor(traces: &[Trace], lr: f64, l2: f64) -> PredictorEval {
use firstpass_core::{PassPredictor, Verdict};
let mut model = PassPredictor::new(lr, l2);
let mut pairs: Vec<(f64, bool)> = Vec::new();
let mut se = 0.0_f64;
for t in traces {
for a in &t.attempts {
let y = match a.verdict {
Verdict::Pass => true,
Verdict::Fail => false,
Verdict::Abstain => continue,
};
let p = model.predict(&t.request.features, a.rung);
let target = f64::from(u8::from(y));
se += (p - target) * (p - target);
pairs.push((p, y));
model.update(&t.request.features, a.rung, y);
}
}
let n = pairs.len();
PredictorEval {
n,
auc: predictor_auc(&pairs),
brier: if n == 0 { 0.0 } else { se / n as f64 },
}
}
pub fn summarize_evals(traces: &[Trace]) -> EvalsSummary {
let mut s = EvalsSummary {
enforce_traces: 0,
escalations: 0,
gates: std::collections::BTreeMap::new(),
served_by_rung: std::collections::BTreeMap::new(),
};
for t in traces {
if t.mode != Mode::Enforce {
continue;
}
s.enforce_traces += 1;
s.escalations += u64::from(t.final_.escalations);
if let Some(rung) = t.final_.served_rung {
*s.served_by_rung.entry(rung).or_insert(0) += 1;
}
for attempt in &t.attempts {
for g in &attempt.gates {
let e = s.gates.entry(g.gate_id.clone()).or_insert((0, 0, 0));
match g.verdict {
firstpass_core::Verdict::Pass => e.0 += 1,
firstpass_core::Verdict::Fail => e.1 += 1,
firstpass_core::Verdict::Abstain => e.2 += 1,
}
}
}
}
s
}
#[must_use]
pub fn format_evals(s: &EvalsSummary) -> String {
if s.enforce_traces == 0 {
return "no enforce traces yet — route some traffic in enforce mode first".to_owned();
}
let mut out = format!(
"enforce traces: {} · escalations: {}\n",
s.enforce_traces, s.escalations
);
out.push_str("gates (pass / fail / abstain):\n");
for (id, (p, f, a)) in &s.gates {
out.push_str(&format!(" {id:<24} {p:>6} / {f:>6} / {a:>6}\n"));
}
out.push_str("served by rung:\n");
for (rung, n) in &s.served_by_rung {
out.push_str(&format!(" rung {rung:<2} {n:>6}\n"));
}
out.trim_end().to_owned()
}
#[derive(Debug, serde::Serialize)]
pub struct RouteExplanation {
pub trace_id: String,
pub mode: String,
pub policy: String,
pub served_model: Option<String>,
pub served_rung: Option<u32>,
pub escalations: u32,
pub total_cost_usd: f64,
pub baseline_usd: f64,
pub savings_usd: f64,
pub attempts: Vec<AttemptExplanation>,
pub summary: String,
}
#[derive(Debug, serde::Serialize)]
pub struct AttemptExplanation {
pub rung: u32,
pub model: String,
pub verdict: String,
pub gates: Vec<(String, String)>,
}
fn verdict_str(v: firstpass_core::Verdict) -> &'static str {
match v {
firstpass_core::Verdict::Pass => "pass",
firstpass_core::Verdict::Fail => "fail",
firstpass_core::Verdict::Abstain => "abstain",
}
}
#[must_use]
pub fn explain_trace(t: &Trace) -> RouteExplanation {
let attempts: Vec<AttemptExplanation> = t
.attempts
.iter()
.map(|a| AttemptExplanation {
rung: a.rung,
model: a.model.clone(),
verdict: verdict_str(a.verdict).to_owned(),
gates: a
.gates
.iter()
.map(|g| (g.gate_id.clone(), verdict_str(g.verdict).to_owned()))
.collect(),
})
.collect();
let served_model = t
.final_
.served_rung
.and_then(|r| t.attempts.iter().find(|a| a.rung == r))
.map(|a| a.model.clone());
let summary = match &served_model {
Some(m) => format!(
"served {m} at rung {} after {} escalation(s); spent ${:.4} vs ${:.4} always-top (saved ${:.4})",
t.final_.served_rung.unwrap_or(0),
t.final_.escalations,
t.final_.total_cost_usd,
t.final_.counterfactual_baseline_usd,
t.final_.savings_usd
),
None => "no rung served (all attempts failed the gate or errored)".to_owned(),
};
RouteExplanation {
trace_id: t.trace_id.to_string(),
mode: match t.mode {
Mode::Enforce => "enforce",
Mode::Observe => "observe",
}
.to_owned(),
policy: t.policy.id.clone(),
served_model,
served_rung: t.final_.served_rung,
escalations: t.final_.escalations,
total_cost_usd: t.final_.total_cost_usd,
baseline_usd: t.final_.counterfactual_baseline_usd,
savings_usd: t.final_.savings_usd,
attempts,
summary,
}
}
#[derive(Debug, serde::Serialize)]
pub struct VerifyReport {
pub receipts: usize,
pub valid: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub broken_at: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub detail: Option<String>,
}
#[must_use]
pub fn verify_receipts(traces: &[Trace]) -> VerifyReport {
match firstpass_core::verify_chain(traces, firstpass_core::GENESIS_HASH) {
Ok(()) => VerifyReport {
receipts: traces.len(),
valid: true,
broken_at: None,
detail: None,
},
Err(firstpass_core::Error::ChainBroken { index, detail }) => VerifyReport {
receipts: traces.len(),
valid: false,
broken_at: Some(index),
detail: Some(detail),
},
Err(e) => VerifyReport {
receipts: traces.len(),
valid: false,
broken_at: None,
detail: Some(e.to_string()),
},
}
}
pub fn parse_receipt_jsonl(text: &str) -> Result<Vec<Trace>, String> {
let mut traces = Vec::new();
for (i, line) in text.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
let t: Trace = serde_json::from_str(line).map_err(|e| format!("line {}: {e}", i + 1))?;
traces.push(t);
}
Ok(traces)
}
#[must_use]
#[derive(Debug, Clone, serde::Serialize)]
pub struct TrainingRow {
pub trace_id: String,
pub features: firstpass_core::Features,
pub start_rung: u32,
pub served_rung: Option<u32>,
pub served_passed: bool,
pub cost_usd: f64,
pub propensity: Option<f64>,
pub explore: bool,
pub policy_id: String,
}
impl TrainingRow {
pub fn from_trace(t: &Trace) -> Self {
let served_rung = t.final_.served_rung;
Self {
trace_id: t.trace_id.to_string(),
features: t.request.features.clone(),
start_rung: t.attempts.first().map_or(0, |a| a.rung),
served_rung,
served_passed: served_rung.is_some_and(|r| {
t.attempts
.iter()
.any(|a| a.rung == r && a.verdict == firstpass_core::Verdict::Pass)
}),
cost_usd: t.final_.total_cost_usd,
propensity: t.policy.propensity,
explore: t.policy.explore,
policy_id: t.policy.id.clone(),
}
}
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct OverheadSummary {
pub n: usize,
pub skipped_concurrent: usize,
pub p50_ms: u64,
pub p95_ms: u64,
pub p99_ms: u64,
pub max_ms: u64,
}
#[must_use]
pub fn summarize_overhead(traces: &[Trace]) -> OverheadSummary {
let mut v: Vec<u64> = Vec::with_capacity(traces.len());
let mut skipped = 0usize;
for t in traces {
match t.overhead_ms() {
Some(ms) => v.push(ms),
None => skipped += 1,
}
}
v.sort_unstable();
let pct = |p: f64| -> u64 {
if v.is_empty() {
return 0;
}
let rank = (p * v.len() as f64).ceil() as usize;
v[rank.clamp(1, v.len()) - 1]
};
OverheadSummary {
n: v.len(),
skipped_concurrent: skipped,
p50_ms: pct(0.50),
p95_ms: pct(0.95),
p99_ms: pct(0.99),
max_ms: v.last().copied().unwrap_or(0),
}
}
#[must_use]
pub fn format_overhead(s: &OverheadSummary) -> String {
if s.n == 0 {
return format!(
"no receipts with a measurable overhead ({} skipped as concurrent).\n\
Route some traffic first, or run with speculation off to measure it.\n",
s.skipped_concurrent
);
}
let mut lines = vec![
"Firstpass overhead — time added beyond the provider calls and gates it ran".to_owned(),
String::new(),
format!(" p50 {:>6} ms", s.p50_ms),
format!(" p95 {:>6} ms", s.p95_ms),
format!(" p99 {:>6} ms", s.p99_ms),
format!(" max {:>6} ms", s.max_ms),
String::new(),
format!(" over {} receipt(s)", s.n),
];
if s.p95_ms == 0 {
lines.push(
" 0 ms is the floor, not a missing measurement: receipts record whole".to_owned(),
);
lines.push(
" milliseconds, so the proxy's own work stayed under 1 ms per request.".to_owned(),
);
}
if s.skipped_concurrent > 0 {
lines.push(format!(
" {} skipped: speculative rungs run concurrently, so subtracting dispatched",
s.skipped_concurrent
));
lines.push(
" work from wall-clock is meaningless there — excluded, not floored to 0.".to_owned(),
);
}
lines.push(String::new());
lines.join("\n")
}
#[must_use]
pub fn export_training_jsonl(traces: &[Trace]) -> String {
let mut out = String::new();
for t in traces {
if let Ok(line) = serde_json::to_string(&TrainingRow::from_trace(t)) {
out.push_str(&line);
out.push('\n');
}
}
out
}
pub fn export_receipts_jsonl(traces: &[Trace]) -> String {
let mut out = String::new();
for t in traces {
if let Ok(line) = serde_json::to_string(t) {
out.push_str(&line);
out.push('\n');
}
}
out
}
#[derive(Debug, serde::Serialize)]
pub struct SavingsSummary {
pub traces: usize,
pub enforce_traces: usize,
pub spent_usd: f64,
pub gate_usd: f64,
pub baseline_usd: f64,
pub savings_usd: f64,
pub savings_pct: f64,
}
#[must_use]
pub fn summarize_savings(traces: &[Trace]) -> SavingsSummary {
let mut s = SavingsSummary {
traces: traces.len(),
enforce_traces: 0,
spent_usd: 0.0,
gate_usd: 0.0,
baseline_usd: 0.0,
savings_usd: 0.0,
savings_pct: 0.0,
};
for t in traces {
if t.mode == Mode::Enforce {
s.enforce_traces += 1;
}
s.spent_usd += t.final_.total_cost_usd;
s.gate_usd += t.final_.gate_cost_usd;
s.baseline_usd += t.final_.counterfactual_baseline_usd;
}
s.savings_usd = s.baseline_usd - s.spent_usd;
if s.baseline_usd > 0.0 {
s.savings_pct = s.savings_usd / s.baseline_usd;
}
s
}
#[must_use]
pub fn format_savings(s: &SavingsSummary) -> String {
if s.traces == 0 {
return "no traces recorded yet — route some traffic through the proxy first".to_owned();
}
format!(
"traces: {} ({} enforce)\nspent: ${:.4} (gates ${:.4})\nbaseline: ${:.4} (always top rung)\nsavings: ${:.4} ({:.1}%)",
s.traces,
s.enforce_traces,
s.spent_usd,
s.gate_usd,
s.baseline_usd,
s.savings_usd,
s.savings_pct * 100.0
)
}
#[must_use]
pub fn format_traces(traces: &[Trace], limit: usize) -> String {
let mut lines: Vec<String> = traces
.iter()
.rev()
.take(limit)
.filter_map(|t| serde_json::to_string(t).ok())
.collect();
lines.reverse();
if lines.is_empty() {
"no traces recorded yet".to_owned()
} else {
lines.join("\n")
}
}
#[cfg(test)]
mod tests {
use super::*;
fn config_with(toml: Option<&str>, db_path: &str) -> ProxyConfig {
ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_MODE" if toml.is_some() => Some("enforce".to_owned()),
"FIRSTPASS_CONFIG_TOML" => toml.map(str::to_owned),
"FIRSTPASS_DB" => Some(db_path.to_owned()),
_ => None,
})
.unwrap()
}
#[test]
fn doctor_flags_a_ladder_rung_whose_provider_key_is_missing() {
let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"groq/llama-3.3-70b-versatile\"]\n\
[[price]]\nmodel = \"groq/llama-3.3-70b-versatile\"\n\
input_per_mtok = 0.59\noutput_per_mtok = 0.79\n";
let config = config_with(Some(toml), "/tmp/doctor-groq.db");
let report = doctor(&config, |k| match k {
"PATH" => Some("/usr/bin".to_owned()),
_ => None, });
let groq = report
.checks
.iter()
.find(|c| c.name == "provider:groq")
.expect("groq rung should be checked");
assert_eq!(groq.status, CheckStatus::Fail);
assert!(groq.detail.contains("GROQ_API_KEY"), "{}", groq.detail);
assert!(
!report.healthy(),
"a rung that cannot authenticate is not healthy"
);
}
#[test]
fn doctor_stays_quiet_about_providers_the_ladder_does_not_use() {
let toml = "[[route]]\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\n";
let config = config_with(Some(toml), "/tmp/doctor-quiet.db");
let report = doctor(&config, |k| match k {
"PATH" => Some("/usr/bin".to_owned()),
"ANTHROPIC_API_KEY" => Some("sk-test".to_owned()),
_ => None,
});
assert!(
!report
.checks
.iter()
.any(|c| c.name.starts_with("provider:")),
"unused providers must not be reported: {:?}",
report.checks.iter().map(|c| &c.name).collect::<Vec<_>>()
);
}
#[test]
fn command_on_path_finds_real_and_rejects_fake() {
let path = std::env::var("PATH").ok();
assert!(command_on_path("sh", path.as_deref()), "sh is on PATH");
assert!(!command_on_path(
"firstpass-definitely-not-a-real-binary",
path.as_deref()
));
assert!(!command_on_path("/nonexistent/abs/path", None));
}
#[test]
fn doctor_flags_a_missing_gate_binary() {
let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"good\", \"bad\"]\n\
[[gate]]\nid = \"good\"\ncmd = [\"sh\"]\n\
[[gate]]\nid = \"bad\"\ncmd = [\"firstpass-nope-not-real\"]\n";
let db = std::env::temp_dir().join("firstpass-doctor-test.db");
let config = config_with(Some(toml), db.to_str().unwrap());
let report = doctor(&config, |k| match k {
"PATH" => std::env::var("PATH").ok(),
_ => None,
});
assert!(
!report.healthy(),
"a missing gate binary must fail the report"
);
let bad = report.checks.iter().find(|c| c.name == "gate:bad").unwrap();
assert_eq!(bad.status, CheckStatus::Fail);
let good = report
.checks
.iter()
.find(|c| c.name == "gate:good")
.unwrap();
assert_eq!(good.status, CheckStatus::Ok);
let key = report
.checks
.iter()
.find(|c| c.name == "anthropic-key")
.unwrap();
assert_eq!(key.status, CheckStatus::Warn);
}
#[test]
fn doctor_is_healthy_for_a_plain_observe_setup() {
let db = std::env::temp_dir().join("firstpass-doctor-ok.db");
let config = config_with(None, db.to_str().unwrap());
let report = doctor(&config, |k| {
(k == "ANTHROPIC_API_KEY").then(|| "sk-test".to_owned())
});
assert!(report.healthy(), "{}", report.render());
}
#[test]
fn format_traces_handles_empty() {
assert_eq!(format_traces(&[], 10), "no traces recorded yet");
}
#[test]
fn savings_summary_empty_and_nonempty() {
let empty = summarize_savings(&[]);
assert_eq!(empty.traces, 0);
assert!(format_savings(&empty).contains("no traces"));
}
#[test]
fn evals_summary_empty_zero_state() {
let s = summarize_evals(&[]);
assert_eq!(s.enforce_traces, 0);
assert!(format_evals(&s).contains("no enforce traces"));
}
#[test]
fn verify_and_export_round_trip_detects_tampering() {
use firstpass_core::{
Attempt, Features, FinalOutcome, GENESIS_HASH, PolicyRef, RequestInfo, ServedFrom,
TaskKind, Verdict,
};
let mk = |prev: &str, session: &str| Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: prev.to_owned(),
tenant_id: "default".to_owned(),
session_id: session.to_owned(),
ts: jiff::Timestamp::UNIX_EPOCH,
mode: Mode::Observe,
policy: PolicyRef {
id: "observe-passthrough@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "deadbeef".to_owned(),
features: Features::new(TaskKind::Other),
},
attempts: vec![Attempt {
rung: 0,
model: "anthropic/claude-haiku-4-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
cost_usd: 0.001,
latency_ms: 12,
gates: vec![],
verdict: Verdict::Pass,
}],
deferred: Vec::new(),
final_: FinalOutcome {
served_rung: Some(0),
served_from: ServedFrom::Attempt,
total_cost_usd: 0.001,
gate_cost_usd: 0.0,
total_latency_ms: 12,
escalations: 0,
counterfactual_baseline_usd: 0.001,
savings_usd: 0.0,
cache_source: None,
},
probe: None,
rollout: None,
shadow: None,
route_ix: None,
predicted_pass: None,
elastic: None,
};
let t0 = mk(GENESIS_HASH, "s0");
let t1 = mk(&t0.hash().unwrap(), "s1");
let t2 = mk(&t1.hash().unwrap(), "s2");
let chain = vec![t0, t1, t2];
let report = verify_receipts(&chain);
assert!(report.valid, "intact chain must verify: {report:?}");
assert_eq!(report.receipts, 3);
let jsonl = export_receipts_jsonl(&chain);
assert_eq!(jsonl.lines().count(), 3);
let reparsed = parse_receipt_jsonl(&jsonl).expect("export must re-parse");
assert!(
verify_receipts(&reparsed).valid,
"round-tripped chain must verify"
);
let mut tampered = reparsed;
tampered[1].final_.total_cost_usd = 999.0; let bad = verify_receipts(&tampered);
assert!(!bad.valid, "a mutated receipt must break the chain");
assert_eq!(
bad.broken_at,
Some(2),
"the break surfaces at the next link"
);
let mut reordered = parse_receipt_jsonl(&jsonl).unwrap();
reordered.swap(0, 1);
assert!(
!verify_receipts(&reordered).valid,
"reordering breaks the chain"
);
}
#[test]
fn parse_receipt_jsonl_reports_bad_line() {
let err = parse_receipt_jsonl("not json\n").unwrap_err();
assert!(err.starts_with("line 1:"), "must name the bad line: {err}");
assert!(
parse_receipt_jsonl("\n\n").unwrap().is_empty(),
"blank lines skipped"
);
}
fn trace_with_timing(total_ms: u64, attempts: &[(u64, u64)]) -> Trace {
use firstpass_core::{
Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
ServedFrom, TaskKind, Verdict,
};
Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: "default".to_owned(),
session_id: "s".to_owned(),
ts: jiff::Timestamp::UNIX_EPOCH,
mode: Mode::Enforce,
policy: PolicyRef {
id: "static-ladder@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "x".to_owned(),
features: Features::new(TaskKind::CodeEdit),
},
attempts: attempts
.iter()
.enumerate()
.map(|(i, (model_ms, gate_ms))| Attempt {
rung: i as u32,
model: "anthropic/claude-haiku-4-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 1,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 1,
cost_usd: 0.0,
latency_ms: *model_ms,
gates: vec![GateResult::deterministic(
"non-empty",
Verdict::Pass,
*gate_ms,
)],
verdict: Verdict::Pass,
})
.collect(),
final_: FinalOutcome {
served_rung: Some(0),
served_from: ServedFrom::Attempt,
total_cost_usd: 0.0,
gate_cost_usd: 0.0,
total_latency_ms: total_ms,
escalations: 0,
counterfactual_baseline_usd: 0.0,
savings_usd: 0.0,
cache_source: None,
},
deferred: vec![],
predicted_pass: None,
probe: None,
elastic: None,
rollout: None,
shadow: None,
route_ix: None,
}
}
#[test]
fn overhead_is_wall_clock_minus_the_work_actually_dispatched() {
let t = trace_with_timing(500, &[(400, 60)]);
assert_eq!(t.overhead_ms(), Some(40));
let t = trace_with_timing(1000, &[(400, 50), (500, 20)]);
assert_eq!(t.overhead_ms(), Some(30));
}
#[test]
fn a_concurrent_receipt_is_excluded_rather_than_floored_to_zero() {
let t = trace_with_timing(600, &[(400, 50), (500, 50)]);
assert_eq!(t.overhead_ms(), None, "must not report a made-up 0");
let s = summarize_overhead(&[t, trace_with_timing(500, &[(400, 60)])]);
assert_eq!(s.n, 1, "only the measurable one counts");
assert_eq!(
s.skipped_concurrent, 1,
"and the skip is reported, not hidden"
);
assert_eq!(s.p50_ms, 40);
}
#[test]
fn percentiles_are_observed_values_not_interpolated_ones() {
let traces: Vec<Trace> = [10u64, 20, 30, 40, 100]
.iter()
.map(|ms| trace_with_timing(100 + ms, &[(100, 0)]))
.collect();
let s = summarize_overhead(&traces);
assert_eq!(s.n, 5);
assert_eq!(s.p50_ms, 30);
assert_eq!(s.p95_ms, 100);
assert_eq!(s.p99_ms, 100);
assert_eq!(s.max_ms, 100);
for v in [s.p50_ms, s.p95_ms, s.p99_ms] {
assert!([10, 20, 30, 40, 100].contains(&v), "{v} was never observed");
}
}
#[test]
fn no_receipts_reports_that_rather_than_zeros() {
let s = summarize_overhead(&[]);
assert_eq!(s.n, 0);
assert!(
format_overhead(&s).contains("no receipts"),
"{}",
format_overhead(&s)
);
}
#[test]
fn training_rows_carry_the_logged_propensity() {
use firstpass_core::{
Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
ServedFrom, TaskKind, Verdict,
};
let mk = |propensity: Option<f64>, explore: bool| Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: "default".to_owned(),
session_id: "s".to_owned(),
ts: jiff::Timestamp::UNIX_EPOCH,
mode: Mode::Enforce,
policy: PolicyRef {
id: "bandit@v1+eps".to_owned(),
explore,
propensity,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "x".to_owned(),
features: Features::new(TaskKind::CodeEdit),
},
attempts: vec![Attempt {
rung: 1,
model: "anthropic/claude-sonnet-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
cost_usd: 0.004,
latency_ms: 7,
gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
verdict: Verdict::Pass,
}],
final_: FinalOutcome {
served_rung: Some(1),
served_from: ServedFrom::Attempt,
total_cost_usd: 0.004,
gate_cost_usd: 0.0,
total_latency_ms: 7,
escalations: 0,
counterfactual_baseline_usd: 0.02,
savings_usd: 0.016,
cache_source: None,
},
deferred: vec![],
predicted_pass: None,
probe: None,
elastic: None,
rollout: None,
shadow: None,
route_ix: None,
};
let rows = export_training_jsonl(&[mk(Some(0.25), true), mk(None, false)]);
let lines: Vec<&str> = rows.lines().collect();
assert_eq!(lines.len(), 2);
let a: serde_json::Value = serde_json::from_str(lines[0]).expect("row is json");
assert_eq!(a["propensity"], 0.25);
assert_eq!(a["explore"], true);
assert_eq!(a["start_rung"], 1);
assert_eq!(a["served_rung"], 1);
assert_eq!(a["served_passed"], true);
assert_eq!(a["policy_id"], "bandit@v1+eps");
let b: serde_json::Value = serde_json::from_str(lines[1]).expect("row is json");
assert!(b["propensity"].is_null(), "got {}", b["propensity"]);
}
#[test]
fn explain_trace_summarizes_served_and_escalation() {
use firstpass_core::{
Attempt, Features, FinalOutcome, GENESIS_HASH, GateResult, PolicyRef, RequestInfo,
ServedFrom, TaskKind, Verdict,
};
let t = Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: "default".to_owned(),
session_id: "s".to_owned(),
ts: jiff::Timestamp::UNIX_EPOCH,
mode: Mode::Enforce,
policy: PolicyRef {
id: "static-ladder@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "x".to_owned(),
features: Features::new(TaskKind::Other),
},
attempts: vec![
Attempt {
rung: 0,
model: "anthropic/claude-haiku-4-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 1,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 1,
cost_usd: 0.001,
latency_ms: 5,
gates: vec![GateResult::deterministic("non-empty", Verdict::Fail, 0)],
verdict: Verdict::Fail,
},
Attempt {
rung: 1,
model: "anthropic/claude-sonnet-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 1,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 1,
cost_usd: 0.01,
latency_ms: 9,
gates: vec![GateResult::deterministic("non-empty", Verdict::Pass, 0)],
verdict: Verdict::Pass,
},
],
deferred: Vec::new(),
final_: FinalOutcome {
served_rung: Some(1),
served_from: ServedFrom::Attempt,
total_cost_usd: 0.011,
gate_cost_usd: 0.0,
total_latency_ms: 14,
escalations: 1,
counterfactual_baseline_usd: 0.02,
savings_usd: 0.009,
cache_source: None,
},
probe: None,
rollout: None,
shadow: None,
route_ix: None,
predicted_pass: None,
elastic: None,
};
let ex = explain_trace(&t);
assert_eq!(
ex.served_model.as_deref(),
Some("anthropic/claude-sonnet-5")
);
assert_eq!(ex.escalations, 1);
assert_eq!(ex.attempts.len(), 2);
assert_eq!(ex.attempts[0].verdict, "fail");
assert_eq!(
ex.attempts[1].gates[0],
("non-empty".to_owned(), "pass".to_owned())
);
assert!(ex.summary.contains("served anthropic/claude-sonnet-5"));
}
#[test]
fn evaluate_predictor_empty_and_signal() {
let e = evaluate_predictor(&[], 0.05, 1e-4);
assert_eq!(e.n, 0);
assert!(e.auc.is_none());
}
#[test]
fn detect_channel_reads_real_install_paths() {
use std::path::Path;
let cases = [
("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
(
"/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
Channel::Homebrew,
),
(
"/home/linuxbrew/.linuxbrew/bin/firstpass",
Channel::Homebrew,
),
(
"/Users/u/.venv/lib/python3.12/site-packages/firstpass/firstpass",
Channel::Python,
),
(
"/Users/u/.cache/uv/archive-v0/DQ4usH_XuqNyywgHKB8qB/bin/firstpass",
Channel::Python,
),
(
"/Users/u/.local/share/uv/tools/firstpass/bin/firstpass",
Channel::Python,
),
(
"/usr/local/lib/node_modules/@dshakesnotbot/firstpass-proxy/bin/firstpass",
Channel::Npm,
),
(r"C:\Users\u\.cargo\bin\firstpass.exe", Channel::Cargo),
];
for (path, want) in cases {
assert_eq!(
detect_channel(Path::new(path), false, false),
want,
"misread {path}"
);
}
assert_eq!(
detect_channel(Path::new("/Users/u/.local/bin/firstpass"), true, false),
Channel::Dist
);
assert_eq!(
detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
Channel::Unknown
);
assert_eq!(
detect_channel(Path::new("/usr/local/bin/firstpass"), true, true),
Channel::Docker
);
}
#[test]
fn package_manager_paths_win_over_the_self_updater() {
use std::path::Path;
for (path, want) in [
(
"/opt/homebrew/Cellar/firstpass-proxy/0.2.2/bin/firstpass",
Channel::Homebrew,
),
("/Users/u/.venv/bin/firstpass", Channel::Python),
("/Users/u/.cargo/bin/firstpass", Channel::Cargo),
] {
assert_eq!(detect_channel(Path::new(path), true, false), want, "{path}");
}
}
#[test]
fn upgrade_report_runs_only_our_own_channel() {
assert!(upgrade_report(Channel::Dist, "0.2.2").contains("upgrading in place"));
for c in [
Channel::Homebrew,
Channel::Python,
Channel::Npm,
Channel::Docker,
Channel::Cargo,
] {
let r = upgrade_report(c, "0.2.2");
assert!(
!r.contains("upgrading in place"),
"{c:?} must not self-update"
);
assert!(
r.contains("owned by its package manager"),
"{c:?} must explain why"
);
assert!(
r.contains(c.upgrade_command().expect("has a command")),
"{c:?} must print its exact command"
);
}
let unknown = upgrade_report(Channel::Unknown, "0.2.2");
assert!(unknown.contains("brew upgrade") && unknown.contains("docker pull"));
assert!(
unknown.contains("0.2.2"),
"always states the running version"
);
}
#[test]
fn pipx_install_is_recognised_from_its_real_path() {
use std::path::Path;
let real = "/Users/u/.local/pipx/venvs/firstpass/bin/firstpass";
assert_eq!(detect_channel(Path::new(real), false, false), Channel::Pipx);
assert_eq!(
Channel::Pipx.upgrade_command(),
Some("pipx upgrade firstpass")
);
assert_eq!(
detect_channel(Path::new("/Users/u/.local/bin/firstpass"), false, false),
Channel::Unknown
);
assert_eq!(
detect_channel(
Path::new(
"/Users/u/.local/pipx/venvs/firstpass/lib/python3.14/site-packages/x/firstpass"
),
false,
false
),
Channel::Pipx
);
}
}