use std::io::{BufRead, BufReader, Seek, SeekFrom, Write};
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use firstpass_core::{DeferredVerdict, GENESIS_HASH, Score, Trace, Verdict};
use rusqlite::{Connection, OptionalExtension as _};
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use crate::config::ReceiptsMode;
fn connect(db_path: impl AsRef<Path>) -> Result<Connection, StoreError> {
let conn = Connection::open(db_path.as_ref())?;
conn.busy_timeout(Duration::from_secs(5))?;
Ok(conn)
}
#[derive(Debug, thiserror::Error)]
pub enum StoreError {
#[error("sqlite error: {0}")]
Sqlite(#[from] rusqlite::Error),
#[error("trace error: {0}")]
Trace(#[from] firstpass_core::Error),
#[error("json error: {0}")]
Json(#[from] serde_json::Error),
#[error("spill I/O error: {0}")]
Io(#[from] std::io::Error),
}
pub type SpillHandle = Arc<std::sync::Mutex<std::fs::File>>;
pub(crate) fn spill_path(db_path: &Path) -> std::path::PathBuf {
let mut s = db_path.as_os_str().to_owned();
s.push(".spill.jsonl");
std::path::PathBuf::from(s)
}
pub fn append_to_spill(handle: &SpillHandle, trace: &Trace) -> Result<(), StoreError> {
let line = serde_json::to_string(trace)?;
let mut guard = handle.lock().unwrap_or_else(|e| e.into_inner());
writeln!(guard, "{line}")?;
guard.sync_data()?;
Ok(())
}
pub type TraceSender = mpsc::Sender<Trace>;
pub const TRACE_CHANNEL_CAP: usize = 8192;
pub fn open(db_path: impl AsRef<Path>) -> Result<(TraceSender, JoinHandle<()>), StoreError> {
let (tx, _spill, handle) = open_with_receipts(db_path, ReceiptsMode::BestEffort)?;
Ok((tx, handle))
}
pub fn open_with_receipts(
db_path: impl AsRef<Path>,
receipts_mode: ReceiptsMode,
) -> Result<(TraceSender, Option<SpillHandle>, JoinHandle<()>), StoreError> {
let db_path = db_path.as_ref();
let conn = connect(db_path)?;
migrate(&conn)?;
let spill = if receipts_mode == ReceiptsMode::Durable {
let path = spill_path(db_path);
let file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.read(true)
.open(&path)?;
Some(Arc::new(std::sync::Mutex::new(file)))
} else {
None
};
let (tx, rx) = mpsc::channel::<Trace>(TRACE_CHANNEL_CAP);
let spill_writer = spill.clone();
let handle = tokio::task::spawn_blocking(move || writer_loop(conn, rx, spill_writer));
Ok((tx, spill, handle))
}
fn migrate(conn: &Connection) -> Result<(), StoreError> {
conn.pragma_update(None, "journal_mode", "WAL")?;
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS traces (
seq INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id TEXT NOT NULL,
ts TEXT NOT NULL,
prev_hash TEXT NOT NULL,
hash TEXT NOT NULL,
tenant TEXT NOT NULL,
session TEXT NOT NULL,
body TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS traces_session_idx ON traces(session);
-- Tenant-scoped reads (ADR 0004 §D3) filter on `tenant`; index it with `seq` so a
-- per-tenant scan stays ordered and cheap. Existing rows keep their `tenant` value.
CREATE INDEX IF NOT EXISTS traces_tenant_seq_idx ON traces(tenant, seq);
-- Deferred verdicts live in their OWN table, keyed by trace_id. They are NEVER folded
-- into the sealed, hashed `traces.body`, so a late outcome can't alter a past record and
-- the tamper-evident chain stays valid. They are merged onto a trace only on read.
CREATE TABLE IF NOT EXISTS deferred_verdicts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trace_id TEXT NOT NULL,
gate_id TEXT NOT NULL,
verdict TEXT NOT NULL,
score REAL,
reported_at TEXT NOT NULL,
reporter TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS deferred_trace_idx ON deferred_verdicts(trace_id);
-- Paired-shadow experiment pairs (ADR 0014 Stage 2): one row per paired evaluation.
-- Gate verdicts from the incumbent (x_inc) and challenger (x_ch) are stored here so the
-- experiment can be rebuilt deterministically at startup or CLI replay: arrival order is
-- the row id, and replaying in that order yields the same e-process trajectory.
CREATE TABLE IF NOT EXISTS experiment_pairs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant TEXT NOT NULL,
route_ix INTEGER NOT NULL,
fingerprint TEXT NOT NULL,
trace_id TEXT NOT NULL,
x_inc REAL NOT NULL,
x_ch REAL NOT NULL,
challenger_cost_usd REAL NOT NULL,
ch_error INTEGER NOT NULL DEFAULT 0,
ch_truncated INTEGER NOT NULL DEFAULT 0,
ts TEXT NOT NULL,
-- DEFAULT 2 = InLogStatus::Unknown: a missed write surfaces as Unknown,
-- not as InLog (1) or NotInLog (0), so tests can detect missing explicit writes.
-- Production code always writes this field explicitly.
in_log INTEGER NOT NULL DEFAULT 2
);
CREATE INDEX IF NOT EXISTS exp_pairs_tenant_idx ON experiment_pairs(tenant, route_ix, fingerprint);",
)?;
let has_column = |name: &str| -> Result<bool, rusqlite::Error> {
let mut stmt = conn.prepare("PRAGMA table_info(experiment_pairs)")?;
let mut rows = stmt.query([])?;
while let Some(row) = rows.next()? {
let col: String = row.get(1)?;
if col == name {
return Ok(true);
}
}
Ok(false)
};
if !has_column("ch_error")? {
conn.execute_batch(
"ALTER TABLE experiment_pairs ADD COLUMN ch_error INTEGER NOT NULL DEFAULT 0;",
)?;
}
if !has_column("ch_truncated")? {
conn.execute_batch(
"ALTER TABLE experiment_pairs ADD COLUMN ch_truncated INTEGER NOT NULL DEFAULT 0;",
)?;
}
if !has_column("in_log")? {
conn.execute_batch(
"ALTER TABLE experiment_pairs ADD COLUMN in_log INTEGER NOT NULL DEFAULT 2;",
)?;
}
Ok(())
}
fn write_trace(conn: &Connection, trace: &mut Trace, head: &mut String) {
trace.prev_hash = head.clone();
let hash = match trace.hash() {
Ok(h) => h,
Err(err) => {
tracing::error!(%err, trace_id = %trace.trace_id, "trace writer: hash failed, dropping");
return;
}
};
if let Err(err) = insert(conn, trace, &hash) {
tracing::error!(%err, trace_id = %trace.trace_id, "trace writer: insert failed, dropping");
return;
}
*head = hash;
}
fn drain_spill(conn: &Connection, spill: &SpillHandle, head: &mut String) {
let mut guard = spill.lock().unwrap_or_else(|e| e.into_inner());
if let Err(e) = guard.seek(SeekFrom::Start(0)) {
tracing::error!(%e, "drain_spill: seek failed");
return;
}
let lines: Vec<String> = {
let reader = BufReader::new(&*guard);
reader
.lines()
.filter_map(|l| match l {
Ok(s) if !s.trim().is_empty() => Some(s),
Ok(_) => None,
Err(e) => {
tracing::error!(%e, "drain_spill: read error");
None
}
})
.collect()
};
if lines.is_empty() {
return;
}
let n = lines.len();
for line in &lines {
let mut trace: Trace = match serde_json::from_str(line) {
Ok(t) => t,
Err(e) => {
tracing::error!(%e, "drain_spill: bad JSON line, skipping");
continue;
}
};
write_trace(conn, &mut trace, head);
}
tracing::info!(drained = n, "spill file drained into trace store");
if let Err(e) = guard.seek(SeekFrom::Start(0)) {
tracing::error!(%e, "drain_spill: post-drain seek failed");
return;
}
if let Err(e) = guard.set_len(0) {
tracing::error!(%e, "drain_spill: truncate failed — spill file may replay on next boot");
}
}
fn writer_loop(conn: Connection, mut rx: mpsc::Receiver<Trace>, spill: Option<SpillHandle>) {
let mut head = match current_head(&conn) {
Ok(h) => h,
Err(err) => {
tracing::error!(%err, "trace writer: failed to load chain head, stopping");
return;
}
};
if let Some(ref s) = spill {
drain_spill(&conn, s, &mut head);
}
loop {
let Some(mut trace) = rx.blocking_recv() else {
break;
};
write_trace(&conn, &mut trace, &mut head);
loop {
match rx.try_recv() {
Ok(mut t) => write_trace(&conn, &mut t, &mut head),
Err(mpsc::error::TryRecvError::Empty) => {
if let Some(ref s) = spill {
drain_spill(&conn, s, &mut head);
}
break;
}
Err(mpsc::error::TryRecvError::Disconnected) => return,
}
}
}
if let Some(ref s) = spill {
drain_spill(&conn, s, &mut head);
}
}
fn current_head(conn: &Connection) -> Result<String, StoreError> {
let mut stmt = conn.prepare("SELECT hash FROM traces ORDER BY seq DESC LIMIT 1")?;
let mut rows = stmt.query([])?;
match rows.next()? {
Some(row) => Ok(row.get(0)?),
None => Ok(GENESIS_HASH.to_owned()),
}
}
fn insert(conn: &Connection, trace: &Trace, hash: &str) -> Result<(), StoreError> {
let body = serde_json::to_string(trace)?;
conn.execute(
"INSERT INTO traces (trace_id, ts, prev_hash, hash, tenant, session, body)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
rusqlite::params![
trace.trace_id.to_string(),
trace.ts.to_string(),
trace.prev_hash,
hash,
trace.tenant_id,
trace.session_id,
body,
],
)?;
Ok(())
}
pub fn load_all_traces(db_path: impl AsRef<Path>) -> Result<Vec<Trace>, StoreError> {
let conn = connect(db_path.as_ref())?;
let mut stmt = conn.prepare("SELECT body FROM traces ORDER BY seq ASC")?;
let rows = stmt.query_map([], |row| row.get::<_, String>(0))?;
let mut traces = Vec::new();
for row in rows {
traces.push(serde_json::from_str(&row?)?);
}
Ok(traces)
}
pub fn load_tenant_traces(
db_path: impl AsRef<Path>,
tenant: &str,
) -> Result<Vec<Trace>, StoreError> {
let conn = connect(db_path.as_ref())?;
let mut stmt = conn.prepare("SELECT body FROM traces WHERE tenant = ?1 ORDER BY seq ASC")?;
let rows = stmt.query_map([tenant], |row| row.get::<_, String>(0))?;
let mut traces = Vec::new();
for row in rows {
traces.push(serde_json::from_str(&row?)?);
}
Ok(traces)
}
pub fn load_recent_tenant_traces(
db_path: impl AsRef<Path>,
tenant: &str,
limit: usize,
) -> Result<Vec<Trace>, StoreError> {
let conn = connect(db_path.as_ref())?;
let mut stmt =
conn.prepare("SELECT body FROM traces WHERE tenant = ?1 ORDER BY seq DESC LIMIT ?2")?;
let limit = i64::try_from(limit).unwrap_or(i64::MAX);
let rows = stmt.query_map(rusqlite::params![tenant, limit], |row| {
row.get::<_, String>(0)
})?;
let mut traces = Vec::new();
for row in rows {
traces.push(serde_json::from_str(&row?)?);
}
Ok(traces)
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct ReceiptsSummary {
pub requests: u64,
pub spent_usd: f64,
pub baseline_usd: f64,
pub first_pass: u64,
pub escalated: u64,
pub gates: [u64; 3],
pub truncated: u64,
}
impl ReceiptsSummary {
fn merged(&self, d: &Self) -> Self {
Self {
requests: self.requests + d.requests,
spent_usd: self.spent_usd + d.spent_usd,
baseline_usd: self.baseline_usd + d.baseline_usd,
first_pass: self.first_pass + d.first_pass,
escalated: self.escalated + d.escalated,
gates: [
self.gates[0] + d.gates[0],
self.gates[1] + d.gates[1],
self.gates[2] + d.gates[2],
],
truncated: self.truncated + d.truncated,
}
}
}
type SummaryCache = std::collections::HashMap<(std::path::PathBuf, String), (i64, ReceiptsSummary)>;
static SUMMARY_CACHE: std::sync::LazyLock<std::sync::Mutex<SummaryCache>> =
std::sync::LazyLock::new(Default::default);
pub fn tenant_receipts_summary(
db_path: impl AsRef<Path>,
tenant: &str,
) -> Result<ReceiptsSummary, StoreError> {
let conn = connect(db_path.as_ref())?;
let key = (db_path.as_ref().to_path_buf(), tenant.to_owned());
let hi: i64 = conn.query_row(
"SELECT coalesce(max(seq), 0) FROM traces WHERE tenant = ?1",
[tenant],
|r| r.get(0),
)?;
let cached = SUMMARY_CACHE
.lock()
.ok()
.and_then(|c| c.get(&key).cloned())
.filter(|(seen, _)| *seen <= hi);
let (after, base) = cached.unwrap_or_default();
if after == hi {
return Ok(base);
}
let total = base.merged(&summary_between(&conn, tenant, after, hi)?);
if let Ok(mut c) = SUMMARY_CACHE.lock() {
if c.len() >= 4096 {
c.clear();
}
c.insert(key, (hi, total.clone()));
}
Ok(total)
}
fn summary_between(
conn: &Connection,
tenant: &str,
after: i64,
hi: i64,
) -> Result<ReceiptsSummary, StoreError> {
let mut s = conn.query_row(
"SELECT count(*),
total(json_extract(body, '$.final.total_cost_usd')),
total(json_extract(body, '$.final.counterfactual_baseline_usd')),
coalesce(sum(json_extract(body, '$.final.served_rung') = 0), 0),
coalesce(sum(json_extract(body, '$.final.escalations') > 0), 0)
FROM traces WHERE tenant = ?1 AND seq > ?2 AND seq <= ?3",
rusqlite::params![tenant, after, hi],
|r| {
Ok(ReceiptsSummary {
requests: u64::try_from(r.get::<_, i64>(0)?).unwrap_or(0),
spent_usd: r.get(1)?,
baseline_usd: r.get(2)?,
first_pass: u64::try_from(r.get::<_, i64>(3)?).unwrap_or(0),
escalated: u64::try_from(r.get::<_, i64>(4)?).unwrap_or(0),
gates: [0; 3],
truncated: 0,
})
},
)?;
let truncated: i64 = conn.query_row(
"SELECT count(*) FROM traces t, json_each(t.body, '$.attempts') a
WHERE t.tenant = ?1 AND t.seq > ?2 AND t.seq <= ?3
AND json_extract(a.value, '$.truncated') = 1",
rusqlite::params![tenant, after, hi],
|r| r.get(0),
)?;
s.truncated = u64::try_from(truncated).unwrap_or(0);
let mut stmt = conn.prepare(
"SELECT json_extract(g.value, '$.verdict'), count(*)
FROM traces t, json_each(t.body, '$.attempts') a, json_each(a.value, '$.gates') g
WHERE t.tenant = ?1 AND t.seq > ?2 AND t.seq <= ?3 GROUP BY 1",
)?;
let rows = stmt.query_map(rusqlite::params![tenant, after, hi], |r| {
Ok((r.get::<_, Option<String>>(0)?, r.get::<_, i64>(1)?))
})?;
for row in rows {
let (verdict, n) = row?;
let slot = match verdict.as_deref() {
Some("pass") => 0,
Some("fail") => 1,
Some("abstain") => 2,
_ => continue,
};
s.gates[slot] += u64::try_from(n).unwrap_or(0);
}
Ok(s)
}
pub fn trace_exists(
db_path: impl AsRef<Path>,
tenant: &str,
trace_id: &str,
) -> Result<bool, StoreError> {
let conn = connect(db_path.as_ref())?;
let n: i64 = conn.query_row(
"SELECT COUNT(1) FROM traces WHERE tenant = ?1 AND trace_id = ?2",
[tenant, trace_id],
|row| row.get(0),
)?;
Ok(n > 0)
}
pub fn trace_cache_source(
db_path: impl AsRef<Path>,
tenant: &str,
trace_id: &str,
) -> Result<Option<Option<uuid::Uuid>>, StoreError> {
let conn = connect(db_path.as_ref())?;
let body: Option<String> = conn
.query_row(
"SELECT body FROM traces WHERE tenant = ?1 AND trace_id = ?2",
[tenant, trace_id],
|row| row.get(0),
)
.optional()?;
Ok(body.map(|b: String| {
serde_json::from_str::<firstpass_core::Trace>(&b)
.ok()
.and_then(|t| t.final_.cache_source)
}))
}
pub fn trace_route(
db_path: impl AsRef<Path>,
tenant: &str,
trace_id: &str,
) -> Result<Option<Option<u32>>, StoreError> {
let conn = connect(db_path.as_ref())?;
let body: Option<String> = conn
.query_row(
"SELECT body FROM traces WHERE tenant = ?1 AND trace_id = ?2",
[tenant, trace_id],
|row| row.get(0),
)
.optional()?;
Ok(body.map(|b: String| {
serde_json::from_str::<firstpass_core::Trace>(&b)
.ok()
.and_then(|t| t.route_ix)
}))
}
pub fn append_deferred(
db_path: impl AsRef<Path>,
trace_id: &str,
v: &DeferredVerdict,
) -> Result<(), StoreError> {
let conn = connect(db_path.as_ref())?;
conn.execute(
"INSERT INTO deferred_verdicts (trace_id, gate_id, verdict, score, reported_at, reporter)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
rusqlite::params![
trace_id,
v.gate_id,
v.verdict.as_str(),
v.score.map(Score::value),
v.reported_at.to_string(),
v.reporter,
],
)?;
Ok(())
}
pub fn load_deferred(
db_path: impl AsRef<Path>,
trace_id: &str,
) -> Result<Vec<DeferredVerdict>, StoreError> {
let conn = connect(db_path.as_ref())?;
let mut stmt = conn.prepare(
"SELECT gate_id, verdict, score, reported_at, reporter
FROM deferred_verdicts WHERE trace_id = ?1 ORDER BY id ASC",
)?;
let rows = stmt.query_map([trace_id], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, Option<f64>>(2)?,
row.get::<_, String>(3)?,
row.get::<_, String>(4)?,
))
})?;
let mut out = Vec::new();
for row in rows {
let (gate_id, verdict_s, score, reported_s, reporter) = row?;
let verdict = match verdict_s.as_str() {
"pass" => Verdict::Pass,
"fail" => Verdict::Fail,
"abstain" => Verdict::Abstain,
other => {
tracing::warn!(verdict = %other, %trace_id, "skipping deferred row with bad verdict");
continue;
}
};
let Ok(reported_at) = reported_s.parse::<jiff::Timestamp>() else {
tracing::warn!(%trace_id, "skipping deferred row with bad timestamp");
continue;
};
out.push(DeferredVerdict {
gate_id,
verdict,
score: score.and_then(|s| Score::new(s).ok()),
reported_at,
reporter,
});
}
Ok(out)
}
pub fn load_all_deferred(
db_path: impl AsRef<Path>,
) -> Result<Vec<(uuid::Uuid, Verdict)>, StoreError> {
let conn = connect(db_path.as_ref())?;
let mut stmt =
conn.prepare("SELECT trace_id, verdict FROM deferred_verdicts ORDER BY id ASC")?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?;
let mut out = Vec::new();
for row in rows {
let (id, verdict) = row?;
let verdict = match verdict.as_str() {
"pass" => Verdict::Pass,
"fail" => Verdict::Fail,
"abstain" => Verdict::Abstain,
_ => continue,
};
if let Ok(id) = id.parse::<uuid::Uuid>() {
out.push((id, verdict));
}
}
Ok(out)
}
pub fn trace_experiment(
db_path: impl AsRef<Path>,
tenant: &str,
trace_id: &str,
) -> Result<Option<(String, usize, firstpass_core::trace::ExperimentRecord, f64)>, StoreError> {
let conn = connect(db_path.as_ref())?;
let body: Option<String> = conn
.query_row(
"SELECT body FROM traces WHERE tenant = ?1 AND trace_id = ?2",
[tenant, trace_id],
|row| row.get(0),
)
.optional()?;
let Some(body) = body else { return Ok(None) };
let trace: Trace = serde_json::from_str(&body)?;
let cost_usd = trace.final_.total_cost_usd;
Ok(trace.experiment.map(|rec| {
(
trace.session_id,
trace.route_ix.map_or(0, |r| r as usize),
rec,
cost_usd,
)
}))
}
pub fn load_trace_view(
db_path: impl AsRef<Path>,
tenant: &str,
trace_id: &str,
) -> Result<Option<Trace>, StoreError> {
let conn = connect(db_path.as_ref())?;
let body: Option<String> = conn
.query_row(
"SELECT body FROM traces WHERE tenant = ?1 AND trace_id = ?2",
[tenant, trace_id],
|row| row.get(0),
)
.ok();
let Some(body) = body else { return Ok(None) };
let mut trace: Trace = serde_json::from_str(&body)?;
trace.deferred = load_deferred(db_path, trace_id)?;
Ok(Some(trace))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum InLogStatus {
NotInLog = 0,
InLog = 1,
Unknown = 2,
}
impl InLogStatus {
fn from_i64(v: i64) -> Self {
match v {
0 => Self::NotInLog,
1 => Self::InLog,
_ => Self::Unknown,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ExperimentPairRow {
pub tenant: String,
pub route_ix: usize,
pub fingerprint: String,
pub trace_id: uuid::Uuid,
pub x_inc: f64,
pub x_ch: f64,
pub challenger_cost_usd: f64,
pub ch_error: bool,
pub ch_truncated: u64,
pub ts: jiff::Timestamp,
pub in_log: InLogStatus,
}
pub fn append_experiment_pair(
db_path: impl AsRef<std::path::Path>,
row: &ExperimentPairRow,
) -> Result<(), StoreError> {
let conn = connect(db_path.as_ref())?;
conn.execute(
"INSERT INTO experiment_pairs \
(tenant, route_ix, fingerprint, trace_id, x_inc, x_ch, challenger_cost_usd, \
ch_error, ch_truncated, ts, in_log) \
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
rusqlite::params![
row.tenant,
row.route_ix as i64,
row.fingerprint,
row.trace_id.to_string(),
row.x_inc,
row.x_ch,
row.challenger_cost_usd,
i64::from(row.ch_error),
row.ch_truncated as i64,
row.ts.to_string(),
row.in_log as i64,
],
)?;
Ok(())
}
pub fn load_all_experiment_pairs(
db_path: impl AsRef<std::path::Path>,
) -> Result<Vec<ExperimentPairRow>, StoreError> {
let conn = connect(db_path.as_ref())?;
let mut stmt = conn.prepare(
"SELECT tenant, route_ix, fingerprint, trace_id, x_inc, x_ch, challenger_cost_usd, \
ch_error, ch_truncated, ts, in_log \
FROM experiment_pairs ORDER BY id ASC",
)?;
let rows = stmt.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, i64>(1)?,
row.get::<_, String>(2)?,
row.get::<_, String>(3)?,
row.get::<_, f64>(4)?,
row.get::<_, f64>(5)?,
row.get::<_, f64>(6)?,
row.get::<_, i64>(7)?,
row.get::<_, i64>(8)?,
row.get::<_, String>(9)?,
row.get::<_, i64>(10)?,
))
})?;
let mut out = Vec::new();
for row in rows {
let (
tenant,
route_ix,
fingerprint,
trace_id_s,
x_inc,
x_ch,
ch_cost,
ch_error_i,
ch_truncated_i,
ts_s,
in_log_i,
) = row?;
let Ok(trace_id) = trace_id_s.parse::<uuid::Uuid>() else {
tracing::warn!(
%trace_id_s,
"experiment_pairs: skipping row with unparseable trace_id"
);
continue;
};
let Ok(ts) = ts_s.parse::<jiff::Timestamp>() else {
tracing::warn!(
%trace_id_s,
"experiment_pairs: skipping row with unparseable ts"
);
continue;
};
out.push(ExperimentPairRow {
tenant,
route_ix: route_ix as usize,
fingerprint,
trace_id,
x_inc,
x_ch,
challenger_cost_usd: ch_cost,
ch_error: ch_error_i != 0,
ch_truncated: u64::try_from(ch_truncated_i).unwrap_or(0),
ts,
in_log: InLogStatus::from_i64(in_log_i),
});
}
Ok(out)
}
pub fn ensure_migrated(db_path: impl AsRef<Path>) -> Result<(), StoreError> {
let conn = connect(db_path)?;
migrate(&conn)
}
#[cfg(test)]
mod tests {
use firstpass_core::{
Attempt, Features, FinalOutcome, GENESIS_HASH, PolicyRef, RequestInfo, ServedFrom,
TaskKind, Verdict, verify_chain,
};
use super::*;
fn sample_trace(tenant: &str, session: &str) -> Trace {
let attempt = Attempt {
rung: 0,
model: "claude-haiku-4-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
cost_usd: 0.001,
latency_ms: 12,
gates: vec![],
verdict: Verdict::Pass,
truncated: false,
};
let mut trace = Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: tenant.to_owned(),
session_id: session.to_owned(),
ts: jiff::Timestamp::now(),
mode: firstpass_core::Mode::Observe,
policy: PolicyRef {
id: "observe-passthrough@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "deadbeef".to_owned(),
features: Features::new(TaskKind::Other),
},
attempts: vec![attempt],
deferred: Vec::new(),
final_: FinalOutcome {
served_rung: Some(0),
served_from: ServedFrom::Attempt,
total_cost_usd: 0.001,
gate_cost_usd: 0.0,
total_latency_ms: 12,
escalations: 0,
counterfactual_baseline_usd: 0.001,
savings_usd: 0.0,
cache_source: None,
},
probe: None,
rollout: None,
shadow: None,
route_ix: None,
predicted_pass: None,
decision_prior: None,
elastic: None,
risk: None,
experiment: None,
upstream_provider: String::new(),
};
trace.recompute_savings();
trace
}
#[tokio::test]
async fn summary_aggregate_matches_a_rust_fold_and_page_is_newest_first() {
use firstpass_core::GateResult;
let db_path =
std::env::temp_dir().join(format!("firstpass-store-sum-{}.db", uuid::Uuid::now_v7()));
let gate = |v: Verdict| GateResult {
gate_id: "tests".to_owned(),
verdict: v,
score: None,
cost_usd: 0.0,
ms: 1,
reason: None,
evidence_ref: None,
};
let mut mine = Vec::new();
for i in 0..23u32 {
let mut t = sample_trace("tenant-a", &format!("s{i}"));
t.final_.total_cost_usd = 0.01 * f64::from(i + 1);
t.final_.counterfactual_baseline_usd = 0.05;
t.final_.escalations = i % 3;
t.final_.served_rung = match i % 5 {
0 => None,
k => Some(k % 3), };
t.attempts[0].gates = (0..i % 4)
.map(|k| {
gate([Verdict::Pass, Verdict::Fail, Verdict::Abstain][(k + i) as usize % 3])
})
.collect();
t.attempts[0].truncated = i % 3 == 0;
if i % 2 == 0 {
let mut second = t.attempts[0].clone();
second.rung = 1;
second.gates = vec![gate(Verdict::Pass)];
second.truncated = i % 4 == 0;
t.attempts.push(second);
}
mine.push(t);
}
let (tx, handle) = open(&db_path).unwrap();
for (i, t) in mine.iter().enumerate() {
tx.send(t.clone()).await.unwrap();
if i % 4 == 0 {
let mut other = sample_trace("tenant-b", "x");
other.final_.total_cost_usd = 100.0;
other.attempts[0].gates = vec![gate(Verdict::Fail)];
other.attempts[0].truncated = true;
tx.send(other).await.unwrap();
}
}
drop(tx);
handle.await.unwrap();
let mut want = ReceiptsSummary {
requests: mine.len() as u64,
..Default::default()
};
for t in &mine {
want.spent_usd += t.final_.total_cost_usd;
want.baseline_usd += t.final_.counterfactual_baseline_usd;
want.first_pass += u64::from(t.final_.served_rung == Some(0));
want.escalated += u64::from(t.final_.escalations > 0);
want.truncated += t.attempts.iter().filter(|a| a.truncated).count() as u64;
for g in t.attempts.iter().flat_map(|a| &a.gates) {
want.gates[match g.verdict {
Verdict::Pass => 0,
Verdict::Fail => 1,
Verdict::Abstain => 2,
}] += 1;
}
}
let got = tenant_receipts_summary(&db_path, "tenant-a").unwrap();
assert_eq!(got.requests, want.requests);
assert_eq!(got.first_pass, want.first_pass);
assert_eq!(got.escalated, want.escalated);
assert_eq!(got.gates, want.gates);
assert_eq!(
got.truncated, want.truncated,
"only this tenant's cut-off attempts"
);
assert!(want.truncated > 0);
assert!(want.first_pass > 0 && want.escalated > 0 && want.gates.iter().all(|&n| n > 0));
assert!(
(got.spent_usd - want.spent_usd).abs() < 1e-9,
"{got:?} {want:?}"
);
assert!((got.baseline_usd - want.baseline_usd).abs() < 1e-9);
assert_eq!(
tenant_receipts_summary(&db_path, "nobody").unwrap(),
ReceiptsSummary::default()
);
let page = load_recent_tenant_traces(&db_path, "tenant-a", 5).unwrap();
let ids: Vec<_> = page.iter().map(|t| t.trace_id).collect();
let newest: Vec<_> = mine.iter().rev().take(5).map(|t| t.trace_id).collect();
assert_eq!(ids, newest);
let (tx, handle) = open(&db_path).unwrap();
for _ in 0..4 {
let mut t = sample_trace("tenant-a", "late");
t.final_.served_rung = Some(0);
t.attempts[0].gates = vec![gate(Verdict::Abstain)];
tx.send(t).await.unwrap();
}
drop(tx);
handle.await.unwrap();
let again = tenant_receipts_summary(&db_path, "tenant-a").unwrap();
assert_eq!(again.requests, want.requests + 4);
assert_eq!(again.first_pass, want.first_pass + 4);
assert_eq!(
again.gates,
[want.gates[0], want.gates[1], want.gates[2] + 4]
);
assert_eq!(
tenant_receipts_summary(&db_path, "tenant-a").unwrap(),
again
);
let _ = std::fs::remove_file(&db_path);
let (tx, handle) = open(&db_path).unwrap();
tx.send(sample_trace("tenant-a", "fresh")).await.unwrap();
drop(tx);
handle.await.unwrap();
assert_eq!(
tenant_receipts_summary(&db_path, "tenant-a")
.unwrap()
.requests,
1
);
let _ = std::fs::remove_file(&db_path);
}
#[tokio::test]
async fn writer_assigns_prev_hash_and_forms_a_valid_chain() {
let db_path =
std::env::temp_dir().join(format!("firstpass-store-test-{}.db", uuid::Uuid::now_v7()));
let (tx, handle) = open(&db_path).unwrap();
tx.try_send(sample_trace("tenant-a", "session-1")).unwrap();
tx.try_send(sample_trace("tenant-a", "session-1")).unwrap();
drop(tx);
handle.await.unwrap();
let traces = load_all_traces(&db_path).unwrap();
assert_eq!(traces.len(), 2);
assert_eq!(traces[0].prev_hash, GENESIS_HASH);
assert_eq!(traces[1].prev_hash, traces[0].hash().unwrap());
verify_chain(&traces, GENESIS_HASH).unwrap();
let _ = std::fs::remove_file(&db_path);
}
#[tokio::test]
async fn tenant_scoped_reads_never_cross_the_boundary() {
let db_path =
std::env::temp_dir().join(format!("firstpass-isolation-{}.db", uuid::Uuid::now_v7()));
let (tx, handle) = open(&db_path).unwrap();
let a0 = sample_trace("tenant-a", "sa-0");
let a1 = sample_trace("tenant-a", "sa-1");
let b0 = sample_trace("tenant-b", "sb-0");
let (a0_id, a1_id, b0_id) = (
a0.trace_id.to_string(),
a1.trace_id.to_string(),
b0.trace_id.to_string(),
);
tx.try_send(a0).unwrap();
tx.try_send(b0).unwrap();
tx.try_send(a1).unwrap();
drop(tx);
handle.await.unwrap();
let a_traces = load_tenant_traces(&db_path, "tenant-a").unwrap();
assert_eq!(a_traces.len(), 2, "A must see only A's traces");
assert!(a_traces.iter().all(|t| t.tenant_id == "tenant-a"));
let b_traces = load_tenant_traces(&db_path, "tenant-b").unwrap();
assert_eq!(b_traces.len(), 1, "B must see only B's trace");
assert!(b_traces.iter().all(|t| t.tenant_id == "tenant-b"));
assert!(trace_exists(&db_path, "tenant-a", &a0_id).unwrap());
assert!(!trace_exists(&db_path, "tenant-a", &b0_id).unwrap());
assert!(trace_exists(&db_path, "tenant-b", &b0_id).unwrap());
assert!(!trace_exists(&db_path, "tenant-b", &a1_id).unwrap());
assert!(
load_trace_view(&db_path, "tenant-a", &a0_id)
.unwrap()
.is_some()
);
assert!(
load_trace_view(&db_path, "tenant-a", &b0_id)
.unwrap()
.is_none()
);
assert!(
load_trace_view(&db_path, "tenant-b", &a0_id)
.unwrap()
.is_none()
);
assert!(load_tenant_traces(&db_path, "ghost").unwrap().is_empty());
let all = load_all_traces(&db_path).unwrap();
assert_eq!(all.len(), 3);
verify_chain(&all, GENESIS_HASH).unwrap();
let _ = std::fs::remove_file(&db_path);
}
#[tokio::test]
async fn deferred_verdicts_attach_on_read_without_breaking_the_chain() {
let db_path = std::env::temp_dir().join(format!(
"firstpass-deferred-test-{}.db",
uuid::Uuid::now_v7()
));
let (tx, handle) = open(&db_path).unwrap();
let t0 = sample_trace("acme", "run-1");
let t1 = sample_trace("acme", "run-1");
let (id0, id1) = (t0.trace_id.to_string(), t1.trace_id.to_string());
tx.try_send(t0).unwrap();
tx.try_send(t1).unwrap();
drop(tx);
handle.await.unwrap();
let dv = DeferredVerdict {
gate_id: "tests".to_owned(),
verdict: Verdict::Pass,
score: Some(Score::new(1.0).unwrap()),
reported_at: jiff::Timestamp::now(),
reporter: "ci".to_owned(),
};
append_deferred(&db_path, &id0, &dv).unwrap();
assert!(trace_exists(&db_path, "acme", &id0).unwrap());
assert!(!trace_exists(&db_path, "acme", "no-such-trace").unwrap());
assert!(!trace_exists(&db_path, "other-tenant", &id0).unwrap());
let view = load_trace_view(&db_path, "acme", &id0).unwrap().unwrap();
assert_eq!(view.deferred.len(), 1);
assert_eq!(view.deferred[0].gate_id, "tests");
assert_eq!(view.deferred[0].verdict, Verdict::Pass);
assert!(
load_trace_view(&db_path, "acme", &id1)
.unwrap()
.unwrap()
.deferred
.is_empty()
);
assert!(
load_trace_view(&db_path, "other-tenant", &id0)
.unwrap()
.is_none()
);
let traces = load_all_traces(&db_path).unwrap();
assert!(
traces.iter().all(|t| t.deferred.is_empty()),
"sealed records stay deferred-free"
);
verify_chain(&traces, GENESIS_HASH).unwrap();
let _ = std::fs::remove_file(&db_path);
}
#[test]
fn durable_append_to_spill_writes_valid_json_line() {
let db_path =
std::env::temp_dir().join(format!("firstpass-spill-write-{}.db", uuid::Uuid::now_v7()));
let spill_p = spill_path(&db_path);
let file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.read(true)
.open(&spill_p)
.unwrap();
let handle = Arc::new(std::sync::Mutex::new(file));
let t = sample_trace("t", "s");
append_to_spill(&handle, &t).unwrap();
let content = std::fs::read_to_string(&spill_p).unwrap();
let lines: Vec<&str> = content.lines().collect();
assert_eq!(lines.len(), 1, "one line per spilled trace");
let parsed: Trace = serde_json::from_str(lines[0]).unwrap();
assert_eq!(parsed.trace_id, t.trace_id);
let _ = std::fs::remove_file(&spill_p);
}
#[tokio::test]
async fn drain_on_boot_restores_spilled_traces_and_chain_is_valid() {
let db_path =
std::env::temp_dir().join(format!("firstpass-drain-boot-{}.db", uuid::Uuid::now_v7()));
let spill_p = spill_path(&db_path);
let t1 = sample_trace("t", "s1");
let t2 = sample_trace("t", "s2");
{
let file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.read(true)
.open(&spill_p)
.unwrap();
let h = Arc::new(std::sync::Mutex::new(file));
append_to_spill(&h, &t1).unwrap();
append_to_spill(&h, &t2).unwrap();
}
let (tx, _spill, handle) = open_with_receipts(&db_path, ReceiptsMode::Durable).unwrap();
drop(tx); handle.await.unwrap();
let traces = load_all_traces(&db_path).unwrap();
assert_eq!(traces.len(), 2, "both spilled traces recovered");
verify_chain(&traces, GENESIS_HASH).unwrap();
let remaining = std::fs::read_to_string(&spill_p).unwrap();
assert!(
remaining.trim().is_empty(),
"spill file must be empty after drain"
);
let _ = std::fs::remove_file(&db_path);
let _ = std::fs::remove_file(&spill_p);
}
#[tokio::test]
async fn spilled_traces_land_before_later_channel_traces() {
let db_path =
std::env::temp_dir().join(format!("firstpass-spill-order-{}.db", uuid::Uuid::now_v7()));
let spill_p = spill_path(&db_path);
let spilled = sample_trace("t", "spilled");
{
let file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.read(true)
.open(&spill_p)
.unwrap();
let h = Arc::new(std::sync::Mutex::new(file));
append_to_spill(&h, &spilled).unwrap();
}
let (tx, _spill, handle) = open_with_receipts(&db_path, ReceiptsMode::Durable).unwrap();
let channel_trace = sample_trace("t", "channel");
tx.try_send(channel_trace.clone()).unwrap();
drop(tx);
handle.await.unwrap();
let traces = load_all_traces(&db_path).unwrap();
assert_eq!(traces.len(), 2, "spilled + channel trace");
assert_eq!(
traces[0].trace_id, spilled.trace_id,
"spilled trace must precede channel trace"
);
assert_eq!(
traces[1].trace_id, channel_trace.trace_id,
"channel trace must follow spilled trace"
);
verify_chain(&traces, GENESIS_HASH).unwrap();
let _ = std::fs::remove_file(&db_path);
let _ = std::fs::remove_file(&spill_p);
}
#[tokio::test]
async fn experiment_pairs_migration_adds_columns_to_old_schema() {
let db_path = std::env::temp_dir().join(format!(
"firstpass-pairs-migrate-{}.db",
uuid::Uuid::now_v7()
));
{
let conn = rusqlite::Connection::open(&db_path).unwrap();
conn.execute_batch(
"CREATE TABLE experiment_pairs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant TEXT NOT NULL,
route_ix INTEGER NOT NULL,
fingerprint TEXT NOT NULL,
trace_id TEXT NOT NULL,
x_inc REAL NOT NULL,
x_ch REAL NOT NULL,
challenger_cost_usd REAL NOT NULL,
ts TEXT NOT NULL
);",
)
.unwrap();
conn.execute(
"INSERT INTO experiment_pairs \
(tenant, route_ix, fingerprint, trace_id, x_inc, x_ch, challenger_cost_usd, ts) \
VALUES ('t', 0, 'fp', '00000000-0000-0000-0000-000000000001', 0.5, 0.5, 0.01, \
'2026-01-01T00:00:00Z')",
[],
)
.unwrap();
}
let (tx, handle) = open(&db_path).unwrap();
drop(tx);
handle.await.unwrap();
let rows = load_all_experiment_pairs(&db_path).unwrap();
assert_eq!(rows.len(), 1, "old row must be readable after migration");
assert!(!rows[0].ch_error, "old row defaults ch_error = false");
assert_eq!(rows[0].ch_truncated, 0, "old row defaults ch_truncated = 0");
assert_eq!(
rows[0].in_log,
InLogStatus::Unknown,
"legacy row defaults to Unknown"
);
let _ = std::fs::remove_file(&db_path);
}
#[test]
fn ensure_migrated_makes_pre_in_log_db_loadable() {
let db_path = std::env::temp_dir().join(format!(
"firstpass-ensure-migrated-{}.db",
uuid::Uuid::now_v7()
));
{
let conn = rusqlite::Connection::open(&db_path).unwrap();
conn.execute_batch(
"CREATE TABLE experiment_pairs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant TEXT NOT NULL,
route_ix INTEGER NOT NULL,
fingerprint TEXT NOT NULL,
trace_id TEXT NOT NULL,
x_inc REAL NOT NULL,
x_ch REAL NOT NULL,
challenger_cost_usd REAL NOT NULL,
ch_error INTEGER NOT NULL DEFAULT 0,
ch_truncated INTEGER NOT NULL DEFAULT 0,
ts TEXT NOT NULL
);",
)
.unwrap();
conn.execute(
"INSERT INTO experiment_pairs \
(tenant, route_ix, fingerprint, trace_id, x_inc, x_ch, challenger_cost_usd, ts) \
VALUES ('t', 0, 'fp', '00000000-0000-0000-0000-000000000002', 0.8, 0.4, 0.05, \
'2026-06-01T00:00:00Z')",
[],
)
.unwrap();
}
ensure_migrated(&db_path).expect("ensure_migrated must succeed");
let rows = load_all_experiment_pairs(&db_path).expect("load must succeed after migration");
assert_eq!(rows.len(), 1, "pre-migration row must be readable");
assert_eq!(
rows[0].in_log,
InLogStatus::Unknown,
"pre-in_log row defaults to Unknown after migration"
);
let _ = std::fs::remove_file(&db_path);
}
}