Skip to main content

firstpass_proxy/
store.rs

1//! Tamper-evident trace storage: a background SQLite writer fed by a bounded channel.
2//!
3//! The hot request path never blocks on disk: it `try_send`s a [`Trace`] down a bounded channel
4//! and returns immediately (dropping the trace if the writer has fallen far enough behind to fill
5//! the buffer — bounded memory over a guaranteed write). A single background task owns the SQLite
6//! connection, assigns each trace's `prev_hash` from the current chain head, computes its `hash`,
7//! and appends it (SPEC §9: the hash chain is only meaningful if every writer agrees on chain
8//! order, which a single writer task guarantees for free).
9//!
10//! In [`crate::config::ReceiptsMode::Durable`] mode the channel-full path spills traces to
11//! `<db_path>.spill.jsonl` (one JSON line per trace, synced to disk) instead of dropping them.
12//! The writer drains the spill file at startup and whenever the channel empties, inserting spilled
13//! traces BEFORE new channel arrivals so the hash chain stays append-only and valid.
14
15use std::io::{BufRead, BufReader, Seek, SeekFrom, Write};
16use std::path::Path;
17use std::sync::Arc;
18use std::time::Duration;
19
20use firstpass_core::{DeferredVerdict, GENESIS_HASH, Score, Trace, Verdict};
21use rusqlite::{Connection, OptionalExtension as _};
22use tokio::sync::mpsc;
23use tokio::task::JoinHandle;
24
25use crate::config::ReceiptsMode;
26
27/// Open a connection with WAL + a busy timeout, so the background writer and short-lived
28/// feedback/read connections can share the file without "database is locked" errors.
29fn connect(db_path: impl AsRef<Path>) -> Result<Connection, StoreError> {
30    let conn = Connection::open(db_path.as_ref())?;
31    conn.busy_timeout(Duration::from_secs(5))?;
32    Ok(conn)
33}
34
35/// Errors from the trace store.
36#[derive(Debug, thiserror::Error)]
37pub enum StoreError {
38    /// The SQLite database could not be opened, migrated, or queried.
39    #[error("sqlite error: {0}")]
40    Sqlite(#[from] rusqlite::Error),
41    /// A trace could not be hashed or (de)serialized.
42    #[error("trace error: {0}")]
43    Trace(#[from] firstpass_core::Error),
44    /// A stored row was not valid trace JSON.
45    #[error("json error: {0}")]
46    Json(#[from] serde_json::Error),
47    /// An I/O error from the spill file.
48    #[error("spill I/O error: {0}")]
49    Io(#[from] std::io::Error),
50}
51
52/// A shared, mutex-guarded handle to the spill file used in durable mode.
53///
54/// Multiple async tasks may hit a full channel simultaneously; the `Mutex` serialises their
55/// appends without data loss.
56///
57/// ponytail: global Mutex over the file — upgrade to a dedicated spill-writer task if append
58/// throughput under sustained backpressure becomes the bottleneck.
59pub type SpillHandle = Arc<std::sync::Mutex<std::fs::File>>;
60
61/// Derive the spill-file path from the main DB path: `<db_path>.spill.jsonl`.
62pub(crate) fn spill_path(db_path: &Path) -> std::path::PathBuf {
63    let mut s = db_path.as_os_str().to_owned();
64    s.push(".spill.jsonl");
65    std::path::PathBuf::from(s)
66}
67
68/// Serialize `trace` as one JSON line and append it to the spill file, flushing to disk.
69///
70/// Called from the hot async path only when the writer channel is full (durable mode). This
71/// blocks the calling tokio task on the disk write — the deliberate tradeoff of durable mode;
72/// it only fires under sustained backpressure.
73///
74/// # Errors
75/// Returns [`StoreError::Json`] on serialization failure, [`StoreError::Io`] on disk failure.
76pub fn append_to_spill(handle: &SpillHandle, trace: &Trace) -> Result<(), StoreError> {
77    let line = serde_json::to_string(trace)?;
78    // Recover from a poisoned mutex rather than failing: a previous writer's panic doesn't
79    // corrupt the file, so we can safely continue appending.
80    let mut guard = handle.lock().unwrap_or_else(|e| e.into_inner());
81    writeln!(guard, "{line}")?;
82    // sync_data flushes the write to stable storage — the whole point of durable mode.
83    guard.sync_data()?;
84    Ok(())
85}
86
87/// Sending half of the trace channel; cheap to clone, safe to share across request handlers.
88/// Sending is fire-and-forget via `try_send` — the hot path never awaits the writer, and a bounded
89/// buffer means a stalled writer sheds load (drops traces) instead of growing memory without limit.
90pub type TraceSender = mpsc::Sender<Trace>;
91
92/// Trace buffer depth. Deep enough to absorb normal write bursts; bounded so a wedged writer (disk
93/// stall) can't OOM the process — excess traces are dropped with a warning, not queued forever.
94pub const TRACE_CHANNEL_CAP: usize = 8192;
95
96/// Open (creating if needed) the SQLite trace database in best-effort mode, migrate its schema,
97/// and spawn the background writer task.
98///
99/// Returns a [`TraceSender`] for the hot path and the writer's [`JoinHandle`]. The writer
100/// exits cleanly once every clone of the sender is dropped.
101///
102/// This is the backward-compatible convenience wrapper. Use [`open_with_receipts`] when you need
103/// durable (never-drop) mode.
104///
105/// # Errors
106/// Returns [`StoreError::Sqlite`] if the database cannot be opened or migrated.
107pub fn open(db_path: impl AsRef<Path>) -> Result<(TraceSender, JoinHandle<()>), StoreError> {
108    let (tx, _spill, handle) = open_with_receipts(db_path, ReceiptsMode::BestEffort)?;
109    Ok((tx, handle))
110}
111
112/// Open the SQLite trace database with the given receipts mode, migrate its schema, and spawn
113/// the background writer task.
114///
115/// Returns a [`TraceSender`], an optional [`SpillHandle`] (present only in durable mode, for use
116/// by [`append_to_spill`] on channel-full), and the writer's [`JoinHandle`].
117///
118/// In [`ReceiptsMode::Durable`] mode the writer drains any existing spill file at startup and
119/// whenever the channel empties, inserting spilled traces BEFORE new channel arrivals so the hash
120/// chain stays append-only and valid across crashes and restarts.
121///
122/// # Errors
123/// Returns [`StoreError::Sqlite`] if the database cannot be opened or migrated, or
124/// [`StoreError::Io`] if the spill file cannot be created in durable mode.
125pub fn open_with_receipts(
126    db_path: impl AsRef<Path>,
127    receipts_mode: ReceiptsMode,
128) -> Result<(TraceSender, Option<SpillHandle>, JoinHandle<()>), StoreError> {
129    let db_path = db_path.as_ref();
130    let conn = connect(db_path)?;
131    migrate(&conn)?;
132
133    let spill = if receipts_mode == ReceiptsMode::Durable {
134        let path = spill_path(db_path);
135        let file = std::fs::OpenOptions::new()
136            .create(true)
137            .append(true)
138            .read(true)
139            .open(&path)?;
140        Some(Arc::new(std::sync::Mutex::new(file)))
141    } else {
142        None
143    };
144
145    let (tx, rx) = mpsc::channel::<Trace>(TRACE_CHANNEL_CAP);
146    let spill_writer = spill.clone();
147    let handle = tokio::task::spawn_blocking(move || writer_loop(conn, rx, spill_writer));
148    Ok((tx, spill, handle))
149}
150
151fn migrate(conn: &Connection) -> Result<(), StoreError> {
152    // WAL: lets the background writer and short-lived feedback/read connections share the file
153    // concurrently. `journal_mode` is persisted in the file header once set by any connection.
154    conn.pragma_update(None, "journal_mode", "WAL")?;
155    conn.execute_batch(
156        "CREATE TABLE IF NOT EXISTS traces (
157            seq INTEGER PRIMARY KEY AUTOINCREMENT,
158            trace_id TEXT NOT NULL,
159            ts TEXT NOT NULL,
160            prev_hash TEXT NOT NULL,
161            hash TEXT NOT NULL,
162            tenant TEXT NOT NULL,
163            session TEXT NOT NULL,
164            body TEXT NOT NULL
165        );
166        CREATE INDEX IF NOT EXISTS traces_session_idx ON traces(session);
167        -- Tenant-scoped reads (ADR 0004 §D3) filter on `tenant`; index it with `seq` so a
168        -- per-tenant scan stays ordered and cheap. Existing rows keep their `tenant` value.
169        CREATE INDEX IF NOT EXISTS traces_tenant_seq_idx ON traces(tenant, seq);
170        -- Deferred verdicts live in their OWN table, keyed by trace_id. They are NEVER folded
171        -- into the sealed, hashed `traces.body`, so a late outcome can't alter a past record and
172        -- the tamper-evident chain stays valid. They are merged onto a trace only on read.
173        CREATE TABLE IF NOT EXISTS deferred_verdicts (
174            id INTEGER PRIMARY KEY AUTOINCREMENT,
175            trace_id TEXT NOT NULL,
176            gate_id TEXT NOT NULL,
177            verdict TEXT NOT NULL,
178            score REAL,
179            reported_at TEXT NOT NULL,
180            reporter TEXT NOT NULL
181        );
182        CREATE INDEX IF NOT EXISTS deferred_trace_idx ON deferred_verdicts(trace_id);",
183    )?;
184    Ok(())
185}
186
187/// Assign `prev_hash`, compute `hash`, and insert one trace. Updates `head` on success.
188/// Logs and skips on hash or insert failure — one bad record must not wedge the pipeline.
189fn write_trace(conn: &Connection, trace: &mut Trace, head: &mut String) {
190    trace.prev_hash = head.clone();
191    let hash = match trace.hash() {
192        Ok(h) => h,
193        Err(err) => {
194            tracing::error!(%err, trace_id = %trace.trace_id, "trace writer: hash failed, dropping");
195            return;
196        }
197    };
198    if let Err(err) = insert(conn, trace, &hash) {
199        tracing::error!(%err, trace_id = %trace.trace_id, "trace writer: insert failed, dropping");
200        return;
201    }
202    *head = hash;
203}
204
205/// Drain the spill file into the store, inserting traces in file order BEFORE any new channel
206/// arrivals. Truncates the file after a successful drain. Errors on individual lines are logged
207/// and skipped so one bad spill line can't stall the recovery.
208///
209/// ponytail: reads the entire file into memory; the spill file is bounded by the channel size
210/// and trace sizes, so this is fine in practice. Stream-parse if traces grow very large.
211fn drain_spill(conn: &Connection, spill: &SpillHandle, head: &mut String) {
212    // Hold the lock for the entire drain: prevents concurrent spill writers from appending
213    // new lines mid-drain, which would corrupt ordering.
214    let mut guard = spill.lock().unwrap_or_else(|e| e.into_inner());
215
216    // Seek to start before reading.
217    if let Err(e) = guard.seek(SeekFrom::Start(0)) {
218        tracing::error!(%e, "drain_spill: seek failed");
219        return;
220    }
221
222    // Collect all non-empty lines while holding the lock (reader borrows from guard).
223    let lines: Vec<String> = {
224        let reader = BufReader::new(&*guard);
225        reader
226            .lines()
227            .filter_map(|l| match l {
228                Ok(s) if !s.trim().is_empty() => Some(s),
229                Ok(_) => None,
230                Err(e) => {
231                    tracing::error!(%e, "drain_spill: read error");
232                    None
233                }
234            })
235            .collect()
236    };
237
238    if lines.is_empty() {
239        return;
240    }
241
242    let n = lines.len();
243    for line in &lines {
244        let mut trace: Trace = match serde_json::from_str(line) {
245            Ok(t) => t,
246            Err(e) => {
247                tracing::error!(%e, "drain_spill: bad JSON line, skipping");
248                continue;
249            }
250        };
251        write_trace(conn, &mut trace, head);
252    }
253    tracing::info!(drained = n, "spill file drained into trace store");
254
255    // Truncate: all lines have been processed (failures are skipped and logged above).
256    if let Err(e) = guard.seek(SeekFrom::Start(0)) {
257        tracing::error!(%e, "drain_spill: post-drain seek failed");
258        return;
259    }
260    if let Err(e) = guard.set_len(0) {
261        tracing::error!(%e, "drain_spill: truncate failed — spill file may replay on next boot");
262    }
263}
264
265/// The writer's main loop: runs on a blocking-pool thread for the lifetime of the store.
266/// Never panics on a bad trace — logs and drops it, so one malformed record can't wedge the
267/// whole audit pipeline.
268///
269/// In durable mode (`spill` is `Some`): drains the spill file at startup and whenever the
270/// channel empties, so spilled traces always land BEFORE later channel traces in the chain.
271fn writer_loop(conn: Connection, mut rx: mpsc::Receiver<Trace>, spill: Option<SpillHandle>) {
272    let mut head = match current_head(&conn) {
273        Ok(h) => h,
274        Err(err) => {
275            tracing::error!(%err, "trace writer: failed to load chain head, stopping");
276            return;
277        }
278    };
279
280    // Startup drain: recover any traces spilled during a previous run's backpressure.
281    if let Some(ref s) = spill {
282        drain_spill(&conn, s, &mut head);
283    }
284
285    loop {
286        let Some(mut trace) = rx.blocking_recv() else {
287            break;
288        };
289        write_trace(&conn, &mut trace, &mut head);
290
291        // Fast-drain: pull any immediately available channel items, then drain spill when
292        // the channel is momentarily empty so spilled traces land before the next wave.
293        loop {
294            match rx.try_recv() {
295                Ok(mut t) => write_trace(&conn, &mut t, &mut head),
296                Err(mpsc::error::TryRecvError::Empty) => {
297                    if let Some(ref s) = spill {
298                        drain_spill(&conn, s, &mut head);
299                    }
300                    break;
301                }
302                Err(mpsc::error::TryRecvError::Disconnected) => return,
303            }
304        }
305    }
306
307    // Channel closed: one final spill drain to capture any last-moment spills.
308    if let Some(ref s) = spill {
309        drain_spill(&conn, s, &mut head);
310    }
311}
312
313fn current_head(conn: &Connection) -> Result<String, StoreError> {
314    let mut stmt = conn.prepare("SELECT hash FROM traces ORDER BY seq DESC LIMIT 1")?;
315    let mut rows = stmt.query([])?;
316    match rows.next()? {
317        Some(row) => Ok(row.get(0)?),
318        None => Ok(GENESIS_HASH.to_owned()),
319    }
320}
321
322fn insert(conn: &Connection, trace: &Trace, hash: &str) -> Result<(), StoreError> {
323    let body = serde_json::to_string(trace)?;
324    conn.execute(
325        "INSERT INTO traces (trace_id, ts, prev_hash, hash, tenant, session, body)
326         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
327        rusqlite::params![
328            trace.trace_id.to_string(),
329            trace.ts.to_string(),
330            trace.prev_hash,
331            hash,
332            trace.tenant_id,
333            trace.session_id,
334            body,
335        ],
336    )?;
337    Ok(())
338}
339
340/// Load every trace from the database in insertion (chain) order — used by tests and
341/// operators to verify the hash chain with [`firstpass_core::verify_chain`].
342///
343/// **Operator-wide** read: every trace across ALL tenants, in `seq` order.
344///
345/// This deliberately crosses tenant boundaries and must stay reserved for operator-scoped work
346/// where a global view is intrinsic — namely verifying the single hash-chain, which spans every
347/// tenant's traces in one sequence (ADR 0004 §D3). For tenant-facing reads use
348/// [`load_tenant_traces`].
349///
350/// # Errors
351/// Returns [`StoreError::Sqlite`] on a database error, or [`StoreError::Json`] if a stored
352/// row is not valid trace JSON.
353pub fn load_all_traces(db_path: impl AsRef<Path>) -> Result<Vec<Trace>, StoreError> {
354    let conn = connect(db_path.as_ref())?;
355    let mut stmt = conn.prepare("SELECT body FROM traces ORDER BY seq ASC")?;
356    let rows = stmt.query_map([], |row| row.get::<_, String>(0))?;
357    let mut traces = Vec::new();
358    for row in rows {
359        traces.push(serde_json::from_str(&row?)?);
360    }
361    Ok(traces)
362}
363
364/// **Tenant-scoped** read: only traces owned by `tenant`, in `seq` order (ADR 0004 §D3). Tenant A
365/// can never see tenant B's traces through this path.
366///
367/// # Errors
368/// Returns [`StoreError::Sqlite`] on a database error, or [`StoreError::Json`] if a stored
369/// row is not valid trace JSON.
370pub fn load_tenant_traces(
371    db_path: impl AsRef<Path>,
372    tenant: &str,
373) -> Result<Vec<Trace>, StoreError> {
374    let conn = connect(db_path.as_ref())?;
375    let mut stmt = conn.prepare("SELECT body FROM traces WHERE tenant = ?1 ORDER BY seq ASC")?;
376    let rows = stmt.query_map([tenant], |row| row.get::<_, String>(0))?;
377    let mut traces = Vec::new();
378    for row in rows {
379        traces.push(serde_json::from_str(&row?)?);
380    }
381    Ok(traces)
382}
383
384/// Whether a trace with `trace_id` exists **and is owned by `tenant`** — used to reject feedback
385/// for unknown traces and, crucially, to deny cross-tenant feedback (ADR 0004 §D3/§D4). A trace
386/// owned by another tenant is indistinguishable from a non-existent one here, so the caller can
387/// return a `404` with no existence oracle.
388///
389/// # Errors
390/// Returns [`StoreError::Sqlite`] on a database error.
391pub fn trace_exists(
392    db_path: impl AsRef<Path>,
393    tenant: &str,
394    trace_id: &str,
395) -> Result<bool, StoreError> {
396    let conn = connect(db_path.as_ref())?;
397    let n: i64 = conn.query_row(
398        "SELECT COUNT(1) FROM traces WHERE tenant = ?1 AND trace_id = ?2",
399        [tenant, trace_id],
400        |row| row.get(0),
401    )?;
402    Ok(n > 0)
403}
404
405/// The route index recorded on `trace_id`, scoped to `tenant`.
406///
407/// `Ok(None)` means the trace is not visible to this tenant (or does not exist) — the same
408/// ownership check [`trace_exists`] makes, so this cannot be used to probe another tenant's
409/// traces. `Ok(Some(None))` means the trace exists but predates route recording.
410///
411/// # Errors
412/// On a store read failure.
413/// The decision a trace's answer was originally proven by, if it was itself a cache replay.
414///
415/// A cache hit is recorded under a NEW trace id, with `cache_source` naming the decision whose gate
416/// passed. So feedback arriving on a replay must be applied to that source, not to the replay's own
417/// id — otherwise a Fail reported against a replayed answer retracts nothing and the disproven
418/// entry keeps serving until TTL.
419///
420/// Returns `Ok(None)` for an unknown trace, `Ok(Some(None))` for one that ran the ladder itself.
421///
422/// # Errors
423/// [`StoreError`] when the database cannot be opened or queried.
424pub fn trace_cache_source(
425    db_path: impl AsRef<Path>,
426    tenant: &str,
427    trace_id: &str,
428) -> Result<Option<Option<uuid::Uuid>>, StoreError> {
429    let conn = connect(db_path.as_ref())?;
430    let body: Option<String> = conn
431        .query_row(
432            "SELECT body FROM traces WHERE tenant = ?1 AND trace_id = ?2",
433            [tenant, trace_id],
434            |row| row.get(0),
435        )
436        .optional()?;
437    Ok(body.map(|b: String| {
438        serde_json::from_str::<firstpass_core::Trace>(&b)
439            .ok()
440            .and_then(|t| t.final_.cache_source)
441    }))
442}
443
444pub fn trace_route(
445    db_path: impl AsRef<Path>,
446    tenant: &str,
447    trace_id: &str,
448) -> Result<Option<Option<u32>>, StoreError> {
449    let conn = connect(db_path.as_ref())?;
450    let body: Option<String> = conn
451        .query_row(
452            "SELECT body FROM traces WHERE tenant = ?1 AND trace_id = ?2",
453            [tenant, trace_id],
454            |row| row.get(0),
455        )
456        .optional()?;
457    Ok(body.map(|b: String| {
458        serde_json::from_str::<firstpass_core::Trace>(&b)
459            .ok()
460            .and_then(|t| t.route_ix)
461    }))
462}
463
464/// Append a deferred verdict for `trace_id` (a downstream outcome or async gate result). This
465/// writes ONLY to the `deferred_verdicts` table; the sealed trace and its hash are untouched, so
466/// the audit chain remains verifiable (SPEC §8.3.4 — the outcome-feedback loop).
467///
468/// # Errors
469/// Returns [`StoreError::Sqlite`] on a database error.
470pub fn append_deferred(
471    db_path: impl AsRef<Path>,
472    trace_id: &str,
473    v: &DeferredVerdict,
474) -> Result<(), StoreError> {
475    let conn = connect(db_path.as_ref())?;
476    conn.execute(
477        "INSERT INTO deferred_verdicts (trace_id, gate_id, verdict, score, reported_at, reporter)
478         VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
479        rusqlite::params![
480            trace_id,
481            v.gate_id,
482            v.verdict.as_str(),
483            v.score.map(Score::value),
484            v.reported_at.to_string(),
485            v.reporter,
486        ],
487    )?;
488    Ok(())
489}
490
491/// Load the deferred verdicts recorded for `trace_id`, oldest first. Malformed stored rows are
492/// skipped (logged), never fatal — a corrupt late outcome must not break reading the trace.
493///
494/// # Errors
495/// Returns [`StoreError::Sqlite`] on a database error.
496pub fn load_deferred(
497    db_path: impl AsRef<Path>,
498    trace_id: &str,
499) -> Result<Vec<DeferredVerdict>, StoreError> {
500    let conn = connect(db_path.as_ref())?;
501    let mut stmt = conn.prepare(
502        "SELECT gate_id, verdict, score, reported_at, reporter
503         FROM deferred_verdicts WHERE trace_id = ?1 ORDER BY id ASC",
504    )?;
505    let rows = stmt.query_map([trace_id], |row| {
506        Ok((
507            row.get::<_, String>(0)?,
508            row.get::<_, String>(1)?,
509            row.get::<_, Option<f64>>(2)?,
510            row.get::<_, String>(3)?,
511            row.get::<_, String>(4)?,
512        ))
513    })?;
514
515    let mut out = Vec::new();
516    for row in rows {
517        let (gate_id, verdict_s, score, reported_s, reporter) = row?;
518        let verdict = match verdict_s.as_str() {
519            "pass" => Verdict::Pass,
520            "fail" => Verdict::Fail,
521            "abstain" => Verdict::Abstain,
522            other => {
523                tracing::warn!(verdict = %other, %trace_id, "skipping deferred row with bad verdict");
524                continue;
525            }
526        };
527        let Ok(reported_at) = reported_s.parse::<jiff::Timestamp>() else {
528            tracing::warn!(%trace_id, "skipping deferred row with bad timestamp");
529            continue;
530        };
531        out.push(DeferredVerdict {
532            gate_id,
533            verdict,
534            score: score.and_then(|s| Score::new(s).ok()),
535            reported_at,
536            reporter,
537        });
538    }
539    Ok(out)
540}
541
542/// Load a single trace by id **scoped to `tenant`**, with its deferred verdicts merged into
543/// `deferred` — the **view** for display/inspection (ADR 0004 §D3). A trace owned by another
544/// tenant returns `None`, exactly like a missing one, so an inspecting agent can never read across
545/// tenants. This is deliberately separate from [`load_all_traces`]: merging deferred verdicts
546/// changes the record, so a merged trace must NOT be fed to `verify_chain` (chain verification
547/// always runs on the sealed bodies from [`load_all_traces`]).
548///
549/// # Errors
550/// Returns [`StoreError::Sqlite`] / [`StoreError::Json`] on database or decode errors.
551pub fn load_trace_view(
552    db_path: impl AsRef<Path>,
553    tenant: &str,
554    trace_id: &str,
555) -> Result<Option<Trace>, StoreError> {
556    let conn = connect(db_path.as_ref())?;
557    let body: Option<String> = conn
558        .query_row(
559            "SELECT body FROM traces WHERE tenant = ?1 AND trace_id = ?2",
560            [tenant, trace_id],
561            |row| row.get(0),
562        )
563        .ok();
564    let Some(body) = body else { return Ok(None) };
565    let mut trace: Trace = serde_json::from_str(&body)?;
566    trace.deferred = load_deferred(db_path, trace_id)?;
567    Ok(Some(trace))
568}
569
570#[cfg(test)]
571mod tests {
572    use firstpass_core::{
573        Attempt, Features, FinalOutcome, GENESIS_HASH, PolicyRef, RequestInfo, ServedFrom,
574        TaskKind, Verdict, verify_chain,
575    };
576
577    use super::*;
578
579    fn sample_trace(tenant: &str, session: &str) -> Trace {
580        let attempt = Attempt {
581            rung: 0,
582            model: "claude-haiku-4-5".to_owned(),
583            provider: "anthropic".to_owned(),
584            in_tokens: 10,
585            cache_write_tokens: 0,
586            cache_read_tokens: 0,
587            out_tokens: 5,
588            cost_usd: 0.001,
589            latency_ms: 12,
590            gates: vec![],
591            verdict: Verdict::Pass,
592        };
593        let mut trace = Trace {
594            trace_id: uuid::Uuid::now_v7(),
595            prev_hash: GENESIS_HASH.to_owned(),
596            tenant_id: tenant.to_owned(),
597            session_id: session.to_owned(),
598            ts: jiff::Timestamp::now(),
599            mode: firstpass_core::Mode::Observe,
600            policy: PolicyRef {
601                id: "observe-passthrough@v0".to_owned(),
602                explore: false,
603                propensity: None,
604                mode_profile: None,
605            },
606            request: RequestInfo {
607                api: "anthropic.messages".to_owned(),
608                prompt_hash: "deadbeef".to_owned(),
609                features: Features::new(TaskKind::Other),
610            },
611            attempts: vec![attempt],
612            deferred: Vec::new(),
613            final_: FinalOutcome {
614                served_rung: Some(0),
615                served_from: ServedFrom::Attempt,
616                total_cost_usd: 0.001,
617                gate_cost_usd: 0.0,
618                total_latency_ms: 12,
619                escalations: 0,
620                counterfactual_baseline_usd: 0.001,
621                savings_usd: 0.0,
622                cache_source: None,
623            },
624            probe: None,
625            rollout: None,
626            shadow: None,
627            route_ix: None,
628            predicted_pass: None,
629            elastic: None,
630        };
631        trace.recompute_savings();
632        trace
633    }
634
635    #[tokio::test]
636    async fn writer_assigns_prev_hash_and_forms_a_valid_chain() {
637        let db_path =
638            std::env::temp_dir().join(format!("firstpass-store-test-{}.db", uuid::Uuid::now_v7()));
639        let (tx, handle) = open(&db_path).unwrap();
640
641        tx.try_send(sample_trace("tenant-a", "session-1")).unwrap();
642        tx.try_send(sample_trace("tenant-a", "session-1")).unwrap();
643        drop(tx);
644        handle.await.unwrap();
645
646        let traces = load_all_traces(&db_path).unwrap();
647        assert_eq!(traces.len(), 2);
648        assert_eq!(traces[0].prev_hash, GENESIS_HASH);
649        assert_eq!(traces[1].prev_hash, traces[0].hash().unwrap());
650        verify_chain(&traces, GENESIS_HASH).unwrap();
651
652        let _ = std::fs::remove_file(&db_path);
653    }
654
655    /// D7 cross-tenant isolation, at the store layer: with rows for tenants A and B, every
656    /// tenant-scoped read for A returns only A's data, and vice-versa. The operator-wide
657    /// [`load_all_traces`] still sees both (for chain verification).
658    #[tokio::test]
659    async fn tenant_scoped_reads_never_cross_the_boundary() {
660        let db_path =
661            std::env::temp_dir().join(format!("firstpass-isolation-{}.db", uuid::Uuid::now_v7()));
662        let (tx, handle) = open(&db_path).unwrap();
663
664        // Two traces for A, one for B.
665        let a0 = sample_trace("tenant-a", "sa-0");
666        let a1 = sample_trace("tenant-a", "sa-1");
667        let b0 = sample_trace("tenant-b", "sb-0");
668        let (a0_id, a1_id, b0_id) = (
669            a0.trace_id.to_string(),
670            a1.trace_id.to_string(),
671            b0.trace_id.to_string(),
672        );
673        tx.try_send(a0).unwrap();
674        tx.try_send(b0).unwrap();
675        tx.try_send(a1).unwrap();
676        drop(tx);
677        handle.await.unwrap();
678
679        // Scoped list: A sees exactly its two, B sees exactly its one.
680        let a_traces = load_tenant_traces(&db_path, "tenant-a").unwrap();
681        assert_eq!(a_traces.len(), 2, "A must see only A's traces");
682        assert!(a_traces.iter().all(|t| t.tenant_id == "tenant-a"));
683        let b_traces = load_tenant_traces(&db_path, "tenant-b").unwrap();
684        assert_eq!(b_traces.len(), 1, "B must see only B's trace");
685        assert!(b_traces.iter().all(|t| t.tenant_id == "tenant-b"));
686
687        // A can prove its own trace exists but cannot see B's, and vice-versa.
688        assert!(trace_exists(&db_path, "tenant-a", &a0_id).unwrap());
689        assert!(!trace_exists(&db_path, "tenant-a", &b0_id).unwrap());
690        assert!(trace_exists(&db_path, "tenant-b", &b0_id).unwrap());
691        assert!(!trace_exists(&db_path, "tenant-b", &a1_id).unwrap());
692
693        // The view is likewise scoped: cross-tenant reads are indistinguishable from a miss.
694        assert!(
695            load_trace_view(&db_path, "tenant-a", &a0_id)
696                .unwrap()
697                .is_some()
698        );
699        assert!(
700            load_trace_view(&db_path, "tenant-a", &b0_id)
701                .unwrap()
702                .is_none()
703        );
704        assert!(
705            load_trace_view(&db_path, "tenant-b", &a0_id)
706                .unwrap()
707                .is_none()
708        );
709
710        // A tenant that owns nothing sees nothing.
711        assert!(load_tenant_traces(&db_path, "ghost").unwrap().is_empty());
712
713        // Operator-wide read still spans both, and the global chain stays valid.
714        let all = load_all_traces(&db_path).unwrap();
715        assert_eq!(all.len(), 3);
716        verify_chain(&all, GENESIS_HASH).unwrap();
717
718        let _ = std::fs::remove_file(&db_path);
719    }
720
721    #[tokio::test]
722    async fn deferred_verdicts_attach_on_read_without_breaking_the_chain() {
723        let db_path = std::env::temp_dir().join(format!(
724            "firstpass-deferred-test-{}.db",
725            uuid::Uuid::now_v7()
726        ));
727        let (tx, handle) = open(&db_path).unwrap();
728        let t0 = sample_trace("acme", "run-1");
729        let t1 = sample_trace("acme", "run-1");
730        let (id0, id1) = (t0.trace_id.to_string(), t1.trace_id.to_string());
731        tx.try_send(t0).unwrap();
732        tx.try_send(t1).unwrap();
733        drop(tx);
734        handle.await.unwrap();
735
736        // A downstream outcome arrives for the first trace (e.g. "tests passed an hour later").
737        let dv = DeferredVerdict {
738            gate_id: "tests".to_owned(),
739            verdict: Verdict::Pass,
740            score: Some(Score::new(1.0).unwrap()),
741            reported_at: jiff::Timestamp::now(),
742            reporter: "ci".to_owned(),
743        };
744        append_deferred(&db_path, &id0, &dv).unwrap();
745        assert!(trace_exists(&db_path, "acme", &id0).unwrap());
746        assert!(!trace_exists(&db_path, "acme", "no-such-trace").unwrap());
747        // Cross-tenant: the same real trace id is invisible to a different tenant.
748        assert!(!trace_exists(&db_path, "other-tenant", &id0).unwrap());
749
750        // The view surfaces the deferred verdict...
751        let view = load_trace_view(&db_path, "acme", &id0).unwrap().unwrap();
752        assert_eq!(view.deferred.len(), 1);
753        assert_eq!(view.deferred[0].gate_id, "tests");
754        assert_eq!(view.deferred[0].verdict, Verdict::Pass);
755        // ...the second trace has none.
756        assert!(
757            load_trace_view(&db_path, "acme", &id1)
758                .unwrap()
759                .unwrap()
760                .deferred
761                .is_empty()
762        );
763        // Cross-tenant: another tenant cannot read this trace's view at all.
764        assert!(
765            load_trace_view(&db_path, "other-tenant", &id0)
766                .unwrap()
767                .is_none()
768        );
769
770        // THE INVARIANT: the sealed bodies are untouched, so the chain still verifies. A late
771        // outcome can never alter a past decision's hash.
772        let traces = load_all_traces(&db_path).unwrap();
773        assert!(
774            traces.iter().all(|t| t.deferred.is_empty()),
775            "sealed records stay deferred-free"
776        );
777        verify_chain(&traces, GENESIS_HASH).unwrap();
778
779        let _ = std::fs::remove_file(&db_path);
780    }
781
782    // ── Durable-receipts tests ────────────────────────────────────────────────
783
784    /// Test (2): `append_to_spill` writes valid JSON lines to the spill file.
785    #[test]
786    fn durable_append_to_spill_writes_valid_json_line() {
787        let db_path =
788            std::env::temp_dir().join(format!("firstpass-spill-write-{}.db", uuid::Uuid::now_v7()));
789        let spill_p = spill_path(&db_path);
790        let file = std::fs::OpenOptions::new()
791            .create(true)
792            .append(true)
793            .read(true)
794            .open(&spill_p)
795            .unwrap();
796        let handle = Arc::new(std::sync::Mutex::new(file));
797
798        let t = sample_trace("t", "s");
799        append_to_spill(&handle, &t).unwrap();
800
801        let content = std::fs::read_to_string(&spill_p).unwrap();
802        let lines: Vec<&str> = content.lines().collect();
803        assert_eq!(lines.len(), 1, "one line per spilled trace");
804        let parsed: Trace = serde_json::from_str(lines[0]).unwrap();
805        assert_eq!(parsed.trace_id, t.trace_id);
806
807        let _ = std::fs::remove_file(&spill_p);
808    }
809
810    /// Test (3): on boot with a pre-populated spill file, the writer drains it and
811    /// `verify_chain` passes over the full store.
812    #[tokio::test]
813    async fn drain_on_boot_restores_spilled_traces_and_chain_is_valid() {
814        let db_path =
815            std::env::temp_dir().join(format!("firstpass-drain-boot-{}.db", uuid::Uuid::now_v7()));
816        let spill_p = spill_path(&db_path);
817
818        // Pre-populate spill file with 2 traces (simulating backpressure from a previous run).
819        let t1 = sample_trace("t", "s1");
820        let t2 = sample_trace("t", "s2");
821        {
822            let file = std::fs::OpenOptions::new()
823                .create(true)
824                .append(true)
825                .read(true)
826                .open(&spill_p)
827                .unwrap();
828            let h = Arc::new(std::sync::Mutex::new(file));
829            append_to_spill(&h, &t1).unwrap();
830            append_to_spill(&h, &t2).unwrap();
831        }
832
833        // Open in durable mode — startup drain fires before any channel traffic.
834        let (tx, _spill, handle) = open_with_receipts(&db_path, ReceiptsMode::Durable).unwrap();
835        drop(tx); // close the channel immediately
836        handle.await.unwrap();
837
838        let traces = load_all_traces(&db_path).unwrap();
839        assert_eq!(traces.len(), 2, "both spilled traces recovered");
840        verify_chain(&traces, GENESIS_HASH).unwrap();
841
842        // Spill file must be empty after a successful drain.
843        let remaining = std::fs::read_to_string(&spill_p).unwrap();
844        assert!(
845            remaining.trim().is_empty(),
846            "spill file must be empty after drain"
847        );
848
849        let _ = std::fs::remove_file(&db_path);
850        let _ = std::fs::remove_file(&spill_p);
851    }
852
853    /// Test (4): spilled traces land before later channel traces in the store, preserving the
854    /// hash chain's append-only ordering.
855    #[tokio::test]
856    async fn spilled_traces_land_before_later_channel_traces() {
857        let db_path =
858            std::env::temp_dir().join(format!("firstpass-spill-order-{}.db", uuid::Uuid::now_v7()));
859        let spill_p = spill_path(&db_path);
860
861        // Pre-populate the spill file (as if these traces were spilled under backpressure).
862        let spilled = sample_trace("t", "spilled");
863        {
864            let file = std::fs::OpenOptions::new()
865                .create(true)
866                .append(true)
867                .read(true)
868                .open(&spill_p)
869                .unwrap();
870            let h = Arc::new(std::sync::Mutex::new(file));
871            append_to_spill(&h, &spilled).unwrap();
872        }
873
874        // Open in durable mode: startup drain inserts the spilled trace first.
875        let (tx, _spill, handle) = open_with_receipts(&db_path, ReceiptsMode::Durable).unwrap();
876        // Send a new channel trace after the store is open (arrives after startup drain).
877        let channel_trace = sample_trace("t", "channel");
878        tx.try_send(channel_trace.clone()).unwrap();
879        drop(tx);
880        handle.await.unwrap();
881
882        let traces = load_all_traces(&db_path).unwrap();
883        assert_eq!(traces.len(), 2, "spilled + channel trace");
884        // Spilled trace must come first (lower seq).
885        assert_eq!(
886            traces[0].trace_id, spilled.trace_id,
887            "spilled trace must precede channel trace"
888        );
889        assert_eq!(
890            traces[1].trace_id, channel_trace.trace_id,
891            "channel trace must follow spilled trace"
892        );
893        verify_chain(&traces, GENESIS_HASH).unwrap();
894
895        let _ = std::fs::remove_file(&db_path);
896        let _ = std::fs::remove_file(&spill_p);
897    }
898}