Skip to main content

firstpass_proxy/
experiment.rs

1//! Live monitor for certified model selection (ADR 0014).
2//!
3//! A route with a `[route.challenger]` serves a randomized fraction of sessions from a second
4//! ladder. This module owns the evidence: one [`Experiment`] per `(tenant, route)`, fed with the
5//! cost of every served experiment request and the first task outcome of every session. Its two
6//! jobs are the halt guard (the serving path asks [`ExperimentMonitor::status`] and stops using a
7//! challenger that has been proven worse) and the report an operator reads before adopting one.
8//!
9//! ## What counts as an outcome
10//!
11//! Feedback with a `pass` or `fail` verdict counts, **whatever its `gate_id`**; `abstain` is
12//! ignored, and only the first counted outcome per session is used. Nothing here can tell a CI
13//! result from a human rating, so operators should post exactly one thing per session: the task
14//! outcome the experiment is meant to compare. Posting several different signals under different
15//! gate ids makes "first one wins" arbitrary.
16//!
17//! ## Configuration identity
18//!
19//! Every receipt carries a fingerprint of the experiment configuration (both ladders, margin,
20//! alpha). Receipts whose fingerprint differs from the current config's are ignored and counted
21//! (`ignored_stale`), so changing either ladder or the margin/alpha starts a fresh experiment
22//! instead of pooling evidence about a different challenger.
23//!
24//! Tenants never share an experiment (ADR 0004): the key is `(tenant, route_ix)` and nothing here
25//! reads across tenants.
26//!
27//! The same state is re-derivable from receipts. [`experiments_from_traces`] replays the stored
28//! traces and deferred verdicts through the very same monitor, so a restart (or the
29//! `firstpass experiment` CLI) lands on the decision the live process would have reached.
30//!
31//! ## Shared state across replicas (opt-in)
32//!
33//! When [`ExperimentMonitor`] is built with a [`ExperimentLogBackend`] (set via
34//! [`ExperimentMonitor::with_log`]), every outcome (live mode) or pair (paired mode) is appended
35//! to a shared log keyed by `(tenant, route_ix, fingerprint)`. Each replica re-reads the log on
36//! every feedback event and on a periodic timer (≤ 5 s), rebuilding its local [`Experiment`] from
37//! the canonical arrival order. Because the e-process is deterministic, all replicas holding the
38//! same log reach the same halt/certify decision. Without a backend the behaviour is unchanged.
39
40use std::collections::{HashMap, HashSet};
41use std::fmt;
42use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
43
44use firstpass_core::{
45    Arm, Config, Experiment, ExperimentReport, ExperimentStatus, Trace, Verdict, select,
46};
47use serde::{Deserialize, Serialize};
48use uuid::Uuid;
49
50/// Upper bound on distinct sessions remembered per experiment, as 8-byte hashes (~8 MB per
51/// million). Past it new sessions' outcomes are ignored rather than evicting old entries: evicting
52/// would let a session count twice, which breaks the pairing argument, while ignoring only slows
53/// the experiment down. A million sessions is far beyond what the test needs to decide.
54const MAX_TRACKED_SESSIONS: usize = 1_000_000;
55
56/// After a cold-start `sync_from_backend` failure, `ensure_synced` returns `false` immediately
57/// for this long without retrying the backend. Prevents a Redis outage from turning N
58/// concurrent first-requests per `(tenant, route)` into N × backend-timeout of user-visible
59/// latency: only the first caller pays the wait; subsequent callers within this window
60/// fast-fail. The cooldown resets once the window expires and the next retry succeeds.
61const SYNC_FAILURE_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(5);
62
63/// `ensure_synced` returns `true` (state is fresh) only while the last successful sync was within
64/// this bound. 30 s = 6 × the 5 s background-sync interval: a single missed tick does not
65/// trigger a re-sync; two consecutive failures do. Once the bound is exceeded, the next
66/// `ensure_synced` call re-attempts a sync (subject to the failure cooldown above).
67const SYNC_STALENESS_BOUND: std::time::Duration = std::time::Duration::from_secs(30);
68
69/// Hard ceiling on the time `ensure_synced` may block waiting for the backend. A hung Redis
70/// connection (e.g., half-open TCP) would otherwise stall live request processing for minutes.
71/// 500 ms is generous (one LRANGE of a small list) but short enough to keep tail latency
72/// acceptable. On timeout, `ensure_synced` fails closed (same as any other backend error).
73const SYNC_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
74
75/// Bound on a shared-log call made off the request path (the paired task's append and read-back,
76/// and the background sync). A half-open connection must not stall the paired task before it
77/// writes its durable row, nor wedge the background sync loop.
78const LOG_OP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
79
80/// A shared-log call bounded by [`LOG_OP_TIMEOUT`]; a timeout is reported as a backend error.
81async fn bounded<T>(
82    fut: impl std::future::Future<Output = Result<T, LogError>>,
83) -> Result<T, LogError> {
84    match tokio::time::timeout(LOG_OP_TIMEOUT, fut).await {
85        Ok(r) => r,
86        Err(_) => Err(LogError::Backend(format!(
87            "timed out after {} ms",
88            LOG_OP_TIMEOUT.as_millis()
89        ))),
90    }
91}
92
93// ── Shared log (optional Redis / in-memory backend) ───────────────────────────────────────────
94
95/// An error returned by [`ExperimentLogBackend`] operations.
96///
97/// Backend failures are not fatal for the request path: evidence is withheld from other replicas
98/// (delaying a decision), but the local replica keeps serving. Callers use this to decide whether
99/// to serve the challenger arm (fail closed when the backend is unreachable).
100#[derive(Debug, thiserror::Error)]
101pub enum LogError {
102    /// The backend (e.g. Redis) returned an error.
103    #[error("experiment log backend error: {0}")]
104    Backend(String),
105}
106
107/// A record appended to the shared experiment log.
108///
109/// Live mode uses `Outcome`; paired mode uses `Pair`. The list for a given
110/// `(tenant, route_ix, fingerprint)` contains only one kind (the mode is fixed per config).
111#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
112#[serde(tag = "kind", rename_all = "snake_case")]
113pub enum LogRecord {
114    /// One session's outcome from live mode.
115    Outcome {
116        arm: Arm,
117        /// Gate verdict in [0, 1].
118        outcome: f64,
119        /// SHA-256 prefix of the session id (for in-log dedup on replay, not primary dedup).
120        session_hash: u64,
121        /// USD cost of this request.
122        cost_usd: f64,
123    },
124    /// One paired shadow result.
125    Pair {
126        x_inc: f64,
127        x_ch: f64,
128        /// Cost of the incumbent request.
129        incumbent_cost_usd: f64,
130        /// Cost of the challenger off-path evaluation.
131        challenger_cost_usd: f64,
132        /// Whether the challenger call ended in an error with no output.
133        /// `#[serde(default)]` so records written by older replicas deserialize to `false`.
134        #[serde(default)]
135        ch_error: bool,
136        /// Number of challenger attempts cut off at the output-token limit.
137        /// `#[serde(default)]` so records written by older replicas deserialize to `0`.
138        #[serde(default)]
139        ch_truncated: u64,
140        /// Unix seconds when this pair was recorded. `0` for records written before this field
141        /// was added (treated as "long ago" by `ShadowLedger::seed`, i.e., not debited today).
142        #[serde(default)]
143        ts_unix: i64,
144        /// The per-process UUID of the replica that wrote this record. Used to skip double-tallying
145        /// health and spend on the originating replica (which already tallied them locally when the
146        /// challenger call completed). Records from older replicas (missing this field) deserialize
147        /// to `Uuid::nil()`, which never matches any live replica's UUID and so are treated as remote.
148        #[serde(default)]
149        replica_id: Uuid,
150        /// The challenger trace-id, used for cross-source dedup: prevents a pair from being
151        /// counted twice when an append times out after Redis applied it (append returns Err but
152        /// the record is in the log). `#[serde(default)]` so records written by older replicas
153        /// deserialize to `Uuid::nil()`, which is never deduplicated.
154        #[serde(default)]
155        trace_id: Uuid,
156        /// The served incumbent trace-id. A replica restarted on the same trace store has already
157        /// counted that request's incumbent cost from its stored receipt, but it has a new
158        /// `replica_id`, so without this its own pre-restart records would look remote and the
159        /// incumbent cost would count twice. `Uuid::nil()` for older records (never matched).
160        #[serde(default)]
161        incumbent_trace_id: Uuid,
162    },
163}
164
165/// A shared log that lets several replicas converge on the same [`Experiment`] state.
166///
167/// Implementors: [`InMemoryExperimentLog`] (tests), and `RedisExperimentLog` (behind the
168/// `redis-cache` feature).
169///
170/// The log for a given experiment is identified by `(tenant, route_ix, fingerprint)`. Multiple
171/// replicas can append concurrently; the authoritative ordering is the list's arrival order (for
172/// Redis: RPUSH index; for the in-memory implementation: insertion position). A halt decision
173/// derived from the list at position `N` holds forever — re-reading the list can only add more
174/// evidence, never retract it.
175#[async_trait::async_trait]
176pub trait ExperimentLogBackend: Send + Sync + fmt::Debug {
177    /// Append `record` to the list. Returns the new list length (1-based), or an error if the
178    /// backend is unreachable. A failure means the pair/outcome will not reach other replicas,
179    /// but local accounting (health, ledger) was already performed by the caller.
180    async fn append(
181        &self,
182        tenant: &str,
183        route_ix: usize,
184        fingerprint: &str,
185        record: LogRecord,
186    ) -> Result<u64, LogError>;
187
188    /// Return records starting at `offset` (0-based). Each element is `Some(record)` for a valid
189    /// entry or `None` for a malformed one (failed to deserialize). The list length equals the
190    /// number of positions consumed: a `None` at index `i` still advances `offset` by 1. Empty
191    /// if the list has ≤ `offset` entries. Returns an error if the backend is unreachable; callers
192    /// use this to distinguish "truly empty" from "unreachable" and fail closed.
193    async fn load_since(
194        &self,
195        tenant: &str,
196        route_ix: usize,
197        fingerprint: &str,
198        offset: u64,
199    ) -> Result<Vec<Option<LogRecord>>, LogError>;
200
201    /// Atomically claim a session for this experiment (live mode session dedup across replicas).
202    ///
203    /// Returns `true` if the session was newly claimed (this replica should count the outcome),
204    /// `false` if another replica already claimed it. Claimants are permanent: once claimed the
205    /// result is final regardless of which replica appended it.
206    async fn claim_session(
207        &self,
208        tenant: &str,
209        route_ix: usize,
210        fingerprint: &str,
211        session_id: &str,
212    ) -> bool;
213}
214
215/// In-process log for tests (and the no-Redis single-replica path, where it is never used).
216///
217/// Two `ExperimentMonitor` instances that hold `Arc`s to the same `InMemoryExperimentLog` will
218/// converge to the same `Experiment` state after syncing, exactly as two remote replicas sharing
219/// a Redis-backed log would.
220#[derive(Debug, Default)]
221pub struct InMemoryExperimentLog {
222    lists: Mutex<HashMap<(String, usize, String), Vec<LogRecord>>>,
223    sessions: Mutex<HashSet<(String, usize, String, String)>>,
224}
225
226impl InMemoryExperimentLog {
227    /// A fresh empty log.
228    #[must_use]
229    pub fn new() -> Self {
230        Self::default()
231    }
232}
233
234#[async_trait::async_trait]
235impl ExperimentLogBackend for InMemoryExperimentLog {
236    async fn append(
237        &self,
238        tenant: &str,
239        route_ix: usize,
240        fingerprint: &str,
241        record: LogRecord,
242    ) -> Result<u64, LogError> {
243        let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
244        let mut lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
245        let list = lists.entry(key).or_default();
246        list.push(record);
247        Ok(list.len() as u64)
248    }
249
250    async fn load_since(
251        &self,
252        tenant: &str,
253        route_ix: usize,
254        fingerprint: &str,
255        offset: u64,
256    ) -> Result<Vec<Option<LogRecord>>, LogError> {
257        let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
258        let lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
259        match lists.get(&key) {
260            None => Ok(vec![]),
261            Some(list) => {
262                let start = offset as usize;
263                if start >= list.len() {
264                    Ok(vec![])
265                } else {
266                    // In-memory records are always valid (never malformed).
267                    Ok(list[start..].iter().map(|r| Some(r.clone())).collect())
268                }
269            }
270        }
271    }
272
273    async fn claim_session(
274        &self,
275        tenant: &str,
276        route_ix: usize,
277        fingerprint: &str,
278        session_id: &str,
279    ) -> bool {
280        let key = (
281            tenant.to_owned(),
282            route_ix,
283            fingerprint.to_owned(),
284            session_id.to_owned(),
285        );
286        let mut sessions = self.sessions.lock().unwrap_or_else(PoisonError::into_inner);
287        sessions.insert(key)
288    }
289}
290
291// ── Redis backend (behind `redis-cache` feature) ─────────────────────────────────────────────
292
293#[cfg(feature = "redis-cache")]
294pub use redis_log::RedisExperimentLog;
295
296#[cfg(feature = "redis-cache")]
297mod redis_log {
298    use super::{ExperimentLogBackend, LogRecord};
299    use std::fmt;
300
301    // Use tokio::sync::Mutex so the guard is Send and can cross await points.
302    use tokio::sync::Mutex;
303
304    const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
305
306    /// Strips credentials from a Redis URL so it can appear in log lines safely.
307    fn redact(url: &str) -> String {
308        let Some((scheme, rest)) = url.split_once("://") else {
309            return "<malformed redis url>".to_owned();
310        };
311        let host = match rest.rfind('@') {
312            Some(i) => &rest[i + 1..],
313            None => rest,
314        };
315        format!("{scheme}://{host}")
316    }
317
318    /// Redis-backed experiment log. Each experiment lives in a list at
319    /// `firstpass:exp:{tenant}:{route_ix}:{fingerprint}` and sessions are claimed via
320    /// `SET NX firstpass:exp:sess:{tenant}:{route_ix}:{fingerprint}:{session_id}`.
321    pub struct RedisExperimentLog {
322        /// `tokio::sync::Mutex` so the guard is `Send` and can be held across `.await`.
323        conn: Mutex<redis::aio::ConnectionManager>,
324    }
325
326    impl fmt::Debug for RedisExperimentLog {
327        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
328            f.debug_struct("RedisExperimentLog").finish_non_exhaustive()
329        }
330    }
331
332    impl RedisExperimentLog {
333        /// Connect to `url`, timing out after 5 s.
334        ///
335        /// # Errors
336        /// A malformed URL, an unreachable server, or a timeout. Surfaced at startup so an operator
337        /// who configured Redis is told immediately rather than discovering it via a missing halt.
338        pub async fn connect(url: &str) -> Result<Self, String> {
339            let safe = redact(url);
340            let client = redis::Client::open(url).map_err(|e| format!("redis url {safe}: {e}"))?;
341            let conn =
342                tokio::time::timeout(CONNECT_TIMEOUT, redis::aio::ConnectionManager::new(client))
343                    .await
344                    .map_err(|_| format!("redis url {safe}: connect timed out"))?
345                    .map_err(|e| format!("redis url {safe}: {e}"))?;
346            Ok(Self {
347                conn: Mutex::new(conn),
348            })
349        }
350
351        fn list_key(tenant: &str, route_ix: usize, fingerprint: &str) -> String {
352            format!("firstpass:exp:{tenant}:{route_ix}:{fingerprint}")
353        }
354
355        fn session_key(
356            tenant: &str,
357            route_ix: usize,
358            fingerprint: &str,
359            session_id: &str,
360        ) -> String {
361            format!("firstpass:exp:sess:{tenant}:{route_ix}:{fingerprint}:{session_id}")
362        }
363    }
364
365    #[async_trait::async_trait]
366    impl ExperimentLogBackend for RedisExperimentLog {
367        async fn append(
368            &self,
369            tenant: &str,
370            route_ix: usize,
371            fingerprint: &str,
372            record: LogRecord,
373        ) -> Result<u64, super::LogError> {
374            let key = Self::list_key(tenant, route_ix, fingerprint);
375            let json = serde_json::to_string(&record)
376                .map_err(|e| super::LogError::Backend(format!("serialize: {e}")))?;
377            let mut conn = self.conn.lock().await;
378            use redis::AsyncCommands as _;
379            conn.rpush::<_, _, u64>(&key, json)
380                .await
381                .map_err(|e| super::LogError::Backend(format!("RPUSH {key}: {e}")))
382        }
383
384        async fn load_since(
385            &self,
386            tenant: &str,
387            route_ix: usize,
388            fingerprint: &str,
389            offset: u64,
390        ) -> Result<Vec<Option<super::LogRecord>>, super::LogError> {
391            let key = Self::list_key(tenant, route_ix, fingerprint);
392            let mut conn = self.conn.lock().await;
393            use redis::AsyncCommands as _;
394            // Saturating cast: a list with > isize::MAX entries is unreachable in practice
395            // (Redis is bounded by memory long before that), but the cast must not wrap to a
396            // negative index (which LRANGE would interpret as "from the end").
397            let start = isize::try_from(offset).unwrap_or(isize::MAX);
398            let raw: Vec<String> = conn
399                .lrange(&key, start, -1)
400                .await
401                .map_err(|e| super::LogError::Backend(format!("LRANGE {key}: {e}")))?;
402            // Preserve positions: None for malformed entries so sync_from_backend advances
403            // remote_log_offset over them and does not re-read them on the next sync.
404            Ok(raw
405                .iter()
406                .map(|s| match serde_json::from_str(s) {
407                    Ok(r) => Some(r),
408                    Err(e) => {
409                        tracing::warn!(
410                            %e,
411                            "experiment log: malformed record; position consumed, evidence skipped"
412                        );
413                        None
414                    }
415                })
416                .collect())
417        }
418
419        async fn claim_session(
420            &self,
421            tenant: &str,
422            route_ix: usize,
423            fingerprint: &str,
424            session_id: &str,
425        ) -> bool {
426            let key = Self::session_key(tenant, route_ix, fingerprint, session_id);
427            let mut conn = self.conn.lock().await;
428            use redis::AsyncCommands as _;
429            // SET key "1" NX EX 86400 — expires after 1 day so the set doesn't grow unboundedly.
430            match conn
431                .set_options::<_, _, ()>(
432                    &key,
433                    "1",
434                    redis::SetOptions::default()
435                        .with_expiration(redis::SetExpiry::EX(86_400))
436                        .conditional_set(redis::ExistenceCheck::NX),
437                )
438                .await
439            {
440                Ok(()) => true,  // NX succeeded: this replica claimed the session
441                Err(_) => false, // key already exists: another replica claimed it
442            }
443        }
444    }
445}
446
447/// One experiment plus the sessions that already contributed an outcome.
448#[derive(Debug)]
449struct Entry {
450    experiment: Experiment,
451    /// Receipts seen from a previous experiment configuration and therefore not counted.
452    ignored_stale: u64,
453    /// Errors and cut-off answers per arm, for the operator to judge what the pairs left out.
454    health: Health,
455    /// Hashes of sessions whose outcome was already counted. A 64-bit collision merely drops one
456    /// legitimate outcome, which costs power and never validity.
457    sessions: HashSet<u64>,
458    saturated_logged: bool,
459    /// Challenger trace-ids whose evidence (e-process pair) has already been applied.
460    ///
461    /// Prevents a pair from being counted twice when an append times out after Redis applied
462    /// the record (the append returns `Err` but the record is in the log). The apply-once
463    /// rule covers three paths:
464    /// - `apply_pair_locally` called by `spawn_paired_challenger` after a SQLite write
465    ///   (inserts the id so a later log-sync sees it as already applied).
466    /// - SQLite replay at startup (`monitor_from_traces` inserts ids for `NotInLog` rows).
467    /// - Log sync in `apply_log_record` (skips evidence if id already present).
468    ///
469    /// A nil UUID (`Uuid::nil()`) is never inserted and never checked (old records without
470    /// a trace_id are always applied; their dedup depends on the e-process being ordered and
471    /// idempotent by offset, not by id).
472    ///
473    /// Capped at `MAX_TRACKED_SESSIONS` with the same saturation policy: once full, new
474    /// pairs are skipped rather than evicting old ids (evicting would let a pair count twice).
475    applied_pair_ids: HashSet<Uuid>,
476    /// Incumbent trace-ids whose cost was counted from stored receipts at startup. A log record
477    /// naming one does not count the incumbent cost again; matched ids are removed, so the set
478    /// only shrinks after startup. Capped at `MAX_TRACKED_SESSIONS` (beyond that, a pre-restart
479    /// own record can still double-count the incumbent cost: biased toward "no saving", safe).
480    locally_costed: HashSet<Uuid>,
481    /// How many records have been consumed from the shared log so far (incremental LRANGE).
482    remote_log_offset: u64,
483    /// Whether a `tracing::warn!` has already been emitted for a failed cold-start sync on this
484    /// entry. Guards against per-request log spam when the backend is persistently unreachable.
485    sync_warn_logged: bool,
486    /// Tokio instant of the last failed `sync_from_backend` cold-start attempt. Used by
487    /// `ensure_synced` to enforce the failure cooldown: callers that arrive within
488    /// `SYNC_FAILURE_COOLDOWN` of the last failure return `false` immediately without
489    /// re-issuing a backend read, so a Redis outage cannot turn N concurrent requests into
490    /// N × backend-timeout of user-visible latency.
491    sync_failed_at: Option<tokio::time::Instant>,
492    /// Tokio instant of the last *successful* `sync_from_backend`. Used by `ensure_synced` to
493    /// enforce the staleness bound: state is considered fresh only while this is within
494    /// `SYNC_STALENESS_BOUND`. Beyond it, a new sync is attempted; callers that arrive during a
495    /// Redis outage will keep failing closed until connectivity is restored.
496    last_synced_at: Option<tokio::time::Instant>,
497}
498
499fn session_hash(session_id: &str) -> u64 {
500    // SHA-256, not `DefaultHasher`: std reserves the right to change its algorithm between
501    // releases, and the dedup set rebuilt from receipts after an upgrade must agree with the one
502    // the live path built before it.
503    use sha2::{Digest, Sha256};
504    let d = Sha256::digest(session_id.as_bytes());
505    u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]])
506}
507
508/// Gauge value for [`ExperimentStatus`].
509fn status_code(s: ExperimentStatus) -> f64 {
510    match s {
511        ExperimentStatus::Undecided => 0.0,
512        ExperimentStatus::Certified => 1.0,
513        ExperimentStatus::Halted => 2.0,
514    }
515}
516
517#[derive(Debug)]
518struct Params {
519    margin: f64,
520    alpha: f64,
521    fingerprint: String,
522}
523
524/// Type alias to keep the `sync_gates` field type below clippy's complexity threshold.
525type SyncGates = Mutex<HashMap<(String, usize), Arc<tokio::sync::Mutex<()>>>>;
526
527/// Per-`(tenant, route)` certified-selection state. Cheap to share behind an `Arc`.
528#[derive(Debug)]
529pub struct ExperimentMonitor {
530    /// `(margin, alpha)` per route index that has a challenger.
531    params: HashMap<usize, Params>,
532    /// Set when the trace store could not be read at startup: the history needed to know whether a
533    /// challenger was already halted is missing, so no challenger is served (fail closed).
534    unavailable: bool,
535    /// tenant -> route -> entry. Nested so a lookup by `&str` never allocates on the request path.
536    entries: Mutex<HashMap<String, HashMap<usize, Entry>>>,
537    /// Optional shared log that synchronises evidence across replicas (Redis or in-memory).
538    log: Option<Arc<dyn ExperimentLogBackend>>,
539    /// Optional paired-challenger spend ledger. When present, `apply_log_record` seeds cluster
540    /// spend into it so the daily cap is cluster-wide rather than per-process.
541    paired_ledger: Option<Arc<crate::shadow::ShadowLedger>>,
542    /// Random UUID for this process instance. Stamped on every `LogRecord::Pair` this replica
543    /// writes so that `apply_log_record` can skip double-tallying health and ledger for own
544    /// records (which were already tallied locally when the challenger call completed).
545    replica_id: Uuid,
546    /// Per-`(tenant, route_ix)` gate for coalescing concurrent cold-start syncs.
547    ///
548    /// All concurrent first-request callers for a fresh entry block on the same
549    /// `tokio::sync::Mutex`. Only one calls the backend (LRANGE); subsequent waiters re-check
550    /// `last_synced_at` and return immediately once the winner has synced. The outer `Mutex` protects
551    /// the map; the inner async `Mutex` serialises the cold-start burst without holding the outer
552    /// lock across an await.
553    sync_gates: SyncGates,
554}
555
556impl Default for ExperimentMonitor {
557    /// An inert monitor with no experiments configured: every lookup returns `false`/unknown, no
558    /// log is attached, and no ledger is attached.  Used in demo/test paths that construct an
559    /// `AppState` without a real `Config` (e.g. `Arc::default()`).
560    fn default() -> Self {
561        Self {
562            params: HashMap::new(),
563            unavailable: false,
564            entries: Mutex::new(HashMap::new()),
565            log: None,
566            paired_ledger: None,
567            replica_id: Uuid::now_v7(),
568            sync_gates: Mutex::new(HashMap::new()),
569        }
570    }
571}
572
573impl ExperimentMonitor {
574    /// A monitor for every route in `config` that has a challenger. Routes without one are ignored
575    /// by every method, so a deployment with no experiments pays nothing.
576    #[must_use]
577    pub fn from_config(config: &Config) -> Self {
578        let params = config
579            .routes
580            .iter()
581            .enumerate()
582            .filter_map(|(i, r)| {
583                r.challenger.as_ref().map(|c| {
584                    (
585                        i,
586                        Params {
587                            margin: c.margin,
588                            alpha: c.alpha,
589                            fingerprint: c.fingerprint(&r.ladder),
590                        },
591                    )
592                })
593            })
594            .collect();
595        Self {
596            params,
597            unavailable: false,
598            entries: Mutex::default(),
599            log: None,
600            paired_ledger: None,
601            replica_id: Uuid::now_v7(),
602            sync_gates: Mutex::default(),
603        }
604    }
605
606    /// A monitor that refuses to serve any challenger. Used when the history could not be read.
607    #[must_use]
608    pub fn unavailable(config: &Config) -> Self {
609        Self {
610            unavailable: true,
611            ..Self::from_config(config)
612        }
613    }
614
615    /// Attach a shared log backend. Every `observe_outcome`/`observe_pair` call will append to
616    /// the backend, and `sync_from_backend` will replay newly-arrived records from it. Returns
617    /// `self` for method-chaining.
618    #[must_use]
619    pub fn with_log(mut self, log: Arc<dyn ExperimentLogBackend>) -> Self {
620        self.log = Some(log);
621        self
622    }
623
624    /// Attach a paired-challenger spend ledger. When set, `apply_log_record` seeds cluster spend
625    /// into it so `may_spend` reflects all replicas' spend, not just this process's. Returns
626    /// `self` for method-chaining. Only effective when a log backend is also attached.
627    #[must_use]
628    pub fn with_paired_ledger(mut self, ledger: Arc<crate::shadow::ShadowLedger>) -> Self {
629        self.paired_ledger = Some(ledger);
630        self
631    }
632
633    /// Whether a shared log backend is currently attached.
634    #[must_use]
635    pub fn has_log(&self) -> bool {
636        self.log.is_some()
637    }
638
639    /// Perform one `sync_from_backend` the first time a `(tenant, route)` entry is used in this
640    /// process. Returns `true` when the entry is synced (either it already was, or the sync just
641    /// succeeded). Returns `false` when the log backend is unreachable; callers should fail closed
642    /// for this request (serve incumbent / skip evaluation).
643    ///
644    /// **Why this is needed.** `build_paired_ledger` starts empty when `shared_log=true`, and
645    /// `start_sync_task` only syncs entries already in the map. A brand-new replica (or a
646    /// `(tenant, route)` pair not yet seen on this instance) would check `challenger_blocked` and
647    /// `may_spend` against nothing, silently bypassing a cluster-level halt decision or the daily
648    /// spend cap. One synchronous sync before the first check closes that window.
649    ///
650    /// **Burst coalescing.** A per-entry `tokio::sync::Mutex` gate serialises concurrent
651    /// first-request callers so exactly one LRANGE is issued per burst. Subsequent waiters
652    /// re-check `last_synced_at` after acquiring the gate and return immediately. `last_synced_at` is
653    /// set only on success; on failure it stays false (so the next burst retries), but a warning
654    /// is logged only once per entry to avoid spam during a persistent outage.
655    ///
656    /// Only the first request per `(tenant, route_ix)` per process pays one backend LRANGE.
657    pub async fn ensure_synced(&self, tenant: &str, route_ix: usize, fingerprint: &str) -> bool {
658        if self.log.is_none() {
659            return true; // no log = always "synced" (local is authoritative)
660        }
661
662        // ── Quick path: inspect current state without acquiring the async gate ──────────────
663        // Done first so the common steady-state case never touches sync_gates.
664        {
665            let mut map = self.lock();
666            match self.entry(&mut map, tenant, route_ix) {
667                None => return true, // no challenger on this route
668                Some(e) => {
669                    // State is fresh: last successful sync was within the staleness bound.
670                    // 30 s = 6 × the 5 s background interval, so one missed tick is fine.
671                    if e.last_synced_at
672                        .is_some_and(|t| t.elapsed() <= SYNC_STALENESS_BOUND)
673                    {
674                        return true;
675                    }
676                    // Cooldown: a recent failure means Redis is likely still down.
677                    // Return false *before* touching sync_gates to avoid lock contention
678                    // during an outage (Issue 6a: fast-fail before acquiring the gate).
679                    if e.sync_failed_at
680                        .is_some_and(|t| t.elapsed() < SYNC_FAILURE_COOLDOWN)
681                    {
682                        return false;
683                    }
684                    // Otherwise: state is stale (or never synced) and not in cooldown.
685                    // Fall through to the gated sync path below.
686                }
687            }
688        }
689
690        // Get or create the per-entry coalescing gate. We clone the Arc while holding the
691        // outer (sync) lock, then drop the lock before awaiting the async gate.
692        let gate = {
693            let mut gates = self
694                .sync_gates
695                .lock()
696                .unwrap_or_else(PoisonError::into_inner);
697            gates
698                .entry((tenant.to_owned(), route_ix))
699                .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
700                .clone()
701        };
702
703        // All concurrent callers block here on the same gate.
704        // Only one calls the backend; the others re-check after the winner finishes.
705        let _guard = gate.lock().await;
706
707        // Re-check after acquiring the gate: a concurrent caller may have synced while we
708        // waited, or the failure cooldown may now be active.
709        {
710            let map = self.lock();
711            if let Some(entry) = map.get(tenant).and_then(|r| r.get(&route_ix)) {
712                if entry
713                    .last_synced_at
714                    .is_some_and(|t| t.elapsed() <= SYNC_STALENESS_BOUND)
715                {
716                    return true;
717                }
718                if entry
719                    .sync_failed_at
720                    .is_some_and(|t| t.elapsed() < SYNC_FAILURE_COOLDOWN)
721                {
722                    return false;
723                }
724            }
725        }
726
727        // Perform the sync with a hard timeout so a hung Redis cannot stall live requests.
728        // On timeout we fail closed (same as any backend error) and activate the cooldown.
729        let sync_result = tokio::time::timeout(
730            SYNC_TIMEOUT,
731            self.sync_from_backend(tenant, route_ix, fingerprint),
732        )
733        .await;
734
735        match sync_result {
736            Ok(Ok(())) => {
737                // sync_from_backend sets last_synced_at on success.
738                // Once synced, remove the gate so subsequent callers take the quick path
739                // instead of re-acquiring an Arc that will never be needed again (Issue 6b).
740                let mut gates = self
741                    .sync_gates
742                    .lock()
743                    .unwrap_or_else(PoisonError::into_inner);
744                gates.remove(&(tenant.to_owned(), route_ix));
745                true
746            }
747            Ok(Err(e)) => {
748                let mut map = self.lock();
749                if let Some(entry) = self.entry(&mut map, tenant, route_ix) {
750                    entry.sync_failed_at = Some(tokio::time::Instant::now());
751                    if !entry.sync_warn_logged {
752                        entry.sync_warn_logged = true;
753                        tracing::warn!(
754                            tenant,
755                            route = route_ix,
756                            error = %e,
757                            "shared log unreachable; halt guard and spend cap may use stale \
758                             state — serving incumbent only for this request"
759                        );
760                    }
761                }
762                false
763            }
764            Err(_elapsed) => {
765                // Backend timed out: fail closed + activate cooldown.
766                let mut map = self.lock();
767                if let Some(entry) = self.entry(&mut map, tenant, route_ix) {
768                    entry.sync_failed_at = Some(tokio::time::Instant::now());
769                    if !entry.sync_warn_logged {
770                        entry.sync_warn_logged = true;
771                        tracing::warn!(
772                            tenant,
773                            route = route_ix,
774                            "shared log sync timed out (>{} ms); serving incumbent only",
775                            SYNC_TIMEOUT.as_millis()
776                        );
777                    }
778                }
779                false
780            }
781        }
782    }
783
784    /// The fingerprint receipts for `route_ix` must carry to count, if it has a challenger.
785    #[must_use]
786    pub fn fingerprint(&self, route_ix: usize) -> Option<&str> {
787        self.params.get(&route_ix).map(|p| p.fingerprint.as_str())
788    }
789
790    /// Whether the serving path must NOT use the challenger for `(tenant, route_ix)`: the
791    /// experiment is halted, or its history was unreadable at startup.
792    #[must_use]
793    pub fn challenger_blocked(&self, tenant: &str, route_ix: usize) -> bool {
794        self.unavailable || self.status(tenant, route_ix) == ExperimentStatus::Halted
795    }
796
797    /// Whether any route runs an experiment.
798    #[must_use]
799    pub fn is_active(&self) -> bool {
800        !self.params.is_empty()
801    }
802
803    fn lock(&self) -> MutexGuard<'_, HashMap<String, HashMap<usize, Entry>>> {
804        // A poisoned lock only means another thread panicked mid-update of plain counters; the
805        // state is still usable, and the halt guard must not turn into a request-path panic.
806        self.entries.lock().unwrap_or_else(PoisonError::into_inner)
807    }
808
809    fn entry<'a>(
810        &self,
811        map: &'a mut HashMap<String, HashMap<usize, Entry>>,
812        tenant: &str,
813        route_ix: usize,
814    ) -> Option<&'a mut Entry> {
815        let p = self.params.get(&route_ix)?;
816        let (margin, alpha) = (p.margin, p.alpha);
817        if !map.contains_key(tenant) {
818            map.insert(tenant.to_owned(), HashMap::new());
819        }
820        let routes = map.get_mut(tenant)?;
821        if let std::collections::hash_map::Entry::Vacant(v) = routes.entry(route_ix) {
822            match Experiment::new(margin, alpha) {
823                Ok(experiment) => {
824                    v.insert(Entry {
825                        experiment,
826                        ignored_stale: 0,
827                        health: Health::default(),
828                        sessions: HashSet::new(),
829                        saturated_logged: false,
830                        applied_pair_ids: HashSet::new(),
831                        locally_costed: HashSet::new(),
832                        remote_log_offset: 0,
833                        sync_warn_logged: false,
834                        sync_failed_at: None,
835                        last_synced_at: None,
836                    });
837                }
838                Err(e) => {
839                    // Unreachable for a parsed config; fail open (no experiment) rather than panic.
840                    tracing::warn!(route = route_ix, %e, "experiment parameters rejected");
841                    return None;
842                }
843            }
844        }
845        routes.get_mut(&route_ix)
846    }
847
848    /// Where the experiment for `(tenant, route_ix)` stands. [`ExperimentStatus::Undecided`] when
849    /// nothing has been observed or the route has no challenger. The serving path stops using the
850    /// challenger when this is [`ExperimentStatus::Halted`].
851    #[must_use]
852    pub fn status(&self, tenant: &str, route_ix: usize) -> ExperimentStatus {
853        self.lock()
854            .get(tenant)
855            .and_then(|r| r.get(&route_ix))
856            .map_or(ExperimentStatus::Undecided, |e| e.experiment.status())
857    }
858
859    /// Record the cost of one served request of `arm`. Call with `trace.final_.total_cost_usd`.
860    ///
861    /// `fingerprint` is the receipt's; a mismatch with the current config counts the receipt in
862    /// `ignored_stale` and records nothing.
863    pub fn observe_cost(
864        &self,
865        tenant: &str,
866        route_ix: usize,
867        fingerprint: &str,
868        arm: Arm,
869        usd: f64,
870    ) {
871        let current = self.fingerprint(route_ix) == Some(fingerprint);
872        let mut map = self.lock();
873        if let Some(e) = self.entry(&mut map, tenant, route_ix) {
874            if current {
875                e.experiment.observe_cost(arm, usd);
876            } else {
877                e.ignored_stale += 1;
878            }
879        }
880    }
881
882    /// Tally one request of `arm` that errored and/or produced answers cut off at the output-token
883    /// limit (`truncated` counts attempts). Stale fingerprints are ignored: `observe_cost` already
884    /// counts the receipt once.
885    pub fn observe_health(
886        &self,
887        tenant: &str,
888        route_ix: usize,
889        fingerprint: &str,
890        arm: Arm,
891        errored: bool,
892        truncated: u64,
893    ) {
894        if (!errored && truncated == 0) || self.fingerprint(route_ix) != Some(fingerprint) {
895            return;
896        }
897        let mut map = self.lock();
898        if let Some(e) = self.entry(&mut map, tenant, route_ix) {
899            let h = match arm {
900                Arm::Incumbent => &mut e.health.incumbent,
901                Arm::Challenger => &mut e.health.challenger,
902            };
903            h.errors += u64::from(errored);
904            h.truncated += truncated;
905        }
906    }
907
908    /// Record a session's task outcome. Returns whether it was counted.
909    ///
910    /// Only the **first** clear outcome per session counts; later ones are ignored. The test pairs
911    /// the k-th incumbent outcome with the k-th challenger outcome and treats sessions as
912    /// independent draws, so several correlated outcomes from one long agent run would both
913    /// overweight that session and break the independence the error bound rests on.
914    ///
915    /// `Pass` is 1.0, `Fail` is 0.0, `Abstain` carries no information and is ignored.
916    pub fn observe_outcome(
917        &self,
918        tenant: &str,
919        route_ix: usize,
920        fingerprint: &str,
921        arm: Arm,
922        session_id: &str,
923        verdict: Verdict,
924    ) -> bool {
925        // A stale receipt is counted once, by `observe_cost`; here it is just not evidence.
926        if self.fingerprint(route_ix) != Some(fingerprint) {
927            return false;
928        }
929        let outcome = match verdict {
930            Verdict::Pass => 1.0,
931            Verdict::Fail => 0.0,
932            Verdict::Abstain => return false,
933        };
934        let mut map = self.lock();
935        let Some(e) = self.entry(&mut map, tenant, route_ix) else {
936            return false;
937        };
938        let key = session_hash(session_id);
939        if e.sessions.contains(&key) {
940            return false;
941        }
942        if e.sessions.len() >= MAX_TRACKED_SESSIONS {
943            if !e.saturated_logged {
944                e.saturated_logged = true;
945                tracing::warn!(
946                    tenant,
947                    route = route_ix,
948                    "experiment session ledger is full; new sessions' outcomes are ignored"
949                );
950            }
951            return false;
952        }
953        let before = e.experiment.status();
954        if !e.experiment.observe_outcome(arm, outcome) {
955            return false;
956        }
957        e.sessions.insert(key);
958        let after = e.experiment.status();
959        if before != after {
960            match after {
961                ExperimentStatus::Halted => tracing::error!(
962                    tenant,
963                    route = route_ix,
964                    "EXPERIMENT HALTED: the challenger ladder is worse than the incumbent by more \
965                     than the margin — it is no longer served"
966                ),
967                ExperimentStatus::Certified => tracing::info!(
968                    tenant,
969                    route = route_ix,
970                    "experiment certified: the challenger is within the margin of the incumbent; \
971                     adopting it is a human decision (`firstpass experiment`)"
972                ),
973                ExperimentStatus::Undecided => {}
974            }
975            Self::publish(&map, route_ix);
976        }
977        true
978    }
979
980    /// Export the route's status as `firstpass_experiment_status{route}` (0 undecided, 1 certified,
981    /// 2 halted). The label is the route index — bounded by the config, never by traffic — and
982    /// tenants are deliberately not a label (unbounded in a hosted deployment), so the value is the
983    /// worst status across tenants: a halt anywhere is visible.
984    fn publish(map: &HashMap<String, HashMap<usize, Entry>>, route_ix: usize) {
985        let worst = map
986            .values()
987            .filter_map(|r| r.get(&route_ix))
988            .map(|e| status_code(e.experiment.status()))
989            .fold(0.0, f64::max);
990        metrics::gauge!("firstpass_experiment_status", "route" => route_ix.to_string()).set(worst);
991    }
992
993    /// A snapshot of every experiment, ordered by `(tenant, route_ix)`.
994    #[must_use]
995    pub fn reports(&self) -> Vec<Snapshot> {
996        let map = self.lock();
997        let mut out: Vec<_> = map
998            .iter()
999            .flat_map(|(t, routes)| {
1000                routes.iter().map(move |(ix, e)| Snapshot {
1001                    tenant: t.clone(),
1002                    route_ix: *ix,
1003                    report: e.experiment.report(),
1004                    ignored_stale: e.ignored_stale,
1005                    health: e.health,
1006                })
1007            })
1008            .collect();
1009        out.sort_by(|a, b| (&a.tenant, a.route_ix).cmp(&(&b.tenant, b.route_ix)));
1010        out
1011    }
1012
1013    // ── Shared-log async methods ──────────────────────────────────────────────────────────────
1014
1015    /// Apply a single [`LogRecord`] from the shared backend to local in-memory state.
1016    ///
1017    /// All records arriving through `sync_from_backend` already carry the correct fingerprint
1018    /// (they were keyed by it). This check is defense-in-depth. Returns `true` if the experiment
1019    /// status changed so the caller can call `Self::publish`.
1020    fn apply_log_record(
1021        &self,
1022        map: &mut HashMap<String, HashMap<usize, Entry>>,
1023        tenant: &str,
1024        route_ix: usize,
1025        fingerprint: &str,
1026        record: &LogRecord,
1027    ) -> bool {
1028        if self.fingerprint(route_ix) != Some(fingerprint) {
1029            if let Some(e) = self.entry(map, tenant, route_ix) {
1030                e.ignored_stale += 1;
1031            }
1032            return false;
1033        }
1034        match record {
1035            LogRecord::Outcome {
1036                arm,
1037                outcome,
1038                session_hash: hash,
1039                ..
1040            } => {
1041                let Some(e) = self.entry(map, tenant, route_ix) else {
1042                    return false;
1043                };
1044                if e.sessions.contains(hash) {
1045                    return false; // idempotent re-apply on overlapping LRANGE
1046                }
1047                if e.sessions.len() >= MAX_TRACKED_SESSIONS {
1048                    return false;
1049                }
1050                let before = e.experiment.status();
1051                if !e.experiment.observe_outcome(*arm, *outcome) {
1052                    return false;
1053                }
1054                e.sessions.insert(*hash);
1055                before != e.experiment.status()
1056            }
1057            LogRecord::Pair {
1058                x_inc,
1059                x_ch,
1060                incumbent_cost_usd,
1061                challenger_cost_usd,
1062                ch_error,
1063                ch_truncated,
1064                ts_unix,
1065                replica_id: record_replica_id,
1066                trace_id: record_trace_id,
1067                incumbent_trace_id,
1068            } => {
1069                let Some(e) = self.entry(map, tenant, route_ix) else {
1070                    return false;
1071                };
1072                // Dedup by trace_id: a pair whose append timed out after Redis applied it
1073                // may have already been counted locally (in record_pair's failure path or
1074                // replayed from SQLite at startup). Skip evidence if the id is already known.
1075                // Nil UUID (old records without trace_id) bypasses dedup — always applied.
1076                let already_applied =
1077                    !record_trace_id.is_nil() && e.applied_pair_ids.contains(record_trace_id);
1078                let before = e.experiment.status();
1079                if !already_applied {
1080                    if e.applied_pair_ids.len() >= MAX_TRACKED_SESSIONS {
1081                        // Saturated: skip evidence rather than evicting (evicting lets a pair
1082                        // count twice, breaking the pairing argument).
1083                        if !e.saturated_logged {
1084                            e.saturated_logged = true;
1085                            tracing::warn!(
1086                                tenant,
1087                                route = route_ix,
1088                                "experiment pair-id set is full; new pairs' evidence is skipped"
1089                            );
1090                        }
1091                    } else {
1092                        // Evidence (e-process wealth) is applied for ALL non-deduped records —
1093                        // own and remote — because in shared-log mode the log is the canonical
1094                        // source of evidence for all replicas.
1095                        if e.experiment.observe_pair(*x_inc, *x_ch) {
1096                            // The incumbent's cost is already counted on the serving path; only
1097                            // the off-path challenger call is new spend.
1098                            e.experiment
1099                                .observe_cost(Arm::Challenger, *challenger_cost_usd);
1100                        }
1101                        if !record_trace_id.is_nil() {
1102                            e.applied_pair_ids.insert(*record_trace_id);
1103                        }
1104                    }
1105                }
1106                // Health and spend for remote records are nested inside `!already_applied`:
1107                // if `already_applied` is true, the originating replica counted them locally
1108                // before restart (via SQLite replay in `monitor_from_traces` which rebuilds
1109                // health from persisted ch_error/ch_truncated and spend via build_paired_ledger).
1110                // Applying them again here would double-count health and spend.
1111                //
1112                // Skipping own records (is_own) prevents double-counting on the live path:
1113                // health and spend were tallied in `spawn_paired_challenger` before append.
1114                let is_own = *record_replica_id == self.replica_id;
1115                if !already_applied && !is_own {
1116                    let costed_here =
1117                        !incumbent_trace_id.is_nil() && e.locally_costed.remove(incumbent_trace_id);
1118                    // The originating replica counted the incumbent's cost when it served the
1119                    // request; every other replica learns it only from the log. Without this a
1120                    // cluster's incumbent looks ~N times cheaper and a certified, cheaper
1121                    // challenger would be reported "certified, no saving" instead of "adopt".
1122                    if !costed_here {
1123                        e.experiment
1124                            .observe_cost(Arm::Incumbent, *incumbent_cost_usd);
1125                    }
1126                    if *ch_error || *ch_truncated > 0 {
1127                        e.health.challenger.errors += u64::from(*ch_error);
1128                        e.health.challenger.truncated += *ch_truncated;
1129                    }
1130                    if let Some(ledger) = &self.paired_ledger {
1131                        let now = jiff::Timestamp::now();
1132                        let pair_ts = jiff::Timestamp::from_second(*ts_unix)
1133                            .unwrap_or(jiff::Timestamp::UNIX_EPOCH);
1134                        ledger.seed(tenant, route_ix, *challenger_cost_usd, pair_ts, now);
1135                    }
1136                }
1137                before != e.experiment.status()
1138            }
1139        }
1140    }
1141
1142    /// Read records appended to the shared log since the last sync and apply them locally.
1143    ///
1144    /// Each call is cheap when there are no new records: one LRANGE that returns empty.
1145    /// Call this after every feedback event (within the feedback handler, not on the hot
1146    /// serve path) and from a background timer so all replicas stay ≤ 5 s stale.
1147    ///
1148    /// **Concurrent-call safety.** `record_pair` and `start_sync_task` both call this
1149    /// concurrently on the same `(tenant, route_ix)`. Both may snapshot the same
1150    /// `remote_log_offset` before either updates it, load the same slice from the backend,
1151    /// and then race to apply it. The fix is two-part:
1152    /// 1. After re-acquiring the lock, skip any record whose absolute index
1153    ///    `offset + i` is already covered by the current `remote_log_offset` (a concurrent
1154    ///    call advanced it while we were awaiting `load_since`).
1155    /// 2. Only ever advance `remote_log_offset` — never rewind it — so a slower
1156    ///    concurrent call that loaded a shorter slice cannot wind the cursor back.
1157    ///
1158    /// No-op (returns `Ok`) when no log backend is configured.
1159    /// Returns `Err` if the backend is unreachable; callers should treat a failure as
1160    /// "state unknown" and fail closed (do not serve the challenger for this request).
1161    pub async fn sync_from_backend(
1162        &self,
1163        tenant: &str,
1164        route_ix: usize,
1165        fingerprint: &str,
1166    ) -> Result<(), LogError> {
1167        let Some(log) = &self.log else {
1168            return Ok(());
1169        };
1170        // Snapshot offset before dropping the lock to await I/O.
1171        let offset = {
1172            let mut map = self.lock();
1173            self.entry(&mut map, tenant, route_ix)
1174                .map_or(0, |e| e.remote_log_offset)
1175        };
1176        // Lock is dropped here. Another sync may advance `remote_log_offset` while we await.
1177        let records = log
1178            .load_since(tenant, route_ix, fingerprint, offset)
1179            .await?;
1180        if records.is_empty() {
1181            // Successful read (log reachable), just nothing new. Mark synced so a background
1182            // sync spares the first explicit request from a cold-start LRANGE.
1183            let mut map = self.lock();
1184            if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
1185                e.last_synced_at = Some(tokio::time::Instant::now());
1186                // A recovered log re-arms the once-per-outage warning for the next outage.
1187                e.sync_warn_logged = false;
1188                e.sync_failed_at = None;
1189            }
1190            return Ok(());
1191        }
1192        let mut map = self.lock();
1193        for (i, opt_rec) in records.iter().enumerate() {
1194            let abs_ix = offset + i as u64;
1195            // Skip records already applied by a concurrent sync that ran ahead of us.
1196            let current_offset = map
1197                .get(tenant)
1198                .and_then(|r| r.get(&route_ix))
1199                .map_or(0, |e| e.remote_log_offset);
1200            if abs_ix < current_offset {
1201                continue;
1202            }
1203            // Malformed entries (None): advance the offset but apply nothing.
1204            let changed = if let Some(rec) = opt_rec {
1205                self.apply_log_record(&mut map, tenant, route_ix, fingerprint, rec)
1206            } else {
1207                false
1208            };
1209            // Monotonic advance: never rewind the offset even if a concurrent call loaded
1210            // a shorter prefix.  Update regardless of whether the record was applied (a
1211            // stale-fingerprint record still must not be re-read on the next sync).
1212            if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
1213                e.remote_log_offset = e.remote_log_offset.max(abs_ix + 1);
1214            }
1215            if changed {
1216                let after = map
1217                    .get(tenant)
1218                    .and_then(|r| r.get(&route_ix))
1219                    .map(|e| e.experiment.status());
1220                match after {
1221                    Some(ExperimentStatus::Halted) => tracing::error!(
1222                        tenant,
1223                        route = route_ix,
1224                        "EXPERIMENT HALTED (shared log): challenger worse than incumbent"
1225                    ),
1226                    Some(ExperimentStatus::Certified) => tracing::info!(
1227                        tenant,
1228                        route = route_ix,
1229                        "experiment certified (shared log)"
1230                    ),
1231                    _ => {}
1232                }
1233                Self::publish(&map, route_ix);
1234            }
1235        }
1236        // Mark synced: a successful background sync spares the first explicit request from
1237        // a cold-start LRANGE and marks the state as fresh for the staleness bound.
1238        if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
1239            e.last_synced_at = Some(tokio::time::Instant::now());
1240            // A recovered log re-arms the once-per-outage warning for the next outage.
1241            e.sync_warn_logged = false;
1242            e.sync_failed_at = None;
1243        }
1244        Ok(())
1245    }
1246
1247    /// Record an outcome to the shared log (if configured), then sync from the log.
1248    ///
1249    /// In single-replica mode (no log), falls back to the synchronous [`Self::observe_outcome`].
1250    /// `cost_usd` is stored in the log so other replicas can tally costs without needing the
1251    /// full trace. Returns `true` when the outcome was counted.
1252    #[allow(clippy::too_many_arguments)]
1253    pub async fn record_outcome(
1254        &self,
1255        tenant: &str,
1256        route_ix: usize,
1257        fingerprint: &str,
1258        arm: Arm,
1259        session_id: &str,
1260        verdict: Verdict,
1261        cost_usd: f64,
1262    ) -> bool {
1263        // Filter Abstain before claim_session: an abstain carries no evidence, and burning the
1264        // session slot would prevent a later non-Abstain verdict for the same session from being
1265        // counted across replicas.
1266        let outcome = match verdict {
1267            Verdict::Pass => 1.0,
1268            Verdict::Fail => 0.0,
1269            Verdict::Abstain => return false,
1270        };
1271        let Some(log) = self.log.as_ref().cloned() else {
1272            return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
1273        };
1274        // Cross-replica session dedup: only one replica may claim a given session.
1275        if !log
1276            .claim_session(tenant, route_ix, fingerprint, session_id)
1277            .await
1278        {
1279            return false;
1280        }
1281        if let Err(e) = log
1282            .append(
1283                tenant,
1284                route_ix,
1285                fingerprint,
1286                LogRecord::Outcome {
1287                    arm,
1288                    outcome,
1289                    session_hash: session_hash(session_id),
1290                    cost_usd,
1291                },
1292            )
1293            .await
1294        {
1295            tracing::warn!(
1296                tenant,
1297                route = route_ix,
1298                error = %e,
1299                "outcome log append failed; evidence will not reach other replicas"
1300            );
1301            // Fall back to local state so at least this replica counts the outcome.
1302            return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
1303        }
1304        // Pull in any records other replicas appended since last sync, plus the one we
1305        // just appended (so local state is always up-to-date after an append).
1306        // A sync failure is non-fatal: we'll catch up on the next background sync.
1307        let _ = self.sync_from_backend(tenant, route_ix, fingerprint).await;
1308        true
1309    }
1310
1311    /// Apply a paired shadow result directly to local in-memory state.
1312    ///
1313    /// **Call only after the pair has been durably recorded** (written to SQLite as `NotInLog`).
1314    /// This ordering guarantee is what `spawn_paired_challenger` enforces: the SQLite row is the
1315    /// only durable record for local-only pairs, and the evidence must not exist in memory without
1316    /// it — otherwise a crash between the evidence apply and the write lets a halted challenger
1317    /// run again on restart (it has evidence but no persisted state to replay the halt from).
1318    ///
1319    /// Deduplicates by `trace_id` so a pair is never counted twice even if this method is called
1320    /// from both the local path and the log-sync path. `Uuid::nil()` bypasses dedup.
1321    ///
1322    /// Returns `true` if the pair was accepted by the underlying [`Experiment`].
1323    #[allow(clippy::too_many_arguments)]
1324    pub fn apply_pair_locally(
1325        &self,
1326        tenant: &str,
1327        route_ix: usize,
1328        fingerprint: &str,
1329        x_inc: f64,
1330        x_ch: f64,
1331        challenger_cost_usd: f64,
1332        trace_id: Uuid,
1333    ) -> bool {
1334        if self.fingerprint(route_ix) != Some(fingerprint) {
1335            return false;
1336        }
1337        let mut map = self.lock();
1338        let Some(e) = self.entry(&mut map, tenant, route_ix) else {
1339            return false;
1340        };
1341        // Cross-source dedup: skip if this trace_id was already counted (e.g., via a concurrent
1342        // log sync that saw the same record, or a previous local apply call).
1343        let already_applied = !trace_id.is_nil() && e.applied_pair_ids.contains(&trace_id);
1344        if already_applied {
1345            return false;
1346        }
1347        if e.applied_pair_ids.len() >= MAX_TRACKED_SESSIONS {
1348            if !e.saturated_logged {
1349                e.saturated_logged = true;
1350                tracing::warn!(
1351                    tenant,
1352                    route = route_ix,
1353                    "experiment pair-id set is full; new pairs' evidence is skipped"
1354                );
1355            }
1356            return false;
1357        }
1358        let before = e.experiment.status();
1359        let ok = e.experiment.observe_pair(x_inc, x_ch);
1360        if ok {
1361            // Without this the challenger's $/request reads n/a, so a certified cheaper
1362            // challenger could never be recommended for adoption.
1363            e.experiment
1364                .observe_cost(Arm::Challenger, challenger_cost_usd);
1365        }
1366        if !trace_id.is_nil() {
1367            e.applied_pair_ids.insert(trace_id);
1368        }
1369        if ok && before != e.experiment.status() {
1370            match e.experiment.status() {
1371                ExperimentStatus::Halted => tracing::error!(
1372                    tenant,
1373                    route = route_ix,
1374                    "EXPERIMENT HALTED: paired challenger worse than incumbent (local evidence)"
1375                ),
1376                ExperimentStatus::Certified => tracing::info!(
1377                    tenant,
1378                    route = route_ix,
1379                    "experiment certified (paired, local evidence): challenger within margin"
1380                ),
1381                ExperimentStatus::Undecided => {}
1382            }
1383            Self::publish(&map, route_ix);
1384        }
1385        ok
1386    }
1387
1388    /// Try to append a paired shadow result to the shared log (if configured), then sync.
1389    ///
1390    /// **Durability contract for `spawn_paired_challenger`:**
1391    ///
1392    /// - Returns `true` → the record is in the shared log (the durable record). Evidence is
1393    ///   applied via `sync_from_backend` inside this call. Caller writes the SQLite row as
1394    ///   `InLog` after this returns (the row is just a startup-rebuild cache; it can follow
1395    ///   the log write safely).
1396    /// - Returns `false` → NOT in the log (no log configured, or the append failed). Evidence
1397    ///   is NOT applied here. The caller must write the SQLite row as `NotInLog` **first**,
1398    ///   then call [`Self::apply_pair_locally`] only on success. This ordering ensures the
1399    ///   durable record exists before evidence is in memory: a crash between the two would
1400    ///   leave no evidence, which is the safe direction (a missed pair only slows the
1401    ///   experiment; evidence without a durable record could mislead a halt).
1402    ///
1403    /// `ch_error`, `ch_truncated`, and `ts_unix` are forwarded into [`LogRecord::Pair`] so other
1404    /// replicas can rebuild challenger health and seed the paired-spend ledger.
1405    ///
1406    /// `trace_id` is used for cross-source dedup: if Redis applied the record but the append
1407    /// returned `Err` (timeout), the next sync sees the same trace_id and skips evidence it
1408    /// already counted locally. Pass `Uuid::nil()` in tests not exercising dedup.
1409    #[allow(clippy::too_many_arguments)]
1410    pub async fn record_pair(
1411        &self,
1412        tenant: &str,
1413        route_ix: usize,
1414        fingerprint: &str,
1415        x_inc: f64,
1416        x_ch: f64,
1417        incumbent_cost_usd: f64,
1418        challenger_cost_usd: f64,
1419        ch_error: bool,
1420        ch_truncated: u64,
1421        ts_unix: i64,
1422        trace_id: Uuid,
1423        incumbent_trace_id: Uuid,
1424    ) -> bool {
1425        if self.fingerprint(route_ix) != Some(fingerprint) {
1426            return false;
1427        }
1428        let Some(log) = self.log.as_ref().cloned() else {
1429            // No shared log: this replica is the only one; evidence is NOT applied here.
1430            // The caller (spawn_paired_challenger) writes SQLite first, then calls
1431            // apply_pair_locally so the durable record precedes the in-memory evidence.
1432            return false;
1433        };
1434        // Validate before appending: an invalid record would permanently pollute the log.
1435        if !x_inc.is_finite()
1436            || !(0.0..=1.0).contains(&x_inc)
1437            || !x_ch.is_finite()
1438            || !(0.0..=1.0).contains(&x_ch)
1439        {
1440            return false;
1441        }
1442        // Health and ledger were already tallied locally in `spawn_paired_challenger` (always,
1443        // regardless of log state). The caller's local debit is the single source of truth;
1444        // `apply_log_record` will skip re-tallying for own records (matched by replica_id).
1445        // An append failure means the evidence will not reach other replicas, which only delays
1446        // a decision — it does not affect local accounting.
1447        if let Err(e) = bounded(log.append(
1448            tenant,
1449            route_ix,
1450            fingerprint,
1451            LogRecord::Pair {
1452                x_inc,
1453                x_ch,
1454                incumbent_cost_usd,
1455                challenger_cost_usd,
1456                ch_error,
1457                ch_truncated,
1458                ts_unix,
1459                replica_id: self.replica_id,
1460                trace_id,
1461                incumbent_trace_id,
1462            },
1463        ))
1464        .await
1465        {
1466            tracing::warn!(
1467                tenant,
1468                route = route_ix,
1469                error = %e,
1470                "paired log append failed; evidence withheld until SQLite write succeeds \
1471                 (local spend and health were already tallied)"
1472            );
1473            // NOT in the log: caller writes SQLite (NotInLog) then calls apply_pair_locally.
1474            return false;
1475        }
1476        // Sync to apply own record immediately (so this replica sees its own evidence) and
1477        // to pick up any remote records. If the read-back fails, apply this pair locally now:
1478        // the record IS in the log, so the later successful sync will see the same trace_id and
1479        // skip it (no double count), while a decisive halt is not delayed in the meantime.
1480        if bounded(self.sync_from_backend(tenant, route_ix, fingerprint))
1481            .await
1482            .is_err()
1483        {
1484            self.apply_pair_locally(
1485                tenant,
1486                route_ix,
1487                fingerprint,
1488                x_inc,
1489                x_ch,
1490                challenger_cost_usd,
1491                trace_id,
1492            );
1493        }
1494        true
1495    }
1496
1497    /// Spawn a background task that calls `sync_from_backend` for every known experiment at
1498    /// most every `interval` (spec: ≤ 5 s). The task exits silently when the `Arc` becomes
1499    /// the last reference. No-op when no log backend is configured.
1500    pub fn start_sync_task(self: &Arc<Self>, interval: std::time::Duration) {
1501        if self.log.is_none() {
1502            return;
1503        }
1504        let monitor = Arc::clone(self);
1505        tokio::spawn(async move {
1506            let mut ticker = tokio::time::interval(interval);
1507            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1508            loop {
1509                ticker.tick().await;
1510                // Collect (tenant, route_ix, fingerprint) tuples while holding the lock,
1511                // then drop the lock before each async sync call.
1512                let keys: Vec<(String, usize, String)> = {
1513                    let map = monitor.lock();
1514                    let mut v = Vec::new();
1515                    for (tenant, routes) in map.iter() {
1516                        for &route_ix in routes.keys() {
1517                            if let Some(p) = monitor.params.get(&route_ix) {
1518                                v.push((tenant.clone(), route_ix, p.fingerprint.clone()));
1519                            }
1520                        }
1521                    }
1522                    v
1523                };
1524                for (tenant, route_ix, fingerprint) in keys {
1525                    if let Err(e) =
1526                        bounded(monitor.sync_from_backend(&tenant, route_ix, &fingerprint)).await
1527                    {
1528                        // Record the failure time so `ensure_synced` sees the backend as
1529                        // unavailable and transitions to stale → fail-closed once the
1530                        // staleness bound is exceeded. Without this update the staleness
1531                        // bound would tick down against `last_synced_at` from the last
1532                        // success but the cooldown would never activate, causing every
1533                        // `ensure_synced` caller to re-attempt the failing backend.
1534                        let mut map = monitor.lock();
1535                        if let Some(e_entry) =
1536                            map.get_mut(&tenant).and_then(|r| r.get_mut(&route_ix))
1537                        {
1538                            e_entry.sync_failed_at = Some(tokio::time::Instant::now());
1539                        }
1540                        tracing::warn!(
1541                            tenant,
1542                            route = route_ix,
1543                            error = %e,
1544                            "background experiment sync failed"
1545                        );
1546                    }
1547                }
1548            }
1549        });
1550    }
1551}
1552
1553/// One experiment as the monitor holds it.
1554#[derive(Debug, Clone, PartialEq)]
1555pub struct Snapshot {
1556    /// Tenant the experiment belongs to.
1557    pub tenant: String,
1558    /// Index of the route in the config.
1559    pub route_ix: usize,
1560    /// The test's state and arm totals.
1561    pub report: ExperimentReport,
1562    /// Receipts from a previous experiment configuration that were ignored.
1563    pub ignored_stale: u64,
1564    /// Errors and cut-off answers per arm.
1565    pub health: Health,
1566}
1567
1568/// Requests an arm could not answer cleanly. Neither kind is random: errors (timeouts above all)
1569/// and output-limit cut-offs cluster on long, hard requests. An incumbent error drops its pair, so
1570/// many of them mean the pass rates describe an easier workload than the route serves; a
1571/// challenger error counts as a failed pair. Truncated answers are graded as they are, so a high
1572/// count says the route's `max_tokens` may be deciding the comparison.
1573#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
1574pub struct ArmHealth {
1575    /// Requests that ended in an error, with no output to grade.
1576    pub errors: u64,
1577    /// Attempts whose answer stopped at the output-token limit.
1578    pub truncated: u64,
1579}
1580
1581/// [`ArmHealth`] for both arms. For a **paired** challenger the counts are persisted in
1582/// `experiment_pairs` and in [`LogRecord::Pair`], so they survive restarts and are shared across
1583/// replicas when a shared log backend is attached.
1584#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
1585pub struct Health {
1586    /// The incumbent arm (rebuilt from receipts at startup).
1587    pub incumbent: ArmHealth,
1588    /// The challenger arm.
1589    pub challenger: ArmHealth,
1590}
1591
1592/// Number of attempts in `t` that stopped at the output-token limit.
1593#[must_use]
1594pub fn truncated_attempts(t: &Trace) -> u64 {
1595    t.attempts.iter().filter(|a| a.truncated).count() as u64
1596}
1597
1598/// Whether a trace may be used for rung-indexed or threshold analyses (`ope`, `calibrate`).
1599/// Challenger-arm traces are excluded: their rungs index a different ladder and their gate scores
1600/// relate to correctness differently, so pooling them would silently mix two models' behaviour.
1601#[must_use]
1602pub fn is_incumbent_trace(t: &Trace) -> bool {
1603    t.experiment
1604        .as_ref()
1605        .is_none_or(|e| e.arm != Arm::Challenger)
1606}
1607
1608/// Rebuild a monitor from receipts.
1609///
1610/// `traces` are in chain order; `deferred` are `(trace_id, verdict)` in the order the feedback
1611/// arrived (the store's insertion order); `pairs` are paired-mode records in insertion order.
1612/// Costs are tallied per trace and outcomes are replayed in arrival order with the same
1613/// first-outcome-per-session rule as the live path. For sequential feedback the result is exactly
1614/// what the live monitor held. Under concurrent feedback the store's insertion order and the
1615/// monitor's lock order can differ, so the pairing — and with it `decided_at_pair` and the
1616/// e-values — may differ slightly from the live run. The replay is then the authoritative record
1617/// (it is what an auditor can re-derive), and both remain valid e-processes because arm
1618/// assignment is randomized independently of arrival order. Traces without an experiment record
1619/// (other routes, other eras) are skipped; deferred verdicts for unknown traces are ignored.
1620#[must_use]
1621pub fn monitor_from_traces(
1622    traces: &[Trace],
1623    deferred: &[(Uuid, Verdict)],
1624    pairs: &[crate::store::ExperimentPairRow],
1625    config: &Config,
1626) -> ExperimentMonitor {
1627    let monitor = ExperimentMonitor::from_config(config);
1628    if !monitor.is_active() {
1629        return monitor;
1630    }
1631    let mut by_id: HashMap<Uuid, &Trace> = HashMap::new();
1632    for t in traces {
1633        let Some(rec) = &t.experiment else { continue };
1634        // Matches the feedback path, which attributes a trace with no recorded route to route 0.
1635        let route_ix = t.route_ix.map_or(0, |r| r as usize);
1636        monitor.observe_cost(
1637            &t.tenant_id,
1638            route_ix,
1639            &rec.fingerprint,
1640            rec.arm,
1641            t.final_.total_cost_usd,
1642        );
1643        monitor.observe_health(
1644            &t.tenant_id,
1645            route_ix,
1646            &rec.fingerprint,
1647            rec.arm,
1648            t.final_.served_from == firstpass_core::ServedFrom::Error,
1649            truncated_attempts(t),
1650        );
1651        if rec.arm == Arm::Incumbent {
1652            let mut map = monitor.lock();
1653            if let Some(e) = monitor.entry(&mut map, &t.tenant_id, route_ix)
1654                && e.locally_costed.len() < MAX_TRACKED_SESSIONS
1655            {
1656                e.locally_costed.insert(t.trace_id);
1657            }
1658        }
1659        by_id.insert(t.trace_id, t);
1660    }
1661    for (id, verdict) in deferred {
1662        let Some(t) = by_id.get(id) else { continue };
1663        let Some(rec) = &t.experiment else { continue };
1664        let route_ix = t.route_ix.map_or(0, |r| r as usize);
1665        monitor.observe_outcome(
1666            &t.tenant_id,
1667            route_ix,
1668            &rec.fingerprint,
1669            rec.arm,
1670            &t.session_id,
1671            *verdict,
1672        );
1673    }
1674    // Replay paired-mode evidence in insertion order. Each pair bypasses FIFO queuing
1675    // (both outcomes arrived simultaneously in the off-path task), which is why they land
1676    // in `experiment_pairs` rather than the deferred-verdict table.
1677    //
1678    // After applying evidence, register the trace_id in applied_pair_ids so that when the
1679    // same pair later arrives from the shared log (e.g. an append that timed out but Redis
1680    // applied it) sync_from_backend skips it and does not double-count.
1681    for row in pairs {
1682        let mut map = monitor.lock();
1683        if let Some(e) = monitor.entry(&mut map, &row.tenant, row.route_ix) {
1684            if monitor.fingerprint(row.route_ix) == Some(row.fingerprint.as_str()) {
1685                // The same pair stored twice (the table has no unique trace_id) counts once.
1686                if !row.trace_id.is_nil() && e.applied_pair_ids.contains(&row.trace_id) {
1687                    continue;
1688                }
1689                if e.experiment.observe_pair(row.x_inc, row.x_ch) {
1690                    e.experiment
1691                        .observe_cost(Arm::Challenger, row.challenger_cost_usd);
1692                }
1693                // Register trace_id for cross-source dedup (prevents double-counting if the
1694                // same pair arrives from the shared log on the first sync_from_backend).
1695                if !row.trace_id.is_nil() {
1696                    e.applied_pair_ids.insert(row.trace_id);
1697                }
1698                // Rebuild challenger health from the persisted fields.
1699                if row.ch_error || row.ch_truncated > 0 {
1700                    e.health.challenger.errors += u64::from(row.ch_error);
1701                    e.health.challenger.truncated += row.ch_truncated;
1702                }
1703            } else {
1704                e.ignored_stale += 1;
1705            }
1706        }
1707    }
1708    monitor
1709}
1710
1711/// The monitor to serve with, rebuilt from the trace store at startup.
1712///
1713/// Only touches the store when some route has a challenger. **Fails closed on the challenger**: if
1714/// the store cannot be read, the proxy still starts but serves the incumbent only
1715/// ([`ExperimentMonitor::unavailable`]), because a challenger already proven worse must never be
1716/// served again just because its history could not be read.
1717#[must_use]
1718pub fn build_monitor(db_path: &str, config: &Config) -> ExperimentMonitor {
1719    build_monitor_with(db_path, config, false)
1720}
1721
1722/// [`build_monitor`], told whether a shared experiment log will be attached.
1723///
1724/// With a shared log, the log is the cluster-wide record of every paired evaluation, and the
1725/// sync task reads it from the start on boot. Replaying the local `experiment_pairs` table as
1726/// well would apply each pair twice (pairs, unlike live outcomes, carry no session to dedup on),
1727/// doubling the e-values and the challenger's spend. So pairs then only register their
1728/// experiment, and the evidence itself comes from the log on the first sync.
1729#[must_use]
1730pub fn build_monitor_with(db_path: &str, config: &Config, shared_log: bool) -> ExperimentMonitor {
1731    let empty = ExperimentMonitor::from_config(config);
1732    if !empty.is_active() {
1733        return empty;
1734    }
1735    let loaded = crate::store::load_all_traces(db_path)
1736        .and_then(|t| crate::store::load_all_deferred(db_path).map(|d| (t, d)))
1737        .and_then(|(t, d)| crate::store::load_all_experiment_pairs(db_path).map(|p| (t, d, p)));
1738    match loaded {
1739        Ok((traces, deferred, pairs)) => {
1740            // With a shared log, the restart rules for evidence are (see InLogStatus docs):
1741            //   NotInLog → replay evidence locally (written during an outage; not in Redis).
1742            //   InLog    → do NOT replay evidence; the log delivers it on the first sync.
1743            //   Unknown  → do NOT replay evidence; we cannot know if it's in the log,
1744            //              so we conservatively withhold it (delays a decision — safe).
1745            // Without a shared log: all pairs replay normally (original behaviour).
1746            let offline_pairs: Vec<crate::store::ExperimentPairRow>;
1747            let replayed: &[crate::store::ExperimentPairRow] = if shared_log {
1748                offline_pairs = pairs
1749                    .iter()
1750                    .filter(|r| r.in_log == crate::store::InLogStatus::NotInLog)
1751                    .cloned()
1752                    .collect();
1753                &offline_pairs
1754            } else {
1755                &pairs
1756            };
1757            let m = monitor_from_traces(&traces, &deferred, replayed, config);
1758            if shared_log {
1759                let mut map = m.lock();
1760                for row in &pairs {
1761                    let _ = m.entry(&mut map, &row.tenant, row.route_ix);
1762                }
1763            }
1764            for s in m.reports() {
1765                tracing::info!(
1766                    tenant = %s.tenant,
1767                    route = s.route_ix,
1768                    status = ?s.report.status,
1769                    pairs = s.report.pairs,
1770                    ignored_stale = s.ignored_stale,
1771                    "experiment restored from receipts"
1772                );
1773                ExperimentMonitor::publish(&m.lock(), s.route_ix);
1774            }
1775            m
1776        }
1777        Err(e) => {
1778            tracing::error!(
1779                %e,
1780                "could not read the trace store, so it is unknown whether a challenger was already \
1781                 halted: challenger arms are NOT served (incumbent only) until restart with a \
1782                 readable store"
1783            );
1784            ExperimentMonitor::unavailable(config)
1785        }
1786    }
1787}
1788
1789/// Replay receipts into per-`(tenant, route)` snapshots. See [`monitor_from_traces`].
1790#[must_use]
1791pub fn experiments_from_traces(
1792    traces: &[Trace],
1793    deferred: &[(Uuid, Verdict)],
1794    pairs: &[crate::store::ExperimentPairRow],
1795    config: &Config,
1796) -> Vec<Snapshot> {
1797    monitor_from_traces(traces, deferred, pairs, config).reports()
1798}
1799
1800/// What an operator should do with an experiment's current state.
1801#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
1802#[serde(rename_all = "snake_case")]
1803pub enum Recommendation {
1804    /// Certified, and the challenger costs less per request: adopt it (by editing the config).
1805    Adopt,
1806    /// Certified, but the challenger is not measurably cheaper: nothing to gain.
1807    CertifiedNoSaving,
1808    /// Proven worse by more than the margin; no longer served.
1809    ChallengerStopped,
1810    /// Not enough evidence yet.
1811    KeepRunning,
1812}
1813
1814/// The recommendation for a report. Cost is compared as dollars per served request on each arm;
1815/// missing cost data on either arm counts as "no saving" because adoption must not rest on a
1816/// number that was never measured.
1817#[must_use]
1818pub fn recommend(r: &ExperimentReport) -> Recommendation {
1819    match r.status {
1820        select::Status::Undecided => Recommendation::KeepRunning,
1821        select::Status::Halted => Recommendation::ChallengerStopped,
1822        select::Status::Certified => {
1823            match (
1824                r.challenger.cost_per_request(),
1825                r.incumbent.cost_per_request(),
1826            ) {
1827                (Some(c), Some(i)) if c < i => Recommendation::Adopt,
1828                _ => Recommendation::CertifiedNoSaving,
1829            }
1830        }
1831    }
1832}
1833
1834/// One experiment as the CLI presents it.
1835#[derive(Debug, Clone, Serialize)]
1836pub struct ExperimentView {
1837    /// Tenant the experiment belongs to.
1838    pub tenant: String,
1839    /// Index of the route in the config.
1840    pub route_ix: usize,
1841    /// The test's state and arm totals.
1842    pub report: ExperimentReport,
1843    /// Receipts from a previous experiment configuration that were ignored.
1844    pub ignored_stale: u64,
1845    /// Errors and cut-off answers per arm.
1846    pub health: Health,
1847    /// What to do about it.
1848    pub recommendation: Recommendation,
1849    /// The challenger ladder under test, so `adopt` is actionable without re-reading the config.
1850    pub challenger_ladder: Vec<String>,
1851}
1852
1853/// Pair each report with its recommendation and the challenger ladder from `config`.
1854#[must_use]
1855pub fn views(reports: Vec<Snapshot>, config: &Config) -> Vec<ExperimentView> {
1856    reports
1857        .into_iter()
1858        .map(|s| ExperimentView {
1859            recommendation: recommend(&s.report),
1860            ignored_stale: s.ignored_stale,
1861            health: s.health,
1862            challenger_ladder: config
1863                .routes
1864                .get(s.route_ix)
1865                .and_then(|r| r.challenger.as_ref())
1866                .map(|c| c.ladder.clone())
1867                .unwrap_or_default(),
1868            tenant: s.tenant,
1869            route_ix: s.route_ix,
1870            report: s.report,
1871        })
1872        .collect()
1873}
1874
1875fn fmt_opt(v: Option<f64>, f: impl Fn(f64) -> String) -> String {
1876    v.map_or_else(|| "n/a".to_owned(), f)
1877}
1878
1879/// Human-readable report, one block per `(tenant, route)`.
1880#[must_use]
1881pub fn render(views: &[ExperimentView]) -> String {
1882    use std::fmt::Write as _;
1883    if views.is_empty() {
1884        return "no experiment data yet — the config has no [route.challenger], or no experiment \
1885                traffic has been recorded\n"
1886            .to_owned();
1887    }
1888    let mut out = String::new();
1889    for v in views {
1890        let r = &v.report;
1891        let _ = writeln!(
1892            out,
1893            "tenant {} · route {} · status: {:?}",
1894            v.tenant, v.route_ix, r.status
1895        );
1896        let _ = writeln!(
1897            out,
1898            "  pairs: {} (margin {}, alpha {})",
1899            r.pairs, r.margin, r.alpha
1900        );
1901        for (name, a) in [("incumbent ", &r.incumbent), ("challenger", &r.challenger)] {
1902            let _ = writeln!(
1903                out,
1904                "  {name}: success {} over {} sessions · ${} per request over {} requests",
1905                fmt_opt(a.success_rate(), |s| format!("{:.1}%", s * 100.0)),
1906                a.outcomes,
1907                fmt_opt(a.cost_per_request(), |c| format!("{c:.6}")),
1908                a.requests,
1909            );
1910        }
1911        let _ = writeln!(
1912            out,
1913            "  e-value to certify: {:.3} · e-value to halt: {:.3} (decide at {:.1})",
1914            r.e_certify,
1915            r.e_halt,
1916            1.0 / r.alpha
1917        );
1918        if v.ignored_stale > 0 {
1919            let _ = writeln!(
1920                out,
1921                "  {} receipts from a previous experiment configuration ignored",
1922                v.ignored_stale
1923            );
1924        }
1925        for (name, h) in [
1926            ("incumbent", &v.health.incumbent),
1927            ("challenger", &v.health.challenger),
1928        ] {
1929            if h.errors > 0 || h.truncated > 0 {
1930                let _ = writeln!(
1931                    out,
1932                    "  {name}: {} errors, {} answers cut off at max_tokens (neither is random: \
1933                     check them before trusting the rates)",
1934                    h.errors, h.truncated
1935                );
1936            }
1937        }
1938        match v.recommendation {
1939            Recommendation::Adopt => {
1940                let ladder = v
1941                    .challenger_ladder
1942                    .iter()
1943                    .map(|m| format!("{m:?}"))
1944                    .collect::<Vec<_>>()
1945                    .join(", ");
1946                let _ = writeln!(
1947                    out,
1948                    "  ADOPT: set this route's ladder to [{ladder}]\n\n    ladder = [{ladder}]\n"
1949                );
1950            }
1951            Recommendation::CertifiedNoSaving => {
1952                let _ = writeln!(out, "  certified, but no saving — keep incumbent");
1953            }
1954            Recommendation::ChallengerStopped => {
1955                let _ = writeln!(out, "  challenger stopped: worse by more than the margin");
1956            }
1957            Recommendation::KeepRunning => {
1958                let _ = writeln!(out, "  keep running");
1959            }
1960        }
1961    }
1962    out
1963}
1964
1965#[cfg(test)]
1966mod tests {
1967    use firstpass_core::trace::ExperimentRecord;
1968    use firstpass_core::{
1969        Features, FinalOutcome, GENESIS_HASH, Mode, PolicyRef, RequestInfo, ServedFrom, TaskKind,
1970    };
1971
1972    use super::*;
1973
1974    const CFG: &str = "[[route]]\nmatch = {}\nmode = \"enforce\"\n\
1975        ladder = [\"anthropic/claude-sonnet-5\"]\n\
1976        [route.challenger]\nladder = [\"anthropic/claude-haiku-4-5\"]\npercent = 50.0\n";
1977
1978    fn config() -> Config {
1979        Config::parse(CFG).expect("test config parses")
1980    }
1981
1982    /// The fingerprint the current test config stamps on its receipts.
1983    fn fp() -> String {
1984        let c = config();
1985        c.routes[0]
1986            .challenger
1987            .as_ref()
1988            .expect("challenger")
1989            .fingerprint(&c.routes[0].ladder)
1990    }
1991
1992    fn trace(tenant: &str, session: &str, arm: Arm, cost: f64) -> Trace {
1993        Trace {
1994            trace_id: Uuid::now_v7(),
1995            prev_hash: GENESIS_HASH.to_owned(),
1996            tenant_id: tenant.to_owned(),
1997            session_id: session.to_owned(),
1998            ts: jiff::Timestamp::now(),
1999            mode: Mode::Enforce,
2000            policy: PolicyRef {
2001                id: "static-ladder@v0".to_owned(),
2002                explore: false,
2003                propensity: None,
2004                mode_profile: None,
2005            },
2006            request: RequestInfo {
2007                api: "anthropic.messages".to_owned(),
2008                prompt_hash: "h".to_owned(),
2009                features: Features::new(TaskKind::Other),
2010            },
2011            attempts: vec![],
2012            deferred: vec![],
2013            final_: FinalOutcome {
2014                served_rung: Some(0),
2015                served_from: ServedFrom::Attempt,
2016                total_cost_usd: cost,
2017                gate_cost_usd: 0.0,
2018                total_latency_ms: 1,
2019                escalations: 0,
2020                counterfactual_baseline_usd: cost,
2021                savings_usd: 0.0,
2022                cache_source: None,
2023            },
2024            probe: None,
2025            rollout: None,
2026            shadow: None,
2027            route_ix: Some(0),
2028            predicted_pass: None,
2029            decision_prior: None,
2030            elastic: None,
2031            risk: None,
2032            experiment: Some(ExperimentRecord {
2033                arm,
2034                percent: 50.0,
2035                bucket: 0,
2036                fingerprint: fp(),
2037            }),
2038            upstream_provider: String::new(),
2039        }
2040    }
2041
2042    /// Errors and cut-off answers are rebuilt from receipts, per arm, and receipts from an older
2043    /// experiment configuration contribute nothing.
2044    #[test]
2045    fn arm_health_is_rebuilt_from_receipts() {
2046        let attempt = |truncated| firstpass_core::trace::Attempt {
2047            rung: 0,
2048            model: "anthropic/claude-sonnet-5".to_owned(),
2049            provider: "anthropic".to_owned(),
2050            in_tokens: 1,
2051            cache_write_tokens: 0,
2052            cache_read_tokens: 0,
2053            out_tokens: 1,
2054            cost_usd: 0.0,
2055            latency_ms: 1,
2056            gates: vec![],
2057            verdict: Verdict::Pass,
2058            truncated,
2059        };
2060        let mut errored = trace("t", "a", Arm::Incumbent, 0.01);
2061        errored.final_.served_from = ServedFrom::Error;
2062        let mut cut = trace("t", "b", Arm::Incumbent, 0.01);
2063        cut.attempts = vec![attempt(true), attempt(true), attempt(false)];
2064        let mut ch_cut = trace("t", "c", Arm::Challenger, 0.01);
2065        ch_cut.attempts = vec![attempt(true)];
2066        let mut stale = trace("t", "d", Arm::Incumbent, 0.01);
2067        stale.final_.served_from = ServedFrom::Error;
2068        if let Some(rec) = stale.experiment.as_mut() {
2069            rec.fingerprint = "an-older-config".to_owned();
2070        }
2071        let snaps = experiments_from_traces(&[errored, cut, ch_cut, stale], &[], &[], &config());
2072        let h = snaps[0].health;
2073        assert_eq!(
2074            h.incumbent,
2075            ArmHealth {
2076                errors: 1,
2077                truncated: 2
2078            },
2079            "{h:?}"
2080        );
2081        assert_eq!(
2082            h.challenger,
2083            ArmHealth {
2084                errors: 0,
2085                truncated: 1
2086            },
2087            "{h:?}"
2088        );
2089        assert_eq!(snaps[0].ignored_stale, 1);
2090    }
2091
2092    #[test]
2093    fn first_outcome_per_session_counts_and_abstain_is_ignored() {
2094        let m = ExperimentMonitor::from_config(&config());
2095        assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Abstain));
2096        assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Pass));
2097        // Later outcomes from the same session are dropped, whatever they say.
2098        assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Fail));
2099        assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s2", Verdict::Fail));
2100        let r = &m.reports()[0].report;
2101        assert_eq!(r.incumbent.outcomes, 2);
2102        assert!((r.incumbent.outcome_sum - 1.0).abs() < f64::EPSILON);
2103    }
2104
2105    #[test]
2106    fn routes_without_a_challenger_and_unknown_tenants_are_inert() {
2107        let m = ExperimentMonitor::from_config(&config());
2108        assert!(!m.observe_outcome("t", 7, &fp(), Arm::Incumbent, "s", Verdict::Pass));
2109        assert_eq!(m.status("nobody", 0), ExperimentStatus::Undecided);
2110        assert!(m.reports().is_empty());
2111        let off = ExperimentMonitor::default();
2112        assert!(!off.is_active());
2113        off.observe_cost("t", 0, &fp(), Arm::Incumbent, 1.0);
2114        assert!(off.reports().is_empty());
2115    }
2116
2117    #[test]
2118    fn tenants_never_share_an_experiment() {
2119        let m = ExperimentMonitor::from_config(&config());
2120        for i in 0..400 {
2121            m.observe_outcome(
2122                "a",
2123                0,
2124                &fp(),
2125                Arm::Incumbent,
2126                &format!("i{i}"),
2127                Verdict::Pass,
2128            );
2129            m.observe_outcome(
2130                "a",
2131                0,
2132                &fp(),
2133                Arm::Challenger,
2134                &format!("c{i}"),
2135                Verdict::Fail,
2136            );
2137        }
2138        assert_eq!(m.status("a", 0), ExperimentStatus::Halted);
2139        assert_eq!(m.status("b", 0), ExperimentStatus::Undecided);
2140    }
2141
2142    #[test]
2143    fn a_clearly_worse_challenger_halts() {
2144        let m = ExperimentMonitor::from_config(&config());
2145        let mut at = None;
2146        for i in 0..500 {
2147            m.observe_outcome(
2148                "t",
2149                0,
2150                &fp(),
2151                Arm::Incumbent,
2152                &format!("i{i}"),
2153                Verdict::Pass,
2154            );
2155            m.observe_outcome(
2156                "t",
2157                0,
2158                &fp(),
2159                Arm::Challenger,
2160                &format!("c{i}"),
2161                Verdict::Fail,
2162            );
2163            if at.is_none() && m.status("t", 0) == ExperimentStatus::Halted {
2164                at = Some(i);
2165            }
2166        }
2167        assert!(at.is_some(), "never halted");
2168        assert_eq!(m.status("t", 0), ExperimentStatus::Halted);
2169    }
2170
2171    #[test]
2172    fn replay_matches_live_with_session_dedup_and_halt() {
2173        let cfg = config();
2174        let mut traces = Vec::new();
2175        let mut deferred = Vec::new();
2176        for i in 0..400 {
2177            let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
2178            let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
2179            deferred.push((inc.trace_id, Verdict::Pass));
2180            deferred.push((chl.trace_id, Verdict::Fail));
2181            // A second trace in the same session whose outcome contradicts the first: ignored.
2182            let again = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
2183            deferred.push((again.trace_id, Verdict::Pass));
2184            traces.extend([inc, chl, again]);
2185        }
2186        let reports = experiments_from_traces(&traces, &deferred, &[], &cfg);
2187        assert_eq!(reports.len(), 1);
2188        let snap = &reports[0];
2189        let r = &snap.report;
2190        assert_eq!((snap.tenant.as_str(), snap.route_ix), ("t", 0));
2191        assert_eq!(snap.ignored_stale, 0);
2192        assert_eq!(r.status, select::Status::Halted);
2193        // Dedup: one counted outcome per session, and the counted one is the first (Fail = 0).
2194        assert_eq!(r.challenger.outcomes, 400);
2195        assert!(r.challenger.outcome_sum.abs() < f64::EPSILON);
2196        // Cost is per request, not per session: the extra trace in each session still cost money.
2197        assert_eq!(r.challenger.requests, 800);
2198        assert!((r.challenger.cost_per_request().unwrap_or(0.0) - 0.02).abs() < 1e-12);
2199        assert!((r.incumbent.cost_per_request().unwrap_or(0.0) - 0.10).abs() < 1e-12);
2200        assert_eq!(recommend(r), Recommendation::ChallengerStopped);
2201
2202        // The live monitor, fed the same events, reaches the same decision.
2203        let live = ExperimentMonitor::from_config(&cfg);
2204        for t in &traces {
2205            let rec = t.experiment.as_ref().expect("recorded");
2206            live.observe_cost(
2207                &t.tenant_id,
2208                0,
2209                &rec.fingerprint,
2210                rec.arm,
2211                t.final_.total_cost_usd,
2212            );
2213        }
2214        for (id, v) in &deferred {
2215            let t = traces.iter().find(|t| t.trace_id == *id).expect("trace");
2216            let rec = t.experiment.as_ref().expect("recorded");
2217            live.observe_outcome("t", 0, &rec.fingerprint, rec.arm, &t.session_id, *v);
2218        }
2219        assert_eq!(live.reports()[0], *snap);
2220        // Replaying the same receipts under the same config is exactly reproducible.
2221        assert_eq!(
2222            experiments_from_traces(&traces, &deferred, &[], &cfg),
2223            reports
2224        );
2225    }
2226
2227    #[test]
2228    fn replay_ignores_unrecorded_traces_and_unknown_deferred() {
2229        let cfg = config();
2230        let mut plain = trace("t", "s", Arm::Incumbent, 0.1);
2231        plain.experiment = None;
2232        let deferred = [
2233            (plain.trace_id, Verdict::Pass),
2234            (Uuid::now_v7(), Verdict::Pass),
2235        ];
2236        assert!(experiments_from_traces(&[plain], &deferred, &[], &cfg).is_empty());
2237        // A config with no challenger rebuilds nothing at all.
2238        let bare = Config::parse(
2239            "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
2240        )
2241        .expect("parses");
2242        let t = trace("t", "s", Arm::Incumbent, 0.1);
2243        assert!(experiments_from_traces(&[t], &[], &[], &bare).is_empty());
2244    }
2245
2246    fn view_with(status: select::Status, inc_cost: f64, ch_cost: f64) -> ExperimentView {
2247        let mut exp = Experiment::new(0.05, 0.05).expect("valid");
2248        exp.observe_cost(Arm::Incumbent, inc_cost);
2249        exp.observe_cost(Arm::Challenger, ch_cost);
2250        let mut report = exp.report();
2251        report.status = status;
2252        ExperimentView {
2253            tenant: "t".to_owned(),
2254            route_ix: 0,
2255            recommendation: recommend(&report),
2256            ignored_stale: 0,
2257            health: Health::default(),
2258            report,
2259            challenger_ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
2260        }
2261    }
2262
2263    #[test]
2264    fn recommendation_and_rendering_cover_every_state() {
2265        let adopt = view_with(select::Status::Certified, 0.10, 0.02);
2266        assert_eq!(adopt.recommendation, Recommendation::Adopt);
2267        let text = render(&[adopt]);
2268        assert!(
2269            text.contains("ADOPT: set this route's ladder to [\"anthropic/claude-haiku-4-5\"]"),
2270            "{text}"
2271        );
2272        assert!(text.contains("ladder = [\"anthropic/claude-haiku-4-5\"]"));
2273
2274        let dear = view_with(select::Status::Certified, 0.02, 0.10);
2275        assert_eq!(dear.recommendation, Recommendation::CertifiedNoSaving);
2276        assert!(render(&[dear]).contains("certified, but no saving — keep incumbent"));
2277
2278        let halted = view_with(select::Status::Halted, 0.10, 0.02);
2279        assert!(render(&[halted]).contains("challenger stopped: worse by more than the margin"));
2280
2281        let open = view_with(select::Status::Undecided, 0.10, 0.02);
2282        assert_eq!(open.recommendation, Recommendation::KeepRunning);
2283        assert!(render(&[open]).contains("keep running"));
2284        assert!(render(&[]).contains("no experiment data"));
2285    }
2286
2287    #[test]
2288    fn certified_without_cost_data_is_not_adopted() {
2289        let exp = Experiment::new(0.05, 0.05).expect("valid");
2290        let mut report = exp.report();
2291        report.status = select::Status::Certified;
2292        assert_eq!(recommend(&report), Recommendation::CertifiedNoSaving);
2293    }
2294
2295    /// Changing the challenger ladder (or margin/alpha) starts a fresh experiment: receipts from
2296    /// the old configuration are ignored and counted, never pooled.
2297    #[test]
2298    fn changing_the_challenger_ladder_resets_the_experiment() {
2299        let old = config();
2300        let mut traces = Vec::new();
2301        let mut deferred = Vec::new();
2302        for i in 0..300 {
2303            let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
2304            let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
2305            deferred.push((inc.trace_id, Verdict::Pass));
2306            deferred.push((chl.trace_id, Verdict::Fail));
2307            traces.extend([inc, chl]);
2308        }
2309        let before = experiments_from_traces(&traces, &deferred, &[], &old);
2310        assert_eq!(before[0].report.status, select::Status::Halted);
2311        assert_eq!(before[0].ignored_stale, 0);
2312
2313        let new =
2314            Config::parse(&CFG.replace("claude-haiku-4-5", "claude-opus-4-8")).expect("parses");
2315        let after = experiments_from_traces(&traces, &deferred, &[], &new);
2316        assert_eq!(after[0].report.status, select::Status::Undecided);
2317        assert_eq!(after[0].report.pairs, 0);
2318        assert_eq!(after[0].report.challenger.requests, 0);
2319        assert_eq!(after[0].ignored_stale, 600);
2320        let text = render(&views(after, &new));
2321        assert!(
2322            text.contains("600 receipts from a previous experiment configuration ignored"),
2323            "{text}"
2324        );
2325
2326        // margin and alpha are part of the identity too.
2327        let margin = Config::parse(&format!("{CFG}margin = 0.04\n")).expect("parses");
2328        assert_eq!(
2329            experiments_from_traces(&traces, &deferred, &[], &margin)[0].ignored_stale,
2330            600
2331        );
2332        // percent is not: ramping the split keeps the evidence.
2333        let ramped = Config::parse(&CFG.replace("50.0", "30.0")).expect("parses");
2334        assert_eq!(
2335            experiments_from_traces(&traces, &deferred, &[], &ramped)[0].report,
2336            before[0].report
2337        );
2338    }
2339
2340    #[test]
2341    fn challenger_traces_are_not_incumbent_traces() {
2342        assert!(is_incumbent_trace(&trace("t", "s", Arm::Incumbent, 0.1)));
2343        assert!(!is_incumbent_trace(&trace("t", "s", Arm::Challenger, 0.1)));
2344        let mut plain = trace("t", "s", Arm::Challenger, 0.1);
2345        plain.experiment = None;
2346        assert!(is_incumbent_trace(&plain));
2347    }
2348
2349    /// An unreadable store must never re-open a halted challenger: the monitor is built blocked.
2350    #[tokio::test]
2351    async fn unreadable_store_fails_closed_on_the_challenger() {
2352        let cfg = config();
2353        let ok = ExperimentMonitor::from_config(&cfg);
2354        assert!(!ok.challenger_blocked("t", 0));
2355        let blocked = ExperimentMonitor::unavailable(&cfg);
2356        assert!(blocked.challenger_blocked("t", 0));
2357        assert!(blocked.challenger_blocked("any-other-tenant", 0));
2358
2359        // A path that is a directory cannot be opened as a database.
2360        let dir = std::env::temp_dir();
2361        let m = build_monitor(&dir.to_string_lossy(), &cfg);
2362        assert!(
2363            m.challenger_blocked("t", 0),
2364            "unreadable store served a challenger"
2365        );
2366
2367        // A readable (fresh) store is not blocked.
2368        let db = dir.join(format!("firstpass-exp-build-{}.db", Uuid::now_v7()));
2369        let (tx, handle) = crate::store::open(&db).expect("open");
2370        drop(tx);
2371        handle.await.expect("writer");
2372        let m = build_monitor(&db.to_string_lossy(), &cfg);
2373        assert!(!m.challenger_blocked("t", 0));
2374        let _ = std::fs::remove_file(&db);
2375
2376        // No challenger configured: nothing to block, store never touched.
2377        let bare = Config::parse(
2378            "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
2379        )
2380        .expect("parses");
2381        assert!(!build_monitor(&dir.to_string_lossy(), &bare).challenger_blocked("t", 0));
2382    }
2383
2384    /// A paired evaluation's challenger spend must reach the report on every path that applies a
2385    /// pair: direct (single replica), synced from the shared log, and rebuilt from the store at
2386    /// startup. Without it the challenger's $/request is n/a and a certified, cheaper paired
2387    /// challenger can never be recommended for adoption.
2388    #[tokio::test]
2389    async fn paired_challenger_cost_reaches_the_report_on_every_path() {
2390        let cfg = config();
2391        let fp_str = fp();
2392        let challenger = |m: &ExperimentMonitor| {
2393            let r = m.reports();
2394            assert_eq!(r.len(), 1);
2395            let c = r[0].report.challenger;
2396            (c.requests, c.cost_usd, r[0].report.pairs)
2397        };
2398        let close = |(n, usd, pairs): (u64, f64, u64)| {
2399            assert_eq!((n, pairs), (3, 3));
2400            assert!((usd - 0.03).abs() < 1e-12, "{usd}");
2401        };
2402
2403        // Direct (no-log) path: record_pair returns false and does not apply evidence;
2404        // the caller uses apply_pair_locally after a successful SQLite write.
2405        let direct = ExperimentMonitor::from_config(&cfg);
2406        for _ in 0..3 {
2407            let ok = direct
2408                .record_pair(
2409                    "t",
2410                    0,
2411                    &fp_str,
2412                    1.0,
2413                    1.0,
2414                    0.10,
2415                    0.01,
2416                    false,
2417                    0,
2418                    0,
2419                    Uuid::nil(),
2420                    Uuid::nil(),
2421                )
2422                .await;
2423            assert!(
2424                !ok,
2425                "no log → record_pair returns false (evidence via apply_pair_locally)"
2426            );
2427            direct.apply_pair_locally("t", 0, &fp_str, 1.0, 1.0, 0.01, Uuid::nil());
2428        }
2429        close(challenger(&direct));
2430
2431        let shared = Arc::new(InMemoryExperimentLog::new());
2432        let a = ExperimentMonitor::from_config(&cfg)
2433            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2434        let b = ExperimentMonitor::from_config(&cfg)
2435            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2436        for _ in 0..3 {
2437            a.record_pair(
2438                "t",
2439                0,
2440                &fp_str,
2441                1.0,
2442                1.0,
2443                0.10,
2444                0.01,
2445                false,
2446                0,
2447                0,
2448                Uuid::nil(),
2449                Uuid::nil(),
2450            )
2451            .await;
2452        }
2453        let _ = b.sync_from_backend("t", 0, &fp_str).await;
2454        close(challenger(&a));
2455        close(challenger(&b));
2456
2457        let db =
2458            std::env::temp_dir().join(format!("firstpass-paircost-{}.db", uuid::Uuid::now_v7()));
2459        let (tx, handle) = crate::store::open(&db).unwrap();
2460        drop(tx);
2461        handle.await.unwrap();
2462        for _ in 0..3 {
2463            crate::store::append_experiment_pair(
2464                &db,
2465                &crate::store::ExperimentPairRow {
2466                    tenant: "t".to_owned(),
2467                    route_ix: 0,
2468                    fingerprint: fp_str.clone(),
2469                    trace_id: uuid::Uuid::now_v7(),
2470                    x_inc: 1.0,
2471                    x_ch: 1.0,
2472                    challenger_cost_usd: 0.01,
2473                    ch_error: false,
2474                    ch_truncated: 0,
2475                    ts: jiff::Timestamp::now(),
2476                    in_log: crate::store::InLogStatus::InLog,
2477                },
2478            )
2479            .unwrap();
2480        }
2481        close(challenger(&build_monitor(&db.to_string_lossy(), &cfg)));
2482
2483        // Restart with a shared log: the same three pairs are in the store AND the log. They
2484        // must count once, not twice (pairs carry no session to dedup on).
2485        let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
2486            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2487        let _ = restarted.sync_from_backend("t", 0, &fp_str).await;
2488        close(challenger(&restarted));
2489        let _ = std::fs::remove_file(&db);
2490    }
2491
2492    // ── Shared-log tests (InMemoryExperimentLog) ─────────────────────────────────────────────
2493
2494    /// Two monitors sharing one log converge to the same state after syncing.
2495    ///
2496    /// Mutation test: if `sync_from_backend` does not apply records, replica_b stays Undecided
2497    /// even after 400 pairs of Pass/Fail evidence, so the final `Halted` assertion fails.
2498    #[tokio::test]
2499    async fn two_replicas_sharing_one_log_converge_after_sync() {
2500        let shared = Arc::new(InMemoryExperimentLog::new());
2501        let cfg = config();
2502        let fp_str = fp();
2503
2504        let replica_a = ExperimentMonitor::from_config(&cfg)
2505            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2506        let replica_b = ExperimentMonitor::from_config(&cfg)
2507            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2508
2509        // Feed all evidence through replica A (it appends to the shared log).
2510        for i in 0..400 {
2511            replica_a
2512                .record_outcome(
2513                    "t",
2514                    0,
2515                    &fp_str,
2516                    Arm::Incumbent,
2517                    &format!("i{i}"),
2518                    Verdict::Pass,
2519                    0.10,
2520                )
2521                .await;
2522            replica_a
2523                .record_outcome(
2524                    "t",
2525                    0,
2526                    &fp_str,
2527                    Arm::Challenger,
2528                    &format!("c{i}"),
2529                    Verdict::Fail,
2530                    0.02,
2531                )
2532                .await;
2533        }
2534
2535        // Replica B has not received any feedback directly: still Undecided.
2536        // (Mutation check: disabling sync_from_backend makes the next assertion the failing one.)
2537        assert_eq!(
2538            replica_b.status("t", 0),
2539            ExperimentStatus::Undecided,
2540            "replica_b should not have any evidence before syncing"
2541        );
2542
2543        // After one sync pass, replica B reads the shared log and reaches the same decision.
2544        let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
2545        assert_eq!(
2546            replica_b.status("t", 0),
2547            ExperimentStatus::Halted,
2548            "replica_b must halt after syncing from shared log"
2549        );
2550        assert!(
2551            replica_b.challenger_blocked("t", 0),
2552            "halted experiment must block the challenger on the synced replica"
2553        );
2554
2555        // Both replicas agree on pair counts and e-values.
2556        let reps_a = replica_a.reports();
2557        let reps_b = replica_b.reports();
2558        assert_eq!(reps_a.len(), 1);
2559        assert_eq!(reps_b.len(), 1);
2560        assert_eq!(reps_a[0].report.status, reps_b[0].report.status);
2561        assert_eq!(reps_a[0].report.pairs, reps_b[0].report.pairs);
2562    }
2563
2564    /// A halt observed via one replica blocks the challenger on the other.
2565    ///
2566    /// Specifically: replica A drives the experiment to Halted; replica B has not synced yet,
2567    /// so it does NOT block. After one sync, replica B also blocks.
2568    ///
2569    /// Mutation test: if `challenger_blocked` does not consult status (always returns false),
2570    /// the first `assert!(!replica_b.challenger_blocked(...))` passes trivially, but the
2571    /// second one (`assert!(... after sync)`) fails — verifying the block gate actually changed.
2572    #[tokio::test]
2573    async fn halt_on_one_replica_blocks_challenger_on_other_after_sync() {
2574        let shared = Arc::new(InMemoryExperimentLog::new());
2575        let cfg = config();
2576        let fp_str = fp();
2577
2578        let replica_a = ExperimentMonitor::from_config(&cfg)
2579            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2580        let replica_b = ExperimentMonitor::from_config(&cfg)
2581            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2582
2583        // B has not synced yet — it does not know about the halt.
2584        assert!(
2585            !replica_b.challenger_blocked("t", 0),
2586            "should be unblocked before any evidence"
2587        );
2588
2589        // Drive A to halted.
2590        let mut halted = false;
2591        for i in 0..500 {
2592            replica_a
2593                .record_outcome(
2594                    "t",
2595                    0,
2596                    &fp_str,
2597                    Arm::Incumbent,
2598                    &format!("i{i}"),
2599                    Verdict::Pass,
2600                    0.10,
2601                )
2602                .await;
2603            replica_a
2604                .record_outcome(
2605                    "t",
2606                    0,
2607                    &fp_str,
2608                    Arm::Challenger,
2609                    &format!("c{i}"),
2610                    Verdict::Fail,
2611                    0.02,
2612                )
2613                .await;
2614            if replica_a.status("t", 0) == ExperimentStatus::Halted {
2615                halted = true;
2616                break;
2617            }
2618        }
2619        assert!(halted, "experiment never halted on replica A");
2620
2621        // B still does not know.
2622        assert!(
2623            !replica_b.challenger_blocked("t", 0),
2624            "replica_b should not block before syncing"
2625        );
2626
2627        // After sync, B learns of the halt and blocks.
2628        let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
2629        assert!(
2630            replica_b.challenger_blocked("t", 0),
2631            "replica_b must block after syncing the halt from the shared log"
2632        );
2633    }
2634
2635    /// Cross-replica session dedup: `claim_session` prevents the same session from being
2636    /// counted twice even when two replicas receive the same feedback.
2637    ///
2638    /// Mutation test: if `claim_session` always returns `true`, both replicas count the
2639    /// same session, the pair count doubles, and the equality assertion fails.
2640    #[tokio::test]
2641    async fn cross_replica_session_dedup_prevents_double_counting() {
2642        let shared = Arc::new(InMemoryExperimentLog::new());
2643        let cfg = config();
2644        let fp_str = fp();
2645
2646        let replica_a = ExperimentMonitor::from_config(&cfg)
2647            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2648        let replica_b = ExperimentMonitor::from_config(&cfg)
2649            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2650
2651        // Both replicas independently receive the same feedback for the same session.
2652        replica_a
2653            .record_outcome(
2654                "t",
2655                0,
2656                &fp_str,
2657                Arm::Incumbent,
2658                "session-1",
2659                Verdict::Pass,
2660                0.10,
2661            )
2662            .await;
2663        replica_b
2664            .record_outcome(
2665                "t",
2666                0,
2667                &fp_str,
2668                Arm::Incumbent,
2669                "session-1",
2670                Verdict::Pass,
2671                0.10,
2672            )
2673            .await;
2674
2675        // Sync both so they agree on the log.
2676        let _ = replica_a.sync_from_backend("t", 0, &fp_str).await;
2677        let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
2678
2679        // Only one outcome should have been counted (not two).
2680        let reps_a = replica_a.reports();
2681        let reps_b = replica_b.reports();
2682        assert_eq!(
2683            reps_a[0].report.incumbent.outcomes, 1,
2684            "session should be counted exactly once"
2685        );
2686        assert_eq!(
2687            reps_b[0].report.incumbent.outcomes, 1,
2688            "both replicas should agree on outcome count"
2689        );
2690    }
2691
2692    /// Abstain must not burn the session slot: if a session's first feedback is Abstain,
2693    /// a subsequent Pass/Fail for the same session must still be accepted.
2694    ///
2695    /// Mutation test: if Abstain is filtered AFTER `claim_session` (the pre-fix behaviour),
2696    /// Abstain claims the slot and the subsequent Pass returns `false`; the `== 1` assertion
2697    /// below fails.
2698    #[tokio::test]
2699    async fn abstain_does_not_burn_session_slot_before_claim() {
2700        let shared = Arc::new(InMemoryExperimentLog::new());
2701        let cfg = config();
2702        let fp_str = fp();
2703
2704        let monitor = ExperimentMonitor::from_config(&cfg)
2705            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2706
2707        // First feedback for this session is Abstain — must not claim the session slot.
2708        let counted = monitor
2709            .record_outcome(
2710                "t",
2711                0,
2712                &fp_str,
2713                Arm::Incumbent,
2714                "session-abstain",
2715                Verdict::Abstain,
2716                0.10,
2717            )
2718            .await;
2719        assert!(!counted, "Abstain must not be counted");
2720
2721        // Second feedback for the SAME session: Pass — must be accepted.
2722        let counted = monitor
2723            .record_outcome(
2724                "t",
2725                0,
2726                &fp_str,
2727                Arm::Incumbent,
2728                "session-abstain",
2729                Verdict::Pass,
2730                0.10,
2731            )
2732            .await;
2733        assert!(counted, "Pass after Abstain must be counted");
2734
2735        let reps = monitor.reports();
2736        assert_eq!(
2737            reps[0].report.incumbent.outcomes, 1,
2738            "exactly one outcome must have been counted"
2739        );
2740    }
2741
2742    // ── Redis-backed shared-log tests ────────────────────────────────────────────────────────
2743
2744    /// Connect to the test Redis, wipe all keys whose name contains `unique_tenant`, and return
2745    /// the open connection for subsequent cleanup calls.
2746    #[cfg(feature = "redis-cache")]
2747    async fn redis_cleanup(url: &str, unique_tenant: &str) {
2748        let client = redis::Client::open(url).expect("redis client");
2749        let mut conn = redis::aio::ConnectionManager::new(client)
2750            .await
2751            .expect("redis connection for cleanup");
2752        use redis::AsyncCommands as _;
2753        // SCAN for all experiment keys that carry this tenant in their name.
2754        let pattern = format!("firstpass:exp*{unique_tenant}*");
2755        let keys: Vec<String> = {
2756            let mut out = Vec::new();
2757            let mut cursor: u64 = 0;
2758            loop {
2759                let (next, batch): (u64, Vec<String>) = redis::cmd("SCAN")
2760                    .arg(cursor)
2761                    .arg("MATCH")
2762                    .arg(&pattern)
2763                    .arg("COUNT")
2764                    .arg(100u64)
2765                    .query_async(&mut conn)
2766                    .await
2767                    .expect("SCAN");
2768                out.extend(batch);
2769                cursor = next;
2770                if cursor == 0 {
2771                    break;
2772                }
2773            }
2774            out
2775        };
2776        if !keys.is_empty() {
2777            let _: () = conn.del(keys).await.expect("DEL cleanup");
2778        }
2779    }
2780
2781    /// Two `RedisExperimentLog`-backed monitors on the same key converge to identical reports
2782    /// after sync.
2783    ///
2784    /// Requires `FIRSTPASS_TEST_REDIS` (e.g. `redis://127.0.0.1:56411/15`). **Absent, this
2785    /// panics rather than skipping**: a green suite that silently tested nothing is worse than a
2786    /// failing one.
2787    ///
2788    /// Mutation test: disabling `sync_from_backend` keeps replica_b `Undecided` and the
2789    /// `Halted` assertion below fails.
2790    #[cfg(feature = "redis-cache")]
2791    #[tokio::test]
2792    async fn redis_two_replicas_converge_to_identical_reports_after_sync() {
2793        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2794            panic!(
2795                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2796                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2797                 is worse than a red one."
2798            )
2799        });
2800        // Unique tenant prevents key collisions across concurrent or repeated test runs.
2801        let unique_tenant = format!("t-conv-{}", Uuid::now_v7());
2802        let log_a = Arc::new(
2803            RedisExperimentLog::connect(&url)
2804                .await
2805                .expect("RedisExperimentLog::connect"),
2806        );
2807        let log_b = Arc::new(
2808            RedisExperimentLog::connect(&url)
2809                .await
2810                .expect("RedisExperimentLog::connect"),
2811        );
2812        let cfg = config();
2813        let fp_str = fp();
2814
2815        let replica_a = ExperimentMonitor::from_config(&cfg)
2816            .with_log(Arc::clone(&log_a) as Arc<dyn ExperimentLogBackend>);
2817        let replica_b = ExperimentMonitor::from_config(&cfg)
2818            .with_log(Arc::clone(&log_b) as Arc<dyn ExperimentLogBackend>);
2819
2820        // All evidence through replica A.
2821        for i in 0..400 {
2822            replica_a
2823                .record_outcome(
2824                    &unique_tenant,
2825                    0,
2826                    &fp_str,
2827                    Arm::Incumbent,
2828                    &format!("i{i}"),
2829                    Verdict::Pass,
2830                    0.10,
2831                )
2832                .await;
2833            replica_a
2834                .record_outcome(
2835                    &unique_tenant,
2836                    0,
2837                    &fp_str,
2838                    Arm::Challenger,
2839                    &format!("c{i}"),
2840                    Verdict::Fail,
2841                    0.02,
2842                )
2843                .await;
2844        }
2845
2846        // B has not synced: still Undecided locally.
2847        assert_eq!(
2848            replica_b.status(&unique_tenant, 0),
2849            ExperimentStatus::Undecided,
2850            "replica_b must be Undecided before syncing"
2851        );
2852
2853        // One sync makes B read the full log and reach the same decision.
2854        let _ = replica_b
2855            .sync_from_backend(&unique_tenant, 0, &fp_str)
2856            .await;
2857        assert_eq!(
2858            replica_b.status(&unique_tenant, 0),
2859            ExperimentStatus::Halted,
2860            "replica_b must be Halted after syncing from Redis log"
2861        );
2862
2863        let reps_a = replica_a.reports();
2864        let reps_b = replica_b.reports();
2865        assert_eq!(reps_a.len(), 1);
2866        assert_eq!(reps_b.len(), 1);
2867        assert_eq!(
2868            reps_a[0].report.status, reps_b[0].report.status,
2869            "both replicas must agree on status"
2870        );
2871        assert_eq!(
2872            reps_a[0].report.pairs, reps_b[0].report.pairs,
2873            "both replicas must agree on pair counts"
2874        );
2875
2876        redis_cleanup(&url, &unique_tenant).await;
2877    }
2878
2879    /// `claim_session` prevents the same session from being double-counted across replicas when
2880    /// both receive the same feedback.
2881    ///
2882    /// Requires `FIRSTPASS_TEST_REDIS`. Panics when unset (same rationale as above).
2883    ///
2884    /// Mutation test: if `claim_session` always returns `true`, both replicas append the same
2885    /// session and the outcome count doubles; the `== 1` assertion fails.
2886    #[cfg(feature = "redis-cache")]
2887    #[tokio::test]
2888    async fn redis_cross_replica_session_dedup_via_claim_session_holds() {
2889        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2890            panic!(
2891                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2892                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2893                 is worse than a red one."
2894            )
2895        });
2896        let unique_tenant = format!("t-dedup-{}", Uuid::now_v7());
2897        let log = Arc::new(
2898            RedisExperimentLog::connect(&url)
2899                .await
2900                .expect("RedisExperimentLog::connect"),
2901        );
2902        let cfg = config();
2903        let fp_str = fp();
2904
2905        let replica_a = ExperimentMonitor::from_config(&cfg)
2906            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2907        let replica_b = ExperimentMonitor::from_config(&cfg)
2908            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2909
2910        // Both replicas receive the same session feedback.
2911        replica_a
2912            .record_outcome(
2913                &unique_tenant,
2914                0,
2915                &fp_str,
2916                Arm::Incumbent,
2917                "session-x",
2918                Verdict::Pass,
2919                0.10,
2920            )
2921            .await;
2922        replica_b
2923            .record_outcome(
2924                &unique_tenant,
2925                0,
2926                &fp_str,
2927                Arm::Incumbent,
2928                "session-x",
2929                Verdict::Pass,
2930                0.10,
2931            )
2932            .await;
2933
2934        // Sync both.
2935        let _ = replica_a
2936            .sync_from_backend(&unique_tenant, 0, &fp_str)
2937            .await;
2938        let _ = replica_b
2939            .sync_from_backend(&unique_tenant, 0, &fp_str)
2940            .await;
2941
2942        let reps_a = replica_a.reports();
2943        let reps_b = replica_b.reports();
2944        assert_eq!(
2945            reps_a[0].report.incumbent.outcomes, 1,
2946            "session must be counted exactly once on replica A"
2947        );
2948        assert_eq!(
2949            reps_b[0].report.incumbent.outcomes, 1,
2950            "session must be counted exactly once on replica B"
2951        );
2952
2953        redis_cleanup(&url, &unique_tenant).await;
2954    }
2955
2956    /// A halt driven on replica A is propagated to replica B after one sync, blocking the
2957    /// challenger there too.
2958    ///
2959    /// Requires `FIRSTPASS_TEST_REDIS`. Panics when unset.
2960    ///
2961    /// Mutation test: removing the `sync_from_backend` call means B never learns of the halt,
2962    /// and the `challenger_blocked` assertion at the end fails.
2963    #[cfg(feature = "redis-cache")]
2964    #[tokio::test]
2965    async fn redis_halt_on_one_replica_blocks_challenger_on_other() {
2966        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2967            panic!(
2968                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2969                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2970                 is worse than a red one."
2971            )
2972        });
2973        let unique_tenant = format!("t-halt-{}", Uuid::now_v7());
2974        let log = Arc::new(
2975            RedisExperimentLog::connect(&url)
2976                .await
2977                .expect("RedisExperimentLog::connect"),
2978        );
2979        let cfg = config();
2980        let fp_str = fp();
2981
2982        let replica_a = ExperimentMonitor::from_config(&cfg)
2983            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2984        let replica_b = ExperimentMonitor::from_config(&cfg)
2985            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2986
2987        // B starts unblocked.
2988        assert!(
2989            !replica_b.challenger_blocked(&unique_tenant, 0),
2990            "replica_b must be unblocked before any evidence"
2991        );
2992
2993        // Drive A to Halted.
2994        let mut halted = false;
2995        for i in 0..500 {
2996            replica_a
2997                .record_outcome(
2998                    &unique_tenant,
2999                    0,
3000                    &fp_str,
3001                    Arm::Incumbent,
3002                    &format!("i{i}"),
3003                    Verdict::Pass,
3004                    0.10,
3005                )
3006                .await;
3007            replica_a
3008                .record_outcome(
3009                    &unique_tenant,
3010                    0,
3011                    &fp_str,
3012                    Arm::Challenger,
3013                    &format!("c{i}"),
3014                    Verdict::Fail,
3015                    0.02,
3016                )
3017                .await;
3018            if replica_a.status(&unique_tenant, 0) == ExperimentStatus::Halted {
3019                halted = true;
3020                break;
3021            }
3022        }
3023        assert!(halted, "experiment never halted on replica A");
3024
3025        // B still does not know: it hasn't synced.
3026        assert!(
3027            !replica_b.challenger_blocked(&unique_tenant, 0),
3028            "replica_b must not block before syncing"
3029        );
3030
3031        // After one sync, B learns the halt and blocks.
3032        let _ = replica_b
3033            .sync_from_backend(&unique_tenant, 0, &fp_str)
3034            .await;
3035        assert!(
3036            replica_b.challenger_blocked(&unique_tenant, 0),
3037            "replica_b must block the challenger after syncing the halt from Redis"
3038        );
3039
3040        redis_cleanup(&url, &unique_tenant).await;
3041    }
3042
3043    /// Keys are isolated per tenant: evidence written under `tenant_a` is invisible to a monitor
3044    /// reading under `tenant_b`.
3045    ///
3046    /// Requires `FIRSTPASS_TEST_REDIS`. Panics when unset.
3047    ///
3048    /// Mutation test: if the key omits the tenant and both monitors share one list, B would
3049    /// gain outcomes after sync and the `Undecided` assertion fails.
3050    #[cfg(feature = "redis-cache")]
3051    #[tokio::test]
3052    async fn redis_keys_are_isolated_per_tenant() {
3053        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
3054            panic!(
3055                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
3056                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
3057                 is worse than a red one."
3058            )
3059        });
3060        let run_id = Uuid::now_v7();
3061        let tenant_a = format!("t-iso-a-{run_id}");
3062        let tenant_b = format!("t-iso-b-{run_id}");
3063        let log = Arc::new(
3064            RedisExperimentLog::connect(&url)
3065                .await
3066                .expect("RedisExperimentLog::connect"),
3067        );
3068        let cfg = config();
3069        let fp_str = fp();
3070
3071        let monitor_a = ExperimentMonitor::from_config(&cfg)
3072            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
3073        let monitor_b = ExperimentMonitor::from_config(&cfg)
3074            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
3075
3076        // Feed evidence only under tenant_a.
3077        for i in 0..5 {
3078            monitor_a
3079                .record_outcome(
3080                    &tenant_a,
3081                    0,
3082                    &fp_str,
3083                    Arm::Incumbent,
3084                    &format!("s{i}"),
3085                    Verdict::Pass,
3086                    0.10,
3087                )
3088                .await;
3089        }
3090
3091        // Prove storage-level isolation: load_since for tenant_b returns nothing.
3092        let records = log
3093            .load_since(&tenant_b, 0, &fp_str, 0)
3094            .await
3095            .expect("load_since must not fail on a reachable Redis");
3096        assert!(
3097            records.is_empty(),
3098            "backend must return no records for tenant_b"
3099        );
3100
3101        // monitor_b reads under tenant_b via status() (does not create an entry) and sees
3102        // Undecided, confirming no cross-tenant contamination.
3103        assert_eq!(
3104            monitor_b.status(&tenant_b, 0),
3105            ExperimentStatus::Undecided,
3106            "evidence written under tenant_a must not appear under tenant_b"
3107        );
3108
3109        // monitor_a still has its own evidence.
3110        let reps = monitor_a.reports();
3111        assert_eq!(reps.len(), 1);
3112        assert_eq!(reps[0].report.incumbent.outcomes, 5);
3113
3114        // Clean up both tenants' keys.
3115        redis_cleanup(&url, &tenant_a).await;
3116        redis_cleanup(&url, &tenant_b).await;
3117    }
3118
3119    // ── Challenger health persistence and cluster-wide spend cap ─────────────────────────────
3120
3121    /// An old-format `LogRecord::Pair` JSON (without `ch_error`, `ch_truncated`, `ts_unix`)
3122    /// deserializes without error, with all three fields defaulting to `false`/`0`/`0`.
3123    ///
3124    /// Mutation test: removing `#[serde(default)]` from any of the three fields makes this
3125    /// deserialize fail and the `unwrap` panics — the assertion never runs.
3126    #[test]
3127    fn old_log_record_pair_deserializes_without_new_fields() {
3128        let json = r#"{"kind":"pair","x_inc":1.0,"x_ch":0.5,"incumbent_cost_usd":0.1,"challenger_cost_usd":0.01}"#;
3129        let record: LogRecord = serde_json::from_str(json).unwrap();
3130        match record {
3131            LogRecord::Pair {
3132                ch_error,
3133                ch_truncated,
3134                ts_unix,
3135                x_inc,
3136                x_ch,
3137                replica_id,
3138                ..
3139            } => {
3140                assert!(!ch_error, "ch_error defaults to false");
3141                assert_eq!(ch_truncated, 0, "ch_truncated defaults to 0");
3142                assert_eq!(ts_unix, 0, "ts_unix defaults to 0");
3143                assert!((x_inc - 1.0).abs() < f64::EPSILON);
3144                assert!((x_ch - 0.5).abs() < f64::EPSILON);
3145                // Old records have no replica_id field → default nil UUID, treated as "remote"
3146                // by apply_log_record (health+ledger tallied for them, not skipped).
3147                assert_eq!(
3148                    replica_id,
3149                    uuid::Uuid::nil(),
3150                    "missing replica_id defaults to nil"
3151                );
3152            }
3153            _ => panic!("expected Pair variant"),
3154        }
3155    }
3156
3157    /// Challenger health (`ch_error`, `ch_truncated`) persists to `experiment_pairs` via
3158    /// `ExperimentPairRow` and is rebuilt by `monitor_from_traces` at startup (no shared log).
3159    ///
3160    /// Mutation test: if `monitor_from_traces` does not tally `ch_error`/`ch_truncated` from
3161    /// pair rows, the challenger health stays at 0 and these assertions fail.
3162    #[tokio::test]
3163    async fn pair_health_persists_and_rebuilds_from_store() {
3164        let db =
3165            std::env::temp_dir().join(format!("firstpass-health-persist-{}.db", Uuid::now_v7()));
3166        let (tx, handle) = crate::store::open(&db).unwrap();
3167        drop(tx);
3168        handle.await.unwrap();
3169        let cfg = config();
3170        let fp_str = fp();
3171        // Write a pair with ch_error=true, ch_truncated=2.
3172        crate::store::append_experiment_pair(
3173            &db,
3174            &crate::store::ExperimentPairRow {
3175                tenant: "t".to_owned(),
3176                route_ix: 0,
3177                fingerprint: fp_str.clone(),
3178                trace_id: Uuid::now_v7(),
3179                x_inc: 1.0,
3180                x_ch: 1.0,
3181                challenger_cost_usd: 0.01,
3182                ch_error: true,
3183                ch_truncated: 2,
3184                ts: jiff::Timestamp::now(),
3185                in_log: crate::store::InLogStatus::InLog,
3186            },
3187        )
3188        .unwrap();
3189        // Rebuild without a shared log: pairs are replayed from SQLite.
3190        let m = build_monitor(&db.to_string_lossy(), &cfg);
3191        let reps = m.reports();
3192        assert_eq!(reps.len(), 1, "one experiment entry");
3193        assert_eq!(reps[0].health.challenger.errors, 1, "ch_error rebuilt");
3194        assert_eq!(
3195            reps[0].health.challenger.truncated, 2,
3196            "ch_truncated rebuilt"
3197        );
3198        let _ = std::fs::remove_file(&db);
3199    }
3200
3201    /// Using a shared `InMemoryExperimentLog`, challenger health is counted exactly once per
3202    /// replica view (not double-counted on the originating replica), and replica B's paired
3203    /// ledger reflects replica A's spend after one sync, reaching the cluster cap.
3204    ///
3205    /// Mutation tests (one assertion per invariant):
3206    /// (a) If `apply_log_record` does not tally health → A's health.errors == 0 → assertion fails.
3207    /// (b) If `apply_log_record` does not seed the ledger → A's spent_today == 0.0 → fails.
3208    /// (c) If live `observe_health` were also called (double-count) → A's errors == 2 → fails.
3209    /// (d) If B's sync does not read the ledger → B's spent_today == 0.0 → fails.
3210    /// (e) If cluster cap not enforced → `may_spend` returns true when over cap → fails.
3211    /// A log that is fully down: every append and load_since returns an error.
3212    #[derive(Debug)]
3213    struct DownLog;
3214
3215    #[async_trait::async_trait]
3216    impl ExperimentLogBackend for DownLog {
3217        async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
3218            Err(LogError::Backend("simulated outage".to_owned()))
3219        }
3220        async fn load_since(
3221            &self,
3222            _: &str,
3223            _: usize,
3224            _: &str,
3225            _: u64,
3226        ) -> Result<Vec<Option<LogRecord>>, LogError> {
3227            Err(LogError::Backend("simulated outage".to_owned()))
3228        }
3229        async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
3230            false
3231        }
3232    }
3233
3234    /// A log where append succeeds but load_since always fails (simulates a write-ok/read-fail
3235    /// scenario, e.g. Redis replica lag or a partial outage).
3236    #[derive(Debug)]
3237    struct AppendOnlyLog(InMemoryExperimentLog);
3238
3239    #[async_trait::async_trait]
3240    impl ExperimentLogBackend for AppendOnlyLog {
3241        async fn append(
3242            &self,
3243            tenant: &str,
3244            route_ix: usize,
3245            fingerprint: &str,
3246            record: LogRecord,
3247        ) -> Result<u64, LogError> {
3248            self.0.append(tenant, route_ix, fingerprint, record).await
3249        }
3250        async fn load_since(
3251            &self,
3252            _: &str,
3253            _: usize,
3254            _: &str,
3255            _: u64,
3256        ) -> Result<Vec<Option<LogRecord>>, LogError> {
3257            Err(LogError::Backend("simulated read failure".to_owned()))
3258        }
3259        async fn claim_session(
3260            &self,
3261            tenant: &str,
3262            route_ix: usize,
3263            fingerprint: &str,
3264            session_id: &str,
3265        ) -> bool {
3266            self.0
3267                .claim_session(tenant, route_ix, fingerprint, session_id)
3268                .await
3269        }
3270    }
3271
3272    /// With the shared log fully down, a paired challenger's spend is still counted against the cap
3273    /// and health is still tallied because `spawn_paired_challenger` debits/tallies locally before
3274    /// calling `record_pair`. The test simulates that local accounting path directly.
3275    #[tokio::test]
3276    async fn paired_spend_counts_against_the_cap_when_the_shared_log_is_down() {
3277        let ledger = Arc::new(crate::shadow::ShadowLedger::new());
3278        let m = ExperimentMonitor::from_config(&config())
3279            .with_log(Arc::new(DownLog) as Arc<dyn ExperimentLogBackend>)
3280            .with_paired_ledger(Arc::clone(&ledger));
3281        let now = jiff::Timestamp::now();
3282
3283        // Simulate what spawn_paired_challenger does: ALWAYS debit and tally health locally,
3284        // then call record_pair. The debit/health happen before record_pair regardless of
3285        // whether the log append succeeds.
3286        ledger.debit("t", 0, 0.7, now);
3287        m.observe_health("t", 0, &fp(), Arm::Challenger, true, 2);
3288
3289        let accepted = m
3290            .record_pair(
3291                "t",
3292                0,
3293                &fp(),
3294                1.0,
3295                0.0,
3296                0.1,
3297                0.7,
3298                true,
3299                2,
3300                now.as_second(),
3301                Uuid::nil(),
3302                Uuid::nil(),
3303            )
3304            .await;
3305        assert!(!accepted, "nothing reached the log");
3306        assert!(
3307            (ledger.spent_today("t", 0, now) - 0.7).abs() < 1e-9,
3308            "spend debited locally"
3309        );
3310        assert!(
3311            !ledger.may_spend("t", 0, 0.5, now),
3312            "cap enforced during the outage"
3313        );
3314        let h = m.reports()[0].health.challenger;
3315        assert_eq!((h.errors, h.truncated), (1, 2), "{h:?}");
3316    }
3317
3318    /// When append succeeds but load_since fails, local spend (debited by spawn_paired_challenger)
3319    /// still enforces the cap. The pair record is in the backend (other replicas can read it on
3320    /// their next sync), but this replica cannot read it back immediately.
3321    #[tokio::test]
3322    async fn paired_spend_enforced_when_append_succeeds_but_read_fails() {
3323        let ledger = Arc::new(crate::shadow::ShadowLedger::new());
3324        let m = ExperimentMonitor::from_config(&config())
3325            .with_log(Arc::new(AppendOnlyLog(InMemoryExperimentLog::new()))
3326                as Arc<dyn ExperimentLogBackend>)
3327            .with_paired_ledger(Arc::clone(&ledger));
3328        let now = jiff::Timestamp::now();
3329
3330        // Simulate spawn_paired_challenger: local debit + health first.
3331        ledger.debit("t", 0, 0.6, now);
3332        m.observe_health("t", 0, &fp(), Arm::Challenger, false, 1);
3333
3334        // record_pair: append succeeds, but sync (load_since) fails.
3335        let accepted = m
3336            .record_pair(
3337                "t",
3338                0,
3339                &fp(),
3340                1.0,
3341                1.0,
3342                0.1,
3343                0.6,
3344                false,
3345                1,
3346                now.as_second(),
3347                Uuid::nil(),
3348                Uuid::nil(),
3349            )
3350            .await;
3351        // The pair was appended to the log (returns false only on append failure).
3352        // With AppendOnlyLog, append succeeds but sync returns Err, so record_pair still
3353        // returns false (sync failure propagates the same as before... actually no).
3354        // record_pair returns true when append succeeds, regardless of sync outcome.
3355        // Actually: record_pair ignores the sync error (let _ = sync...) and returns true.
3356        assert!(accepted, "pair was appended to log");
3357
3358        // Local spend (from the explicit debit above) enforces the cap.
3359        assert!(
3360            (ledger.spent_today("t", 0, now) - 0.6).abs() < 1e-9,
3361            "local spend = 0.6"
3362        );
3363        assert!(
3364            !ledger.may_spend("t", 0, 0.5, now),
3365            "cap enforced by local debit even when read-back fails"
3366        );
3367        let h = m.reports()[0].health.challenger;
3368        assert_eq!((h.errors, h.truncated), (0, 1), "{h:?}");
3369    }
3370
3371    /// Two replicas sharing a log count each pair's health and spend exactly once.
3372    ///
3373    /// `spawn_paired_challenger` ALWAYS debits the local ledger and tallies health, then calls
3374    /// `record_pair`. `apply_log_record` skips health and ledger for records bearing the replica's
3375    /// own `replica_id` (preventing double-count on the originating node) but applies them for
3376    /// remote records (different `replica_id`).
3377    ///
3378    /// This test simulates that contract directly:
3379    ///   - A does explicit `debit` + `observe_health` (what `spawn_paired_challenger` does), then
3380    ///     calls `record_pair`.  A's counts stay at the values set by those local calls.
3381    ///   - B calls `sync_from_backend`.  B's counts come from `apply_log_record` (remote record),
3382    ///     matching A's locally-set values exactly.
3383    #[tokio::test]
3384    async fn two_replicas_health_and_spend_count_once() {
3385        let shared = Arc::new(InMemoryExperimentLog::new());
3386        let cfg = config();
3387        let fp_str = fp();
3388        let now_ts = jiff::Timestamp::now();
3389        let now_secs = now_ts.as_second();
3390
3391        let ledger_a = Arc::new(crate::shadow::ShadowLedger::new());
3392        let ledger_b = Arc::new(crate::shadow::ShadowLedger::new());
3393
3394        let replica_a = ExperimentMonitor::from_config(&cfg)
3395            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3396            .with_paired_ledger(Arc::clone(&ledger_a));
3397        let replica_b = ExperimentMonitor::from_config(&cfg)
3398            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3399            .with_paired_ledger(Arc::clone(&ledger_b));
3400
3401        assert!(
3402            replica_a.has_log(),
3403            "log must be attached for this test to be meaningful"
3404        );
3405
3406        // Simulate what spawn_paired_challenger does: ALWAYS debit and tally health locally,
3407        // regardless of whether the shared log is up.  record_pair only propagates evidence to
3408        // other replicas — it does not re-tally on the originating replica.
3409        ledger_a.debit("t", 0, 0.8, now_ts);
3410        replica_a.observe_health("t", 0, &fp_str, Arm::Challenger, true, 1);
3411
3412        let accepted = replica_a
3413            .record_pair(
3414                "t",
3415                0,
3416                &fp_str,
3417                1.0,
3418                0.5,
3419                0.1,
3420                0.8,
3421                true,
3422                1,
3423                now_secs,
3424                Uuid::nil(),
3425                Uuid::nil(),
3426            )
3427            .await;
3428        assert!(accepted, "pair must be accepted");
3429
3430        // A's health comes from the explicit observe_health above, counted exactly once.
3431        // apply_log_record skips own records (same replica_id), so no double-count.
3432        let reps_a = replica_a.reports();
3433        assert_eq!(
3434            reps_a[0].health.challenger.errors, 1,
3435            "A: challenger error counted once, not zero or doubled"
3436        );
3437        assert_eq!(
3438            reps_a[0].health.challenger.truncated, 1,
3439            "A: challenger truncated counted once"
3440        );
3441
3442        // A's ledger comes from the explicit debit above.
3443        let a_spent = ledger_a.spent_today("t", 0, now_ts);
3444        assert!(
3445            (a_spent - 0.8).abs() < 1e-12,
3446            "A: ledger = 0.8 (not doubled or zero); got {a_spent}"
3447        );
3448
3449        // B has not synced yet: no health, no ledger.
3450        assert_eq!(
3451            ledger_b.spent_today("t", 0, now_ts),
3452            0.0,
3453            "B: no spend before sync"
3454        );
3455
3456        // Sync B from the shared log: B sees A's record as "remote" (different replica_id),
3457        // so apply_log_record tallies health and seeds the ledger for B.
3458        let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
3459
3460        let reps_b = replica_b.reports();
3461        assert_eq!(
3462            reps_b[0].health.challenger.errors, 1,
3463            "B: challenger error visible after sync"
3464        );
3465        let b_spent = ledger_b.spent_today("t", 0, now_ts);
3466        assert!(
3467            (b_spent - 0.8).abs() < 1e-12,
3468            "B: ledger reflects A's spend after sync; got {b_spent}"
3469        );
3470
3471        // Add a second pair (via A's local path then record_pair) to push cluster total over cap.
3472        ledger_a.debit("t", 0, 0.3, now_ts);
3473        replica_a.observe_health("t", 0, &fp_str, Arm::Challenger, false, 0);
3474        replica_a
3475            .record_pair(
3476                "t",
3477                0,
3478                &fp_str,
3479                1.0,
3480                0.5,
3481                0.1,
3482                0.3,
3483                false,
3484                0,
3485                now_secs,
3486                Uuid::nil(),
3487                Uuid::nil(),
3488            )
3489            .await;
3490
3491        let a_total = ledger_a.spent_today("t", 0, now_ts);
3492        assert!(
3493            (a_total - 1.1).abs() < 1e-12,
3494            "A: cluster total = 1.1; got {a_total}"
3495        );
3496        assert!(
3497            !ledger_a.may_spend("t", 0, 1.0, now_ts),
3498            "A: cap exhausted (1.1 > 1.0)"
3499        );
3500
3501        // B syncs and also sees the cap exhausted.
3502        let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
3503        assert!(
3504            !ledger_b.may_spend("t", 0, 1.0, now_ts),
3505            "B: cap exhausted from cluster spend"
3506        );
3507    }
3508
3509    // ── Concurrent sync idempotency (Issue 1 fix) ────────────────────────────────────────────
3510
3511    /// A log backend that yields after returning records from `load_since`, allowing the tokio
3512    /// scheduler to run other tasks between the data-load and the re-acquire of the monitor lock.
3513    /// This deterministically surfaces the "two concurrent syncs, same slice" race.
3514    #[derive(Debug)]
3515    struct YieldingLog(InMemoryExperimentLog);
3516
3517    #[async_trait::async_trait]
3518    impl ExperimentLogBackend for YieldingLog {
3519        async fn append(
3520            &self,
3521            tenant: &str,
3522            route_ix: usize,
3523            fingerprint: &str,
3524            record: LogRecord,
3525        ) -> Result<u64, LogError> {
3526            self.0.append(tenant, route_ix, fingerprint, record).await
3527        }
3528
3529        async fn load_since(
3530            &self,
3531            tenant: &str,
3532            route_ix: usize,
3533            fingerprint: &str,
3534            offset: u64,
3535        ) -> Result<Vec<Option<LogRecord>>, LogError> {
3536            let records = self
3537                .0
3538                .load_since(tenant, route_ix, fingerprint, offset)
3539                .await?;
3540            // Yield control so a concurrently-polled task can enter between load and re-lock.
3541            tokio::task::yield_now().await;
3542            Ok(records)
3543        }
3544
3545        async fn claim_session(
3546            &self,
3547            tenant: &str,
3548            route_ix: usize,
3549            fingerprint: &str,
3550            session_id: &str,
3551        ) -> bool {
3552            self.0
3553                .claim_session(tenant, route_ix, fingerprint, session_id)
3554                .await
3555        }
3556    }
3557
3558    /// Two concurrent `sync_from_backend` calls on the same monitor, both loading the same 3
3559    /// records from offset=0, must result in exactly 3 pairs / 3 errors / 3 truncated — never 6.
3560    ///
3561    /// The `YieldingLog` above forces actual task interleaving: both syncs snapshot `offset=0`,
3562    /// call `load_since` (which yields), and then race to apply the same slice. Without the
3563    /// `abs_ix < current_offset` guard and the monotonic-advance, the second sync would re-apply
3564    /// all 3 records and produce 6 pairs (doubling the e-process wealth — a correctness violation).
3565    ///
3566    /// Mutation test: removing the `if abs_ix < current_offset { continue; }` guard makes the
3567    /// second sync apply records 0–2 again. With `ch_error=true` per record, errors go from 3 to
3568    /// 6 and the `assert_eq!(3)` below fails.
3569    #[tokio::test]
3570    async fn concurrent_sync_does_not_double_count_pairs() {
3571        let shared = Arc::new(YieldingLog(InMemoryExperimentLog::new()));
3572        let cfg = config();
3573        let fp_str = fp();
3574
3575        // Append 3 pairs directly (bypassing the monitor, so offset stays at 0 on the monitor).
3576        for _ in 0..3 {
3577            let _ = shared
3578                .append(
3579                    "t",
3580                    0,
3581                    &fp_str,
3582                    LogRecord::Pair {
3583                        x_inc: 1.0,
3584                        x_ch: 1.0,
3585                        incumbent_cost_usd: 0.10,
3586                        challenger_cost_usd: 0.05,
3587                        ch_error: true,
3588                        ch_truncated: 1,
3589                        ts_unix: 0,
3590                        // nil UUID: simulates records from an older replica; treated as remote
3591                        // by apply_log_record (health+ledger are tallied here, not locally).
3592                        replica_id: uuid::Uuid::nil(),
3593                        // nil: no per-pair dedup for this test.
3594                        trace_id: uuid::Uuid::nil(),
3595                        incumbent_trace_id: Uuid::nil(),
3596                    },
3597                )
3598                .await;
3599        }
3600
3601        let monitor = Arc::new(
3602            ExperimentMonitor::from_config(&cfg)
3603                .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>),
3604        );
3605
3606        // Both tasks snapshot `offset=0`, call load_since (which yields to let the other task
3607        // also snapshot offset=0 and call load_since), then race to apply the same 3 records.
3608        let m1 = Arc::clone(&monitor);
3609        let m2 = Arc::clone(&monitor);
3610        let fp1 = fp_str.clone();
3611        let fp2 = fp_str.clone();
3612        tokio::join!(
3613            async move {
3614                let _ = m1.sync_from_backend("t", 0, &fp1).await;
3615            },
3616            async move {
3617                let _ = m2.sync_from_backend("t", 0, &fp2).await;
3618            },
3619        );
3620
3621        let snaps = monitor.reports();
3622        assert_eq!(snaps.len(), 1);
3623        let snap = &snaps[0];
3624        assert_eq!(
3625            snap.report.pairs, 3,
3626            "pairs must be 3, not double-counted: got {}",
3627            snap.report.pairs
3628        );
3629        assert_eq!(
3630            snap.health.challenger.errors, 3,
3631            "errors must be 3, not double-counted: got {}",
3632            snap.health.challenger.errors
3633        );
3634        assert_eq!(
3635            snap.health.challenger.truncated, 3,
3636            "truncated must be 3, not double-counted: got {}",
3637            snap.health.challenger.truncated
3638        );
3639    }
3640
3641    // ── ensure_synced cold-start guard (Issue 2 fix) ─────────────────────────────────────────
3642
3643    /// A replica whose map is empty (cold start) but whose shared log already shows today's spend
3644    /// at the cap must refuse a new evaluation after `ensure_synced`.
3645    ///
3646    /// Mutation test: removing `ensure_synced` or making it a no-op means B starts with an empty
3647    /// ledger, `may_spend` returns true even though the cap is already exhausted, and the
3648    /// `assert!(!may_spend_after_sync)` assertion fails.
3649    #[tokio::test]
3650    async fn ensure_synced_cold_start_respects_cluster_cap() {
3651        let shared = Arc::new(InMemoryExperimentLog::new());
3652        let cfg = config();
3653        let fp_str = fp();
3654        let now_ts = jiff::Timestamp::now();
3655        let cap = 0.5_f64;
3656
3657        let ledger_a = Arc::new(crate::shadow::ShadowLedger::new());
3658        // Replica A already spent the full cap.
3659        // Simulate spawn_paired_challenger: debit locally first, then record to the shared log.
3660        let replica_a = ExperimentMonitor::from_config(&cfg)
3661            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3662            .with_paired_ledger(Arc::clone(&ledger_a));
3663        ledger_a.debit("t", 0, cap, now_ts);
3664        replica_a
3665            .record_pair(
3666                "t",
3667                0,
3668                &fp_str,
3669                1.0,
3670                1.0,
3671                0.10,
3672                cap,
3673                false,
3674                0,
3675                now_ts.as_second(),
3676                Uuid::nil(),
3677                Uuid::nil(),
3678            )
3679            .await;
3680        assert!(
3681            !ledger_a.may_spend("t", 0, cap, now_ts),
3682            "A: cap exhausted after recording one pair"
3683        );
3684
3685        // Replica B starts with an empty map — it has never seen this (tenant, route).
3686        let ledger_b = Arc::new(crate::shadow::ShadowLedger::new());
3687        let replica_b = ExperimentMonitor::from_config(&cfg)
3688            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3689            .with_paired_ledger(Arc::clone(&ledger_b));
3690
3691        // Before ensure_synced: B's ledger is empty → `may_spend` would incorrectly allow spend.
3692        assert!(
3693            ledger_b.may_spend("t", 0, cap, now_ts),
3694            "B: before ensure_synced, empty ledger allows spend (expected)"
3695        );
3696
3697        // After ensure_synced: B syncs from the log and seeds its ledger from A's pair.
3698        // ensure_synced returns true on success; the caller (proxy.rs) uses this to fail-closed.
3699        let synced = replica_b.ensure_synced("t", 0, &fp_str).await;
3700        assert!(
3701            synced,
3702            "ensure_synced must return true when the log is healthy"
3703        );
3704        let may_spend_after_sync = ledger_b.may_spend("t", 0, cap, now_ts);
3705        assert!(
3706            !may_spend_after_sync,
3707            "B: after ensure_synced, cluster spend at cap — must refuse"
3708        );
3709    }
3710
3711    /// A replica whose log already shows a halted experiment must block the challenger after
3712    /// `ensure_synced`, even though it has never seen the experiment locally.
3713    ///
3714    /// Mutation test: removing `ensure_synced` leaves B with `Undecided` (empty map) after the
3715    /// check. `challenger_blocked` returns `false` and the second `assert!` fails.
3716    /// Two requests arriving together on a cold replica both see the cluster halt by the time
3717    /// `ensure_synced` returns: the second must not slip through while the first is still syncing.
3718    #[tokio::test]
3719    async fn ensure_synced_burst_waits_for_synced_state() {
3720        let shared = Arc::new(YieldingLog(InMemoryExperimentLog::new()));
3721        let cfg = config();
3722        let fp_str = fp();
3723        let replica_a = ExperimentMonitor::from_config(&cfg)
3724            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3725        for i in 0..500 {
3726            replica_a
3727                .record_outcome(
3728                    "t",
3729                    0,
3730                    &fp_str,
3731                    Arm::Incumbent,
3732                    &format!("i{i}"),
3733                    Verdict::Pass,
3734                    0.10,
3735                )
3736                .await;
3737            replica_a
3738                .record_outcome(
3739                    "t",
3740                    0,
3741                    &fp_str,
3742                    Arm::Challenger,
3743                    &format!("c{i}"),
3744                    Verdict::Fail,
3745                    0.02,
3746                )
3747                .await;
3748            if replica_a.status("t", 0) == ExperimentStatus::Halted {
3749                break;
3750            }
3751        }
3752        assert_eq!(replica_a.status("t", 0), ExperimentStatus::Halted);
3753
3754        let replica_b = ExperimentMonitor::from_config(&cfg)
3755            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3756        let (first, second) = tokio::join!(
3757            async {
3758                replica_b.ensure_synced("t", 0, &fp_str).await;
3759                replica_b.challenger_blocked("t", 0)
3760            },
3761            async {
3762                replica_b.ensure_synced("t", 0, &fp_str).await;
3763                replica_b.challenger_blocked("t", 0)
3764            },
3765        );
3766        assert!(
3767            first && second,
3768            "both burst requests see the halt: {first} {second}"
3769        );
3770    }
3771
3772    #[tokio::test]
3773    async fn ensure_synced_cold_start_respects_cluster_halt() {
3774        let shared = Arc::new(InMemoryExperimentLog::new());
3775        let cfg = config();
3776        let fp_str = fp();
3777
3778        let replica_a = ExperimentMonitor::from_config(&cfg)
3779            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3780
3781        // Drive experiment to Halted on replica A.
3782        let mut halted = false;
3783        for i in 0..500 {
3784            replica_a
3785                .record_outcome(
3786                    "t",
3787                    0,
3788                    &fp_str,
3789                    Arm::Incumbent,
3790                    &format!("i{i}"),
3791                    Verdict::Pass,
3792                    0.10,
3793                )
3794                .await;
3795            replica_a
3796                .record_outcome(
3797                    "t",
3798                    0,
3799                    &fp_str,
3800                    Arm::Challenger,
3801                    &format!("c{i}"),
3802                    Verdict::Fail,
3803                    0.02,
3804                )
3805                .await;
3806            if replica_a.status("t", 0) == ExperimentStatus::Halted {
3807                halted = true;
3808                break;
3809            }
3810        }
3811        assert!(halted, "experiment never halted on replica A");
3812
3813        // Replica B: cold start — empty map.
3814        let replica_b = ExperimentMonitor::from_config(&cfg)
3815            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3816
3817        // Before ensure_synced: B does not know about the halt.
3818        assert!(
3819            !replica_b.challenger_blocked("t", 0),
3820            "B: before ensure_synced, halt unknown (expected)"
3821        );
3822
3823        // After ensure_synced: B reads the log and sees the Halted status.
3824        let synced = replica_b.ensure_synced("t", 0, &fp_str).await;
3825        assert!(synced, "ensure_synced must succeed with healthy log");
3826        assert!(
3827            replica_b.challenger_blocked("t", 0),
3828            "B: after ensure_synced, halt must block the challenger"
3829        );
3830    }
3831
3832    // ── Issue 1: restart with shared log replays offline pairs ───────────────────────────────
3833
3834    /// On restart with a shared log, `in_log=false` rows (written during a Redis outage) are
3835    /// replayed locally into the monitor. `in_log=true` rows are NOT replayed locally — they
3836    /// come from the log on the first `sync_from_backend` and must not be double-counted.
3837    #[tokio::test]
3838    async fn restart_with_shared_log_replays_offline_pairs_and_skips_log_pairs() {
3839        let db =
3840            std::env::temp_dir().join(format!("firstpass-restart-{}.db", uuid::Uuid::now_v7()));
3841        // Initialize the schema (migrate) before calling append_experiment_pair directly.
3842        let (tx, handle) = crate::store::open(&db).unwrap();
3843        drop(tx);
3844        handle.await.unwrap();
3845        let cfg = config();
3846        let fp_str = fp();
3847        let now_ts = jiff::Timestamp::now();
3848
3849        // NotInLog: written during a Redis outage; must be replayed on restart.
3850        crate::store::append_experiment_pair(
3851            &db,
3852            &crate::store::ExperimentPairRow {
3853                tenant: "t".to_owned(),
3854                route_ix: 0,
3855                fingerprint: fp_str.clone(),
3856                trace_id: uuid::Uuid::now_v7(),
3857                x_inc: 1.0,
3858                x_ch: 0.0,
3859                challenger_cost_usd: 0.5,
3860                ch_error: false,
3861                ch_truncated: 0,
3862                ts: now_ts,
3863                in_log: crate::store::InLogStatus::NotInLog,
3864            },
3865        )
3866        .unwrap();
3867
3868        // InLog: already in Redis; must NOT be replayed locally (comes from sync).
3869        crate::store::append_experiment_pair(
3870            &db,
3871            &crate::store::ExperimentPairRow {
3872                tenant: "t".to_owned(),
3873                route_ix: 0,
3874                fingerprint: fp_str.clone(),
3875                trace_id: uuid::Uuid::now_v7(),
3876                x_inc: 1.0,
3877                x_ch: 0.0,
3878                challenger_cost_usd: 0.3,
3879                ch_error: false,
3880                ch_truncated: 0,
3881                ts: now_ts,
3882                in_log: crate::store::InLogStatus::InLog,
3883            },
3884        )
3885        .unwrap();
3886
3887        // Restart with shared_log=true: only the offline row is replayed locally.
3888        let m = build_monitor_with(&db.to_string_lossy(), &cfg, true);
3889        let reps = m.reports();
3890        assert_eq!(
3891            reps[0].report.pairs, 1,
3892            "only in_log=false row replayed; got {}",
3893            reps[0].report.pairs
3894        );
3895
3896        // Restart with shared_log=false: both rows are replayed (original behaviour).
3897        let m2 = build_monitor_with(&db.to_string_lossy(), &cfg, false);
3898        let reps2 = m2.reports();
3899        assert_eq!(
3900            reps2[0].report.pairs, 2,
3901            "shared_log=false replays all rows; got {}",
3902            reps2[0].report.pairs
3903        );
3904
3905        let _ = std::fs::remove_file(&db);
3906    }
3907
3908    // ── Issue 2: ensure_synced failure cooldown ───────────────────────────────────────────────
3909
3910    /// A log backend whose `load_since` always fails and counts how many times it is called.
3911    #[derive(Debug)]
3912    struct CountingFailLog {
3913        load_count: Arc<std::sync::atomic::AtomicUsize>,
3914    }
3915
3916    #[async_trait::async_trait]
3917    impl ExperimentLogBackend for CountingFailLog {
3918        async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
3919            Err(LogError::Backend("down".to_owned()))
3920        }
3921        async fn load_since(
3922            &self,
3923            _: &str,
3924            _: usize,
3925            _: &str,
3926            _: u64,
3927        ) -> Result<Vec<Option<LogRecord>>, LogError> {
3928            self.load_count
3929                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
3930            Err(LogError::Backend("down".to_owned()))
3931        }
3932        async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
3933            false
3934        }
3935    }
3936
3937    /// A burst of concurrent `ensure_synced` calls when the backend always fails must issue
3938    /// exactly one backend read: the coalescing gate serialises callers, the first one fails
3939    /// and sets the cooldown, and subsequent callers return false immediately without retrying.
3940    #[tokio::test]
3941    async fn ensure_synced_failure_burst_issues_exactly_one_read() {
3942        let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3943        let m = Arc::new(ExperimentMonitor::from_config(&config()).with_log(Arc::new(
3944            CountingFailLog {
3945                load_count: Arc::clone(&load_count),
3946            },
3947        )
3948            as Arc<dyn ExperimentLogBackend>));
3949        let fp_str = fp();
3950
3951        // Concurrent burst of 10 callers on the same (tenant, route).
3952        let handles: Vec<_> = (0..10)
3953            .map(|_| {
3954                let m = Arc::clone(&m);
3955                let fp_str = fp_str.clone();
3956                tokio::spawn(async move { m.ensure_synced("t", 0, &fp_str).await })
3957            })
3958            .collect();
3959        for h in handles {
3960            let result = h.await.unwrap();
3961            assert!(!result, "every caller gets false on backend failure");
3962        }
3963
3964        assert_eq!(
3965            load_count.load(std::sync::atomic::Ordering::SeqCst),
3966            1,
3967            "exactly one backend read issued during a failure burst"
3968        );
3969    }
3970
3971    /// A second call within the cooldown window issues no additional backend read.
3972    #[tokio::test]
3973    async fn ensure_synced_cooldown_skips_retry_within_window() {
3974        let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3975        let m = ExperimentMonitor::from_config(&config())
3976            .with_log(Arc::new(CountingFailLog {
3977                load_count: Arc::clone(&load_count),
3978            }) as Arc<dyn ExperimentLogBackend>);
3979        let fp_str = fp();
3980
3981        // First call: fails, sets cooldown.
3982        assert!(!m.ensure_synced("t", 0, &fp_str).await);
3983        assert_eq!(load_count.load(std::sync::atomic::Ordering::SeqCst), 1);
3984
3985        // Second call within cooldown: returns false immediately, no backend read.
3986        assert!(!m.ensure_synced("t", 0, &fp_str).await);
3987        assert_eq!(
3988            load_count.load(std::sync::atomic::Ordering::SeqCst),
3989            1,
3990            "no additional read within the cooldown window"
3991        );
3992    }
3993
3994    /// After the cooldown expires a retry is attempted (exactly one more read).
3995    #[tokio::test]
3996    async fn ensure_synced_cooldown_allows_retry_after_expiry() {
3997        tokio::time::pause();
3998        let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3999        let m = ExperimentMonitor::from_config(&config())
4000            .with_log(Arc::new(CountingFailLog {
4001                load_count: Arc::clone(&load_count),
4002            }) as Arc<dyn ExperimentLogBackend>);
4003        let fp_str = fp();
4004
4005        // First call: fails, sets cooldown.
4006        assert!(!m.ensure_synced("t", 0, &fp_str).await);
4007        assert_eq!(load_count.load(std::sync::atomic::Ordering::SeqCst), 1);
4008
4009        // Advance past the cooldown.
4010        tokio::time::advance(SYNC_FAILURE_COOLDOWN + std::time::Duration::from_millis(1)).await;
4011
4012        // Next call: cooldown expired, issues one more read.
4013        assert!(!m.ensure_synced("t", 0, &fp_str).await);
4014        assert_eq!(
4015            load_count.load(std::sync::atomic::Ordering::SeqCst),
4016            2,
4017            "one more read after the cooldown expires"
4018        );
4019    }
4020
4021    // ── Durability ordering: SQLite precedes local evidence (Issue 1) ────────────────────────
4022
4023    /// `record_pair` returns false when the log is down and does NOT apply evidence.
4024    /// The caller (`spawn_paired_challenger`) writes SQLite first, then calls
4025    /// `apply_pair_locally`. Here we verify the contract: `record_pair` returns false and
4026    /// the experiment is still Undecided; only `apply_pair_locally` advances the state.
4027    #[tokio::test]
4028    async fn record_pair_does_not_apply_evidence_on_append_failure_caller_does() {
4029        let m = ExperimentMonitor::from_config(&config())
4030            .with_log(Arc::new(DownLog) as Arc<dyn ExperimentLogBackend>);
4031        let fp_str = fp();
4032
4033        let mut pairs_applied = 0u32;
4034        for _ in 0..500 {
4035            if m.challenger_blocked("t", 0) {
4036                break;
4037            }
4038            // record_pair returns false and does NOT apply evidence.
4039            let accepted = m
4040                .record_pair(
4041                    "t",
4042                    0,
4043                    &fp_str,
4044                    1.0,
4045                    0.0,
4046                    0.1,
4047                    0.01,
4048                    false,
4049                    0,
4050                    0,
4051                    Uuid::nil(),
4052                    Uuid::nil(),
4053                )
4054                .await;
4055            assert!(!accepted, "append failing log returns false");
4056            // Caller writes SQLite (not modelled here), then applies evidence.
4057            m.apply_pair_locally("t", 0, &fp_str, 1.0, 0.0, 0.01, Uuid::nil());
4058            pairs_applied += 1;
4059        }
4060
4061        // Evidence was applied locally via apply_pair_locally: enough losing pairs halted.
4062        assert!(
4063            m.challenger_blocked("t", 0),
4064            "experiment must halt locally after {pairs_applied} pairs via apply_pair_locally"
4065        );
4066        assert!(
4067            m.reports()[0].report.pairs > 0,
4068            "pairs counted locally via apply_pair_locally"
4069        );
4070    }
4071
4072    /// `record_pair` in no-log mode returns false and leaves the experiment Undecided.
4073    /// Only `apply_pair_locally` advances state. Mutation check: removing `apply_pair_locally`
4074    /// keeps the experiment Undecided and the assertion fires.
4075    #[tokio::test]
4076    async fn record_pair_no_log_does_not_apply_evidence() {
4077        let m = ExperimentMonitor::from_config(&config()); // no log
4078        let fp_str = fp();
4079
4080        // Call record_pair many times — should never advance evidence without apply_pair_locally.
4081        for _ in 0..10 {
4082            let ok = m
4083                .record_pair(
4084                    "t",
4085                    0,
4086                    &fp_str,
4087                    1.0,
4088                    0.0,
4089                    0.1,
4090                    0.01,
4091                    false,
4092                    0,
4093                    0,
4094                    Uuid::nil(),
4095                    Uuid::nil(),
4096                )
4097                .await;
4098            assert!(!ok, "no log → record_pair returns false");
4099        }
4100        // No entry was created: record_pair in no-log mode returns immediately.
4101        assert_eq!(
4102            m.reports().len(),
4103            0,
4104            "record_pair in no-log mode must not create an entry or advance evidence"
4105        );
4106
4107        // Now apply via apply_pair_locally — entry created and evidence advances.
4108        m.apply_pair_locally("t", 0, &fp_str, 1.0, 0.0, 0.01, Uuid::nil());
4109        assert_eq!(
4110            m.reports()[0].report.pairs,
4111            1,
4112            "apply_pair_locally counted the pair"
4113        );
4114    }
4115
4116    // ── Issue 1b: Unknown tri-state for legacy rows ───────────────────────────────────────────
4117
4118    /// Rows migrated from a schema without `in_log` get `InLogStatus::Unknown` (DEFAULT 2).
4119    /// With shared_log=true, Unknown rows contribute spend (cap is stricter — safe) but NOT
4120    /// evidence (withholding only delays a decision — safe).
4121    #[tokio::test]
4122    async fn unknown_rows_contribute_spend_but_not_evidence_with_shared_log() {
4123        use crate::store::InLogStatus;
4124        let db =
4125            std::env::temp_dir().join(format!("firstpass-unknown-{}.db", uuid::Uuid::now_v7()));
4126        let (tx, handle) = crate::store::open(&db).unwrap();
4127        drop(tx);
4128        handle.await.unwrap();
4129        let cfg = config();
4130        let fp_str = fp();
4131        let now_ts = jiff::Timestamp::now();
4132
4133        // One Unknown row (legacy) and one NotInLog row (outage).
4134        for (in_log, x_ch) in [
4135            (InLogStatus::Unknown, 0.8_f64),
4136            (InLogStatus::NotInLog, 0.6),
4137        ] {
4138            crate::store::append_experiment_pair(
4139                &db,
4140                &crate::store::ExperimentPairRow {
4141                    tenant: "t".to_owned(),
4142                    route_ix: 0,
4143                    fingerprint: fp_str.clone(),
4144                    trace_id: uuid::Uuid::now_v7(),
4145                    x_inc: 1.0,
4146                    x_ch,
4147                    challenger_cost_usd: 0.1,
4148                    ch_error: false,
4149                    ch_truncated: 0,
4150                    ts: now_ts,
4151                    in_log,
4152                },
4153            )
4154            .unwrap();
4155        }
4156
4157        // shared_log=true: only NotInLog row contributes evidence (1 pair); Unknown withheld.
4158        let m = build_monitor_with(&db.to_string_lossy(), &cfg, true);
4159        let reps = m.reports();
4160        assert_eq!(
4161            reps[0].report.pairs, 1,
4162            "shared_log=true: only NotInLog evidence replayed; got {}",
4163            reps[0].report.pairs
4164        );
4165
4166        // shared_log=false: both rows contribute evidence (2 pairs).
4167        let m2 = build_monitor_with(&db.to_string_lossy(), &cfg, false);
4168        let reps2 = m2.reports();
4169        assert_eq!(
4170            reps2[0].report.pairs, 2,
4171            "shared_log=false: all rows replayed; got {}",
4172            reps2[0].report.pairs
4173        );
4174
4175        let _ = std::fs::remove_file(&db);
4176    }
4177
4178    // ── Issue 2: load_since preserves positions of malformed entries ─────────────────────────
4179
4180    /// A log backend that injects a None (malformed) entry between two valid ones.
4181    /// Position-preserving: the returned vec has length 3 (valid, None, valid).
4182    #[derive(Debug)]
4183    struct MalformedMiddleLog {
4184        inner: InMemoryExperimentLog,
4185    }
4186
4187    #[async_trait::async_trait]
4188    impl ExperimentLogBackend for MalformedMiddleLog {
4189        async fn append(
4190            &self,
4191            tenant: &str,
4192            route_ix: usize,
4193            fingerprint: &str,
4194            record: LogRecord,
4195        ) -> Result<u64, LogError> {
4196            self.inner
4197                .append(tenant, route_ix, fingerprint, record)
4198                .await
4199        }
4200
4201        async fn load_since(
4202            &self,
4203            tenant: &str,
4204            route_ix: usize,
4205            fingerprint: &str,
4206            offset: u64,
4207        ) -> Result<Vec<Option<LogRecord>>, LogError> {
4208            // Call inner (which returns Vec<Option<LogRecord>> with all Some).
4209            let mut records = self
4210                .inner
4211                .load_since(tenant, route_ix, fingerprint, offset)
4212                .await?;
4213            // Inject None at index 1 (if the list has at least 2 elements) to simulate a
4214            // malformed entry. This preserves list length so offset arithmetic stays correct.
4215            if records.len() >= 2 {
4216                records[1] = None;
4217            }
4218            Ok(records)
4219        }
4220
4221        async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4222            false
4223        }
4224    }
4225
4226    /// A log with a malformed entry at position 1 must:
4227    /// 1. Apply position 0 and position 2 exactly once each (None at position 1 is skipped).
4228    /// 2. After two consecutive syncs, `remote_log_offset` == 3 (all positions consumed).
4229    /// 3. A third sync reads nothing new (offset already at 3).
4230    #[tokio::test]
4231    async fn malformed_log_entry_advances_offset_and_valid_entries_applied_once() {
4232        let log = Arc::new(MalformedMiddleLog {
4233            inner: InMemoryExperimentLog::new(),
4234        });
4235        let cfg = config();
4236        let fp_str = fp();
4237
4238        // Append 3 pairs directly to the inner log (records 0, 1, 2).
4239        for _ in 0..3 {
4240            let _ = log
4241                .inner
4242                .append(
4243                    "t",
4244                    0,
4245                    &fp_str,
4246                    LogRecord::Pair {
4247                        x_inc: 1.0,
4248                        x_ch: 1.0,
4249                        incumbent_cost_usd: 0.1,
4250                        challenger_cost_usd: 0.05,
4251                        ch_error: false,
4252                        ch_truncated: 0,
4253                        ts_unix: 0,
4254                        replica_id: uuid::Uuid::nil(),
4255                        trace_id: uuid::Uuid::nil(),
4256                        incumbent_trace_id: Uuid::nil(),
4257                    },
4258                )
4259                .await;
4260        }
4261        // The MalformedMiddleLog replaces position 1 with None.
4262
4263        let m = ExperimentMonitor::from_config(&cfg)
4264            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4265
4266        // First sync: applies positions 0 and 2 (valid), skips position 1 (None).
4267        let _ = m.sync_from_backend("t", 0, &fp_str).await;
4268        let pairs_after_first = m.reports()[0].report.pairs;
4269        // 2 valid entries applied (positions 0 and 2); position 1 consumed but not counted.
4270        assert_eq!(
4271            pairs_after_first, 2,
4272            "first sync: 2 valid entries applied, got {pairs_after_first}"
4273        );
4274
4275        // Second sync: offset is already at 3, nothing new.
4276        let _ = m.sync_from_backend("t", 0, &fp_str).await;
4277        let pairs_after_second = m.reports()[0].report.pairs;
4278        assert_eq!(
4279            pairs_after_second, 2,
4280            "second sync: no re-application, pairs still 2, got {pairs_after_second}"
4281        );
4282
4283        // Verify offset equals list length: offset must be 3 (all 3 positions consumed).
4284        let offset = {
4285            let map = m.lock();
4286            map.get("t")
4287                .and_then(|r| r.get(&0))
4288                .map_or(0, |e| e.remote_log_offset)
4289        };
4290        assert_eq!(
4291            offset, 3,
4292            "offset must equal list length after consuming all positions"
4293        );
4294    }
4295
4296    // ── Issue 3: trace_id dedup prevents double-counting on append-timeout ───────────────────
4297
4298    /// A backend that appends successfully (the record IS in the log) but always returns Err
4299    /// from `append` (simulates an append that timed out after Redis applied it).
4300    #[derive(Debug)]
4301    struct PushThenErrorLog {
4302        inner: InMemoryExperimentLog,
4303    }
4304
4305    #[async_trait::async_trait]
4306    impl ExperimentLogBackend for PushThenErrorLog {
4307        async fn append(
4308            &self,
4309            tenant: &str,
4310            route_ix: usize,
4311            fingerprint: &str,
4312            record: LogRecord,
4313        ) -> Result<u64, LogError> {
4314            // Write the record to the inner log (it IS there).
4315            let _ = self
4316                .inner
4317                .append(tenant, route_ix, fingerprint, record)
4318                .await;
4319            // But report failure to the caller (timeout simulation).
4320            Err(LogError::Backend("timeout after write".to_owned()))
4321        }
4322
4323        async fn load_since(
4324            &self,
4325            tenant: &str,
4326            route_ix: usize,
4327            fingerprint: &str,
4328            offset: u64,
4329        ) -> Result<Vec<Option<LogRecord>>, LogError> {
4330            self.inner
4331                .load_since(tenant, route_ix, fingerprint, offset)
4332                .await
4333        }
4334
4335        async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4336            false
4337        }
4338    }
4339
4340    /// When `append` times out after Redis applied it, the evidence must count exactly once:
4341    /// - Caller writes SQLite (NotInLog) then calls `apply_pair_locally`.
4342    /// - `sync_from_backend` sees the same trace_id in the log and skips the evidence dedup.
4343    ///
4344    /// Also covers the restart path (Issue 3): health and ledger are only applied once even
4345    /// when the record was seen at startup from SQLite and later from the log.
4346    #[tokio::test]
4347    async fn trace_id_dedup_prevents_double_count_on_append_timeout() {
4348        let log = Arc::new(PushThenErrorLog {
4349            inner: InMemoryExperimentLog::new(),
4350        });
4351        let cfg = config();
4352        let fp_str = fp();
4353        let tid = Uuid::now_v7();
4354        let now_ts = jiff::Timestamp::now();
4355
4356        let ledger = Arc::new(crate::shadow::ShadowLedger::new());
4357        let m = ExperimentMonitor::from_config(&cfg)
4358            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>)
4359            .with_paired_ledger(Arc::clone(&ledger));
4360
4361        // record_pair: append fails (Err) but the record IS in the log.
4362        // With the new durability ordering, record_pair returns false and does NOT
4363        // apply evidence — the caller (spawn_paired_challenger) does it after SQLite write.
4364        let appended = m
4365            .record_pair(
4366                "t",
4367                0,
4368                &fp_str,
4369                1.0,
4370                1.0,
4371                0.1,
4372                0.05,
4373                false,
4374                1,
4375                now_ts.as_second(),
4376                tid,
4377                Uuid::nil(),
4378            )
4379            .await;
4380        assert!(!appended, "append returned Err → record_pair returns false");
4381        // record_pair returns false and does NOT create an Entry or apply evidence.
4382        // The entry is created lazily by apply_pair_locally or observe_* calls.
4383        assert_eq!(
4384            m.reports().len(),
4385            0,
4386            "no entry created yet by record_pair alone"
4387        );
4388
4389        // Caller simulates the SQLite write then calls apply_pair_locally.
4390        // Also simulates spawn_paired_challenger's local health+ledger tally.
4391        ledger.debit("t", 0, 0.05, now_ts);
4392        m.observe_health("t", 0, &fp_str, Arm::Challenger, false, 1);
4393        m.apply_pair_locally("t", 0, &fp_str, 1.0, 1.0, 0.05, tid);
4394        assert_eq!(
4395            m.reports()[0].report.pairs,
4396            1,
4397            "evidence applied via apply_pair_locally"
4398        );
4399
4400        // Sync: the record is in the log (PushThenErrorLog wrote it). Without dedup,
4401        // apply_log_record would count evidence + health + ledger again.  With dedup:
4402        // - evidence is skipped (trace_id already in applied_pair_ids)
4403        // - health and ledger are also skipped (Issue 3: nested under !already_applied)
4404        let _ = m.sync_from_backend("t", 0, &fp_str).await;
4405        assert_eq!(
4406            m.reports()[0].report.pairs,
4407            1,
4408            "after sync: pairs still 1 (trace_id dedup prevented double-count)"
4409        );
4410        let h = m.reports()[0].health.challenger;
4411        assert_eq!(
4412            (h.errors, h.truncated),
4413            (0, 1),
4414            "health counted exactly once (not re-applied by apply_log_record)"
4415        );
4416        assert!(
4417            (ledger.spent_today("t", 0, now_ts) - 0.05).abs() < 1e-9,
4418            "ledger spend counted exactly once (not re-seeded by apply_log_record)"
4419        );
4420    }
4421
4422    // ── Issue 2: Staleness bound ──────────────────────────────────────────────────────────────
4423
4424    /// `ensure_synced` returns true while the last sync was within `SYNC_STALENESS_BOUND`,
4425    /// and returns false (and attempts a re-sync) once the bound is exceeded.
4426    ///
4427    /// Uses `tokio::time::pause` to advance time without real I/O.
4428    #[tokio::test]
4429    async fn ensure_synced_stale_after_bound_exceeded() {
4430        tokio::time::pause();
4431        let shared = Arc::new(InMemoryExperimentLog::new());
4432        let m = Arc::new(
4433            ExperimentMonitor::from_config(&config())
4434                .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>),
4435        );
4436        let fp_str = fp();
4437
4438        // First sync succeeds: state is fresh.
4439        assert!(
4440            m.ensure_synced("t", 0, &fp_str).await,
4441            "first sync must succeed"
4442        );
4443
4444        // Advance time within the staleness bound (half-way).
4445        tokio::time::advance(SYNC_STALENESS_BOUND / 2).await;
4446        assert!(
4447            m.ensure_synced("t", 0, &fp_str).await,
4448            "still within staleness bound → true"
4449        );
4450
4451        // Advance past the staleness bound.
4452        tokio::time::advance(SYNC_STALENESS_BOUND).await;
4453
4454        // Now state is stale.  The log is reachable (InMemory) so the re-sync succeeds.
4455        assert!(
4456            m.ensure_synced("t", 0, &fp_str).await,
4457            "stale but log reachable → sync succeeds → true"
4458        );
4459
4460        // Swap to a failing log to simulate Redis going down.
4461        // Replace the log: build a new monitor with the failing log but same entry state.
4462        let m_down = Arc::new(ExperimentMonitor::from_config(&config()).with_log(Arc::new(
4463            CountingFailLog {
4464                load_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
4465            },
4466        )
4467            as Arc<dyn ExperimentLogBackend>));
4468        // No prior sync → first ensure_synced attempt fails (backend down).
4469        assert!(
4470            !m_down.ensure_synced("t", 0, &fp_str).await,
4471            "log down on first sync → false"
4472        );
4473        // Cooldown active: second call returns false without re-attempting.
4474        assert!(
4475            !m_down.ensure_synced("t", 0, &fp_str).await,
4476            "cooldown active → false immediately"
4477        );
4478    }
4479
4480    /// After a restart, a pair replayed from the local store (its append had timed out after
4481    /// Redis applied it) and then seen again in the log under the old replica id counts health
4482    /// and spend once: the restarted process has a new replica id, so only the trace_id dedup
4483    /// stops the log copy from being tallied a second time.
4484    #[tokio::test]
4485    async fn restart_counts_a_timed_out_pair_once_across_store_and_log() {
4486        let db =
4487            std::env::temp_dir().join(format!("firstpass-restart-dedup-{}.db", Uuid::now_v7()));
4488        crate::store::ensure_migrated(&db).unwrap();
4489        let log = Arc::new(InMemoryExperimentLog::new());
4490        let cfg = config();
4491        let fp_str = fp();
4492        let tid = Uuid::now_v7();
4493        let now = jiff::Timestamp::now();
4494        // The record IS in the log, written by the previous process (a different replica id).
4495        let before_restart = ExperimentMonitor::from_config(&cfg)
4496            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4497        log.append(
4498            "t",
4499            0,
4500            &fp_str,
4501            LogRecord::Pair {
4502                x_inc: 1.0,
4503                x_ch: 1.0,
4504                incumbent_cost_usd: 0.1,
4505                challenger_cost_usd: 0.05,
4506                ch_error: false,
4507                ch_truncated: 1,
4508                ts_unix: now.as_second(),
4509                replica_id: before_restart.replica_id,
4510                trace_id: tid,
4511                incumbent_trace_id: Uuid::nil(),
4512            },
4513        )
4514        .await
4515        .unwrap();
4516        // ...and in the local store as NotInLog, because the append reported a timeout.
4517        crate::store::append_experiment_pair(
4518            &db,
4519            &crate::store::ExperimentPairRow {
4520                tenant: "t".to_owned(),
4521                route_ix: 0,
4522                fingerprint: fp_str.clone(),
4523                trace_id: tid,
4524                x_inc: 1.0,
4525                x_ch: 1.0,
4526                challenger_cost_usd: 0.05,
4527                ch_error: false,
4528                ch_truncated: 1,
4529                ts: now,
4530                in_log: crate::store::InLogStatus::NotInLog,
4531            },
4532        )
4533        .unwrap();
4534
4535        // Restart: rebuild from the store with a shared log, as run.rs does.
4536        let ledger = Arc::new(crate::shadow::ShadowLedger::new());
4537        ledger.debit("t", 0, 0.05, now); // what build_paired_ledger seeds from the NotInLog row
4538        let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
4539            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>)
4540            .with_paired_ledger(Arc::clone(&ledger));
4541        assert_ne!(restarted.replica_id, before_restart.replica_id);
4542        restarted.sync_from_backend("t", 0, &fp_str).await.unwrap();
4543
4544        let r = &restarted.reports()[0];
4545        assert_eq!(r.report.pairs, 1, "evidence once");
4546        assert_eq!(r.health.challenger.truncated, 1, "health once");
4547        assert!(
4548            (ledger.spent_today("t", 0, now) - 0.05).abs() < 1e-9,
4549            "spend once: {}",
4550            ledger.spent_today("t", 0, now)
4551        );
4552        let _ = std::fs::remove_file(&db);
4553    }
4554
4555    /// A log that can be switched to failing mid-test, counting reads.
4556    #[derive(Debug, Default)]
4557    struct SwitchableLog {
4558        inner: InMemoryExperimentLog,
4559        failing: std::sync::atomic::AtomicBool,
4560        reads: std::sync::atomic::AtomicUsize,
4561    }
4562
4563    #[async_trait::async_trait]
4564    impl ExperimentLogBackend for SwitchableLog {
4565        async fn append(
4566            &self,
4567            t: &str,
4568            r: usize,
4569            f: &str,
4570            rec: LogRecord,
4571        ) -> Result<u64, LogError> {
4572            self.inner.append(t, r, f, rec).await
4573        }
4574        async fn load_since(
4575            &self,
4576            t: &str,
4577            r: usize,
4578            f: &str,
4579            o: u64,
4580        ) -> Result<Vec<Option<LogRecord>>, LogError> {
4581            self.reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4582            if self.failing.load(std::sync::atomic::Ordering::SeqCst) {
4583                return Err(LogError::Backend("down".to_owned()));
4584            }
4585            self.inner.load_since(t, r, f, o).await
4586        }
4587        async fn claim_session(&self, t: &str, r: usize, f: &str, s: &str) -> bool {
4588            self.inner.claim_session(t, r, f, s).await
4589        }
4590    }
4591
4592    /// The append lands but the read-back fails: the pair's evidence is applied locally right
4593    /// away (a halt is not delayed), and when the log recovers the same record is not counted
4594    /// again.
4595    #[tokio::test]
4596    async fn read_back_failure_applies_the_pair_locally_and_recovery_does_not_double_count() {
4597        let log = Arc::new(SwitchableLog::default());
4598        let m = ExperimentMonitor::from_config(&config())
4599            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4600        let fp_str = fp();
4601        log.failing.store(true, std::sync::atomic::Ordering::SeqCst);
4602        let tid = Uuid::now_v7();
4603        let appended = m
4604            .record_pair(
4605                "t",
4606                0,
4607                &fp_str,
4608                1.0,
4609                0.0,
4610                0.1,
4611                0.05,
4612                false,
4613                0,
4614                0,
4615                tid,
4616                Uuid::nil(),
4617            )
4618            .await;
4619        assert!(appended, "the append itself succeeded");
4620        assert_eq!(
4621            m.reports()[0].report.pairs,
4622            1,
4623            "applied locally despite the failed read"
4624        );
4625        log.failing
4626            .store(false, std::sync::atomic::Ordering::SeqCst);
4627        m.sync_from_backend("t", 0, &fp_str).await.unwrap();
4628        assert_eq!(
4629            m.reports()[0].report.pairs,
4630            1,
4631            "recovery sync deduplicated by trace_id"
4632        );
4633    }
4634
4635    /// A shared log whose append never completes (a half-open connection).
4636    #[derive(Debug)]
4637    struct HungAppendLog;
4638
4639    #[async_trait::async_trait]
4640    impl ExperimentLogBackend for HungAppendLog {
4641        async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
4642            std::future::pending().await
4643        }
4644        async fn load_since(
4645            &self,
4646            _: &str,
4647            _: usize,
4648            _: &str,
4649            _: u64,
4650        ) -> Result<Vec<Option<LogRecord>>, LogError> {
4651            Ok(vec![])
4652        }
4653        async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4654            false
4655        }
4656    }
4657
4658    /// A hung append cannot stall the paired task: `record_pair` gives up after the bound and
4659    /// reports "not in the log", so the caller writes the durable NotInLog row.
4660    #[tokio::test]
4661    async fn hung_append_is_bounded_and_reported_as_not_in_log() {
4662        tokio::time::pause();
4663        let m = ExperimentMonitor::from_config(&config()).with_log(Arc::new(HungAppendLog));
4664        let appended = m
4665            .record_pair(
4666                "t",
4667                0,
4668                &fp(),
4669                1.0,
4670                0.0,
4671                0.1,
4672                0.05,
4673                false,
4674                0,
4675                0,
4676                Uuid::now_v7(),
4677                Uuid::nil(),
4678            )
4679            .await;
4680        assert!(!appended, "a timed-out append is not in the log");
4681    }
4682
4683    /// A replica that did not serve the incumbent request still learns its cost from the log, so
4684    /// the incumbent's $/request is not deflated in a cluster.
4685    #[tokio::test]
4686    async fn remote_pairs_carry_the_incumbent_cost() {
4687        let shared = Arc::new(InMemoryExperimentLog::new());
4688        let a = ExperimentMonitor::from_config(&config())
4689            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4690        let b = ExperimentMonitor::from_config(&config())
4691            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4692        assert!(
4693            a.record_pair(
4694                "t",
4695                0,
4696                &fp(),
4697                1.0,
4698                1.0,
4699                0.10,
4700                0.02,
4701                false,
4702                0,
4703                0,
4704                Uuid::now_v7(),
4705                Uuid::nil(),
4706            )
4707            .await
4708        );
4709        b.sync_from_backend("t", 0, &fp()).await.unwrap();
4710        let inc = b.reports()[0].report.incumbent;
4711        assert_eq!(inc.requests, 1, "{inc:?}");
4712        assert!((inc.cost_usd - 0.10).abs() < 1e-12, "{inc:?}");
4713    }
4714
4715    /// A replica restarted on the same trace store counts the incumbent cost of its own
4716    /// pre-restart pairs once (from the stored receipt), even though its new `replica_id` makes
4717    /// those log records look remote.
4718    #[tokio::test]
4719    async fn restart_counts_own_pre_restart_incumbent_cost_once() {
4720        let shared = Arc::new(InMemoryExperimentLog::new());
4721        let before = ExperimentMonitor::from_config(&config())
4722            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4723        let served = trace("t", "s", Arm::Incumbent, 0.10);
4724        assert!(
4725            before
4726                .record_pair(
4727                    "t",
4728                    0,
4729                    &fp(),
4730                    1.0,
4731                    1.0,
4732                    0.10,
4733                    0.02,
4734                    false,
4735                    0,
4736                    0,
4737                    Uuid::now_v7(),
4738                    served.trace_id,
4739                )
4740                .await
4741        );
4742        let after = monitor_from_traces(std::slice::from_ref(&served), &[], &[], &config())
4743            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4744        after.sync_from_backend("t", 0, &fp()).await.unwrap();
4745        let r = &after.reports()[0].report;
4746        assert_eq!(r.pairs, 1, "the pair's evidence still arrives from the log");
4747        assert_eq!(r.incumbent.requests, 1, "{:?}", r.incumbent);
4748        assert!(
4749            (r.incumbent.cost_usd - 0.10).abs() < 1e-12,
4750            "{:?}",
4751            r.incumbent
4752        );
4753    }
4754
4755    /// A pair stored twice in the local table is replayed once at startup.
4756    #[test]
4757    fn startup_replay_counts_a_duplicated_pair_row_once() {
4758        let tid = Uuid::now_v7();
4759        let row = crate::store::ExperimentPairRow {
4760            tenant: "t".to_owned(),
4761            route_ix: 0,
4762            fingerprint: fp(),
4763            trace_id: tid,
4764            x_inc: 1.0,
4765            x_ch: 0.0,
4766            challenger_cost_usd: 0.05,
4767            ch_error: true,
4768            ch_truncated: 0,
4769            ts: jiff::Timestamp::now(),
4770            in_log: crate::store::InLogStatus::NotInLog,
4771        };
4772        let m = monitor_from_traces(&[], &[], &[row.clone(), row], &config());
4773        let r = &m.reports()[0];
4774        assert_eq!(r.report.pairs, 1);
4775        assert_eq!(r.health.challenger.errors, 1);
4776    }
4777
4778    /// On ONE monitor: synced state stays trusted only within the staleness bound. Once the log
4779    /// goes down, a recent sync still counts (no read), but past the bound the stale state is
4780    /// not trusted and `ensure_synced` fails closed.
4781    #[tokio::test]
4782    async fn synced_state_expires_after_the_staleness_bound_when_the_log_is_down() {
4783        tokio::time::pause();
4784        let log = Arc::new(SwitchableLog::default());
4785        let m = ExperimentMonitor::from_config(&config())
4786            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4787        let fp_str = fp();
4788        assert!(
4789            m.ensure_synced("t", 0, &fp_str).await,
4790            "first sync succeeds"
4791        );
4792        log.failing.store(true, std::sync::atomic::Ordering::SeqCst);
4793        let reads = log.reads.load(std::sync::atomic::Ordering::SeqCst);
4794        tokio::time::advance(SYNC_STALENESS_BOUND / 2).await;
4795        assert!(m.ensure_synced("t", 0, &fp_str).await, "fresh: trusted");
4796        assert_eq!(
4797            log.reads.load(std::sync::atomic::Ordering::SeqCst),
4798            reads,
4799            "fresh state needs no read"
4800        );
4801        tokio::time::advance(SYNC_STALENESS_BOUND).await;
4802        assert!(
4803            !m.ensure_synced("t", 0, &fp_str).await,
4804            "stale and the log is down: fail closed"
4805        );
4806    }
4807
4808    // ── Issue 4: sync timeout ────────────────────────────────────────────────────────────────
4809
4810    /// A `load_since` that never completes (pending forever) must not stall `ensure_synced`.
4811    /// With `tokio::time::pause`, the timeout fires instantly after advancing time.
4812    #[tokio::test]
4813    async fn ensure_synced_times_out_on_hung_backend() {
4814        tokio::time::pause();
4815
4816        /// A backend whose `load_since` never completes (simulates a half-open TCP connection).
4817        #[derive(Debug)]
4818        struct HangingLog;
4819
4820        #[async_trait::async_trait]
4821        impl ExperimentLogBackend for HangingLog {
4822            async fn append(
4823                &self,
4824                _: &str,
4825                _: usize,
4826                _: &str,
4827                _: LogRecord,
4828            ) -> Result<u64, LogError> {
4829                Err(LogError::Backend("down".to_owned()))
4830            }
4831            async fn load_since(
4832                &self,
4833                _: &str,
4834                _: usize,
4835                _: &str,
4836                _: u64,
4837            ) -> Result<Vec<Option<LogRecord>>, LogError> {
4838                // Pend forever: never resolves.
4839                std::future::pending::<Result<Vec<Option<LogRecord>>, LogError>>().await
4840            }
4841            async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4842                false
4843            }
4844        }
4845
4846        let m = Arc::new(
4847            ExperimentMonitor::from_config(&config())
4848                .with_log(Arc::new(HangingLog) as Arc<dyn ExperimentLogBackend>),
4849        );
4850        let fp_str = fp();
4851
4852        // ensure_synced wraps the sync in a timeout; the hanging load_since fires the timeout.
4853        // We advance time past SYNC_TIMEOUT so the timeout elapses.
4854        let result = tokio::select! {
4855            r = m.ensure_synced("t", 0, &fp_str) => r,
4856            _ = tokio::time::sleep(SYNC_TIMEOUT + std::time::Duration::from_millis(1)) => {
4857                // Advance the mock clock past the timeout.
4858                tokio::time::advance(SYNC_TIMEOUT + std::time::Duration::from_millis(10)).await;
4859                m.ensure_synced("t", 0, &fp_str).await
4860            }
4861        };
4862        assert!(
4863            !result,
4864            "ensure_synced must return false when load_since hangs (timeout fires)"
4865        );
4866    }
4867}