Skip to main content

firstpass_proxy/
experiment.rs

1//! Live monitor for certified model selection (ADR 0014).
2//!
3//! A route with a `[route.challenger]` serves a randomized fraction of sessions from a second
4//! ladder. This module owns the evidence: one [`Experiment`] per `(tenant, route)`, fed with the
5//! cost of every served experiment request and the first task outcome of every session. Its two
6//! jobs are the halt guard (the serving path asks [`ExperimentMonitor::status`] and stops using a
7//! challenger that has been proven worse) and the report an operator reads before adopting one.
8//!
9//! ## What counts as an outcome
10//!
11//! Feedback with a `pass` or `fail` verdict counts, **whatever its `gate_id`**; `abstain` is
12//! ignored, and only the first counted outcome per session is used. Nothing here can tell a CI
13//! result from a human rating, so operators should post exactly one thing per session: the task
14//! outcome the experiment is meant to compare. Posting several different signals under different
15//! gate ids makes "first one wins" arbitrary.
16//!
17//! ## Configuration identity
18//!
19//! Every receipt carries a fingerprint of the experiment configuration (both ladders, margin,
20//! alpha). Receipts whose fingerprint differs from the current config's are ignored and counted
21//! (`ignored_stale`), so changing either ladder or the margin/alpha starts a fresh experiment
22//! instead of pooling evidence about a different challenger.
23//!
24//! Tenants never share an experiment (ADR 0004): the key is `(tenant, route_ix)` and nothing here
25//! reads across tenants.
26//!
27//! The same state is re-derivable from receipts. [`experiments_from_traces`] replays the stored
28//! traces and deferred verdicts through the very same monitor, so a restart (or the
29//! `firstpass experiment` CLI) lands on the decision the live process would have reached.
30//!
31//! ## Shared state across replicas (opt-in)
32//!
33//! When [`ExperimentMonitor`] is built with a [`ExperimentLogBackend`] (set via
34//! [`ExperimentMonitor::with_log`]), every outcome (live mode) or pair (paired mode) is appended
35//! to a shared log keyed by `(tenant, route_ix, fingerprint)`. Each replica re-reads the log on
36//! every feedback event and on a periodic timer (≤ 5 s), rebuilding its local [`Experiment`] from
37//! the canonical arrival order. Because the e-process is deterministic, all replicas holding the
38//! same log reach the same halt/certify decision. Without a backend the behaviour is unchanged.
39
40use std::collections::{HashMap, HashSet};
41use std::fmt;
42use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
43
44use firstpass_core::{
45    Arm, Config, Experiment, ExperimentReport, ExperimentStatus, Trace, Verdict, select,
46};
47use serde::{Deserialize, Serialize};
48use uuid::Uuid;
49
50/// Upper bound on distinct sessions remembered per experiment, as 8-byte hashes (~8 MB per
51/// million). Past it new sessions' outcomes are ignored rather than evicting old entries: evicting
52/// would let a session count twice, which breaks the pairing argument, while ignoring only slows
53/// the experiment down. A million sessions is far beyond what the test needs to decide.
54const MAX_TRACKED_SESSIONS: usize = 1_000_000;
55
56// ── Shared log (optional Redis / in-memory backend) ───────────────────────────────────────────
57
58/// A record appended to the shared experiment log.
59///
60/// Live mode uses `Outcome`; paired mode uses `Pair`. The list for a given
61/// `(tenant, route_ix, fingerprint)` contains only one kind (the mode is fixed per config).
62#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
63#[serde(tag = "kind", rename_all = "snake_case")]
64pub enum LogRecord {
65    /// One session's outcome from live mode.
66    Outcome {
67        arm: Arm,
68        /// Gate verdict in [0, 1].
69        outcome: f64,
70        /// SHA-256 prefix of the session id (for in-log dedup on replay, not primary dedup).
71        session_hash: u64,
72        /// USD cost of this request.
73        cost_usd: f64,
74    },
75    /// One paired shadow result.
76    Pair {
77        x_inc: f64,
78        x_ch: f64,
79        /// Cost of the incumbent request.
80        incumbent_cost_usd: f64,
81        /// Cost of the challenger off-path evaluation.
82        challenger_cost_usd: f64,
83    },
84}
85
86/// A shared log that lets several replicas converge on the same [`Experiment`] state.
87///
88/// Implementors: [`InMemoryExperimentLog`] (tests), and `RedisExperimentLog` (behind the
89/// `redis-cache` feature).
90///
91/// The log for a given experiment is identified by `(tenant, route_ix, fingerprint)`. Multiple
92/// replicas can append concurrently; the authoritative ordering is the list's arrival order (for
93/// Redis: RPUSH index; for the in-memory implementation: insertion position). A halt decision
94/// derived from the list at position `N` holds forever — re-reading the list can only add more
95/// evidence, never retract it.
96#[async_trait::async_trait]
97pub trait ExperimentLogBackend: Send + Sync + fmt::Debug {
98    /// Append `record` to the list. Returns the new list length (1-based).
99    async fn append(
100        &self,
101        tenant: &str,
102        route_ix: usize,
103        fingerprint: &str,
104        record: LogRecord,
105    ) -> u64;
106
107    /// Return records starting at `offset` (0-based). Empty if the list has ≤ `offset` entries.
108    async fn load_since(
109        &self,
110        tenant: &str,
111        route_ix: usize,
112        fingerprint: &str,
113        offset: u64,
114    ) -> Vec<LogRecord>;
115
116    /// Atomically claim a session for this experiment (live mode session dedup across replicas).
117    ///
118    /// Returns `true` if the session was newly claimed (this replica should count the outcome),
119    /// `false` if another replica already claimed it. Claimants are permanent: once claimed the
120    /// result is final regardless of which replica appended it.
121    async fn claim_session(
122        &self,
123        tenant: &str,
124        route_ix: usize,
125        fingerprint: &str,
126        session_id: &str,
127    ) -> bool;
128}
129
130/// In-process log for tests (and the no-Redis single-replica path, where it is never used).
131///
132/// Two `ExperimentMonitor` instances that hold `Arc`s to the same `InMemoryExperimentLog` will
133/// converge to the same `Experiment` state after syncing, exactly as two remote replicas sharing
134/// a Redis-backed log would.
135#[derive(Debug, Default)]
136pub struct InMemoryExperimentLog {
137    lists: Mutex<HashMap<(String, usize, String), Vec<LogRecord>>>,
138    sessions: Mutex<HashSet<(String, usize, String, String)>>,
139}
140
141impl InMemoryExperimentLog {
142    /// A fresh empty log.
143    #[must_use]
144    pub fn new() -> Self {
145        Self::default()
146    }
147}
148
149#[async_trait::async_trait]
150impl ExperimentLogBackend for InMemoryExperimentLog {
151    async fn append(
152        &self,
153        tenant: &str,
154        route_ix: usize,
155        fingerprint: &str,
156        record: LogRecord,
157    ) -> u64 {
158        let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
159        let mut lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
160        let list = lists.entry(key).or_default();
161        list.push(record);
162        list.len() as u64
163    }
164
165    async fn load_since(
166        &self,
167        tenant: &str,
168        route_ix: usize,
169        fingerprint: &str,
170        offset: u64,
171    ) -> Vec<LogRecord> {
172        let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
173        let lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
174        match lists.get(&key) {
175            None => vec![],
176            Some(list) => {
177                let start = offset as usize;
178                if start >= list.len() {
179                    vec![]
180                } else {
181                    list[start..].to_vec()
182                }
183            }
184        }
185    }
186
187    async fn claim_session(
188        &self,
189        tenant: &str,
190        route_ix: usize,
191        fingerprint: &str,
192        session_id: &str,
193    ) -> bool {
194        let key = (
195            tenant.to_owned(),
196            route_ix,
197            fingerprint.to_owned(),
198            session_id.to_owned(),
199        );
200        let mut sessions = self.sessions.lock().unwrap_or_else(PoisonError::into_inner);
201        sessions.insert(key)
202    }
203}
204
205// ── Redis backend (behind `redis-cache` feature) ─────────────────────────────────────────────
206
207#[cfg(feature = "redis-cache")]
208pub use redis_log::RedisExperimentLog;
209
210#[cfg(feature = "redis-cache")]
211mod redis_log {
212    use super::{ExperimentLogBackend, LogRecord};
213    use std::fmt;
214
215    // Use tokio::sync::Mutex so the guard is Send and can cross await points.
216    use tokio::sync::Mutex;
217
218    const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
219
220    /// Strips credentials from a Redis URL so it can appear in log lines safely.
221    fn redact(url: &str) -> String {
222        let Some((scheme, rest)) = url.split_once("://") else {
223            return "<malformed redis url>".to_owned();
224        };
225        let host = match rest.rfind('@') {
226            Some(i) => &rest[i + 1..],
227            None => rest,
228        };
229        format!("{scheme}://{host}")
230    }
231
232    /// Redis-backed experiment log. Each experiment lives in a list at
233    /// `firstpass:exp:{tenant}:{route_ix}:{fingerprint}` and sessions are claimed via
234    /// `SET NX firstpass:exp:sess:{tenant}:{route_ix}:{fingerprint}:{session_id}`.
235    pub struct RedisExperimentLog {
236        /// `tokio::sync::Mutex` so the guard is `Send` and can be held across `.await`.
237        conn: Mutex<redis::aio::ConnectionManager>,
238    }
239
240    impl fmt::Debug for RedisExperimentLog {
241        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
242            f.debug_struct("RedisExperimentLog").finish_non_exhaustive()
243        }
244    }
245
246    impl RedisExperimentLog {
247        /// Connect to `url`, timing out after 5 s.
248        ///
249        /// # Errors
250        /// A malformed URL, an unreachable server, or a timeout. Surfaced at startup so an operator
251        /// who configured Redis is told immediately rather than discovering it via a missing halt.
252        pub async fn connect(url: &str) -> Result<Self, String> {
253            let safe = redact(url);
254            let client = redis::Client::open(url).map_err(|e| format!("redis url {safe}: {e}"))?;
255            let conn =
256                tokio::time::timeout(CONNECT_TIMEOUT, redis::aio::ConnectionManager::new(client))
257                    .await
258                    .map_err(|_| format!("redis url {safe}: connect timed out"))?
259                    .map_err(|e| format!("redis url {safe}: {e}"))?;
260            Ok(Self {
261                conn: Mutex::new(conn),
262            })
263        }
264
265        fn list_key(tenant: &str, route_ix: usize, fingerprint: &str) -> String {
266            format!("firstpass:exp:{tenant}:{route_ix}:{fingerprint}")
267        }
268
269        fn session_key(
270            tenant: &str,
271            route_ix: usize,
272            fingerprint: &str,
273            session_id: &str,
274        ) -> String {
275            format!("firstpass:exp:sess:{tenant}:{route_ix}:{fingerprint}:{session_id}")
276        }
277    }
278
279    #[async_trait::async_trait]
280    impl ExperimentLogBackend for RedisExperimentLog {
281        async fn append(
282            &self,
283            tenant: &str,
284            route_ix: usize,
285            fingerprint: &str,
286            record: LogRecord,
287        ) -> u64 {
288            let key = Self::list_key(tenant, route_ix, fingerprint);
289            let Ok(json) = serde_json::to_string(&record) else {
290                return 0;
291            };
292            let mut conn = self.conn.lock().await;
293            use redis::AsyncCommands as _;
294            match conn.rpush::<_, _, u64>(&key, json).await {
295                Ok(n) => n,
296                Err(e) => {
297                    tracing::warn!(key, %e, "experiment log: redis RPUSH failed");
298                    0
299                }
300            }
301        }
302
303        async fn load_since(
304            &self,
305            tenant: &str,
306            route_ix: usize,
307            fingerprint: &str,
308            offset: u64,
309        ) -> Vec<LogRecord> {
310            let key = Self::list_key(tenant, route_ix, fingerprint);
311            let mut conn = self.conn.lock().await;
312            use redis::AsyncCommands as _;
313            let raw: Vec<String> = match conn.lrange(&key, offset as isize, -1).await {
314                Ok(v) => v,
315                Err(e) => {
316                    tracing::warn!(key, %e, "experiment log: redis LRANGE failed");
317                    return vec![];
318                }
319            };
320            raw.iter()
321                .filter_map(|s| match serde_json::from_str(s) {
322                    Ok(r) => Some(r),
323                    Err(e) => {
324                        tracing::warn!(%e, "experiment log: skipping malformed log record");
325                        None
326                    }
327                })
328                .collect()
329        }
330
331        async fn claim_session(
332            &self,
333            tenant: &str,
334            route_ix: usize,
335            fingerprint: &str,
336            session_id: &str,
337        ) -> bool {
338            let key = Self::session_key(tenant, route_ix, fingerprint, session_id);
339            let mut conn = self.conn.lock().await;
340            use redis::AsyncCommands as _;
341            // SET key "1" NX EX 86400 — expires after 1 day so the set doesn't grow unboundedly.
342            match conn
343                .set_options::<_, _, ()>(
344                    &key,
345                    "1",
346                    redis::SetOptions::default()
347                        .with_expiration(redis::SetExpiry::EX(86_400))
348                        .conditional_set(redis::ExistenceCheck::NX),
349                )
350                .await
351            {
352                Ok(()) => true,  // NX succeeded: this replica claimed the session
353                Err(_) => false, // key already exists: another replica claimed it
354            }
355        }
356    }
357}
358
359/// One experiment plus the sessions that already contributed an outcome.
360#[derive(Debug)]
361struct Entry {
362    experiment: Experiment,
363    /// Receipts seen from a previous experiment configuration and therefore not counted.
364    ignored_stale: u64,
365    /// Errors and cut-off answers per arm, for the operator to judge what the pairs left out.
366    health: Health,
367    /// Hashes of sessions whose outcome was already counted. A 64-bit collision merely drops one
368    /// legitimate outcome, which costs power and never validity.
369    sessions: HashSet<u64>,
370    saturated_logged: bool,
371    /// How many records have been consumed from the shared log so far (incremental LRANGE).
372    remote_log_offset: u64,
373}
374
375fn session_hash(session_id: &str) -> u64 {
376    // SHA-256, not `DefaultHasher`: std reserves the right to change its algorithm between
377    // releases, and the dedup set rebuilt from receipts after an upgrade must agree with the one
378    // the live path built before it.
379    use sha2::{Digest, Sha256};
380    let d = Sha256::digest(session_id.as_bytes());
381    u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]])
382}
383
384/// Gauge value for [`ExperimentStatus`].
385fn status_code(s: ExperimentStatus) -> f64 {
386    match s {
387        ExperimentStatus::Undecided => 0.0,
388        ExperimentStatus::Certified => 1.0,
389        ExperimentStatus::Halted => 2.0,
390    }
391}
392
393#[derive(Debug)]
394struct Params {
395    margin: f64,
396    alpha: f64,
397    fingerprint: String,
398}
399
400/// Per-`(tenant, route)` certified-selection state. Cheap to share behind an `Arc`.
401#[derive(Debug, Default)]
402pub struct ExperimentMonitor {
403    /// `(margin, alpha)` per route index that has a challenger.
404    params: HashMap<usize, Params>,
405    /// Set when the trace store could not be read at startup: the history needed to know whether a
406    /// challenger was already halted is missing, so no challenger is served (fail closed).
407    unavailable: bool,
408    /// tenant -> route -> entry. Nested so a lookup by `&str` never allocates on the request path.
409    entries: Mutex<HashMap<String, HashMap<usize, Entry>>>,
410    /// Optional shared log that synchronises evidence across replicas (Redis or in-memory).
411    log: Option<Arc<dyn ExperimentLogBackend>>,
412}
413
414impl ExperimentMonitor {
415    /// A monitor for every route in `config` that has a challenger. Routes without one are ignored
416    /// by every method, so a deployment with no experiments pays nothing.
417    #[must_use]
418    pub fn from_config(config: &Config) -> Self {
419        let params = config
420            .routes
421            .iter()
422            .enumerate()
423            .filter_map(|(i, r)| {
424                r.challenger.as_ref().map(|c| {
425                    (
426                        i,
427                        Params {
428                            margin: c.margin,
429                            alpha: c.alpha,
430                            fingerprint: c.fingerprint(&r.ladder),
431                        },
432                    )
433                })
434            })
435            .collect();
436        Self {
437            params,
438            unavailable: false,
439            entries: Mutex::default(),
440            log: None,
441        }
442    }
443
444    /// A monitor that refuses to serve any challenger. Used when the history could not be read.
445    #[must_use]
446    pub fn unavailable(config: &Config) -> Self {
447        Self {
448            unavailable: true,
449            ..Self::from_config(config)
450        }
451    }
452
453    /// Attach a shared log backend. Every `observe_outcome`/`observe_pair` call will append to
454    /// the backend, and `sync_from_backend` will replay newly-arrived records from it. Returns
455    /// `self` for method-chaining.
456    #[must_use]
457    pub fn with_log(mut self, log: Arc<dyn ExperimentLogBackend>) -> Self {
458        self.log = Some(log);
459        self
460    }
461
462    /// The fingerprint receipts for `route_ix` must carry to count, if it has a challenger.
463    #[must_use]
464    pub fn fingerprint(&self, route_ix: usize) -> Option<&str> {
465        self.params.get(&route_ix).map(|p| p.fingerprint.as_str())
466    }
467
468    /// Whether the serving path must NOT use the challenger for `(tenant, route_ix)`: the
469    /// experiment is halted, or its history was unreadable at startup.
470    #[must_use]
471    pub fn challenger_blocked(&self, tenant: &str, route_ix: usize) -> bool {
472        self.unavailable || self.status(tenant, route_ix) == ExperimentStatus::Halted
473    }
474
475    /// Whether any route runs an experiment.
476    #[must_use]
477    pub fn is_active(&self) -> bool {
478        !self.params.is_empty()
479    }
480
481    fn lock(&self) -> MutexGuard<'_, HashMap<String, HashMap<usize, Entry>>> {
482        // A poisoned lock only means another thread panicked mid-update of plain counters; the
483        // state is still usable, and the halt guard must not turn into a request-path panic.
484        self.entries.lock().unwrap_or_else(PoisonError::into_inner)
485    }
486
487    fn entry<'a>(
488        &self,
489        map: &'a mut HashMap<String, HashMap<usize, Entry>>,
490        tenant: &str,
491        route_ix: usize,
492    ) -> Option<&'a mut Entry> {
493        let p = self.params.get(&route_ix)?;
494        let (margin, alpha) = (p.margin, p.alpha);
495        if !map.contains_key(tenant) {
496            map.insert(tenant.to_owned(), HashMap::new());
497        }
498        let routes = map.get_mut(tenant)?;
499        if let std::collections::hash_map::Entry::Vacant(v) = routes.entry(route_ix) {
500            match Experiment::new(margin, alpha) {
501                Ok(experiment) => {
502                    v.insert(Entry {
503                        experiment,
504                        ignored_stale: 0,
505                        health: Health::default(),
506                        sessions: HashSet::new(),
507                        saturated_logged: false,
508                        remote_log_offset: 0,
509                    });
510                }
511                Err(e) => {
512                    // Unreachable for a parsed config; fail open (no experiment) rather than panic.
513                    tracing::warn!(route = route_ix, %e, "experiment parameters rejected");
514                    return None;
515                }
516            }
517        }
518        routes.get_mut(&route_ix)
519    }
520
521    /// Where the experiment for `(tenant, route_ix)` stands. [`ExperimentStatus::Undecided`] when
522    /// nothing has been observed or the route has no challenger. The serving path stops using the
523    /// challenger when this is [`ExperimentStatus::Halted`].
524    #[must_use]
525    pub fn status(&self, tenant: &str, route_ix: usize) -> ExperimentStatus {
526        self.lock()
527            .get(tenant)
528            .and_then(|r| r.get(&route_ix))
529            .map_or(ExperimentStatus::Undecided, |e| e.experiment.status())
530    }
531
532    /// Record the cost of one served request of `arm`. Call with `trace.final_.total_cost_usd`.
533    ///
534    /// `fingerprint` is the receipt's; a mismatch with the current config counts the receipt in
535    /// `ignored_stale` and records nothing.
536    pub fn observe_cost(
537        &self,
538        tenant: &str,
539        route_ix: usize,
540        fingerprint: &str,
541        arm: Arm,
542        usd: f64,
543    ) {
544        let current = self.fingerprint(route_ix) == Some(fingerprint);
545        let mut map = self.lock();
546        if let Some(e) = self.entry(&mut map, tenant, route_ix) {
547            if current {
548                e.experiment.observe_cost(arm, usd);
549            } else {
550                e.ignored_stale += 1;
551            }
552        }
553    }
554
555    /// Tally one request of `arm` that errored and/or produced answers cut off at the output-token
556    /// limit (`truncated` counts attempts). Stale fingerprints are ignored: `observe_cost` already
557    /// counts the receipt once.
558    pub fn observe_health(
559        &self,
560        tenant: &str,
561        route_ix: usize,
562        fingerprint: &str,
563        arm: Arm,
564        errored: bool,
565        truncated: u64,
566    ) {
567        if (!errored && truncated == 0) || self.fingerprint(route_ix) != Some(fingerprint) {
568            return;
569        }
570        let mut map = self.lock();
571        if let Some(e) = self.entry(&mut map, tenant, route_ix) {
572            let h = match arm {
573                Arm::Incumbent => &mut e.health.incumbent,
574                Arm::Challenger => &mut e.health.challenger,
575            };
576            h.errors += u64::from(errored);
577            h.truncated += truncated;
578        }
579    }
580
581    /// Record a session's task outcome. Returns whether it was counted.
582    ///
583    /// Only the **first** clear outcome per session counts; later ones are ignored. The test pairs
584    /// the k-th incumbent outcome with the k-th challenger outcome and treats sessions as
585    /// independent draws, so several correlated outcomes from one long agent run would both
586    /// overweight that session and break the independence the error bound rests on.
587    ///
588    /// `Pass` is 1.0, `Fail` is 0.0, `Abstain` carries no information and is ignored.
589    pub fn observe_outcome(
590        &self,
591        tenant: &str,
592        route_ix: usize,
593        fingerprint: &str,
594        arm: Arm,
595        session_id: &str,
596        verdict: Verdict,
597    ) -> bool {
598        // A stale receipt is counted once, by `observe_cost`; here it is just not evidence.
599        if self.fingerprint(route_ix) != Some(fingerprint) {
600            return false;
601        }
602        let outcome = match verdict {
603            Verdict::Pass => 1.0,
604            Verdict::Fail => 0.0,
605            Verdict::Abstain => return false,
606        };
607        let mut map = self.lock();
608        let Some(e) = self.entry(&mut map, tenant, route_ix) else {
609            return false;
610        };
611        let key = session_hash(session_id);
612        if e.sessions.contains(&key) {
613            return false;
614        }
615        if e.sessions.len() >= MAX_TRACKED_SESSIONS {
616            if !e.saturated_logged {
617                e.saturated_logged = true;
618                tracing::warn!(
619                    tenant,
620                    route = route_ix,
621                    "experiment session ledger is full; new sessions' outcomes are ignored"
622                );
623            }
624            return false;
625        }
626        let before = e.experiment.status();
627        if !e.experiment.observe_outcome(arm, outcome) {
628            return false;
629        }
630        e.sessions.insert(key);
631        let after = e.experiment.status();
632        if before != after {
633            match after {
634                ExperimentStatus::Halted => tracing::error!(
635                    tenant,
636                    route = route_ix,
637                    "EXPERIMENT HALTED: the challenger ladder is worse than the incumbent by more \
638                     than the margin — it is no longer served"
639                ),
640                ExperimentStatus::Certified => tracing::info!(
641                    tenant,
642                    route = route_ix,
643                    "experiment certified: the challenger is within the margin of the incumbent; \
644                     adopting it is a human decision (`firstpass experiment`)"
645                ),
646                ExperimentStatus::Undecided => {}
647            }
648            Self::publish(&map, route_ix);
649        }
650        true
651    }
652
653    /// Export the route's status as `firstpass_experiment_status{route}` (0 undecided, 1 certified,
654    /// 2 halted). The label is the route index — bounded by the config, never by traffic — and
655    /// tenants are deliberately not a label (unbounded in a hosted deployment), so the value is the
656    /// worst status across tenants: a halt anywhere is visible.
657    fn publish(map: &HashMap<String, HashMap<usize, Entry>>, route_ix: usize) {
658        let worst = map
659            .values()
660            .filter_map(|r| r.get(&route_ix))
661            .map(|e| status_code(e.experiment.status()))
662            .fold(0.0, f64::max);
663        metrics::gauge!("firstpass_experiment_status", "route" => route_ix.to_string()).set(worst);
664    }
665
666    /// A snapshot of every experiment, ordered by `(tenant, route_ix)`.
667    #[must_use]
668    pub fn reports(&self) -> Vec<Snapshot> {
669        let map = self.lock();
670        let mut out: Vec<_> = map
671            .iter()
672            .flat_map(|(t, routes)| {
673                routes.iter().map(move |(ix, e)| Snapshot {
674                    tenant: t.clone(),
675                    route_ix: *ix,
676                    report: e.experiment.report(),
677                    ignored_stale: e.ignored_stale,
678                    health: e.health,
679                })
680            })
681            .collect();
682        out.sort_by(|a, b| (&a.tenant, a.route_ix).cmp(&(&b.tenant, b.route_ix)));
683        out
684    }
685
686    // ── Shared-log async methods ──────────────────────────────────────────────────────────────
687
688    /// Apply a single [`LogRecord`] from the shared backend to local in-memory state.
689    ///
690    /// All records arriving through `sync_from_backend` already carry the correct fingerprint
691    /// (they were keyed by it). This check is defense-in-depth. Returns `true` if the experiment
692    /// status changed so the caller can call `Self::publish`.
693    fn apply_log_record(
694        &self,
695        map: &mut HashMap<String, HashMap<usize, Entry>>,
696        tenant: &str,
697        route_ix: usize,
698        fingerprint: &str,
699        record: &LogRecord,
700    ) -> bool {
701        if self.fingerprint(route_ix) != Some(fingerprint) {
702            if let Some(e) = self.entry(map, tenant, route_ix) {
703                e.ignored_stale += 1;
704            }
705            return false;
706        }
707        match record {
708            LogRecord::Outcome {
709                arm,
710                outcome,
711                session_hash: hash,
712                ..
713            } => {
714                let Some(e) = self.entry(map, tenant, route_ix) else {
715                    return false;
716                };
717                if e.sessions.contains(hash) {
718                    return false; // idempotent re-apply on overlapping LRANGE
719                }
720                if e.sessions.len() >= MAX_TRACKED_SESSIONS {
721                    return false;
722                }
723                let before = e.experiment.status();
724                if !e.experiment.observe_outcome(*arm, *outcome) {
725                    return false;
726                }
727                e.sessions.insert(*hash);
728                before != e.experiment.status()
729            }
730            LogRecord::Pair {
731                x_inc,
732                x_ch,
733                challenger_cost_usd,
734                ..
735            } => {
736                let Some(e) = self.entry(map, tenant, route_ix) else {
737                    return false;
738                };
739                let before = e.experiment.status();
740                if e.experiment.observe_pair(*x_inc, *x_ch) {
741                    // The incumbent's cost is already counted on the serving path; only the
742                    // off-path challenger call is new spend.
743                    e.experiment
744                        .observe_cost(Arm::Challenger, *challenger_cost_usd);
745                }
746                before != e.experiment.status()
747            }
748        }
749    }
750
751    /// Read records appended to the shared log since the last sync and apply them locally.
752    ///
753    /// Each call is cheap when there are no new records: one LRANGE that returns empty.
754    /// Call this after every feedback event (within the feedback handler, not on the hot
755    /// serve path) and from a background timer so all replicas stay ≤ 5 s stale.
756    ///
757    /// No-op when no log backend is configured.
758    pub async fn sync_from_backend(&self, tenant: &str, route_ix: usize, fingerprint: &str) {
759        let Some(log) = &self.log else {
760            return;
761        };
762        let offset = {
763            let mut map = self.lock();
764            self.entry(&mut map, tenant, route_ix)
765                .map_or(0, |e| e.remote_log_offset)
766        };
767        let records = log.load_since(tenant, route_ix, fingerprint, offset).await;
768        if records.is_empty() {
769            return;
770        }
771        let mut map = self.lock();
772        for (i, rec) in records.iter().enumerate() {
773            let changed = self.apply_log_record(&mut map, tenant, route_ix, fingerprint, rec);
774            // Update offset regardless of whether the record was applied (stale records
775            // still advance the cursor; we must not re-read them on the next sync).
776            if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
777                e.remote_log_offset = offset + i as u64 + 1;
778            }
779            if changed {
780                let after = map
781                    .get(tenant)
782                    .and_then(|r| r.get(&route_ix))
783                    .map(|e| e.experiment.status());
784                match after {
785                    Some(ExperimentStatus::Halted) => tracing::error!(
786                        tenant,
787                        route = route_ix,
788                        "EXPERIMENT HALTED (shared log): challenger worse than incumbent"
789                    ),
790                    Some(ExperimentStatus::Certified) => tracing::info!(
791                        tenant,
792                        route = route_ix,
793                        "experiment certified (shared log)"
794                    ),
795                    _ => {}
796                }
797                Self::publish(&map, route_ix);
798            }
799        }
800    }
801
802    /// Record an outcome to the shared log (if configured), then sync from the log.
803    ///
804    /// In single-replica mode (no log), falls back to the synchronous [`Self::observe_outcome`].
805    /// `cost_usd` is stored in the log so other replicas can tally costs without needing the
806    /// full trace. Returns `true` when the outcome was counted.
807    #[allow(clippy::too_many_arguments)]
808    pub async fn record_outcome(
809        &self,
810        tenant: &str,
811        route_ix: usize,
812        fingerprint: &str,
813        arm: Arm,
814        session_id: &str,
815        verdict: Verdict,
816        cost_usd: f64,
817    ) -> bool {
818        // Filter Abstain before claim_session: an abstain carries no evidence, and burning the
819        // session slot would prevent a later non-Abstain verdict for the same session from being
820        // counted across replicas.
821        let outcome = match verdict {
822            Verdict::Pass => 1.0,
823            Verdict::Fail => 0.0,
824            Verdict::Abstain => return false,
825        };
826        let Some(log) = self.log.as_ref().cloned() else {
827            return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
828        };
829        // Cross-replica session dedup: only one replica may claim a given session.
830        if !log
831            .claim_session(tenant, route_ix, fingerprint, session_id)
832            .await
833        {
834            return false;
835        }
836        log.append(
837            tenant,
838            route_ix,
839            fingerprint,
840            LogRecord::Outcome {
841                arm,
842                outcome,
843                session_hash: session_hash(session_id),
844                cost_usd,
845            },
846        )
847        .await;
848        // Pull in any records other replicas appended since last sync, plus the one we
849        // just appended (so local state is always up-to-date after an append).
850        self.sync_from_backend(tenant, route_ix, fingerprint).await;
851        true
852    }
853
854    /// Record a paired shadow result to the shared log (if configured), then sync.
855    ///
856    /// In single-replica mode (no log), applies the pair directly and emits the status log.
857    /// Returns `true` if `x_inc` / `x_ch` are valid (finite, in `[0, 1]`) and the pair was
858    /// accepted by the underlying [`Experiment`].
859    #[allow(clippy::too_many_arguments)]
860    pub async fn record_pair(
861        &self,
862        tenant: &str,
863        route_ix: usize,
864        fingerprint: &str,
865        x_inc: f64,
866        x_ch: f64,
867        incumbent_cost_usd: f64,
868        challenger_cost_usd: f64,
869    ) -> bool {
870        if self.fingerprint(route_ix) != Some(fingerprint) {
871            return false;
872        }
873        let Some(log) = self.log.as_ref().cloned() else {
874            // Single-replica path: apply directly and publish.
875            let mut map = self.lock();
876            let Some(e) = self.entry(&mut map, tenant, route_ix) else {
877                return false;
878            };
879            let before = e.experiment.status();
880            let ok = e.experiment.observe_pair(x_inc, x_ch);
881            if ok {
882                // Without this the challenger's $/request reads n/a, and a certified, cheaper
883                // paired challenger could never be recommended for adoption.
884                e.experiment
885                    .observe_cost(Arm::Challenger, challenger_cost_usd);
886            }
887            if ok && before != e.experiment.status() {
888                match e.experiment.status() {
889                    ExperimentStatus::Halted => tracing::error!(
890                        tenant,
891                        route = route_ix,
892                        "EXPERIMENT HALTED: paired challenger worse than incumbent"
893                    ),
894                    ExperimentStatus::Certified => tracing::info!(
895                        tenant,
896                        route = route_ix,
897                        "experiment certified (paired): challenger within margin"
898                    ),
899                    ExperimentStatus::Undecided => {}
900                }
901                Self::publish(&map, route_ix);
902            }
903            return ok;
904        };
905        // Validate before appending: an invalid record would permanently pollute the log.
906        if !x_inc.is_finite()
907            || !(0.0..=1.0).contains(&x_inc)
908            || !x_ch.is_finite()
909            || !(0.0..=1.0).contains(&x_ch)
910        {
911            return false;
912        }
913        log.append(
914            tenant,
915            route_ix,
916            fingerprint,
917            LogRecord::Pair {
918                x_inc,
919                x_ch,
920                incumbent_cost_usd,
921                challenger_cost_usd,
922            },
923        )
924        .await;
925        self.sync_from_backend(tenant, route_ix, fingerprint).await;
926        true
927    }
928
929    /// Spawn a background task that calls `sync_from_backend` for every known experiment at
930    /// most every `interval` (spec: ≤ 5 s). The task exits silently when the `Arc` becomes
931    /// the last reference. No-op when no log backend is configured.
932    pub fn start_sync_task(self: &Arc<Self>, interval: std::time::Duration) {
933        if self.log.is_none() {
934            return;
935        }
936        let monitor = Arc::clone(self);
937        tokio::spawn(async move {
938            let mut ticker = tokio::time::interval(interval);
939            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
940            loop {
941                ticker.tick().await;
942                // Collect (tenant, route_ix, fingerprint) tuples while holding the lock,
943                // then drop the lock before each async sync call.
944                let keys: Vec<(String, usize, String)> = {
945                    let map = monitor.lock();
946                    let mut v = Vec::new();
947                    for (tenant, routes) in map.iter() {
948                        for &route_ix in routes.keys() {
949                            if let Some(p) = monitor.params.get(&route_ix) {
950                                v.push((tenant.clone(), route_ix, p.fingerprint.clone()));
951                            }
952                        }
953                    }
954                    v
955                };
956                for (tenant, route_ix, fingerprint) in keys {
957                    monitor
958                        .sync_from_backend(&tenant, route_ix, &fingerprint)
959                        .await;
960                }
961            }
962        });
963    }
964}
965
966/// One experiment as the monitor holds it.
967#[derive(Debug, Clone, PartialEq)]
968pub struct Snapshot {
969    /// Tenant the experiment belongs to.
970    pub tenant: String,
971    /// Index of the route in the config.
972    pub route_ix: usize,
973    /// The test's state and arm totals.
974    pub report: ExperimentReport,
975    /// Receipts from a previous experiment configuration that were ignored.
976    pub ignored_stale: u64,
977    /// Errors and cut-off answers per arm.
978    pub health: Health,
979}
980
981/// Requests an arm could not answer cleanly. Neither kind is random: errors (timeouts above all)
982/// and output-limit cut-offs cluster on long, hard requests. An incumbent error drops its pair, so
983/// many of them mean the pass rates describe an easier workload than the route serves; a
984/// challenger error counts as a failed pair. Truncated answers are graded as they are, so a high
985/// count says the route's `max_tokens` may be deciding the comparison.
986#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
987pub struct ArmHealth {
988    /// Requests that ended in an error, with no output to grade.
989    pub errors: u64,
990    /// Attempts whose answer stopped at the output-token limit.
991    pub truncated: u64,
992}
993
994/// [`ArmHealth`] for both arms. For a **paired** challenger the counts cover this process's
995/// lifetime only: its off-path answers are not written as receipts, so they cannot be rebuilt.
996#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
997pub struct Health {
998    /// The incumbent arm (rebuilt from receipts at startup).
999    pub incumbent: ArmHealth,
1000    /// The challenger arm.
1001    pub challenger: ArmHealth,
1002}
1003
1004/// Number of attempts in `t` that stopped at the output-token limit.
1005#[must_use]
1006pub fn truncated_attempts(t: &Trace) -> u64 {
1007    t.attempts.iter().filter(|a| a.truncated).count() as u64
1008}
1009
1010/// Whether a trace may be used for rung-indexed or threshold analyses (`ope`, `calibrate`).
1011/// Challenger-arm traces are excluded: their rungs index a different ladder and their gate scores
1012/// relate to correctness differently, so pooling them would silently mix two models' behaviour.
1013#[must_use]
1014pub fn is_incumbent_trace(t: &Trace) -> bool {
1015    t.experiment
1016        .as_ref()
1017        .is_none_or(|e| e.arm != Arm::Challenger)
1018}
1019
1020/// Rebuild a monitor from receipts.
1021///
1022/// `traces` are in chain order; `deferred` are `(trace_id, verdict)` in the order the feedback
1023/// arrived (the store's insertion order); `pairs` are paired-mode records in insertion order.
1024/// Costs are tallied per trace and outcomes are replayed in arrival order with the same
1025/// first-outcome-per-session rule as the live path. For sequential feedback the result is exactly
1026/// what the live monitor held. Under concurrent feedback the store's insertion order and the
1027/// monitor's lock order can differ, so the pairing — and with it `decided_at_pair` and the
1028/// e-values — may differ slightly from the live run. The replay is then the authoritative record
1029/// (it is what an auditor can re-derive), and both remain valid e-processes because arm
1030/// assignment is randomized independently of arrival order. Traces without an experiment record
1031/// (other routes, other eras) are skipped; deferred verdicts for unknown traces are ignored.
1032#[must_use]
1033pub fn monitor_from_traces(
1034    traces: &[Trace],
1035    deferred: &[(Uuid, Verdict)],
1036    pairs: &[crate::store::ExperimentPairRow],
1037    config: &Config,
1038) -> ExperimentMonitor {
1039    let monitor = ExperimentMonitor::from_config(config);
1040    if !monitor.is_active() {
1041        return monitor;
1042    }
1043    let mut by_id: HashMap<Uuid, &Trace> = HashMap::new();
1044    for t in traces {
1045        let Some(rec) = &t.experiment else { continue };
1046        // Matches the feedback path, which attributes a trace with no recorded route to route 0.
1047        let route_ix = t.route_ix.map_or(0, |r| r as usize);
1048        monitor.observe_cost(
1049            &t.tenant_id,
1050            route_ix,
1051            &rec.fingerprint,
1052            rec.arm,
1053            t.final_.total_cost_usd,
1054        );
1055        monitor.observe_health(
1056            &t.tenant_id,
1057            route_ix,
1058            &rec.fingerprint,
1059            rec.arm,
1060            t.final_.served_from == firstpass_core::ServedFrom::Error,
1061            truncated_attempts(t),
1062        );
1063        by_id.insert(t.trace_id, t);
1064    }
1065    for (id, verdict) in deferred {
1066        let Some(t) = by_id.get(id) else { continue };
1067        let Some(rec) = &t.experiment else { continue };
1068        let route_ix = t.route_ix.map_or(0, |r| r as usize);
1069        monitor.observe_outcome(
1070            &t.tenant_id,
1071            route_ix,
1072            &rec.fingerprint,
1073            rec.arm,
1074            &t.session_id,
1075            *verdict,
1076        );
1077    }
1078    // Replay paired-mode evidence in insertion order. Each pair bypasses FIFO queuing
1079    // (both outcomes arrived simultaneously in the off-path task), which is why they land
1080    // in `experiment_pairs` rather than the deferred-verdict table.
1081    for row in pairs {
1082        let mut map = monitor.lock();
1083        if let Some(e) = monitor.entry(&mut map, &row.tenant, row.route_ix) {
1084            if monitor.fingerprint(row.route_ix) == Some(row.fingerprint.as_str()) {
1085                if e.experiment.observe_pair(row.x_inc, row.x_ch) {
1086                    e.experiment
1087                        .observe_cost(Arm::Challenger, row.challenger_cost_usd);
1088                }
1089            } else {
1090                e.ignored_stale += 1;
1091            }
1092        }
1093    }
1094    monitor
1095}
1096
1097/// The monitor to serve with, rebuilt from the trace store at startup.
1098///
1099/// Only touches the store when some route has a challenger. **Fails closed on the challenger**: if
1100/// the store cannot be read, the proxy still starts but serves the incumbent only
1101/// ([`ExperimentMonitor::unavailable`]), because a challenger already proven worse must never be
1102/// served again just because its history could not be read.
1103#[must_use]
1104pub fn build_monitor(db_path: &str, config: &Config) -> ExperimentMonitor {
1105    build_monitor_with(db_path, config, false)
1106}
1107
1108/// [`build_monitor`], told whether a shared experiment log will be attached.
1109///
1110/// With a shared log, the log is the cluster-wide record of every paired evaluation, and the
1111/// sync task reads it from the start on boot. Replaying the local `experiment_pairs` table as
1112/// well would apply each pair twice (pairs, unlike live outcomes, carry no session to dedup on),
1113/// doubling the e-values and the challenger's spend. So pairs then only register their
1114/// experiment, and the evidence itself comes from the log on the first sync.
1115#[must_use]
1116pub fn build_monitor_with(db_path: &str, config: &Config, shared_log: bool) -> ExperimentMonitor {
1117    let empty = ExperimentMonitor::from_config(config);
1118    if !empty.is_active() {
1119        return empty;
1120    }
1121    let loaded = crate::store::load_all_traces(db_path)
1122        .and_then(|t| crate::store::load_all_deferred(db_path).map(|d| (t, d)))
1123        .and_then(|(t, d)| crate::store::load_all_experiment_pairs(db_path).map(|p| (t, d, p)));
1124    match loaded {
1125        Ok((traces, deferred, pairs)) => {
1126            let replayed: &[crate::store::ExperimentPairRow] =
1127                if shared_log { &[] } else { &pairs };
1128            let m = monitor_from_traces(&traces, &deferred, replayed, config);
1129            if shared_log {
1130                let mut map = m.lock();
1131                for row in &pairs {
1132                    let _ = m.entry(&mut map, &row.tenant, row.route_ix);
1133                }
1134            }
1135            for s in m.reports() {
1136                tracing::info!(
1137                    tenant = %s.tenant,
1138                    route = s.route_ix,
1139                    status = ?s.report.status,
1140                    pairs = s.report.pairs,
1141                    ignored_stale = s.ignored_stale,
1142                    "experiment restored from receipts"
1143                );
1144                ExperimentMonitor::publish(&m.lock(), s.route_ix);
1145            }
1146            m
1147        }
1148        Err(e) => {
1149            tracing::error!(
1150                %e,
1151                "could not read the trace store, so it is unknown whether a challenger was already \
1152                 halted: challenger arms are NOT served (incumbent only) until restart with a \
1153                 readable store"
1154            );
1155            ExperimentMonitor::unavailable(config)
1156        }
1157    }
1158}
1159
1160/// Replay receipts into per-`(tenant, route)` snapshots. See [`monitor_from_traces`].
1161#[must_use]
1162pub fn experiments_from_traces(
1163    traces: &[Trace],
1164    deferred: &[(Uuid, Verdict)],
1165    pairs: &[crate::store::ExperimentPairRow],
1166    config: &Config,
1167) -> Vec<Snapshot> {
1168    monitor_from_traces(traces, deferred, pairs, config).reports()
1169}
1170
1171/// What an operator should do with an experiment's current state.
1172#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
1173#[serde(rename_all = "snake_case")]
1174pub enum Recommendation {
1175    /// Certified, and the challenger costs less per request: adopt it (by editing the config).
1176    Adopt,
1177    /// Certified, but the challenger is not measurably cheaper: nothing to gain.
1178    CertifiedNoSaving,
1179    /// Proven worse by more than the margin; no longer served.
1180    ChallengerStopped,
1181    /// Not enough evidence yet.
1182    KeepRunning,
1183}
1184
1185/// The recommendation for a report. Cost is compared as dollars per served request on each arm;
1186/// missing cost data on either arm counts as "no saving" because adoption must not rest on a
1187/// number that was never measured.
1188#[must_use]
1189pub fn recommend(r: &ExperimentReport) -> Recommendation {
1190    match r.status {
1191        select::Status::Undecided => Recommendation::KeepRunning,
1192        select::Status::Halted => Recommendation::ChallengerStopped,
1193        select::Status::Certified => {
1194            match (
1195                r.challenger.cost_per_request(),
1196                r.incumbent.cost_per_request(),
1197            ) {
1198                (Some(c), Some(i)) if c < i => Recommendation::Adopt,
1199                _ => Recommendation::CertifiedNoSaving,
1200            }
1201        }
1202    }
1203}
1204
1205/// One experiment as the CLI presents it.
1206#[derive(Debug, Clone, Serialize)]
1207pub struct ExperimentView {
1208    /// Tenant the experiment belongs to.
1209    pub tenant: String,
1210    /// Index of the route in the config.
1211    pub route_ix: usize,
1212    /// The test's state and arm totals.
1213    pub report: ExperimentReport,
1214    /// Receipts from a previous experiment configuration that were ignored.
1215    pub ignored_stale: u64,
1216    /// Errors and cut-off answers per arm.
1217    pub health: Health,
1218    /// What to do about it.
1219    pub recommendation: Recommendation,
1220    /// The challenger ladder under test, so `adopt` is actionable without re-reading the config.
1221    pub challenger_ladder: Vec<String>,
1222}
1223
1224/// Pair each report with its recommendation and the challenger ladder from `config`.
1225#[must_use]
1226pub fn views(reports: Vec<Snapshot>, config: &Config) -> Vec<ExperimentView> {
1227    reports
1228        .into_iter()
1229        .map(|s| ExperimentView {
1230            recommendation: recommend(&s.report),
1231            ignored_stale: s.ignored_stale,
1232            health: s.health,
1233            challenger_ladder: config
1234                .routes
1235                .get(s.route_ix)
1236                .and_then(|r| r.challenger.as_ref())
1237                .map(|c| c.ladder.clone())
1238                .unwrap_or_default(),
1239            tenant: s.tenant,
1240            route_ix: s.route_ix,
1241            report: s.report,
1242        })
1243        .collect()
1244}
1245
1246fn fmt_opt(v: Option<f64>, f: impl Fn(f64) -> String) -> String {
1247    v.map_or_else(|| "n/a".to_owned(), f)
1248}
1249
1250/// Human-readable report, one block per `(tenant, route)`.
1251#[must_use]
1252pub fn render(views: &[ExperimentView]) -> String {
1253    use std::fmt::Write as _;
1254    if views.is_empty() {
1255        return "no experiment data yet — the config has no [route.challenger], or no experiment \
1256                traffic has been recorded\n"
1257            .to_owned();
1258    }
1259    let mut out = String::new();
1260    for v in views {
1261        let r = &v.report;
1262        let _ = writeln!(
1263            out,
1264            "tenant {} · route {} · status: {:?}",
1265            v.tenant, v.route_ix, r.status
1266        );
1267        let _ = writeln!(
1268            out,
1269            "  pairs: {} (margin {}, alpha {})",
1270            r.pairs, r.margin, r.alpha
1271        );
1272        for (name, a) in [("incumbent ", &r.incumbent), ("challenger", &r.challenger)] {
1273            let _ = writeln!(
1274                out,
1275                "  {name}: success {} over {} sessions · ${} per request over {} requests",
1276                fmt_opt(a.success_rate(), |s| format!("{:.1}%", s * 100.0)),
1277                a.outcomes,
1278                fmt_opt(a.cost_per_request(), |c| format!("{c:.6}")),
1279                a.requests,
1280            );
1281        }
1282        let _ = writeln!(
1283            out,
1284            "  e-value to certify: {:.3} · e-value to halt: {:.3} (decide at {:.1})",
1285            r.e_certify,
1286            r.e_halt,
1287            1.0 / r.alpha
1288        );
1289        if v.ignored_stale > 0 {
1290            let _ = writeln!(
1291                out,
1292                "  {} receipts from a previous experiment configuration ignored",
1293                v.ignored_stale
1294            );
1295        }
1296        for (name, h) in [
1297            ("incumbent", &v.health.incumbent),
1298            ("challenger", &v.health.challenger),
1299        ] {
1300            if h.errors > 0 || h.truncated > 0 {
1301                let _ = writeln!(
1302                    out,
1303                    "  {name}: {} errors, {} answers cut off at max_tokens (neither is random: \
1304                     check them before trusting the rates)",
1305                    h.errors, h.truncated
1306                );
1307            }
1308        }
1309        match v.recommendation {
1310            Recommendation::Adopt => {
1311                let ladder = v
1312                    .challenger_ladder
1313                    .iter()
1314                    .map(|m| format!("{m:?}"))
1315                    .collect::<Vec<_>>()
1316                    .join(", ");
1317                let _ = writeln!(
1318                    out,
1319                    "  ADOPT: set this route's ladder to [{ladder}]\n\n    ladder = [{ladder}]\n"
1320                );
1321            }
1322            Recommendation::CertifiedNoSaving => {
1323                let _ = writeln!(out, "  certified, but no saving — keep incumbent");
1324            }
1325            Recommendation::ChallengerStopped => {
1326                let _ = writeln!(out, "  challenger stopped: worse by more than the margin");
1327            }
1328            Recommendation::KeepRunning => {
1329                let _ = writeln!(out, "  keep running");
1330            }
1331        }
1332    }
1333    out
1334}
1335
1336#[cfg(test)]
1337mod tests {
1338    use firstpass_core::trace::ExperimentRecord;
1339    use firstpass_core::{
1340        Features, FinalOutcome, GENESIS_HASH, Mode, PolicyRef, RequestInfo, ServedFrom, TaskKind,
1341    };
1342
1343    use super::*;
1344
1345    const CFG: &str = "[[route]]\nmatch = {}\nmode = \"enforce\"\n\
1346        ladder = [\"anthropic/claude-sonnet-5\"]\n\
1347        [route.challenger]\nladder = [\"anthropic/claude-haiku-4-5\"]\npercent = 50.0\n";
1348
1349    fn config() -> Config {
1350        Config::parse(CFG).expect("test config parses")
1351    }
1352
1353    /// The fingerprint the current test config stamps on its receipts.
1354    fn fp() -> String {
1355        let c = config();
1356        c.routes[0]
1357            .challenger
1358            .as_ref()
1359            .expect("challenger")
1360            .fingerprint(&c.routes[0].ladder)
1361    }
1362
1363    fn trace(tenant: &str, session: &str, arm: Arm, cost: f64) -> Trace {
1364        Trace {
1365            trace_id: Uuid::now_v7(),
1366            prev_hash: GENESIS_HASH.to_owned(),
1367            tenant_id: tenant.to_owned(),
1368            session_id: session.to_owned(),
1369            ts: jiff::Timestamp::now(),
1370            mode: Mode::Enforce,
1371            policy: PolicyRef {
1372                id: "static-ladder@v0".to_owned(),
1373                explore: false,
1374                propensity: None,
1375                mode_profile: None,
1376            },
1377            request: RequestInfo {
1378                api: "anthropic.messages".to_owned(),
1379                prompt_hash: "h".to_owned(),
1380                features: Features::new(TaskKind::Other),
1381            },
1382            attempts: vec![],
1383            deferred: vec![],
1384            final_: FinalOutcome {
1385                served_rung: Some(0),
1386                served_from: ServedFrom::Attempt,
1387                total_cost_usd: cost,
1388                gate_cost_usd: 0.0,
1389                total_latency_ms: 1,
1390                escalations: 0,
1391                counterfactual_baseline_usd: cost,
1392                savings_usd: 0.0,
1393                cache_source: None,
1394            },
1395            probe: None,
1396            rollout: None,
1397            shadow: None,
1398            route_ix: Some(0),
1399            predicted_pass: None,
1400            decision_prior: None,
1401            elastic: None,
1402            risk: None,
1403            experiment: Some(ExperimentRecord {
1404                arm,
1405                percent: 50.0,
1406                bucket: 0,
1407                fingerprint: fp(),
1408            }),
1409        }
1410    }
1411
1412    /// Errors and cut-off answers are rebuilt from receipts, per arm, and receipts from an older
1413    /// experiment configuration contribute nothing.
1414    #[test]
1415    fn arm_health_is_rebuilt_from_receipts() {
1416        let attempt = |truncated| firstpass_core::trace::Attempt {
1417            rung: 0,
1418            model: "anthropic/claude-sonnet-5".to_owned(),
1419            provider: "anthropic".to_owned(),
1420            in_tokens: 1,
1421            cache_write_tokens: 0,
1422            cache_read_tokens: 0,
1423            out_tokens: 1,
1424            cost_usd: 0.0,
1425            latency_ms: 1,
1426            gates: vec![],
1427            verdict: Verdict::Pass,
1428            truncated,
1429        };
1430        let mut errored = trace("t", "a", Arm::Incumbent, 0.01);
1431        errored.final_.served_from = ServedFrom::Error;
1432        let mut cut = trace("t", "b", Arm::Incumbent, 0.01);
1433        cut.attempts = vec![attempt(true), attempt(true), attempt(false)];
1434        let mut ch_cut = trace("t", "c", Arm::Challenger, 0.01);
1435        ch_cut.attempts = vec![attempt(true)];
1436        let mut stale = trace("t", "d", Arm::Incumbent, 0.01);
1437        stale.final_.served_from = ServedFrom::Error;
1438        if let Some(rec) = stale.experiment.as_mut() {
1439            rec.fingerprint = "an-older-config".to_owned();
1440        }
1441        let snaps = experiments_from_traces(&[errored, cut, ch_cut, stale], &[], &[], &config());
1442        let h = snaps[0].health;
1443        assert_eq!(
1444            h.incumbent,
1445            ArmHealth {
1446                errors: 1,
1447                truncated: 2
1448            },
1449            "{h:?}"
1450        );
1451        assert_eq!(
1452            h.challenger,
1453            ArmHealth {
1454                errors: 0,
1455                truncated: 1
1456            },
1457            "{h:?}"
1458        );
1459        assert_eq!(snaps[0].ignored_stale, 1);
1460    }
1461
1462    #[test]
1463    fn first_outcome_per_session_counts_and_abstain_is_ignored() {
1464        let m = ExperimentMonitor::from_config(&config());
1465        assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Abstain));
1466        assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Pass));
1467        // Later outcomes from the same session are dropped, whatever they say.
1468        assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Fail));
1469        assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s2", Verdict::Fail));
1470        let r = &m.reports()[0].report;
1471        assert_eq!(r.incumbent.outcomes, 2);
1472        assert!((r.incumbent.outcome_sum - 1.0).abs() < f64::EPSILON);
1473    }
1474
1475    #[test]
1476    fn routes_without_a_challenger_and_unknown_tenants_are_inert() {
1477        let m = ExperimentMonitor::from_config(&config());
1478        assert!(!m.observe_outcome("t", 7, &fp(), Arm::Incumbent, "s", Verdict::Pass));
1479        assert_eq!(m.status("nobody", 0), ExperimentStatus::Undecided);
1480        assert!(m.reports().is_empty());
1481        let off = ExperimentMonitor::default();
1482        assert!(!off.is_active());
1483        off.observe_cost("t", 0, &fp(), Arm::Incumbent, 1.0);
1484        assert!(off.reports().is_empty());
1485    }
1486
1487    #[test]
1488    fn tenants_never_share_an_experiment() {
1489        let m = ExperimentMonitor::from_config(&config());
1490        for i in 0..400 {
1491            m.observe_outcome(
1492                "a",
1493                0,
1494                &fp(),
1495                Arm::Incumbent,
1496                &format!("i{i}"),
1497                Verdict::Pass,
1498            );
1499            m.observe_outcome(
1500                "a",
1501                0,
1502                &fp(),
1503                Arm::Challenger,
1504                &format!("c{i}"),
1505                Verdict::Fail,
1506            );
1507        }
1508        assert_eq!(m.status("a", 0), ExperimentStatus::Halted);
1509        assert_eq!(m.status("b", 0), ExperimentStatus::Undecided);
1510    }
1511
1512    #[test]
1513    fn a_clearly_worse_challenger_halts() {
1514        let m = ExperimentMonitor::from_config(&config());
1515        let mut at = None;
1516        for i in 0..500 {
1517            m.observe_outcome(
1518                "t",
1519                0,
1520                &fp(),
1521                Arm::Incumbent,
1522                &format!("i{i}"),
1523                Verdict::Pass,
1524            );
1525            m.observe_outcome(
1526                "t",
1527                0,
1528                &fp(),
1529                Arm::Challenger,
1530                &format!("c{i}"),
1531                Verdict::Fail,
1532            );
1533            if at.is_none() && m.status("t", 0) == ExperimentStatus::Halted {
1534                at = Some(i);
1535            }
1536        }
1537        assert!(at.is_some(), "never halted");
1538        assert_eq!(m.status("t", 0), ExperimentStatus::Halted);
1539    }
1540
1541    #[test]
1542    fn replay_matches_live_with_session_dedup_and_halt() {
1543        let cfg = config();
1544        let mut traces = Vec::new();
1545        let mut deferred = Vec::new();
1546        for i in 0..400 {
1547            let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
1548            let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
1549            deferred.push((inc.trace_id, Verdict::Pass));
1550            deferred.push((chl.trace_id, Verdict::Fail));
1551            // A second trace in the same session whose outcome contradicts the first: ignored.
1552            let again = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
1553            deferred.push((again.trace_id, Verdict::Pass));
1554            traces.extend([inc, chl, again]);
1555        }
1556        let reports = experiments_from_traces(&traces, &deferred, &[], &cfg);
1557        assert_eq!(reports.len(), 1);
1558        let snap = &reports[0];
1559        let r = &snap.report;
1560        assert_eq!((snap.tenant.as_str(), snap.route_ix), ("t", 0));
1561        assert_eq!(snap.ignored_stale, 0);
1562        assert_eq!(r.status, select::Status::Halted);
1563        // Dedup: one counted outcome per session, and the counted one is the first (Fail = 0).
1564        assert_eq!(r.challenger.outcomes, 400);
1565        assert!(r.challenger.outcome_sum.abs() < f64::EPSILON);
1566        // Cost is per request, not per session: the extra trace in each session still cost money.
1567        assert_eq!(r.challenger.requests, 800);
1568        assert!((r.challenger.cost_per_request().unwrap_or(0.0) - 0.02).abs() < 1e-12);
1569        assert!((r.incumbent.cost_per_request().unwrap_or(0.0) - 0.10).abs() < 1e-12);
1570        assert_eq!(recommend(r), Recommendation::ChallengerStopped);
1571
1572        // The live monitor, fed the same events, reaches the same decision.
1573        let live = ExperimentMonitor::from_config(&cfg);
1574        for t in &traces {
1575            let rec = t.experiment.as_ref().expect("recorded");
1576            live.observe_cost(
1577                &t.tenant_id,
1578                0,
1579                &rec.fingerprint,
1580                rec.arm,
1581                t.final_.total_cost_usd,
1582            );
1583        }
1584        for (id, v) in &deferred {
1585            let t = traces.iter().find(|t| t.trace_id == *id).expect("trace");
1586            let rec = t.experiment.as_ref().expect("recorded");
1587            live.observe_outcome("t", 0, &rec.fingerprint, rec.arm, &t.session_id, *v);
1588        }
1589        assert_eq!(live.reports()[0], *snap);
1590        // Replaying the same receipts under the same config is exactly reproducible.
1591        assert_eq!(
1592            experiments_from_traces(&traces, &deferred, &[], &cfg),
1593            reports
1594        );
1595    }
1596
1597    #[test]
1598    fn replay_ignores_unrecorded_traces_and_unknown_deferred() {
1599        let cfg = config();
1600        let mut plain = trace("t", "s", Arm::Incumbent, 0.1);
1601        plain.experiment = None;
1602        let deferred = [
1603            (plain.trace_id, Verdict::Pass),
1604            (Uuid::now_v7(), Verdict::Pass),
1605        ];
1606        assert!(experiments_from_traces(&[plain], &deferred, &[], &cfg).is_empty());
1607        // A config with no challenger rebuilds nothing at all.
1608        let bare = Config::parse(
1609            "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
1610        )
1611        .expect("parses");
1612        let t = trace("t", "s", Arm::Incumbent, 0.1);
1613        assert!(experiments_from_traces(&[t], &[], &[], &bare).is_empty());
1614    }
1615
1616    fn view_with(status: select::Status, inc_cost: f64, ch_cost: f64) -> ExperimentView {
1617        let mut exp = Experiment::new(0.05, 0.05).expect("valid");
1618        exp.observe_cost(Arm::Incumbent, inc_cost);
1619        exp.observe_cost(Arm::Challenger, ch_cost);
1620        let mut report = exp.report();
1621        report.status = status;
1622        ExperimentView {
1623            tenant: "t".to_owned(),
1624            route_ix: 0,
1625            recommendation: recommend(&report),
1626            ignored_stale: 0,
1627            health: Health::default(),
1628            report,
1629            challenger_ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
1630        }
1631    }
1632
1633    #[test]
1634    fn recommendation_and_rendering_cover_every_state() {
1635        let adopt = view_with(select::Status::Certified, 0.10, 0.02);
1636        assert_eq!(adopt.recommendation, Recommendation::Adopt);
1637        let text = render(&[adopt]);
1638        assert!(
1639            text.contains("ADOPT: set this route's ladder to [\"anthropic/claude-haiku-4-5\"]"),
1640            "{text}"
1641        );
1642        assert!(text.contains("ladder = [\"anthropic/claude-haiku-4-5\"]"));
1643
1644        let dear = view_with(select::Status::Certified, 0.02, 0.10);
1645        assert_eq!(dear.recommendation, Recommendation::CertifiedNoSaving);
1646        assert!(render(&[dear]).contains("certified, but no saving — keep incumbent"));
1647
1648        let halted = view_with(select::Status::Halted, 0.10, 0.02);
1649        assert!(render(&[halted]).contains("challenger stopped: worse by more than the margin"));
1650
1651        let open = view_with(select::Status::Undecided, 0.10, 0.02);
1652        assert_eq!(open.recommendation, Recommendation::KeepRunning);
1653        assert!(render(&[open]).contains("keep running"));
1654        assert!(render(&[]).contains("no experiment data"));
1655    }
1656
1657    #[test]
1658    fn certified_without_cost_data_is_not_adopted() {
1659        let exp = Experiment::new(0.05, 0.05).expect("valid");
1660        let mut report = exp.report();
1661        report.status = select::Status::Certified;
1662        assert_eq!(recommend(&report), Recommendation::CertifiedNoSaving);
1663    }
1664
1665    /// Changing the challenger ladder (or margin/alpha) starts a fresh experiment: receipts from
1666    /// the old configuration are ignored and counted, never pooled.
1667    #[test]
1668    fn changing_the_challenger_ladder_resets_the_experiment() {
1669        let old = config();
1670        let mut traces = Vec::new();
1671        let mut deferred = Vec::new();
1672        for i in 0..300 {
1673            let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
1674            let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
1675            deferred.push((inc.trace_id, Verdict::Pass));
1676            deferred.push((chl.trace_id, Verdict::Fail));
1677            traces.extend([inc, chl]);
1678        }
1679        let before = experiments_from_traces(&traces, &deferred, &[], &old);
1680        assert_eq!(before[0].report.status, select::Status::Halted);
1681        assert_eq!(before[0].ignored_stale, 0);
1682
1683        let new =
1684            Config::parse(&CFG.replace("claude-haiku-4-5", "claude-opus-4-8")).expect("parses");
1685        let after = experiments_from_traces(&traces, &deferred, &[], &new);
1686        assert_eq!(after[0].report.status, select::Status::Undecided);
1687        assert_eq!(after[0].report.pairs, 0);
1688        assert_eq!(after[0].report.challenger.requests, 0);
1689        assert_eq!(after[0].ignored_stale, 600);
1690        let text = render(&views(after, &new));
1691        assert!(
1692            text.contains("600 receipts from a previous experiment configuration ignored"),
1693            "{text}"
1694        );
1695
1696        // margin and alpha are part of the identity too.
1697        let margin = Config::parse(&format!("{CFG}margin = 0.04\n")).expect("parses");
1698        assert_eq!(
1699            experiments_from_traces(&traces, &deferred, &[], &margin)[0].ignored_stale,
1700            600
1701        );
1702        // percent is not: ramping the split keeps the evidence.
1703        let ramped = Config::parse(&CFG.replace("50.0", "30.0")).expect("parses");
1704        assert_eq!(
1705            experiments_from_traces(&traces, &deferred, &[], &ramped)[0].report,
1706            before[0].report
1707        );
1708    }
1709
1710    #[test]
1711    fn challenger_traces_are_not_incumbent_traces() {
1712        assert!(is_incumbent_trace(&trace("t", "s", Arm::Incumbent, 0.1)));
1713        assert!(!is_incumbent_trace(&trace("t", "s", Arm::Challenger, 0.1)));
1714        let mut plain = trace("t", "s", Arm::Challenger, 0.1);
1715        plain.experiment = None;
1716        assert!(is_incumbent_trace(&plain));
1717    }
1718
1719    /// An unreadable store must never re-open a halted challenger: the monitor is built blocked.
1720    #[tokio::test]
1721    async fn unreadable_store_fails_closed_on_the_challenger() {
1722        let cfg = config();
1723        let ok = ExperimentMonitor::from_config(&cfg);
1724        assert!(!ok.challenger_blocked("t", 0));
1725        let blocked = ExperimentMonitor::unavailable(&cfg);
1726        assert!(blocked.challenger_blocked("t", 0));
1727        assert!(blocked.challenger_blocked("any-other-tenant", 0));
1728
1729        // A path that is a directory cannot be opened as a database.
1730        let dir = std::env::temp_dir();
1731        let m = build_monitor(&dir.to_string_lossy(), &cfg);
1732        assert!(
1733            m.challenger_blocked("t", 0),
1734            "unreadable store served a challenger"
1735        );
1736
1737        // A readable (fresh) store is not blocked.
1738        let db = dir.join(format!("firstpass-exp-build-{}.db", Uuid::now_v7()));
1739        let (tx, handle) = crate::store::open(&db).expect("open");
1740        drop(tx);
1741        handle.await.expect("writer");
1742        let m = build_monitor(&db.to_string_lossy(), &cfg);
1743        assert!(!m.challenger_blocked("t", 0));
1744        let _ = std::fs::remove_file(&db);
1745
1746        // No challenger configured: nothing to block, store never touched.
1747        let bare = Config::parse(
1748            "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
1749        )
1750        .expect("parses");
1751        assert!(!build_monitor(&dir.to_string_lossy(), &bare).challenger_blocked("t", 0));
1752    }
1753
1754    /// A paired evaluation's challenger spend must reach the report on every path that applies a
1755    /// pair: direct (single replica), synced from the shared log, and rebuilt from the store at
1756    /// startup. Without it the challenger's $/request is n/a and a certified, cheaper paired
1757    /// challenger can never be recommended for adoption.
1758    #[tokio::test]
1759    async fn paired_challenger_cost_reaches_the_report_on_every_path() {
1760        let cfg = config();
1761        let fp_str = fp();
1762        let challenger = |m: &ExperimentMonitor| {
1763            let r = m.reports();
1764            assert_eq!(r.len(), 1);
1765            let c = r[0].report.challenger;
1766            (c.requests, c.cost_usd, r[0].report.pairs)
1767        };
1768        let close = |(n, usd, pairs): (u64, f64, u64)| {
1769            assert_eq!((n, pairs), (3, 3));
1770            assert!((usd - 0.03).abs() < 1e-12, "{usd}");
1771        };
1772
1773        let direct = ExperimentMonitor::from_config(&cfg);
1774        for _ in 0..3 {
1775            assert!(
1776                direct
1777                    .record_pair("t", 0, &fp_str, 1.0, 1.0, 0.10, 0.01)
1778                    .await
1779            );
1780        }
1781        close(challenger(&direct));
1782
1783        let shared = Arc::new(InMemoryExperimentLog::new());
1784        let a = ExperimentMonitor::from_config(&cfg)
1785            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1786        let b = ExperimentMonitor::from_config(&cfg)
1787            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1788        for _ in 0..3 {
1789            a.record_pair("t", 0, &fp_str, 1.0, 1.0, 0.10, 0.01).await;
1790        }
1791        b.sync_from_backend("t", 0, &fp_str).await;
1792        close(challenger(&a));
1793        close(challenger(&b));
1794
1795        let db =
1796            std::env::temp_dir().join(format!("firstpass-paircost-{}.db", uuid::Uuid::now_v7()));
1797        let (tx, handle) = crate::store::open(&db).unwrap();
1798        drop(tx);
1799        handle.await.unwrap();
1800        for _ in 0..3 {
1801            crate::store::append_experiment_pair(
1802                &db,
1803                &crate::store::ExperimentPairRow {
1804                    tenant: "t".to_owned(),
1805                    route_ix: 0,
1806                    fingerprint: fp_str.clone(),
1807                    trace_id: uuid::Uuid::now_v7(),
1808                    x_inc: 1.0,
1809                    x_ch: 1.0,
1810                    challenger_cost_usd: 0.01,
1811                    ts: jiff::Timestamp::now(),
1812                },
1813            )
1814            .unwrap();
1815        }
1816        close(challenger(&build_monitor(&db.to_string_lossy(), &cfg)));
1817
1818        // Restart with a shared log: the same three pairs are in the store AND the log. They
1819        // must count once, not twice (pairs carry no session to dedup on).
1820        let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
1821            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1822        restarted.sync_from_backend("t", 0, &fp_str).await;
1823        close(challenger(&restarted));
1824        let _ = std::fs::remove_file(&db);
1825    }
1826
1827    // ── Shared-log tests (InMemoryExperimentLog) ─────────────────────────────────────────────
1828
1829    /// Two monitors sharing one log converge to the same state after syncing.
1830    ///
1831    /// Mutation test: if `sync_from_backend` does not apply records, replica_b stays Undecided
1832    /// even after 400 pairs of Pass/Fail evidence, so the final `Halted` assertion fails.
1833    #[tokio::test]
1834    async fn two_replicas_sharing_one_log_converge_after_sync() {
1835        let shared = Arc::new(InMemoryExperimentLog::new());
1836        let cfg = config();
1837        let fp_str = fp();
1838
1839        let replica_a = ExperimentMonitor::from_config(&cfg)
1840            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1841        let replica_b = ExperimentMonitor::from_config(&cfg)
1842            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1843
1844        // Feed all evidence through replica A (it appends to the shared log).
1845        for i in 0..400 {
1846            replica_a
1847                .record_outcome(
1848                    "t",
1849                    0,
1850                    &fp_str,
1851                    Arm::Incumbent,
1852                    &format!("i{i}"),
1853                    Verdict::Pass,
1854                    0.10,
1855                )
1856                .await;
1857            replica_a
1858                .record_outcome(
1859                    "t",
1860                    0,
1861                    &fp_str,
1862                    Arm::Challenger,
1863                    &format!("c{i}"),
1864                    Verdict::Fail,
1865                    0.02,
1866                )
1867                .await;
1868        }
1869
1870        // Replica B has not received any feedback directly: still Undecided.
1871        // (Mutation check: disabling sync_from_backend makes the next assertion the failing one.)
1872        assert_eq!(
1873            replica_b.status("t", 0),
1874            ExperimentStatus::Undecided,
1875            "replica_b should not have any evidence before syncing"
1876        );
1877
1878        // After one sync pass, replica B reads the shared log and reaches the same decision.
1879        replica_b.sync_from_backend("t", 0, &fp_str).await;
1880        assert_eq!(
1881            replica_b.status("t", 0),
1882            ExperimentStatus::Halted,
1883            "replica_b must halt after syncing from shared log"
1884        );
1885        assert!(
1886            replica_b.challenger_blocked("t", 0),
1887            "halted experiment must block the challenger on the synced replica"
1888        );
1889
1890        // Both replicas agree on pair counts and e-values.
1891        let reps_a = replica_a.reports();
1892        let reps_b = replica_b.reports();
1893        assert_eq!(reps_a.len(), 1);
1894        assert_eq!(reps_b.len(), 1);
1895        assert_eq!(reps_a[0].report.status, reps_b[0].report.status);
1896        assert_eq!(reps_a[0].report.pairs, reps_b[0].report.pairs);
1897    }
1898
1899    /// A halt observed via one replica blocks the challenger on the other.
1900    ///
1901    /// Specifically: replica A drives the experiment to Halted; replica B has not synced yet,
1902    /// so it does NOT block. After one sync, replica B also blocks.
1903    ///
1904    /// Mutation test: if `challenger_blocked` does not consult status (always returns false),
1905    /// the first `assert!(!replica_b.challenger_blocked(...))` passes trivially, but the
1906    /// second one (`assert!(... after sync)`) fails — verifying the block gate actually changed.
1907    #[tokio::test]
1908    async fn halt_on_one_replica_blocks_challenger_on_other_after_sync() {
1909        let shared = Arc::new(InMemoryExperimentLog::new());
1910        let cfg = config();
1911        let fp_str = fp();
1912
1913        let replica_a = ExperimentMonitor::from_config(&cfg)
1914            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1915        let replica_b = ExperimentMonitor::from_config(&cfg)
1916            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1917
1918        // B has not synced yet — it does not know about the halt.
1919        assert!(
1920            !replica_b.challenger_blocked("t", 0),
1921            "should be unblocked before any evidence"
1922        );
1923
1924        // Drive A to halted.
1925        let mut halted = false;
1926        for i in 0..500 {
1927            replica_a
1928                .record_outcome(
1929                    "t",
1930                    0,
1931                    &fp_str,
1932                    Arm::Incumbent,
1933                    &format!("i{i}"),
1934                    Verdict::Pass,
1935                    0.10,
1936                )
1937                .await;
1938            replica_a
1939                .record_outcome(
1940                    "t",
1941                    0,
1942                    &fp_str,
1943                    Arm::Challenger,
1944                    &format!("c{i}"),
1945                    Verdict::Fail,
1946                    0.02,
1947                )
1948                .await;
1949            if replica_a.status("t", 0) == ExperimentStatus::Halted {
1950                halted = true;
1951                break;
1952            }
1953        }
1954        assert!(halted, "experiment never halted on replica A");
1955
1956        // B still does not know.
1957        assert!(
1958            !replica_b.challenger_blocked("t", 0),
1959            "replica_b should not block before syncing"
1960        );
1961
1962        // After sync, B learns of the halt and blocks.
1963        replica_b.sync_from_backend("t", 0, &fp_str).await;
1964        assert!(
1965            replica_b.challenger_blocked("t", 0),
1966            "replica_b must block after syncing the halt from the shared log"
1967        );
1968    }
1969
1970    /// Cross-replica session dedup: `claim_session` prevents the same session from being
1971    /// counted twice even when two replicas receive the same feedback.
1972    ///
1973    /// Mutation test: if `claim_session` always returns `true`, both replicas count the
1974    /// same session, the pair count doubles, and the equality assertion fails.
1975    #[tokio::test]
1976    async fn cross_replica_session_dedup_prevents_double_counting() {
1977        let shared = Arc::new(InMemoryExperimentLog::new());
1978        let cfg = config();
1979        let fp_str = fp();
1980
1981        let replica_a = ExperimentMonitor::from_config(&cfg)
1982            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1983        let replica_b = ExperimentMonitor::from_config(&cfg)
1984            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1985
1986        // Both replicas independently receive the same feedback for the same session.
1987        replica_a
1988            .record_outcome(
1989                "t",
1990                0,
1991                &fp_str,
1992                Arm::Incumbent,
1993                "session-1",
1994                Verdict::Pass,
1995                0.10,
1996            )
1997            .await;
1998        replica_b
1999            .record_outcome(
2000                "t",
2001                0,
2002                &fp_str,
2003                Arm::Incumbent,
2004                "session-1",
2005                Verdict::Pass,
2006                0.10,
2007            )
2008            .await;
2009
2010        // Sync both so they agree on the log.
2011        replica_a.sync_from_backend("t", 0, &fp_str).await;
2012        replica_b.sync_from_backend("t", 0, &fp_str).await;
2013
2014        // Only one outcome should have been counted (not two).
2015        let reps_a = replica_a.reports();
2016        let reps_b = replica_b.reports();
2017        assert_eq!(
2018            reps_a[0].report.incumbent.outcomes, 1,
2019            "session should be counted exactly once"
2020        );
2021        assert_eq!(
2022            reps_b[0].report.incumbent.outcomes, 1,
2023            "both replicas should agree on outcome count"
2024        );
2025    }
2026
2027    /// Abstain must not burn the session slot: if a session's first feedback is Abstain,
2028    /// a subsequent Pass/Fail for the same session must still be accepted.
2029    ///
2030    /// Mutation test: if Abstain is filtered AFTER `claim_session` (the pre-fix behaviour),
2031    /// Abstain claims the slot and the subsequent Pass returns `false`; the `== 1` assertion
2032    /// below fails.
2033    #[tokio::test]
2034    async fn abstain_does_not_burn_session_slot_before_claim() {
2035        let shared = Arc::new(InMemoryExperimentLog::new());
2036        let cfg = config();
2037        let fp_str = fp();
2038
2039        let monitor = ExperimentMonitor::from_config(&cfg)
2040            .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2041
2042        // First feedback for this session is Abstain — must not claim the session slot.
2043        let counted = monitor
2044            .record_outcome(
2045                "t",
2046                0,
2047                &fp_str,
2048                Arm::Incumbent,
2049                "session-abstain",
2050                Verdict::Abstain,
2051                0.10,
2052            )
2053            .await;
2054        assert!(!counted, "Abstain must not be counted");
2055
2056        // Second feedback for the SAME session: Pass — must be accepted.
2057        let counted = monitor
2058            .record_outcome(
2059                "t",
2060                0,
2061                &fp_str,
2062                Arm::Incumbent,
2063                "session-abstain",
2064                Verdict::Pass,
2065                0.10,
2066            )
2067            .await;
2068        assert!(counted, "Pass after Abstain must be counted");
2069
2070        let reps = monitor.reports();
2071        assert_eq!(
2072            reps[0].report.incumbent.outcomes, 1,
2073            "exactly one outcome must have been counted"
2074        );
2075    }
2076
2077    // ── Redis-backed shared-log tests ────────────────────────────────────────────────────────
2078
2079    /// Connect to the test Redis, wipe all keys whose name contains `unique_tenant`, and return
2080    /// the open connection for subsequent cleanup calls.
2081    #[cfg(feature = "redis-cache")]
2082    async fn redis_cleanup(url: &str, unique_tenant: &str) {
2083        let client = redis::Client::open(url).expect("redis client");
2084        let mut conn = redis::aio::ConnectionManager::new(client)
2085            .await
2086            .expect("redis connection for cleanup");
2087        use redis::AsyncCommands as _;
2088        // SCAN for all experiment keys that carry this tenant in their name.
2089        let pattern = format!("firstpass:exp*{unique_tenant}*");
2090        let keys: Vec<String> = {
2091            let mut out = Vec::new();
2092            let mut cursor: u64 = 0;
2093            loop {
2094                let (next, batch): (u64, Vec<String>) = redis::cmd("SCAN")
2095                    .arg(cursor)
2096                    .arg("MATCH")
2097                    .arg(&pattern)
2098                    .arg("COUNT")
2099                    .arg(100u64)
2100                    .query_async(&mut conn)
2101                    .await
2102                    .expect("SCAN");
2103                out.extend(batch);
2104                cursor = next;
2105                if cursor == 0 {
2106                    break;
2107                }
2108            }
2109            out
2110        };
2111        if !keys.is_empty() {
2112            let _: () = conn.del(keys).await.expect("DEL cleanup");
2113        }
2114    }
2115
2116    /// Two `RedisExperimentLog`-backed monitors on the same key converge to identical reports
2117    /// after sync.
2118    ///
2119    /// Requires `FIRSTPASS_TEST_REDIS` (e.g. `redis://127.0.0.1:56411/15`). **Absent, this
2120    /// panics rather than skipping**: a green suite that silently tested nothing is worse than a
2121    /// failing one.
2122    ///
2123    /// Mutation test: disabling `sync_from_backend` keeps replica_b `Undecided` and the
2124    /// `Halted` assertion below fails.
2125    #[cfg(feature = "redis-cache")]
2126    #[tokio::test]
2127    async fn redis_two_replicas_converge_to_identical_reports_after_sync() {
2128        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2129            panic!(
2130                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2131                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2132                 is worse than a red one."
2133            )
2134        });
2135        // Unique tenant prevents key collisions across concurrent or repeated test runs.
2136        let unique_tenant = format!("t-conv-{}", Uuid::now_v7());
2137        let log_a = Arc::new(
2138            RedisExperimentLog::connect(&url)
2139                .await
2140                .expect("RedisExperimentLog::connect"),
2141        );
2142        let log_b = Arc::new(
2143            RedisExperimentLog::connect(&url)
2144                .await
2145                .expect("RedisExperimentLog::connect"),
2146        );
2147        let cfg = config();
2148        let fp_str = fp();
2149
2150        let replica_a = ExperimentMonitor::from_config(&cfg)
2151            .with_log(Arc::clone(&log_a) as Arc<dyn ExperimentLogBackend>);
2152        let replica_b = ExperimentMonitor::from_config(&cfg)
2153            .with_log(Arc::clone(&log_b) as Arc<dyn ExperimentLogBackend>);
2154
2155        // All evidence through replica A.
2156        for i in 0..400 {
2157            replica_a
2158                .record_outcome(
2159                    &unique_tenant,
2160                    0,
2161                    &fp_str,
2162                    Arm::Incumbent,
2163                    &format!("i{i}"),
2164                    Verdict::Pass,
2165                    0.10,
2166                )
2167                .await;
2168            replica_a
2169                .record_outcome(
2170                    &unique_tenant,
2171                    0,
2172                    &fp_str,
2173                    Arm::Challenger,
2174                    &format!("c{i}"),
2175                    Verdict::Fail,
2176                    0.02,
2177                )
2178                .await;
2179        }
2180
2181        // B has not synced: still Undecided locally.
2182        assert_eq!(
2183            replica_b.status(&unique_tenant, 0),
2184            ExperimentStatus::Undecided,
2185            "replica_b must be Undecided before syncing"
2186        );
2187
2188        // One sync makes B read the full log and reach the same decision.
2189        replica_b
2190            .sync_from_backend(&unique_tenant, 0, &fp_str)
2191            .await;
2192        assert_eq!(
2193            replica_b.status(&unique_tenant, 0),
2194            ExperimentStatus::Halted,
2195            "replica_b must be Halted after syncing from Redis log"
2196        );
2197
2198        let reps_a = replica_a.reports();
2199        let reps_b = replica_b.reports();
2200        assert_eq!(reps_a.len(), 1);
2201        assert_eq!(reps_b.len(), 1);
2202        assert_eq!(
2203            reps_a[0].report.status, reps_b[0].report.status,
2204            "both replicas must agree on status"
2205        );
2206        assert_eq!(
2207            reps_a[0].report.pairs, reps_b[0].report.pairs,
2208            "both replicas must agree on pair counts"
2209        );
2210
2211        redis_cleanup(&url, &unique_tenant).await;
2212    }
2213
2214    /// `claim_session` prevents the same session from being double-counted across replicas when
2215    /// both receive the same feedback.
2216    ///
2217    /// Requires `FIRSTPASS_TEST_REDIS`. Panics when unset (same rationale as above).
2218    ///
2219    /// Mutation test: if `claim_session` always returns `true`, both replicas append the same
2220    /// session and the outcome count doubles; the `== 1` assertion fails.
2221    #[cfg(feature = "redis-cache")]
2222    #[tokio::test]
2223    async fn redis_cross_replica_session_dedup_via_claim_session_holds() {
2224        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2225            panic!(
2226                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2227                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2228                 is worse than a red one."
2229            )
2230        });
2231        let unique_tenant = format!("t-dedup-{}", Uuid::now_v7());
2232        let log = Arc::new(
2233            RedisExperimentLog::connect(&url)
2234                .await
2235                .expect("RedisExperimentLog::connect"),
2236        );
2237        let cfg = config();
2238        let fp_str = fp();
2239
2240        let replica_a = ExperimentMonitor::from_config(&cfg)
2241            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2242        let replica_b = ExperimentMonitor::from_config(&cfg)
2243            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2244
2245        // Both replicas receive the same session feedback.
2246        replica_a
2247            .record_outcome(
2248                &unique_tenant,
2249                0,
2250                &fp_str,
2251                Arm::Incumbent,
2252                "session-x",
2253                Verdict::Pass,
2254                0.10,
2255            )
2256            .await;
2257        replica_b
2258            .record_outcome(
2259                &unique_tenant,
2260                0,
2261                &fp_str,
2262                Arm::Incumbent,
2263                "session-x",
2264                Verdict::Pass,
2265                0.10,
2266            )
2267            .await;
2268
2269        // Sync both.
2270        replica_a
2271            .sync_from_backend(&unique_tenant, 0, &fp_str)
2272            .await;
2273        replica_b
2274            .sync_from_backend(&unique_tenant, 0, &fp_str)
2275            .await;
2276
2277        let reps_a = replica_a.reports();
2278        let reps_b = replica_b.reports();
2279        assert_eq!(
2280            reps_a[0].report.incumbent.outcomes, 1,
2281            "session must be counted exactly once on replica A"
2282        );
2283        assert_eq!(
2284            reps_b[0].report.incumbent.outcomes, 1,
2285            "session must be counted exactly once on replica B"
2286        );
2287
2288        redis_cleanup(&url, &unique_tenant).await;
2289    }
2290
2291    /// A halt driven on replica A is propagated to replica B after one sync, blocking the
2292    /// challenger there too.
2293    ///
2294    /// Requires `FIRSTPASS_TEST_REDIS`. Panics when unset.
2295    ///
2296    /// Mutation test: removing the `sync_from_backend` call means B never learns of the halt,
2297    /// and the `challenger_blocked` assertion at the end fails.
2298    #[cfg(feature = "redis-cache")]
2299    #[tokio::test]
2300    async fn redis_halt_on_one_replica_blocks_challenger_on_other() {
2301        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2302            panic!(
2303                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2304                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2305                 is worse than a red one."
2306            )
2307        });
2308        let unique_tenant = format!("t-halt-{}", Uuid::now_v7());
2309        let log = Arc::new(
2310            RedisExperimentLog::connect(&url)
2311                .await
2312                .expect("RedisExperimentLog::connect"),
2313        );
2314        let cfg = config();
2315        let fp_str = fp();
2316
2317        let replica_a = ExperimentMonitor::from_config(&cfg)
2318            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2319        let replica_b = ExperimentMonitor::from_config(&cfg)
2320            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2321
2322        // B starts unblocked.
2323        assert!(
2324            !replica_b.challenger_blocked(&unique_tenant, 0),
2325            "replica_b must be unblocked before any evidence"
2326        );
2327
2328        // Drive A to Halted.
2329        let mut halted = false;
2330        for i in 0..500 {
2331            replica_a
2332                .record_outcome(
2333                    &unique_tenant,
2334                    0,
2335                    &fp_str,
2336                    Arm::Incumbent,
2337                    &format!("i{i}"),
2338                    Verdict::Pass,
2339                    0.10,
2340                )
2341                .await;
2342            replica_a
2343                .record_outcome(
2344                    &unique_tenant,
2345                    0,
2346                    &fp_str,
2347                    Arm::Challenger,
2348                    &format!("c{i}"),
2349                    Verdict::Fail,
2350                    0.02,
2351                )
2352                .await;
2353            if replica_a.status(&unique_tenant, 0) == ExperimentStatus::Halted {
2354                halted = true;
2355                break;
2356            }
2357        }
2358        assert!(halted, "experiment never halted on replica A");
2359
2360        // B still does not know: it hasn't synced.
2361        assert!(
2362            !replica_b.challenger_blocked(&unique_tenant, 0),
2363            "replica_b must not block before syncing"
2364        );
2365
2366        // After one sync, B learns the halt and blocks.
2367        replica_b
2368            .sync_from_backend(&unique_tenant, 0, &fp_str)
2369            .await;
2370        assert!(
2371            replica_b.challenger_blocked(&unique_tenant, 0),
2372            "replica_b must block the challenger after syncing the halt from Redis"
2373        );
2374
2375        redis_cleanup(&url, &unique_tenant).await;
2376    }
2377
2378    /// Keys are isolated per tenant: evidence written under `tenant_a` is invisible to a monitor
2379    /// reading under `tenant_b`.
2380    ///
2381    /// Requires `FIRSTPASS_TEST_REDIS`. Panics when unset.
2382    ///
2383    /// Mutation test: if the key omits the tenant and both monitors share one list, B would
2384    /// gain outcomes after sync and the `Undecided` assertion fails.
2385    #[cfg(feature = "redis-cache")]
2386    #[tokio::test]
2387    async fn redis_keys_are_isolated_per_tenant() {
2388        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2389            panic!(
2390                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2391                 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2392                 is worse than a red one."
2393            )
2394        });
2395        let run_id = Uuid::now_v7();
2396        let tenant_a = format!("t-iso-a-{run_id}");
2397        let tenant_b = format!("t-iso-b-{run_id}");
2398        let log = Arc::new(
2399            RedisExperimentLog::connect(&url)
2400                .await
2401                .expect("RedisExperimentLog::connect"),
2402        );
2403        let cfg = config();
2404        let fp_str = fp();
2405
2406        let monitor_a = ExperimentMonitor::from_config(&cfg)
2407            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2408        let monitor_b = ExperimentMonitor::from_config(&cfg)
2409            .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2410
2411        // Feed evidence only under tenant_a.
2412        for i in 0..5 {
2413            monitor_a
2414                .record_outcome(
2415                    &tenant_a,
2416                    0,
2417                    &fp_str,
2418                    Arm::Incumbent,
2419                    &format!("s{i}"),
2420                    Verdict::Pass,
2421                    0.10,
2422                )
2423                .await;
2424        }
2425
2426        // Prove storage-level isolation: load_since for tenant_b returns nothing.
2427        let records = log.load_since(&tenant_b, 0, &fp_str, 0).await;
2428        assert!(
2429            records.is_empty(),
2430            "backend must return no records for tenant_b"
2431        );
2432
2433        // monitor_b reads under tenant_b via status() (does not create an entry) and sees
2434        // Undecided, confirming no cross-tenant contamination.
2435        assert_eq!(
2436            monitor_b.status(&tenant_b, 0),
2437            ExperimentStatus::Undecided,
2438            "evidence written under tenant_a must not appear under tenant_b"
2439        );
2440
2441        // monitor_a still has its own evidence.
2442        let reps = monitor_a.reports();
2443        assert_eq!(reps.len(), 1);
2444        assert_eq!(reps[0].report.incumbent.outcomes, 5);
2445
2446        // Clean up both tenants' keys.
2447        redis_cleanup(&url, &tenant_a).await;
2448        redis_cleanup(&url, &tenant_b).await;
2449    }
2450}