use std::collections::{HashMap, HashSet};
use std::fmt;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use firstpass_core::{
Arm, Config, Experiment, ExperimentReport, ExperimentStatus, Trace, Verdict, select,
};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
const MAX_TRACKED_SESSIONS: usize = 1_000_000;
const SYNC_FAILURE_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(5);
const SYNC_STALENESS_BOUND: std::time::Duration = std::time::Duration::from_secs(30);
const SYNC_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
const LOG_OP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
async fn bounded<T>(
fut: impl std::future::Future<Output = Result<T, LogError>>,
) -> Result<T, LogError> {
match tokio::time::timeout(LOG_OP_TIMEOUT, fut).await {
Ok(r) => r,
Err(_) => Err(LogError::Backend(format!(
"timed out after {} ms",
LOG_OP_TIMEOUT.as_millis()
))),
}
}
#[derive(Debug, thiserror::Error)]
pub enum LogError {
#[error("experiment log backend error: {0}")]
Backend(String),
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum LogRecord {
Outcome {
arm: Arm,
outcome: f64,
session_hash: u64,
cost_usd: f64,
},
Pair {
x_inc: f64,
x_ch: f64,
incumbent_cost_usd: f64,
challenger_cost_usd: f64,
#[serde(default)]
ch_error: bool,
#[serde(default)]
ch_truncated: u64,
#[serde(default)]
ts_unix: i64,
#[serde(default)]
replica_id: Uuid,
#[serde(default)]
trace_id: Uuid,
#[serde(default)]
incumbent_trace_id: Uuid,
},
}
#[async_trait::async_trait]
pub trait ExperimentLogBackend: Send + Sync + fmt::Debug {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: LogRecord,
) -> Result<u64, LogError>;
async fn load_since(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
offset: u64,
) -> Result<Vec<Option<LogRecord>>, LogError>;
async fn claim_session(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
session_id: &str,
) -> bool;
}
#[derive(Debug, Default)]
pub struct InMemoryExperimentLog {
lists: Mutex<HashMap<(String, usize, String), Vec<LogRecord>>>,
sessions: Mutex<HashSet<(String, usize, String, String)>>,
}
impl InMemoryExperimentLog {
#[must_use]
pub fn new() -> Self {
Self::default()
}
}
#[async_trait::async_trait]
impl ExperimentLogBackend for InMemoryExperimentLog {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: LogRecord,
) -> Result<u64, LogError> {
let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
let mut lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
let list = lists.entry(key).or_default();
list.push(record);
Ok(list.len() as u64)
}
async fn load_since(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
offset: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
let lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
match lists.get(&key) {
None => Ok(vec![]),
Some(list) => {
let start = offset as usize;
if start >= list.len() {
Ok(vec![])
} else {
Ok(list[start..].iter().map(|r| Some(r.clone())).collect())
}
}
}
}
async fn claim_session(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
session_id: &str,
) -> bool {
let key = (
tenant.to_owned(),
route_ix,
fingerprint.to_owned(),
session_id.to_owned(),
);
let mut sessions = self.sessions.lock().unwrap_or_else(PoisonError::into_inner);
sessions.insert(key)
}
}
#[cfg(feature = "redis-cache")]
pub use redis_log::RedisExperimentLog;
#[cfg(feature = "redis-cache")]
mod redis_log {
use super::{ExperimentLogBackend, LogRecord};
use std::fmt;
use tokio::sync::Mutex;
const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
fn redact(url: &str) -> String {
let Some((scheme, rest)) = url.split_once("://") else {
return "<malformed redis url>".to_owned();
};
let host = match rest.rfind('@') {
Some(i) => &rest[i + 1..],
None => rest,
};
format!("{scheme}://{host}")
}
pub struct RedisExperimentLog {
conn: Mutex<redis::aio::ConnectionManager>,
}
impl fmt::Debug for RedisExperimentLog {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RedisExperimentLog").finish_non_exhaustive()
}
}
impl RedisExperimentLog {
pub async fn connect(url: &str) -> Result<Self, String> {
let safe = redact(url);
let client = redis::Client::open(url).map_err(|e| format!("redis url {safe}: {e}"))?;
let conn =
tokio::time::timeout(CONNECT_TIMEOUT, redis::aio::ConnectionManager::new(client))
.await
.map_err(|_| format!("redis url {safe}: connect timed out"))?
.map_err(|e| format!("redis url {safe}: {e}"))?;
Ok(Self {
conn: Mutex::new(conn),
})
}
fn list_key(tenant: &str, route_ix: usize, fingerprint: &str) -> String {
format!("firstpass:exp:{tenant}:{route_ix}:{fingerprint}")
}
fn session_key(
tenant: &str,
route_ix: usize,
fingerprint: &str,
session_id: &str,
) -> String {
format!("firstpass:exp:sess:{tenant}:{route_ix}:{fingerprint}:{session_id}")
}
}
#[async_trait::async_trait]
impl ExperimentLogBackend for RedisExperimentLog {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: LogRecord,
) -> Result<u64, super::LogError> {
let key = Self::list_key(tenant, route_ix, fingerprint);
let json = serde_json::to_string(&record)
.map_err(|e| super::LogError::Backend(format!("serialize: {e}")))?;
let mut conn = self.conn.lock().await;
use redis::AsyncCommands as _;
conn.rpush::<_, _, u64>(&key, json)
.await
.map_err(|e| super::LogError::Backend(format!("RPUSH {key}: {e}")))
}
async fn load_since(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
offset: u64,
) -> Result<Vec<Option<super::LogRecord>>, super::LogError> {
let key = Self::list_key(tenant, route_ix, fingerprint);
let mut conn = self.conn.lock().await;
use redis::AsyncCommands as _;
let start = isize::try_from(offset).unwrap_or(isize::MAX);
let raw: Vec<String> = conn
.lrange(&key, start, -1)
.await
.map_err(|e| super::LogError::Backend(format!("LRANGE {key}: {e}")))?;
Ok(raw
.iter()
.map(|s| match serde_json::from_str(s) {
Ok(r) => Some(r),
Err(e) => {
tracing::warn!(
%e,
"experiment log: malformed record; position consumed, evidence skipped"
);
None
}
})
.collect())
}
async fn claim_session(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
session_id: &str,
) -> bool {
let key = Self::session_key(tenant, route_ix, fingerprint, session_id);
let mut conn = self.conn.lock().await;
use redis::AsyncCommands as _;
match conn
.set_options::<_, _, ()>(
&key,
"1",
redis::SetOptions::default()
.with_expiration(redis::SetExpiry::EX(86_400))
.conditional_set(redis::ExistenceCheck::NX),
)
.await
{
Ok(()) => true, Err(_) => false, }
}
}
}
#[derive(Debug)]
struct Entry {
experiment: Experiment,
ignored_stale: u64,
health: Health,
sessions: HashSet<u64>,
saturated_logged: bool,
applied_pair_ids: HashSet<Uuid>,
locally_costed: HashSet<Uuid>,
remote_log_offset: u64,
sync_warn_logged: bool,
sync_failed_at: Option<tokio::time::Instant>,
last_synced_at: Option<tokio::time::Instant>,
}
fn session_hash(session_id: &str) -> u64 {
use sha2::{Digest, Sha256};
let d = Sha256::digest(session_id.as_bytes());
u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]])
}
fn status_code(s: ExperimentStatus) -> f64 {
match s {
ExperimentStatus::Undecided => 0.0,
ExperimentStatus::Certified => 1.0,
ExperimentStatus::Halted => 2.0,
}
}
#[derive(Debug)]
struct Params {
margin: f64,
alpha: f64,
fingerprint: String,
}
type SyncGates = Mutex<HashMap<(String, usize), Arc<tokio::sync::Mutex<()>>>>;
#[derive(Debug)]
pub struct ExperimentMonitor {
params: HashMap<usize, Params>,
unavailable: bool,
entries: Mutex<HashMap<String, HashMap<usize, Entry>>>,
log: Option<Arc<dyn ExperimentLogBackend>>,
paired_ledger: Option<Arc<crate::shadow::ShadowLedger>>,
replica_id: Uuid,
sync_gates: SyncGates,
}
impl Default for ExperimentMonitor {
fn default() -> Self {
Self {
params: HashMap::new(),
unavailable: false,
entries: Mutex::new(HashMap::new()),
log: None,
paired_ledger: None,
replica_id: Uuid::now_v7(),
sync_gates: Mutex::new(HashMap::new()),
}
}
}
impl ExperimentMonitor {
#[must_use]
pub fn from_config(config: &Config) -> Self {
let params = config
.routes
.iter()
.enumerate()
.filter_map(|(i, r)| {
r.challenger.as_ref().map(|c| {
(
i,
Params {
margin: c.margin,
alpha: c.alpha,
fingerprint: c.fingerprint(&r.ladder),
},
)
})
})
.collect();
Self {
params,
unavailable: false,
entries: Mutex::default(),
log: None,
paired_ledger: None,
replica_id: Uuid::now_v7(),
sync_gates: Mutex::default(),
}
}
#[must_use]
pub fn unavailable(config: &Config) -> Self {
Self {
unavailable: true,
..Self::from_config(config)
}
}
#[must_use]
pub fn with_log(mut self, log: Arc<dyn ExperimentLogBackend>) -> Self {
self.log = Some(log);
self
}
#[must_use]
pub fn with_paired_ledger(mut self, ledger: Arc<crate::shadow::ShadowLedger>) -> Self {
self.paired_ledger = Some(ledger);
self
}
#[must_use]
pub fn has_log(&self) -> bool {
self.log.is_some()
}
pub async fn ensure_synced(&self, tenant: &str, route_ix: usize, fingerprint: &str) -> bool {
if self.log.is_none() {
return true; }
{
let mut map = self.lock();
match self.entry(&mut map, tenant, route_ix) {
None => return true, Some(e) => {
if e.last_synced_at
.is_some_and(|t| t.elapsed() <= SYNC_STALENESS_BOUND)
{
return true;
}
if e.sync_failed_at
.is_some_and(|t| t.elapsed() < SYNC_FAILURE_COOLDOWN)
{
return false;
}
}
}
}
let gate = {
let mut gates = self
.sync_gates
.lock()
.unwrap_or_else(PoisonError::into_inner);
gates
.entry((tenant.to_owned(), route_ix))
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
};
let _guard = gate.lock().await;
{
let map = self.lock();
if let Some(entry) = map.get(tenant).and_then(|r| r.get(&route_ix)) {
if entry
.last_synced_at
.is_some_and(|t| t.elapsed() <= SYNC_STALENESS_BOUND)
{
return true;
}
if entry
.sync_failed_at
.is_some_and(|t| t.elapsed() < SYNC_FAILURE_COOLDOWN)
{
return false;
}
}
}
let sync_result = tokio::time::timeout(
SYNC_TIMEOUT,
self.sync_from_backend(tenant, route_ix, fingerprint),
)
.await;
match sync_result {
Ok(Ok(())) => {
let mut gates = self
.sync_gates
.lock()
.unwrap_or_else(PoisonError::into_inner);
gates.remove(&(tenant.to_owned(), route_ix));
true
}
Ok(Err(e)) => {
let mut map = self.lock();
if let Some(entry) = self.entry(&mut map, tenant, route_ix) {
entry.sync_failed_at = Some(tokio::time::Instant::now());
if !entry.sync_warn_logged {
entry.sync_warn_logged = true;
tracing::warn!(
tenant,
route = route_ix,
error = %e,
"shared log unreachable; halt guard and spend cap may use stale \
state — serving incumbent only for this request"
);
}
}
false
}
Err(_elapsed) => {
let mut map = self.lock();
if let Some(entry) = self.entry(&mut map, tenant, route_ix) {
entry.sync_failed_at = Some(tokio::time::Instant::now());
if !entry.sync_warn_logged {
entry.sync_warn_logged = true;
tracing::warn!(
tenant,
route = route_ix,
"shared log sync timed out (>{} ms); serving incumbent only",
SYNC_TIMEOUT.as_millis()
);
}
}
false
}
}
}
#[must_use]
pub fn fingerprint(&self, route_ix: usize) -> Option<&str> {
self.params.get(&route_ix).map(|p| p.fingerprint.as_str())
}
#[must_use]
pub fn challenger_blocked(&self, tenant: &str, route_ix: usize) -> bool {
self.unavailable || self.status(tenant, route_ix) == ExperimentStatus::Halted
}
#[must_use]
pub fn is_active(&self) -> bool {
!self.params.is_empty()
}
fn lock(&self) -> MutexGuard<'_, HashMap<String, HashMap<usize, Entry>>> {
self.entries.lock().unwrap_or_else(PoisonError::into_inner)
}
fn entry<'a>(
&self,
map: &'a mut HashMap<String, HashMap<usize, Entry>>,
tenant: &str,
route_ix: usize,
) -> Option<&'a mut Entry> {
let p = self.params.get(&route_ix)?;
let (margin, alpha) = (p.margin, p.alpha);
if !map.contains_key(tenant) {
map.insert(tenant.to_owned(), HashMap::new());
}
let routes = map.get_mut(tenant)?;
if let std::collections::hash_map::Entry::Vacant(v) = routes.entry(route_ix) {
match Experiment::new(margin, alpha) {
Ok(experiment) => {
v.insert(Entry {
experiment,
ignored_stale: 0,
health: Health::default(),
sessions: HashSet::new(),
saturated_logged: false,
applied_pair_ids: HashSet::new(),
locally_costed: HashSet::new(),
remote_log_offset: 0,
sync_warn_logged: false,
sync_failed_at: None,
last_synced_at: None,
});
}
Err(e) => {
tracing::warn!(route = route_ix, %e, "experiment parameters rejected");
return None;
}
}
}
routes.get_mut(&route_ix)
}
#[must_use]
pub fn status(&self, tenant: &str, route_ix: usize) -> ExperimentStatus {
self.lock()
.get(tenant)
.and_then(|r| r.get(&route_ix))
.map_or(ExperimentStatus::Undecided, |e| e.experiment.status())
}
pub fn observe_cost(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
arm: Arm,
usd: f64,
) {
let current = self.fingerprint(route_ix) == Some(fingerprint);
let mut map = self.lock();
if let Some(e) = self.entry(&mut map, tenant, route_ix) {
if current {
e.experiment.observe_cost(arm, usd);
} else {
e.ignored_stale += 1;
}
}
}
pub fn observe_health(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
arm: Arm,
errored: bool,
truncated: u64,
) {
if (!errored && truncated == 0) || self.fingerprint(route_ix) != Some(fingerprint) {
return;
}
let mut map = self.lock();
if let Some(e) = self.entry(&mut map, tenant, route_ix) {
let h = match arm {
Arm::Incumbent => &mut e.health.incumbent,
Arm::Challenger => &mut e.health.challenger,
};
h.errors += u64::from(errored);
h.truncated += truncated;
}
}
pub fn observe_outcome(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
arm: Arm,
session_id: &str,
verdict: Verdict,
) -> bool {
if self.fingerprint(route_ix) != Some(fingerprint) {
return false;
}
let outcome = match verdict {
Verdict::Pass => 1.0,
Verdict::Fail => 0.0,
Verdict::Abstain => return false,
};
let mut map = self.lock();
let Some(e) = self.entry(&mut map, tenant, route_ix) else {
return false;
};
let key = session_hash(session_id);
if e.sessions.contains(&key) {
return false;
}
if e.sessions.len() >= MAX_TRACKED_SESSIONS {
if !e.saturated_logged {
e.saturated_logged = true;
tracing::warn!(
tenant,
route = route_ix,
"experiment session ledger is full; new sessions' outcomes are ignored"
);
}
return false;
}
let before = e.experiment.status();
if !e.experiment.observe_outcome(arm, outcome) {
return false;
}
e.sessions.insert(key);
let after = e.experiment.status();
if before != after {
match after {
ExperimentStatus::Halted => tracing::error!(
tenant,
route = route_ix,
"EXPERIMENT HALTED: the challenger ladder is worse than the incumbent by more \
than the margin — it is no longer served"
),
ExperimentStatus::Certified => tracing::info!(
tenant,
route = route_ix,
"experiment certified: the challenger is within the margin of the incumbent; \
adopting it is a human decision (`firstpass experiment`)"
),
ExperimentStatus::Undecided => {}
}
Self::publish(&map, route_ix);
}
true
}
fn publish(map: &HashMap<String, HashMap<usize, Entry>>, route_ix: usize) {
let worst = map
.values()
.filter_map(|r| r.get(&route_ix))
.map(|e| status_code(e.experiment.status()))
.fold(0.0, f64::max);
metrics::gauge!("firstpass_experiment_status", "route" => route_ix.to_string()).set(worst);
}
#[must_use]
pub fn reports(&self) -> Vec<Snapshot> {
let map = self.lock();
let mut out: Vec<_> = map
.iter()
.flat_map(|(t, routes)| {
routes.iter().map(move |(ix, e)| Snapshot {
tenant: t.clone(),
route_ix: *ix,
report: e.experiment.report(),
ignored_stale: e.ignored_stale,
health: e.health,
})
})
.collect();
out.sort_by(|a, b| (&a.tenant, a.route_ix).cmp(&(&b.tenant, b.route_ix)));
out
}
fn apply_log_record(
&self,
map: &mut HashMap<String, HashMap<usize, Entry>>,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: &LogRecord,
) -> bool {
if self.fingerprint(route_ix) != Some(fingerprint) {
if let Some(e) = self.entry(map, tenant, route_ix) {
e.ignored_stale += 1;
}
return false;
}
match record {
LogRecord::Outcome {
arm,
outcome,
session_hash: hash,
..
} => {
let Some(e) = self.entry(map, tenant, route_ix) else {
return false;
};
if e.sessions.contains(hash) {
return false; }
if e.sessions.len() >= MAX_TRACKED_SESSIONS {
return false;
}
let before = e.experiment.status();
if !e.experiment.observe_outcome(*arm, *outcome) {
return false;
}
e.sessions.insert(*hash);
before != e.experiment.status()
}
LogRecord::Pair {
x_inc,
x_ch,
incumbent_cost_usd,
challenger_cost_usd,
ch_error,
ch_truncated,
ts_unix,
replica_id: record_replica_id,
trace_id: record_trace_id,
incumbent_trace_id,
} => {
let Some(e) = self.entry(map, tenant, route_ix) else {
return false;
};
let already_applied =
!record_trace_id.is_nil() && e.applied_pair_ids.contains(record_trace_id);
let before = e.experiment.status();
if !already_applied {
if e.applied_pair_ids.len() >= MAX_TRACKED_SESSIONS {
if !e.saturated_logged {
e.saturated_logged = true;
tracing::warn!(
tenant,
route = route_ix,
"experiment pair-id set is full; new pairs' evidence is skipped"
);
}
} else {
if e.experiment.observe_pair(*x_inc, *x_ch) {
e.experiment
.observe_cost(Arm::Challenger, *challenger_cost_usd);
}
if !record_trace_id.is_nil() {
e.applied_pair_ids.insert(*record_trace_id);
}
}
}
let is_own = *record_replica_id == self.replica_id;
if !already_applied && !is_own {
let costed_here =
!incumbent_trace_id.is_nil() && e.locally_costed.remove(incumbent_trace_id);
if !costed_here {
e.experiment
.observe_cost(Arm::Incumbent, *incumbent_cost_usd);
}
if *ch_error || *ch_truncated > 0 {
e.health.challenger.errors += u64::from(*ch_error);
e.health.challenger.truncated += *ch_truncated;
}
if let Some(ledger) = &self.paired_ledger {
let now = jiff::Timestamp::now();
let pair_ts = jiff::Timestamp::from_second(*ts_unix)
.unwrap_or(jiff::Timestamp::UNIX_EPOCH);
ledger.seed(tenant, route_ix, *challenger_cost_usd, pair_ts, now);
}
}
before != e.experiment.status()
}
}
}
pub async fn sync_from_backend(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
) -> Result<(), LogError> {
let Some(log) = &self.log else {
return Ok(());
};
let offset = {
let mut map = self.lock();
self.entry(&mut map, tenant, route_ix)
.map_or(0, |e| e.remote_log_offset)
};
let records = log
.load_since(tenant, route_ix, fingerprint, offset)
.await?;
if records.is_empty() {
let mut map = self.lock();
if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
e.last_synced_at = Some(tokio::time::Instant::now());
e.sync_warn_logged = false;
e.sync_failed_at = None;
}
return Ok(());
}
let mut map = self.lock();
for (i, opt_rec) in records.iter().enumerate() {
let abs_ix = offset + i as u64;
let current_offset = map
.get(tenant)
.and_then(|r| r.get(&route_ix))
.map_or(0, |e| e.remote_log_offset);
if abs_ix < current_offset {
continue;
}
let changed = if let Some(rec) = opt_rec {
self.apply_log_record(&mut map, tenant, route_ix, fingerprint, rec)
} else {
false
};
if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
e.remote_log_offset = e.remote_log_offset.max(abs_ix + 1);
}
if changed {
let after = map
.get(tenant)
.and_then(|r| r.get(&route_ix))
.map(|e| e.experiment.status());
match after {
Some(ExperimentStatus::Halted) => tracing::error!(
tenant,
route = route_ix,
"EXPERIMENT HALTED (shared log): challenger worse than incumbent"
),
Some(ExperimentStatus::Certified) => tracing::info!(
tenant,
route = route_ix,
"experiment certified (shared log)"
),
_ => {}
}
Self::publish(&map, route_ix);
}
}
if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
e.last_synced_at = Some(tokio::time::Instant::now());
e.sync_warn_logged = false;
e.sync_failed_at = None;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub async fn record_outcome(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
arm: Arm,
session_id: &str,
verdict: Verdict,
cost_usd: f64,
) -> bool {
let outcome = match verdict {
Verdict::Pass => 1.0,
Verdict::Fail => 0.0,
Verdict::Abstain => return false,
};
let Some(log) = self.log.as_ref().cloned() else {
return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
};
if !log
.claim_session(tenant, route_ix, fingerprint, session_id)
.await
{
return false;
}
if let Err(e) = log
.append(
tenant,
route_ix,
fingerprint,
LogRecord::Outcome {
arm,
outcome,
session_hash: session_hash(session_id),
cost_usd,
},
)
.await
{
tracing::warn!(
tenant,
route = route_ix,
error = %e,
"outcome log append failed; evidence will not reach other replicas"
);
return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
}
let _ = self.sync_from_backend(tenant, route_ix, fingerprint).await;
true
}
#[allow(clippy::too_many_arguments)]
pub fn apply_pair_locally(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
x_inc: f64,
x_ch: f64,
challenger_cost_usd: f64,
trace_id: Uuid,
) -> bool {
if self.fingerprint(route_ix) != Some(fingerprint) {
return false;
}
let mut map = self.lock();
let Some(e) = self.entry(&mut map, tenant, route_ix) else {
return false;
};
let already_applied = !trace_id.is_nil() && e.applied_pair_ids.contains(&trace_id);
if already_applied {
return false;
}
if e.applied_pair_ids.len() >= MAX_TRACKED_SESSIONS {
if !e.saturated_logged {
e.saturated_logged = true;
tracing::warn!(
tenant,
route = route_ix,
"experiment pair-id set is full; new pairs' evidence is skipped"
);
}
return false;
}
let before = e.experiment.status();
let ok = e.experiment.observe_pair(x_inc, x_ch);
if ok {
e.experiment
.observe_cost(Arm::Challenger, challenger_cost_usd);
}
if !trace_id.is_nil() {
e.applied_pair_ids.insert(trace_id);
}
if ok && before != e.experiment.status() {
match e.experiment.status() {
ExperimentStatus::Halted => tracing::error!(
tenant,
route = route_ix,
"EXPERIMENT HALTED: paired challenger worse than incumbent (local evidence)"
),
ExperimentStatus::Certified => tracing::info!(
tenant,
route = route_ix,
"experiment certified (paired, local evidence): challenger within margin"
),
ExperimentStatus::Undecided => {}
}
Self::publish(&map, route_ix);
}
ok
}
#[allow(clippy::too_many_arguments)]
pub async fn record_pair(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
x_inc: f64,
x_ch: f64,
incumbent_cost_usd: f64,
challenger_cost_usd: f64,
ch_error: bool,
ch_truncated: u64,
ts_unix: i64,
trace_id: Uuid,
incumbent_trace_id: Uuid,
) -> bool {
if self.fingerprint(route_ix) != Some(fingerprint) {
return false;
}
let Some(log) = self.log.as_ref().cloned() else {
return false;
};
if !x_inc.is_finite()
|| !(0.0..=1.0).contains(&x_inc)
|| !x_ch.is_finite()
|| !(0.0..=1.0).contains(&x_ch)
{
return false;
}
if let Err(e) = bounded(log.append(
tenant,
route_ix,
fingerprint,
LogRecord::Pair {
x_inc,
x_ch,
incumbent_cost_usd,
challenger_cost_usd,
ch_error,
ch_truncated,
ts_unix,
replica_id: self.replica_id,
trace_id,
incumbent_trace_id,
},
))
.await
{
tracing::warn!(
tenant,
route = route_ix,
error = %e,
"paired log append failed; evidence withheld until SQLite write succeeds \
(local spend and health were already tallied)"
);
return false;
}
if bounded(self.sync_from_backend(tenant, route_ix, fingerprint))
.await
.is_err()
{
self.apply_pair_locally(
tenant,
route_ix,
fingerprint,
x_inc,
x_ch,
challenger_cost_usd,
trace_id,
);
}
true
}
pub fn start_sync_task(self: &Arc<Self>, interval: std::time::Duration) {
if self.log.is_none() {
return;
}
let monitor = Arc::clone(self);
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
ticker.tick().await;
let keys: Vec<(String, usize, String)> = {
let map = monitor.lock();
let mut v = Vec::new();
for (tenant, routes) in map.iter() {
for &route_ix in routes.keys() {
if let Some(p) = monitor.params.get(&route_ix) {
v.push((tenant.clone(), route_ix, p.fingerprint.clone()));
}
}
}
v
};
for (tenant, route_ix, fingerprint) in keys {
if let Err(e) =
bounded(monitor.sync_from_backend(&tenant, route_ix, &fingerprint)).await
{
let mut map = monitor.lock();
if let Some(e_entry) =
map.get_mut(&tenant).and_then(|r| r.get_mut(&route_ix))
{
e_entry.sync_failed_at = Some(tokio::time::Instant::now());
}
tracing::warn!(
tenant,
route = route_ix,
error = %e,
"background experiment sync failed"
);
}
}
}
});
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Snapshot {
pub tenant: String,
pub route_ix: usize,
pub report: ExperimentReport,
pub ignored_stale: u64,
pub health: Health,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
pub struct ArmHealth {
pub errors: u64,
pub truncated: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
pub struct Health {
pub incumbent: ArmHealth,
pub challenger: ArmHealth,
}
#[must_use]
pub fn truncated_attempts(t: &Trace) -> u64 {
t.attempts.iter().filter(|a| a.truncated).count() as u64
}
#[must_use]
pub fn is_incumbent_trace(t: &Trace) -> bool {
t.experiment
.as_ref()
.is_none_or(|e| e.arm != Arm::Challenger)
}
#[must_use]
pub fn monitor_from_traces(
traces: &[Trace],
deferred: &[(Uuid, Verdict)],
pairs: &[crate::store::ExperimentPairRow],
config: &Config,
) -> ExperimentMonitor {
let monitor = ExperimentMonitor::from_config(config);
if !monitor.is_active() {
return monitor;
}
let mut by_id: HashMap<Uuid, &Trace> = HashMap::new();
for t in traces {
let Some(rec) = &t.experiment else { continue };
let route_ix = t.route_ix.map_or(0, |r| r as usize);
monitor.observe_cost(
&t.tenant_id,
route_ix,
&rec.fingerprint,
rec.arm,
t.final_.total_cost_usd,
);
monitor.observe_health(
&t.tenant_id,
route_ix,
&rec.fingerprint,
rec.arm,
t.final_.served_from == firstpass_core::ServedFrom::Error,
truncated_attempts(t),
);
if rec.arm == Arm::Incumbent {
let mut map = monitor.lock();
if let Some(e) = monitor.entry(&mut map, &t.tenant_id, route_ix)
&& e.locally_costed.len() < MAX_TRACKED_SESSIONS
{
e.locally_costed.insert(t.trace_id);
}
}
by_id.insert(t.trace_id, t);
}
for (id, verdict) in deferred {
let Some(t) = by_id.get(id) else { continue };
let Some(rec) = &t.experiment else { continue };
let route_ix = t.route_ix.map_or(0, |r| r as usize);
monitor.observe_outcome(
&t.tenant_id,
route_ix,
&rec.fingerprint,
rec.arm,
&t.session_id,
*verdict,
);
}
for row in pairs {
let mut map = monitor.lock();
if let Some(e) = monitor.entry(&mut map, &row.tenant, row.route_ix) {
if monitor.fingerprint(row.route_ix) == Some(row.fingerprint.as_str()) {
if !row.trace_id.is_nil() && e.applied_pair_ids.contains(&row.trace_id) {
continue;
}
if e.experiment.observe_pair(row.x_inc, row.x_ch) {
e.experiment
.observe_cost(Arm::Challenger, row.challenger_cost_usd);
}
if !row.trace_id.is_nil() {
e.applied_pair_ids.insert(row.trace_id);
}
if row.ch_error || row.ch_truncated > 0 {
e.health.challenger.errors += u64::from(row.ch_error);
e.health.challenger.truncated += row.ch_truncated;
}
} else {
e.ignored_stale += 1;
}
}
}
monitor
}
#[must_use]
pub fn build_monitor(db_path: &str, config: &Config) -> ExperimentMonitor {
build_monitor_with(db_path, config, false)
}
#[must_use]
pub fn build_monitor_with(db_path: &str, config: &Config, shared_log: bool) -> ExperimentMonitor {
let empty = ExperimentMonitor::from_config(config);
if !empty.is_active() {
return empty;
}
let loaded = crate::store::load_all_traces(db_path)
.and_then(|t| crate::store::load_all_deferred(db_path).map(|d| (t, d)))
.and_then(|(t, d)| crate::store::load_all_experiment_pairs(db_path).map(|p| (t, d, p)));
match loaded {
Ok((traces, deferred, pairs)) => {
let offline_pairs: Vec<crate::store::ExperimentPairRow>;
let replayed: &[crate::store::ExperimentPairRow] = if shared_log {
offline_pairs = pairs
.iter()
.filter(|r| r.in_log == crate::store::InLogStatus::NotInLog)
.cloned()
.collect();
&offline_pairs
} else {
&pairs
};
let m = monitor_from_traces(&traces, &deferred, replayed, config);
if shared_log {
let mut map = m.lock();
for row in &pairs {
let _ = m.entry(&mut map, &row.tenant, row.route_ix);
}
}
for s in m.reports() {
tracing::info!(
tenant = %s.tenant,
route = s.route_ix,
status = ?s.report.status,
pairs = s.report.pairs,
ignored_stale = s.ignored_stale,
"experiment restored from receipts"
);
ExperimentMonitor::publish(&m.lock(), s.route_ix);
}
m
}
Err(e) => {
tracing::error!(
%e,
"could not read the trace store, so it is unknown whether a challenger was already \
halted: challenger arms are NOT served (incumbent only) until restart with a \
readable store"
);
ExperimentMonitor::unavailable(config)
}
}
}
#[must_use]
pub fn experiments_from_traces(
traces: &[Trace],
deferred: &[(Uuid, Verdict)],
pairs: &[crate::store::ExperimentPairRow],
config: &Config,
) -> Vec<Snapshot> {
monitor_from_traces(traces, deferred, pairs, config).reports()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Recommendation {
Adopt,
CertifiedNoSaving,
ChallengerStopped,
KeepRunning,
}
#[must_use]
pub fn recommend(r: &ExperimentReport) -> Recommendation {
match r.status {
select::Status::Undecided => Recommendation::KeepRunning,
select::Status::Halted => Recommendation::ChallengerStopped,
select::Status::Certified => {
match (
r.challenger.cost_per_request(),
r.incumbent.cost_per_request(),
) {
(Some(c), Some(i)) if c < i => Recommendation::Adopt,
_ => Recommendation::CertifiedNoSaving,
}
}
}
}
#[derive(Debug, Clone, Serialize)]
pub struct ExperimentView {
pub tenant: String,
pub route_ix: usize,
pub report: ExperimentReport,
pub ignored_stale: u64,
pub health: Health,
pub recommendation: Recommendation,
pub challenger_ladder: Vec<String>,
}
#[must_use]
pub fn views(reports: Vec<Snapshot>, config: &Config) -> Vec<ExperimentView> {
reports
.into_iter()
.map(|s| ExperimentView {
recommendation: recommend(&s.report),
ignored_stale: s.ignored_stale,
health: s.health,
challenger_ladder: config
.routes
.get(s.route_ix)
.and_then(|r| r.challenger.as_ref())
.map(|c| c.ladder.clone())
.unwrap_or_default(),
tenant: s.tenant,
route_ix: s.route_ix,
report: s.report,
})
.collect()
}
fn fmt_opt(v: Option<f64>, f: impl Fn(f64) -> String) -> String {
v.map_or_else(|| "n/a".to_owned(), f)
}
#[must_use]
pub fn render(views: &[ExperimentView]) -> String {
use std::fmt::Write as _;
if views.is_empty() {
return "no experiment data yet — the config has no [route.challenger], or no experiment \
traffic has been recorded\n"
.to_owned();
}
let mut out = String::new();
for v in views {
let r = &v.report;
let _ = writeln!(
out,
"tenant {} · route {} · status: {:?}",
v.tenant, v.route_ix, r.status
);
let _ = writeln!(
out,
" pairs: {} (margin {}, alpha {})",
r.pairs, r.margin, r.alpha
);
for (name, a) in [("incumbent ", &r.incumbent), ("challenger", &r.challenger)] {
let _ = writeln!(
out,
" {name}: success {} over {} sessions · ${} per request over {} requests",
fmt_opt(a.success_rate(), |s| format!("{:.1}%", s * 100.0)),
a.outcomes,
fmt_opt(a.cost_per_request(), |c| format!("{c:.6}")),
a.requests,
);
}
let _ = writeln!(
out,
" e-value to certify: {:.3} · e-value to halt: {:.3} (decide at {:.1})",
r.e_certify,
r.e_halt,
1.0 / r.alpha
);
if v.ignored_stale > 0 {
let _ = writeln!(
out,
" {} receipts from a previous experiment configuration ignored",
v.ignored_stale
);
}
for (name, h) in [
("incumbent", &v.health.incumbent),
("challenger", &v.health.challenger),
] {
if h.errors > 0 || h.truncated > 0 {
let _ = writeln!(
out,
" {name}: {} errors, {} answers cut off at max_tokens (neither is random: \
check them before trusting the rates)",
h.errors, h.truncated
);
}
}
match v.recommendation {
Recommendation::Adopt => {
let ladder = v
.challenger_ladder
.iter()
.map(|m| format!("{m:?}"))
.collect::<Vec<_>>()
.join(", ");
let _ = writeln!(
out,
" ADOPT: set this route's ladder to [{ladder}]\n\n ladder = [{ladder}]\n"
);
}
Recommendation::CertifiedNoSaving => {
let _ = writeln!(out, " certified, but no saving — keep incumbent");
}
Recommendation::ChallengerStopped => {
let _ = writeln!(out, " challenger stopped: worse by more than the margin");
}
Recommendation::KeepRunning => {
let _ = writeln!(out, " keep running");
}
}
}
out
}
#[cfg(test)]
mod tests {
use firstpass_core::trace::ExperimentRecord;
use firstpass_core::{
Features, FinalOutcome, GENESIS_HASH, Mode, PolicyRef, RequestInfo, ServedFrom, TaskKind,
};
use super::*;
const CFG: &str = "[[route]]\nmatch = {}\nmode = \"enforce\"\n\
ladder = [\"anthropic/claude-sonnet-5\"]\n\
[route.challenger]\nladder = [\"anthropic/claude-haiku-4-5\"]\npercent = 50.0\n";
fn config() -> Config {
Config::parse(CFG).expect("test config parses")
}
fn fp() -> String {
let c = config();
c.routes[0]
.challenger
.as_ref()
.expect("challenger")
.fingerprint(&c.routes[0].ladder)
}
fn trace(tenant: &str, session: &str, arm: Arm, cost: f64) -> Trace {
Trace {
trace_id: Uuid::now_v7(),
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: tenant.to_owned(),
session_id: session.to_owned(),
ts: jiff::Timestamp::now(),
mode: Mode::Enforce,
policy: PolicyRef {
id: "static-ladder@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "h".to_owned(),
features: Features::new(TaskKind::Other),
},
attempts: vec![],
deferred: vec![],
final_: FinalOutcome {
served_rung: Some(0),
served_from: ServedFrom::Attempt,
total_cost_usd: cost,
gate_cost_usd: 0.0,
total_latency_ms: 1,
escalations: 0,
counterfactual_baseline_usd: cost,
savings_usd: 0.0,
cache_source: None,
},
probe: None,
rollout: None,
shadow: None,
route_ix: Some(0),
predicted_pass: None,
decision_prior: None,
elastic: None,
risk: None,
experiment: Some(ExperimentRecord {
arm,
percent: 50.0,
bucket: 0,
fingerprint: fp(),
}),
upstream_provider: String::new(),
}
}
#[test]
fn arm_health_is_rebuilt_from_receipts() {
let attempt = |truncated| firstpass_core::trace::Attempt {
rung: 0,
model: "anthropic/claude-sonnet-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 1,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 1,
cost_usd: 0.0,
latency_ms: 1,
gates: vec![],
verdict: Verdict::Pass,
truncated,
};
let mut errored = trace("t", "a", Arm::Incumbent, 0.01);
errored.final_.served_from = ServedFrom::Error;
let mut cut = trace("t", "b", Arm::Incumbent, 0.01);
cut.attempts = vec![attempt(true), attempt(true), attempt(false)];
let mut ch_cut = trace("t", "c", Arm::Challenger, 0.01);
ch_cut.attempts = vec![attempt(true)];
let mut stale = trace("t", "d", Arm::Incumbent, 0.01);
stale.final_.served_from = ServedFrom::Error;
if let Some(rec) = stale.experiment.as_mut() {
rec.fingerprint = "an-older-config".to_owned();
}
let snaps = experiments_from_traces(&[errored, cut, ch_cut, stale], &[], &[], &config());
let h = snaps[0].health;
assert_eq!(
h.incumbent,
ArmHealth {
errors: 1,
truncated: 2
},
"{h:?}"
);
assert_eq!(
h.challenger,
ArmHealth {
errors: 0,
truncated: 1
},
"{h:?}"
);
assert_eq!(snaps[0].ignored_stale, 1);
}
#[test]
fn first_outcome_per_session_counts_and_abstain_is_ignored() {
let m = ExperimentMonitor::from_config(&config());
assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Abstain));
assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Pass));
assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Fail));
assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s2", Verdict::Fail));
let r = &m.reports()[0].report;
assert_eq!(r.incumbent.outcomes, 2);
assert!((r.incumbent.outcome_sum - 1.0).abs() < f64::EPSILON);
}
#[test]
fn routes_without_a_challenger_and_unknown_tenants_are_inert() {
let m = ExperimentMonitor::from_config(&config());
assert!(!m.observe_outcome("t", 7, &fp(), Arm::Incumbent, "s", Verdict::Pass));
assert_eq!(m.status("nobody", 0), ExperimentStatus::Undecided);
assert!(m.reports().is_empty());
let off = ExperimentMonitor::default();
assert!(!off.is_active());
off.observe_cost("t", 0, &fp(), Arm::Incumbent, 1.0);
assert!(off.reports().is_empty());
}
#[test]
fn tenants_never_share_an_experiment() {
let m = ExperimentMonitor::from_config(&config());
for i in 0..400 {
m.observe_outcome(
"a",
0,
&fp(),
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
);
m.observe_outcome(
"a",
0,
&fp(),
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
);
}
assert_eq!(m.status("a", 0), ExperimentStatus::Halted);
assert_eq!(m.status("b", 0), ExperimentStatus::Undecided);
}
#[test]
fn a_clearly_worse_challenger_halts() {
let m = ExperimentMonitor::from_config(&config());
let mut at = None;
for i in 0..500 {
m.observe_outcome(
"t",
0,
&fp(),
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
);
m.observe_outcome(
"t",
0,
&fp(),
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
);
if at.is_none() && m.status("t", 0) == ExperimentStatus::Halted {
at = Some(i);
}
}
assert!(at.is_some(), "never halted");
assert_eq!(m.status("t", 0), ExperimentStatus::Halted);
}
#[test]
fn replay_matches_live_with_session_dedup_and_halt() {
let cfg = config();
let mut traces = Vec::new();
let mut deferred = Vec::new();
for i in 0..400 {
let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
deferred.push((inc.trace_id, Verdict::Pass));
deferred.push((chl.trace_id, Verdict::Fail));
let again = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
deferred.push((again.trace_id, Verdict::Pass));
traces.extend([inc, chl, again]);
}
let reports = experiments_from_traces(&traces, &deferred, &[], &cfg);
assert_eq!(reports.len(), 1);
let snap = &reports[0];
let r = &snap.report;
assert_eq!((snap.tenant.as_str(), snap.route_ix), ("t", 0));
assert_eq!(snap.ignored_stale, 0);
assert_eq!(r.status, select::Status::Halted);
assert_eq!(r.challenger.outcomes, 400);
assert!(r.challenger.outcome_sum.abs() < f64::EPSILON);
assert_eq!(r.challenger.requests, 800);
assert!((r.challenger.cost_per_request().unwrap_or(0.0) - 0.02).abs() < 1e-12);
assert!((r.incumbent.cost_per_request().unwrap_or(0.0) - 0.10).abs() < 1e-12);
assert_eq!(recommend(r), Recommendation::ChallengerStopped);
let live = ExperimentMonitor::from_config(&cfg);
for t in &traces {
let rec = t.experiment.as_ref().expect("recorded");
live.observe_cost(
&t.tenant_id,
0,
&rec.fingerprint,
rec.arm,
t.final_.total_cost_usd,
);
}
for (id, v) in &deferred {
let t = traces.iter().find(|t| t.trace_id == *id).expect("trace");
let rec = t.experiment.as_ref().expect("recorded");
live.observe_outcome("t", 0, &rec.fingerprint, rec.arm, &t.session_id, *v);
}
assert_eq!(live.reports()[0], *snap);
assert_eq!(
experiments_from_traces(&traces, &deferred, &[], &cfg),
reports
);
}
#[test]
fn replay_ignores_unrecorded_traces_and_unknown_deferred() {
let cfg = config();
let mut plain = trace("t", "s", Arm::Incumbent, 0.1);
plain.experiment = None;
let deferred = [
(plain.trace_id, Verdict::Pass),
(Uuid::now_v7(), Verdict::Pass),
];
assert!(experiments_from_traces(&[plain], &deferred, &[], &cfg).is_empty());
let bare = Config::parse(
"[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
)
.expect("parses");
let t = trace("t", "s", Arm::Incumbent, 0.1);
assert!(experiments_from_traces(&[t], &[], &[], &bare).is_empty());
}
fn view_with(status: select::Status, inc_cost: f64, ch_cost: f64) -> ExperimentView {
let mut exp = Experiment::new(0.05, 0.05).expect("valid");
exp.observe_cost(Arm::Incumbent, inc_cost);
exp.observe_cost(Arm::Challenger, ch_cost);
let mut report = exp.report();
report.status = status;
ExperimentView {
tenant: "t".to_owned(),
route_ix: 0,
recommendation: recommend(&report),
ignored_stale: 0,
health: Health::default(),
report,
challenger_ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
}
}
#[test]
fn recommendation_and_rendering_cover_every_state() {
let adopt = view_with(select::Status::Certified, 0.10, 0.02);
assert_eq!(adopt.recommendation, Recommendation::Adopt);
let text = render(&[adopt]);
assert!(
text.contains("ADOPT: set this route's ladder to [\"anthropic/claude-haiku-4-5\"]"),
"{text}"
);
assert!(text.contains("ladder = [\"anthropic/claude-haiku-4-5\"]"));
let dear = view_with(select::Status::Certified, 0.02, 0.10);
assert_eq!(dear.recommendation, Recommendation::CertifiedNoSaving);
assert!(render(&[dear]).contains("certified, but no saving — keep incumbent"));
let halted = view_with(select::Status::Halted, 0.10, 0.02);
assert!(render(&[halted]).contains("challenger stopped: worse by more than the margin"));
let open = view_with(select::Status::Undecided, 0.10, 0.02);
assert_eq!(open.recommendation, Recommendation::KeepRunning);
assert!(render(&[open]).contains("keep running"));
assert!(render(&[]).contains("no experiment data"));
}
#[test]
fn certified_without_cost_data_is_not_adopted() {
let exp = Experiment::new(0.05, 0.05).expect("valid");
let mut report = exp.report();
report.status = select::Status::Certified;
assert_eq!(recommend(&report), Recommendation::CertifiedNoSaving);
}
#[test]
fn changing_the_challenger_ladder_resets_the_experiment() {
let old = config();
let mut traces = Vec::new();
let mut deferred = Vec::new();
for i in 0..300 {
let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
deferred.push((inc.trace_id, Verdict::Pass));
deferred.push((chl.trace_id, Verdict::Fail));
traces.extend([inc, chl]);
}
let before = experiments_from_traces(&traces, &deferred, &[], &old);
assert_eq!(before[0].report.status, select::Status::Halted);
assert_eq!(before[0].ignored_stale, 0);
let new =
Config::parse(&CFG.replace("claude-haiku-4-5", "claude-opus-4-8")).expect("parses");
let after = experiments_from_traces(&traces, &deferred, &[], &new);
assert_eq!(after[0].report.status, select::Status::Undecided);
assert_eq!(after[0].report.pairs, 0);
assert_eq!(after[0].report.challenger.requests, 0);
assert_eq!(after[0].ignored_stale, 600);
let text = render(&views(after, &new));
assert!(
text.contains("600 receipts from a previous experiment configuration ignored"),
"{text}"
);
let margin = Config::parse(&format!("{CFG}margin = 0.04\n")).expect("parses");
assert_eq!(
experiments_from_traces(&traces, &deferred, &[], &margin)[0].ignored_stale,
600
);
let ramped = Config::parse(&CFG.replace("50.0", "30.0")).expect("parses");
assert_eq!(
experiments_from_traces(&traces, &deferred, &[], &ramped)[0].report,
before[0].report
);
}
#[test]
fn challenger_traces_are_not_incumbent_traces() {
assert!(is_incumbent_trace(&trace("t", "s", Arm::Incumbent, 0.1)));
assert!(!is_incumbent_trace(&trace("t", "s", Arm::Challenger, 0.1)));
let mut plain = trace("t", "s", Arm::Challenger, 0.1);
plain.experiment = None;
assert!(is_incumbent_trace(&plain));
}
#[tokio::test]
async fn unreadable_store_fails_closed_on_the_challenger() {
let cfg = config();
let ok = ExperimentMonitor::from_config(&cfg);
assert!(!ok.challenger_blocked("t", 0));
let blocked = ExperimentMonitor::unavailable(&cfg);
assert!(blocked.challenger_blocked("t", 0));
assert!(blocked.challenger_blocked("any-other-tenant", 0));
let dir = std::env::temp_dir();
let m = build_monitor(&dir.to_string_lossy(), &cfg);
assert!(
m.challenger_blocked("t", 0),
"unreadable store served a challenger"
);
let db = dir.join(format!("firstpass-exp-build-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).expect("open");
drop(tx);
handle.await.expect("writer");
let m = build_monitor(&db.to_string_lossy(), &cfg);
assert!(!m.challenger_blocked("t", 0));
let _ = std::fs::remove_file(&db);
let bare = Config::parse(
"[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
)
.expect("parses");
assert!(!build_monitor(&dir.to_string_lossy(), &bare).challenger_blocked("t", 0));
}
#[tokio::test]
async fn paired_challenger_cost_reaches_the_report_on_every_path() {
let cfg = config();
let fp_str = fp();
let challenger = |m: &ExperimentMonitor| {
let r = m.reports();
assert_eq!(r.len(), 1);
let c = r[0].report.challenger;
(c.requests, c.cost_usd, r[0].report.pairs)
};
let close = |(n, usd, pairs): (u64, f64, u64)| {
assert_eq!((n, pairs), (3, 3));
assert!((usd - 0.03).abs() < 1e-12, "{usd}");
};
let direct = ExperimentMonitor::from_config(&cfg);
for _ in 0..3 {
let ok = direct
.record_pair(
"t",
0,
&fp_str,
1.0,
1.0,
0.10,
0.01,
false,
0,
0,
Uuid::nil(),
Uuid::nil(),
)
.await;
assert!(
!ok,
"no log → record_pair returns false (evidence via apply_pair_locally)"
);
direct.apply_pair_locally("t", 0, &fp_str, 1.0, 1.0, 0.01, Uuid::nil());
}
close(challenger(&direct));
let shared = Arc::new(InMemoryExperimentLog::new());
let a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
for _ in 0..3 {
a.record_pair(
"t",
0,
&fp_str,
1.0,
1.0,
0.10,
0.01,
false,
0,
0,
Uuid::nil(),
Uuid::nil(),
)
.await;
}
let _ = b.sync_from_backend("t", 0, &fp_str).await;
close(challenger(&a));
close(challenger(&b));
let db =
std::env::temp_dir().join(format!("firstpass-paircost-{}.db", uuid::Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
for _ in 0..3 {
crate::store::append_experiment_pair(
&db,
&crate::store::ExperimentPairRow {
tenant: "t".to_owned(),
route_ix: 0,
fingerprint: fp_str.clone(),
trace_id: uuid::Uuid::now_v7(),
x_inc: 1.0,
x_ch: 1.0,
challenger_cost_usd: 0.01,
ch_error: false,
ch_truncated: 0,
ts: jiff::Timestamp::now(),
in_log: crate::store::InLogStatus::InLog,
},
)
.unwrap();
}
close(challenger(&build_monitor(&db.to_string_lossy(), &cfg)));
let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let _ = restarted.sync_from_backend("t", 0, &fp_str).await;
close(challenger(&restarted));
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn two_replicas_sharing_one_log_converge_after_sync() {
let shared = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
for i in 0..400 {
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
0.10,
)
.await;
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
0.02,
)
.await;
}
assert_eq!(
replica_b.status("t", 0),
ExperimentStatus::Undecided,
"replica_b should not have any evidence before syncing"
);
let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
assert_eq!(
replica_b.status("t", 0),
ExperimentStatus::Halted,
"replica_b must halt after syncing from shared log"
);
assert!(
replica_b.challenger_blocked("t", 0),
"halted experiment must block the challenger on the synced replica"
);
let reps_a = replica_a.reports();
let reps_b = replica_b.reports();
assert_eq!(reps_a.len(), 1);
assert_eq!(reps_b.len(), 1);
assert_eq!(reps_a[0].report.status, reps_b[0].report.status);
assert_eq!(reps_a[0].report.pairs, reps_b[0].report.pairs);
}
#[tokio::test]
async fn halt_on_one_replica_blocks_challenger_on_other_after_sync() {
let shared = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
assert!(
!replica_b.challenger_blocked("t", 0),
"should be unblocked before any evidence"
);
let mut halted = false;
for i in 0..500 {
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
0.10,
)
.await;
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
0.02,
)
.await;
if replica_a.status("t", 0) == ExperimentStatus::Halted {
halted = true;
break;
}
}
assert!(halted, "experiment never halted on replica A");
assert!(
!replica_b.challenger_blocked("t", 0),
"replica_b should not block before syncing"
);
let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
assert!(
replica_b.challenger_blocked("t", 0),
"replica_b must block after syncing the halt from the shared log"
);
}
#[tokio::test]
async fn cross_replica_session_dedup_prevents_double_counting() {
let shared = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
"session-1",
Verdict::Pass,
0.10,
)
.await;
replica_b
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
"session-1",
Verdict::Pass,
0.10,
)
.await;
let _ = replica_a.sync_from_backend("t", 0, &fp_str).await;
let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
let reps_a = replica_a.reports();
let reps_b = replica_b.reports();
assert_eq!(
reps_a[0].report.incumbent.outcomes, 1,
"session should be counted exactly once"
);
assert_eq!(
reps_b[0].report.incumbent.outcomes, 1,
"both replicas should agree on outcome count"
);
}
#[tokio::test]
async fn abstain_does_not_burn_session_slot_before_claim() {
let shared = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let monitor = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let counted = monitor
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
"session-abstain",
Verdict::Abstain,
0.10,
)
.await;
assert!(!counted, "Abstain must not be counted");
let counted = monitor
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
"session-abstain",
Verdict::Pass,
0.10,
)
.await;
assert!(counted, "Pass after Abstain must be counted");
let reps = monitor.reports();
assert_eq!(
reps[0].report.incumbent.outcomes, 1,
"exactly one outcome must have been counted"
);
}
#[cfg(feature = "redis-cache")]
async fn redis_cleanup(url: &str, unique_tenant: &str) {
let client = redis::Client::open(url).expect("redis client");
let mut conn = redis::aio::ConnectionManager::new(client)
.await
.expect("redis connection for cleanup");
use redis::AsyncCommands as _;
let pattern = format!("firstpass:exp*{unique_tenant}*");
let keys: Vec<String> = {
let mut out = Vec::new();
let mut cursor: u64 = 0;
loop {
let (next, batch): (u64, Vec<String>) = redis::cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(&pattern)
.arg("COUNT")
.arg(100u64)
.query_async(&mut conn)
.await
.expect("SCAN");
out.extend(batch);
cursor = next;
if cursor == 0 {
break;
}
}
out
};
if !keys.is_empty() {
let _: () = conn.del(keys).await.expect("DEL cleanup");
}
}
#[cfg(feature = "redis-cache")]
#[tokio::test]
async fn redis_two_replicas_converge_to_identical_reports_after_sync() {
let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
panic!(
"set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
is worse than a red one."
)
});
let unique_tenant = format!("t-conv-{}", Uuid::now_v7());
let log_a = Arc::new(
RedisExperimentLog::connect(&url)
.await
.expect("RedisExperimentLog::connect"),
);
let log_b = Arc::new(
RedisExperimentLog::connect(&url)
.await
.expect("RedisExperimentLog::connect"),
);
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log_a) as Arc<dyn ExperimentLogBackend>);
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log_b) as Arc<dyn ExperimentLogBackend>);
for i in 0..400 {
replica_a
.record_outcome(
&unique_tenant,
0,
&fp_str,
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
0.10,
)
.await;
replica_a
.record_outcome(
&unique_tenant,
0,
&fp_str,
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
0.02,
)
.await;
}
assert_eq!(
replica_b.status(&unique_tenant, 0),
ExperimentStatus::Undecided,
"replica_b must be Undecided before syncing"
);
let _ = replica_b
.sync_from_backend(&unique_tenant, 0, &fp_str)
.await;
assert_eq!(
replica_b.status(&unique_tenant, 0),
ExperimentStatus::Halted,
"replica_b must be Halted after syncing from Redis log"
);
let reps_a = replica_a.reports();
let reps_b = replica_b.reports();
assert_eq!(reps_a.len(), 1);
assert_eq!(reps_b.len(), 1);
assert_eq!(
reps_a[0].report.status, reps_b[0].report.status,
"both replicas must agree on status"
);
assert_eq!(
reps_a[0].report.pairs, reps_b[0].report.pairs,
"both replicas must agree on pair counts"
);
redis_cleanup(&url, &unique_tenant).await;
}
#[cfg(feature = "redis-cache")]
#[tokio::test]
async fn redis_cross_replica_session_dedup_via_claim_session_holds() {
let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
panic!(
"set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
is worse than a red one."
)
});
let unique_tenant = format!("t-dedup-{}", Uuid::now_v7());
let log = Arc::new(
RedisExperimentLog::connect(&url)
.await
.expect("RedisExperimentLog::connect"),
);
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
replica_a
.record_outcome(
&unique_tenant,
0,
&fp_str,
Arm::Incumbent,
"session-x",
Verdict::Pass,
0.10,
)
.await;
replica_b
.record_outcome(
&unique_tenant,
0,
&fp_str,
Arm::Incumbent,
"session-x",
Verdict::Pass,
0.10,
)
.await;
let _ = replica_a
.sync_from_backend(&unique_tenant, 0, &fp_str)
.await;
let _ = replica_b
.sync_from_backend(&unique_tenant, 0, &fp_str)
.await;
let reps_a = replica_a.reports();
let reps_b = replica_b.reports();
assert_eq!(
reps_a[0].report.incumbent.outcomes, 1,
"session must be counted exactly once on replica A"
);
assert_eq!(
reps_b[0].report.incumbent.outcomes, 1,
"session must be counted exactly once on replica B"
);
redis_cleanup(&url, &unique_tenant).await;
}
#[cfg(feature = "redis-cache")]
#[tokio::test]
async fn redis_halt_on_one_replica_blocks_challenger_on_other() {
let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
panic!(
"set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
is worse than a red one."
)
});
let unique_tenant = format!("t-halt-{}", Uuid::now_v7());
let log = Arc::new(
RedisExperimentLog::connect(&url)
.await
.expect("RedisExperimentLog::connect"),
);
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
assert!(
!replica_b.challenger_blocked(&unique_tenant, 0),
"replica_b must be unblocked before any evidence"
);
let mut halted = false;
for i in 0..500 {
replica_a
.record_outcome(
&unique_tenant,
0,
&fp_str,
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
0.10,
)
.await;
replica_a
.record_outcome(
&unique_tenant,
0,
&fp_str,
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
0.02,
)
.await;
if replica_a.status(&unique_tenant, 0) == ExperimentStatus::Halted {
halted = true;
break;
}
}
assert!(halted, "experiment never halted on replica A");
assert!(
!replica_b.challenger_blocked(&unique_tenant, 0),
"replica_b must not block before syncing"
);
let _ = replica_b
.sync_from_backend(&unique_tenant, 0, &fp_str)
.await;
assert!(
replica_b.challenger_blocked(&unique_tenant, 0),
"replica_b must block the challenger after syncing the halt from Redis"
);
redis_cleanup(&url, &unique_tenant).await;
}
#[cfg(feature = "redis-cache")]
#[tokio::test]
async fn redis_keys_are_isolated_per_tenant() {
let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
panic!(
"set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
is worse than a red one."
)
});
let run_id = Uuid::now_v7();
let tenant_a = format!("t-iso-a-{run_id}");
let tenant_b = format!("t-iso-b-{run_id}");
let log = Arc::new(
RedisExperimentLog::connect(&url)
.await
.expect("RedisExperimentLog::connect"),
);
let cfg = config();
let fp_str = fp();
let monitor_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
let monitor_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
for i in 0..5 {
monitor_a
.record_outcome(
&tenant_a,
0,
&fp_str,
Arm::Incumbent,
&format!("s{i}"),
Verdict::Pass,
0.10,
)
.await;
}
let records = log
.load_since(&tenant_b, 0, &fp_str, 0)
.await
.expect("load_since must not fail on a reachable Redis");
assert!(
records.is_empty(),
"backend must return no records for tenant_b"
);
assert_eq!(
monitor_b.status(&tenant_b, 0),
ExperimentStatus::Undecided,
"evidence written under tenant_a must not appear under tenant_b"
);
let reps = monitor_a.reports();
assert_eq!(reps.len(), 1);
assert_eq!(reps[0].report.incumbent.outcomes, 5);
redis_cleanup(&url, &tenant_a).await;
redis_cleanup(&url, &tenant_b).await;
}
#[test]
fn old_log_record_pair_deserializes_without_new_fields() {
let json = r#"{"kind":"pair","x_inc":1.0,"x_ch":0.5,"incumbent_cost_usd":0.1,"challenger_cost_usd":0.01}"#;
let record: LogRecord = serde_json::from_str(json).unwrap();
match record {
LogRecord::Pair {
ch_error,
ch_truncated,
ts_unix,
x_inc,
x_ch,
replica_id,
..
} => {
assert!(!ch_error, "ch_error defaults to false");
assert_eq!(ch_truncated, 0, "ch_truncated defaults to 0");
assert_eq!(ts_unix, 0, "ts_unix defaults to 0");
assert!((x_inc - 1.0).abs() < f64::EPSILON);
assert!((x_ch - 0.5).abs() < f64::EPSILON);
assert_eq!(
replica_id,
uuid::Uuid::nil(),
"missing replica_id defaults to nil"
);
}
_ => panic!("expected Pair variant"),
}
}
#[tokio::test]
async fn pair_health_persists_and_rebuilds_from_store() {
let db =
std::env::temp_dir().join(format!("firstpass-health-persist-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let cfg = config();
let fp_str = fp();
crate::store::append_experiment_pair(
&db,
&crate::store::ExperimentPairRow {
tenant: "t".to_owned(),
route_ix: 0,
fingerprint: fp_str.clone(),
trace_id: Uuid::now_v7(),
x_inc: 1.0,
x_ch: 1.0,
challenger_cost_usd: 0.01,
ch_error: true,
ch_truncated: 2,
ts: jiff::Timestamp::now(),
in_log: crate::store::InLogStatus::InLog,
},
)
.unwrap();
let m = build_monitor(&db.to_string_lossy(), &cfg);
let reps = m.reports();
assert_eq!(reps.len(), 1, "one experiment entry");
assert_eq!(reps[0].health.challenger.errors, 1, "ch_error rebuilt");
assert_eq!(
reps[0].health.challenger.truncated, 2,
"ch_truncated rebuilt"
);
let _ = std::fs::remove_file(&db);
}
#[derive(Debug)]
struct DownLog;
#[async_trait::async_trait]
impl ExperimentLogBackend for DownLog {
async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
Err(LogError::Backend("simulated outage".to_owned()))
}
async fn load_since(
&self,
_: &str,
_: usize,
_: &str,
_: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
Err(LogError::Backend("simulated outage".to_owned()))
}
async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
false
}
}
#[derive(Debug)]
struct AppendOnlyLog(InMemoryExperimentLog);
#[async_trait::async_trait]
impl ExperimentLogBackend for AppendOnlyLog {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: LogRecord,
) -> Result<u64, LogError> {
self.0.append(tenant, route_ix, fingerprint, record).await
}
async fn load_since(
&self,
_: &str,
_: usize,
_: &str,
_: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
Err(LogError::Backend("simulated read failure".to_owned()))
}
async fn claim_session(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
session_id: &str,
) -> bool {
self.0
.claim_session(tenant, route_ix, fingerprint, session_id)
.await
}
}
#[tokio::test]
async fn paired_spend_counts_against_the_cap_when_the_shared_log_is_down() {
let ledger = Arc::new(crate::shadow::ShadowLedger::new());
let m = ExperimentMonitor::from_config(&config())
.with_log(Arc::new(DownLog) as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger));
let now = jiff::Timestamp::now();
ledger.debit("t", 0, 0.7, now);
m.observe_health("t", 0, &fp(), Arm::Challenger, true, 2);
let accepted = m
.record_pair(
"t",
0,
&fp(),
1.0,
0.0,
0.1,
0.7,
true,
2,
now.as_second(),
Uuid::nil(),
Uuid::nil(),
)
.await;
assert!(!accepted, "nothing reached the log");
assert!(
(ledger.spent_today("t", 0, now) - 0.7).abs() < 1e-9,
"spend debited locally"
);
assert!(
!ledger.may_spend("t", 0, 0.5, now),
"cap enforced during the outage"
);
let h = m.reports()[0].health.challenger;
assert_eq!((h.errors, h.truncated), (1, 2), "{h:?}");
}
#[tokio::test]
async fn paired_spend_enforced_when_append_succeeds_but_read_fails() {
let ledger = Arc::new(crate::shadow::ShadowLedger::new());
let m = ExperimentMonitor::from_config(&config())
.with_log(Arc::new(AppendOnlyLog(InMemoryExperimentLog::new()))
as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger));
let now = jiff::Timestamp::now();
ledger.debit("t", 0, 0.6, now);
m.observe_health("t", 0, &fp(), Arm::Challenger, false, 1);
let accepted = m
.record_pair(
"t",
0,
&fp(),
1.0,
1.0,
0.1,
0.6,
false,
1,
now.as_second(),
Uuid::nil(),
Uuid::nil(),
)
.await;
assert!(accepted, "pair was appended to log");
assert!(
(ledger.spent_today("t", 0, now) - 0.6).abs() < 1e-9,
"local spend = 0.6"
);
assert!(
!ledger.may_spend("t", 0, 0.5, now),
"cap enforced by local debit even when read-back fails"
);
let h = m.reports()[0].health.challenger;
assert_eq!((h.errors, h.truncated), (0, 1), "{h:?}");
}
#[tokio::test]
async fn two_replicas_health_and_spend_count_once() {
let shared = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let now_ts = jiff::Timestamp::now();
let now_secs = now_ts.as_second();
let ledger_a = Arc::new(crate::shadow::ShadowLedger::new());
let ledger_b = Arc::new(crate::shadow::ShadowLedger::new());
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger_a));
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger_b));
assert!(
replica_a.has_log(),
"log must be attached for this test to be meaningful"
);
ledger_a.debit("t", 0, 0.8, now_ts);
replica_a.observe_health("t", 0, &fp_str, Arm::Challenger, true, 1);
let accepted = replica_a
.record_pair(
"t",
0,
&fp_str,
1.0,
0.5,
0.1,
0.8,
true,
1,
now_secs,
Uuid::nil(),
Uuid::nil(),
)
.await;
assert!(accepted, "pair must be accepted");
let reps_a = replica_a.reports();
assert_eq!(
reps_a[0].health.challenger.errors, 1,
"A: challenger error counted once, not zero or doubled"
);
assert_eq!(
reps_a[0].health.challenger.truncated, 1,
"A: challenger truncated counted once"
);
let a_spent = ledger_a.spent_today("t", 0, now_ts);
assert!(
(a_spent - 0.8).abs() < 1e-12,
"A: ledger = 0.8 (not doubled or zero); got {a_spent}"
);
assert_eq!(
ledger_b.spent_today("t", 0, now_ts),
0.0,
"B: no spend before sync"
);
let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
let reps_b = replica_b.reports();
assert_eq!(
reps_b[0].health.challenger.errors, 1,
"B: challenger error visible after sync"
);
let b_spent = ledger_b.spent_today("t", 0, now_ts);
assert!(
(b_spent - 0.8).abs() < 1e-12,
"B: ledger reflects A's spend after sync; got {b_spent}"
);
ledger_a.debit("t", 0, 0.3, now_ts);
replica_a.observe_health("t", 0, &fp_str, Arm::Challenger, false, 0);
replica_a
.record_pair(
"t",
0,
&fp_str,
1.0,
0.5,
0.1,
0.3,
false,
0,
now_secs,
Uuid::nil(),
Uuid::nil(),
)
.await;
let a_total = ledger_a.spent_today("t", 0, now_ts);
assert!(
(a_total - 1.1).abs() < 1e-12,
"A: cluster total = 1.1; got {a_total}"
);
assert!(
!ledger_a.may_spend("t", 0, 1.0, now_ts),
"A: cap exhausted (1.1 > 1.0)"
);
let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
assert!(
!ledger_b.may_spend("t", 0, 1.0, now_ts),
"B: cap exhausted from cluster spend"
);
}
#[derive(Debug)]
struct YieldingLog(InMemoryExperimentLog);
#[async_trait::async_trait]
impl ExperimentLogBackend for YieldingLog {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: LogRecord,
) -> Result<u64, LogError> {
self.0.append(tenant, route_ix, fingerprint, record).await
}
async fn load_since(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
offset: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
let records = self
.0
.load_since(tenant, route_ix, fingerprint, offset)
.await?;
tokio::task::yield_now().await;
Ok(records)
}
async fn claim_session(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
session_id: &str,
) -> bool {
self.0
.claim_session(tenant, route_ix, fingerprint, session_id)
.await
}
}
#[tokio::test]
async fn concurrent_sync_does_not_double_count_pairs() {
let shared = Arc::new(YieldingLog(InMemoryExperimentLog::new()));
let cfg = config();
let fp_str = fp();
for _ in 0..3 {
let _ = shared
.append(
"t",
0,
&fp_str,
LogRecord::Pair {
x_inc: 1.0,
x_ch: 1.0,
incumbent_cost_usd: 0.10,
challenger_cost_usd: 0.05,
ch_error: true,
ch_truncated: 1,
ts_unix: 0,
replica_id: uuid::Uuid::nil(),
trace_id: uuid::Uuid::nil(),
incumbent_trace_id: Uuid::nil(),
},
)
.await;
}
let monitor = Arc::new(
ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>),
);
let m1 = Arc::clone(&monitor);
let m2 = Arc::clone(&monitor);
let fp1 = fp_str.clone();
let fp2 = fp_str.clone();
tokio::join!(
async move {
let _ = m1.sync_from_backend("t", 0, &fp1).await;
},
async move {
let _ = m2.sync_from_backend("t", 0, &fp2).await;
},
);
let snaps = monitor.reports();
assert_eq!(snaps.len(), 1);
let snap = &snaps[0];
assert_eq!(
snap.report.pairs, 3,
"pairs must be 3, not double-counted: got {}",
snap.report.pairs
);
assert_eq!(
snap.health.challenger.errors, 3,
"errors must be 3, not double-counted: got {}",
snap.health.challenger.errors
);
assert_eq!(
snap.health.challenger.truncated, 3,
"truncated must be 3, not double-counted: got {}",
snap.health.challenger.truncated
);
}
#[tokio::test]
async fn ensure_synced_cold_start_respects_cluster_cap() {
let shared = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let now_ts = jiff::Timestamp::now();
let cap = 0.5_f64;
let ledger_a = Arc::new(crate::shadow::ShadowLedger::new());
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger_a));
ledger_a.debit("t", 0, cap, now_ts);
replica_a
.record_pair(
"t",
0,
&fp_str,
1.0,
1.0,
0.10,
cap,
false,
0,
now_ts.as_second(),
Uuid::nil(),
Uuid::nil(),
)
.await;
assert!(
!ledger_a.may_spend("t", 0, cap, now_ts),
"A: cap exhausted after recording one pair"
);
let ledger_b = Arc::new(crate::shadow::ShadowLedger::new());
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger_b));
assert!(
ledger_b.may_spend("t", 0, cap, now_ts),
"B: before ensure_synced, empty ledger allows spend (expected)"
);
let synced = replica_b.ensure_synced("t", 0, &fp_str).await;
assert!(
synced,
"ensure_synced must return true when the log is healthy"
);
let may_spend_after_sync = ledger_b.may_spend("t", 0, cap, now_ts);
assert!(
!may_spend_after_sync,
"B: after ensure_synced, cluster spend at cap — must refuse"
);
}
#[tokio::test]
async fn ensure_synced_burst_waits_for_synced_state() {
let shared = Arc::new(YieldingLog(InMemoryExperimentLog::new()));
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
for i in 0..500 {
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
0.10,
)
.await;
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
0.02,
)
.await;
if replica_a.status("t", 0) == ExperimentStatus::Halted {
break;
}
}
assert_eq!(replica_a.status("t", 0), ExperimentStatus::Halted);
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let (first, second) = tokio::join!(
async {
replica_b.ensure_synced("t", 0, &fp_str).await;
replica_b.challenger_blocked("t", 0)
},
async {
replica_b.ensure_synced("t", 0, &fp_str).await;
replica_b.challenger_blocked("t", 0)
},
);
assert!(
first && second,
"both burst requests see the halt: {first} {second}"
);
}
#[tokio::test]
async fn ensure_synced_cold_start_respects_cluster_halt() {
let shared = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let replica_a = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let mut halted = false;
for i in 0..500 {
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
0.10,
)
.await;
replica_a
.record_outcome(
"t",
0,
&fp_str,
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
0.02,
)
.await;
if replica_a.status("t", 0) == ExperimentStatus::Halted {
halted = true;
break;
}
}
assert!(halted, "experiment never halted on replica A");
let replica_b = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
assert!(
!replica_b.challenger_blocked("t", 0),
"B: before ensure_synced, halt unknown (expected)"
);
let synced = replica_b.ensure_synced("t", 0, &fp_str).await;
assert!(synced, "ensure_synced must succeed with healthy log");
assert!(
replica_b.challenger_blocked("t", 0),
"B: after ensure_synced, halt must block the challenger"
);
}
#[tokio::test]
async fn restart_with_shared_log_replays_offline_pairs_and_skips_log_pairs() {
let db =
std::env::temp_dir().join(format!("firstpass-restart-{}.db", uuid::Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let cfg = config();
let fp_str = fp();
let now_ts = jiff::Timestamp::now();
crate::store::append_experiment_pair(
&db,
&crate::store::ExperimentPairRow {
tenant: "t".to_owned(),
route_ix: 0,
fingerprint: fp_str.clone(),
trace_id: uuid::Uuid::now_v7(),
x_inc: 1.0,
x_ch: 0.0,
challenger_cost_usd: 0.5,
ch_error: false,
ch_truncated: 0,
ts: now_ts,
in_log: crate::store::InLogStatus::NotInLog,
},
)
.unwrap();
crate::store::append_experiment_pair(
&db,
&crate::store::ExperimentPairRow {
tenant: "t".to_owned(),
route_ix: 0,
fingerprint: fp_str.clone(),
trace_id: uuid::Uuid::now_v7(),
x_inc: 1.0,
x_ch: 0.0,
challenger_cost_usd: 0.3,
ch_error: false,
ch_truncated: 0,
ts: now_ts,
in_log: crate::store::InLogStatus::InLog,
},
)
.unwrap();
let m = build_monitor_with(&db.to_string_lossy(), &cfg, true);
let reps = m.reports();
assert_eq!(
reps[0].report.pairs, 1,
"only in_log=false row replayed; got {}",
reps[0].report.pairs
);
let m2 = build_monitor_with(&db.to_string_lossy(), &cfg, false);
let reps2 = m2.reports();
assert_eq!(
reps2[0].report.pairs, 2,
"shared_log=false replays all rows; got {}",
reps2[0].report.pairs
);
let _ = std::fs::remove_file(&db);
}
#[derive(Debug)]
struct CountingFailLog {
load_count: Arc<std::sync::atomic::AtomicUsize>,
}
#[async_trait::async_trait]
impl ExperimentLogBackend for CountingFailLog {
async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
Err(LogError::Backend("down".to_owned()))
}
async fn load_since(
&self,
_: &str,
_: usize,
_: &str,
_: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
self.load_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Err(LogError::Backend("down".to_owned()))
}
async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
false
}
}
#[tokio::test]
async fn ensure_synced_failure_burst_issues_exactly_one_read() {
let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let m = Arc::new(ExperimentMonitor::from_config(&config()).with_log(Arc::new(
CountingFailLog {
load_count: Arc::clone(&load_count),
},
)
as Arc<dyn ExperimentLogBackend>));
let fp_str = fp();
let handles: Vec<_> = (0..10)
.map(|_| {
let m = Arc::clone(&m);
let fp_str = fp_str.clone();
tokio::spawn(async move { m.ensure_synced("t", 0, &fp_str).await })
})
.collect();
for h in handles {
let result = h.await.unwrap();
assert!(!result, "every caller gets false on backend failure");
}
assert_eq!(
load_count.load(std::sync::atomic::Ordering::SeqCst),
1,
"exactly one backend read issued during a failure burst"
);
}
#[tokio::test]
async fn ensure_synced_cooldown_skips_retry_within_window() {
let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let m = ExperimentMonitor::from_config(&config())
.with_log(Arc::new(CountingFailLog {
load_count: Arc::clone(&load_count),
}) as Arc<dyn ExperimentLogBackend>);
let fp_str = fp();
assert!(!m.ensure_synced("t", 0, &fp_str).await);
assert_eq!(load_count.load(std::sync::atomic::Ordering::SeqCst), 1);
assert!(!m.ensure_synced("t", 0, &fp_str).await);
assert_eq!(
load_count.load(std::sync::atomic::Ordering::SeqCst),
1,
"no additional read within the cooldown window"
);
}
#[tokio::test]
async fn ensure_synced_cooldown_allows_retry_after_expiry() {
tokio::time::pause();
let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let m = ExperimentMonitor::from_config(&config())
.with_log(Arc::new(CountingFailLog {
load_count: Arc::clone(&load_count),
}) as Arc<dyn ExperimentLogBackend>);
let fp_str = fp();
assert!(!m.ensure_synced("t", 0, &fp_str).await);
assert_eq!(load_count.load(std::sync::atomic::Ordering::SeqCst), 1);
tokio::time::advance(SYNC_FAILURE_COOLDOWN + std::time::Duration::from_millis(1)).await;
assert!(!m.ensure_synced("t", 0, &fp_str).await);
assert_eq!(
load_count.load(std::sync::atomic::Ordering::SeqCst),
2,
"one more read after the cooldown expires"
);
}
#[tokio::test]
async fn record_pair_does_not_apply_evidence_on_append_failure_caller_does() {
let m = ExperimentMonitor::from_config(&config())
.with_log(Arc::new(DownLog) as Arc<dyn ExperimentLogBackend>);
let fp_str = fp();
let mut pairs_applied = 0u32;
for _ in 0..500 {
if m.challenger_blocked("t", 0) {
break;
}
let accepted = m
.record_pair(
"t",
0,
&fp_str,
1.0,
0.0,
0.1,
0.01,
false,
0,
0,
Uuid::nil(),
Uuid::nil(),
)
.await;
assert!(!accepted, "append failing log returns false");
m.apply_pair_locally("t", 0, &fp_str, 1.0, 0.0, 0.01, Uuid::nil());
pairs_applied += 1;
}
assert!(
m.challenger_blocked("t", 0),
"experiment must halt locally after {pairs_applied} pairs via apply_pair_locally"
);
assert!(
m.reports()[0].report.pairs > 0,
"pairs counted locally via apply_pair_locally"
);
}
#[tokio::test]
async fn record_pair_no_log_does_not_apply_evidence() {
let m = ExperimentMonitor::from_config(&config()); let fp_str = fp();
for _ in 0..10 {
let ok = m
.record_pair(
"t",
0,
&fp_str,
1.0,
0.0,
0.1,
0.01,
false,
0,
0,
Uuid::nil(),
Uuid::nil(),
)
.await;
assert!(!ok, "no log → record_pair returns false");
}
assert_eq!(
m.reports().len(),
0,
"record_pair in no-log mode must not create an entry or advance evidence"
);
m.apply_pair_locally("t", 0, &fp_str, 1.0, 0.0, 0.01, Uuid::nil());
assert_eq!(
m.reports()[0].report.pairs,
1,
"apply_pair_locally counted the pair"
);
}
#[tokio::test]
async fn unknown_rows_contribute_spend_but_not_evidence_with_shared_log() {
use crate::store::InLogStatus;
let db =
std::env::temp_dir().join(format!("firstpass-unknown-{}.db", uuid::Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let cfg = config();
let fp_str = fp();
let now_ts = jiff::Timestamp::now();
for (in_log, x_ch) in [
(InLogStatus::Unknown, 0.8_f64),
(InLogStatus::NotInLog, 0.6),
] {
crate::store::append_experiment_pair(
&db,
&crate::store::ExperimentPairRow {
tenant: "t".to_owned(),
route_ix: 0,
fingerprint: fp_str.clone(),
trace_id: uuid::Uuid::now_v7(),
x_inc: 1.0,
x_ch,
challenger_cost_usd: 0.1,
ch_error: false,
ch_truncated: 0,
ts: now_ts,
in_log,
},
)
.unwrap();
}
let m = build_monitor_with(&db.to_string_lossy(), &cfg, true);
let reps = m.reports();
assert_eq!(
reps[0].report.pairs, 1,
"shared_log=true: only NotInLog evidence replayed; got {}",
reps[0].report.pairs
);
let m2 = build_monitor_with(&db.to_string_lossy(), &cfg, false);
let reps2 = m2.reports();
assert_eq!(
reps2[0].report.pairs, 2,
"shared_log=false: all rows replayed; got {}",
reps2[0].report.pairs
);
let _ = std::fs::remove_file(&db);
}
#[derive(Debug)]
struct MalformedMiddleLog {
inner: InMemoryExperimentLog,
}
#[async_trait::async_trait]
impl ExperimentLogBackend for MalformedMiddleLog {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: LogRecord,
) -> Result<u64, LogError> {
self.inner
.append(tenant, route_ix, fingerprint, record)
.await
}
async fn load_since(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
offset: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
let mut records = self
.inner
.load_since(tenant, route_ix, fingerprint, offset)
.await?;
if records.len() >= 2 {
records[1] = None;
}
Ok(records)
}
async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
false
}
}
#[tokio::test]
async fn malformed_log_entry_advances_offset_and_valid_entries_applied_once() {
let log = Arc::new(MalformedMiddleLog {
inner: InMemoryExperimentLog::new(),
});
let cfg = config();
let fp_str = fp();
for _ in 0..3 {
let _ = log
.inner
.append(
"t",
0,
&fp_str,
LogRecord::Pair {
x_inc: 1.0,
x_ch: 1.0,
incumbent_cost_usd: 0.1,
challenger_cost_usd: 0.05,
ch_error: false,
ch_truncated: 0,
ts_unix: 0,
replica_id: uuid::Uuid::nil(),
trace_id: uuid::Uuid::nil(),
incumbent_trace_id: Uuid::nil(),
},
)
.await;
}
let m = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
let _ = m.sync_from_backend("t", 0, &fp_str).await;
let pairs_after_first = m.reports()[0].report.pairs;
assert_eq!(
pairs_after_first, 2,
"first sync: 2 valid entries applied, got {pairs_after_first}"
);
let _ = m.sync_from_backend("t", 0, &fp_str).await;
let pairs_after_second = m.reports()[0].report.pairs;
assert_eq!(
pairs_after_second, 2,
"second sync: no re-application, pairs still 2, got {pairs_after_second}"
);
let offset = {
let map = m.lock();
map.get("t")
.and_then(|r| r.get(&0))
.map_or(0, |e| e.remote_log_offset)
};
assert_eq!(
offset, 3,
"offset must equal list length after consuming all positions"
);
}
#[derive(Debug)]
struct PushThenErrorLog {
inner: InMemoryExperimentLog,
}
#[async_trait::async_trait]
impl ExperimentLogBackend for PushThenErrorLog {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: LogRecord,
) -> Result<u64, LogError> {
let _ = self
.inner
.append(tenant, route_ix, fingerprint, record)
.await;
Err(LogError::Backend("timeout after write".to_owned()))
}
async fn load_since(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
offset: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
self.inner
.load_since(tenant, route_ix, fingerprint, offset)
.await
}
async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
false
}
}
#[tokio::test]
async fn trace_id_dedup_prevents_double_count_on_append_timeout() {
let log = Arc::new(PushThenErrorLog {
inner: InMemoryExperimentLog::new(),
});
let cfg = config();
let fp_str = fp();
let tid = Uuid::now_v7();
let now_ts = jiff::Timestamp::now();
let ledger = Arc::new(crate::shadow::ShadowLedger::new());
let m = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger));
let appended = m
.record_pair(
"t",
0,
&fp_str,
1.0,
1.0,
0.1,
0.05,
false,
1,
now_ts.as_second(),
tid,
Uuid::nil(),
)
.await;
assert!(!appended, "append returned Err → record_pair returns false");
assert_eq!(
m.reports().len(),
0,
"no entry created yet by record_pair alone"
);
ledger.debit("t", 0, 0.05, now_ts);
m.observe_health("t", 0, &fp_str, Arm::Challenger, false, 1);
m.apply_pair_locally("t", 0, &fp_str, 1.0, 1.0, 0.05, tid);
assert_eq!(
m.reports()[0].report.pairs,
1,
"evidence applied via apply_pair_locally"
);
let _ = m.sync_from_backend("t", 0, &fp_str).await;
assert_eq!(
m.reports()[0].report.pairs,
1,
"after sync: pairs still 1 (trace_id dedup prevented double-count)"
);
let h = m.reports()[0].health.challenger;
assert_eq!(
(h.errors, h.truncated),
(0, 1),
"health counted exactly once (not re-applied by apply_log_record)"
);
assert!(
(ledger.spent_today("t", 0, now_ts) - 0.05).abs() < 1e-9,
"ledger spend counted exactly once (not re-seeded by apply_log_record)"
);
}
#[tokio::test]
async fn ensure_synced_stale_after_bound_exceeded() {
tokio::time::pause();
let shared = Arc::new(InMemoryExperimentLog::new());
let m = Arc::new(
ExperimentMonitor::from_config(&config())
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>),
);
let fp_str = fp();
assert!(
m.ensure_synced("t", 0, &fp_str).await,
"first sync must succeed"
);
tokio::time::advance(SYNC_STALENESS_BOUND / 2).await;
assert!(
m.ensure_synced("t", 0, &fp_str).await,
"still within staleness bound → true"
);
tokio::time::advance(SYNC_STALENESS_BOUND).await;
assert!(
m.ensure_synced("t", 0, &fp_str).await,
"stale but log reachable → sync succeeds → true"
);
let m_down = Arc::new(ExperimentMonitor::from_config(&config()).with_log(Arc::new(
CountingFailLog {
load_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
},
)
as Arc<dyn ExperimentLogBackend>));
assert!(
!m_down.ensure_synced("t", 0, &fp_str).await,
"log down on first sync → false"
);
assert!(
!m_down.ensure_synced("t", 0, &fp_str).await,
"cooldown active → false immediately"
);
}
#[tokio::test]
async fn restart_counts_a_timed_out_pair_once_across_store_and_log() {
let db =
std::env::temp_dir().join(format!("firstpass-restart-dedup-{}.db", Uuid::now_v7()));
crate::store::ensure_migrated(&db).unwrap();
let log = Arc::new(InMemoryExperimentLog::new());
let cfg = config();
let fp_str = fp();
let tid = Uuid::now_v7();
let now = jiff::Timestamp::now();
let before_restart = ExperimentMonitor::from_config(&cfg)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
log.append(
"t",
0,
&fp_str,
LogRecord::Pair {
x_inc: 1.0,
x_ch: 1.0,
incumbent_cost_usd: 0.1,
challenger_cost_usd: 0.05,
ch_error: false,
ch_truncated: 1,
ts_unix: now.as_second(),
replica_id: before_restart.replica_id,
trace_id: tid,
incumbent_trace_id: Uuid::nil(),
},
)
.await
.unwrap();
crate::store::append_experiment_pair(
&db,
&crate::store::ExperimentPairRow {
tenant: "t".to_owned(),
route_ix: 0,
fingerprint: fp_str.clone(),
trace_id: tid,
x_inc: 1.0,
x_ch: 1.0,
challenger_cost_usd: 0.05,
ch_error: false,
ch_truncated: 1,
ts: now,
in_log: crate::store::InLogStatus::NotInLog,
},
)
.unwrap();
let ledger = Arc::new(crate::shadow::ShadowLedger::new());
ledger.debit("t", 0, 0.05, now); let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>)
.with_paired_ledger(Arc::clone(&ledger));
assert_ne!(restarted.replica_id, before_restart.replica_id);
restarted.sync_from_backend("t", 0, &fp_str).await.unwrap();
let r = &restarted.reports()[0];
assert_eq!(r.report.pairs, 1, "evidence once");
assert_eq!(r.health.challenger.truncated, 1, "health once");
assert!(
(ledger.spent_today("t", 0, now) - 0.05).abs() < 1e-9,
"spend once: {}",
ledger.spent_today("t", 0, now)
);
let _ = std::fs::remove_file(&db);
}
#[derive(Debug, Default)]
struct SwitchableLog {
inner: InMemoryExperimentLog,
failing: std::sync::atomic::AtomicBool,
reads: std::sync::atomic::AtomicUsize,
}
#[async_trait::async_trait]
impl ExperimentLogBackend for SwitchableLog {
async fn append(
&self,
t: &str,
r: usize,
f: &str,
rec: LogRecord,
) -> Result<u64, LogError> {
self.inner.append(t, r, f, rec).await
}
async fn load_since(
&self,
t: &str,
r: usize,
f: &str,
o: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
self.reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if self.failing.load(std::sync::atomic::Ordering::SeqCst) {
return Err(LogError::Backend("down".to_owned()));
}
self.inner.load_since(t, r, f, o).await
}
async fn claim_session(&self, t: &str, r: usize, f: &str, s: &str) -> bool {
self.inner.claim_session(t, r, f, s).await
}
}
#[tokio::test]
async fn read_back_failure_applies_the_pair_locally_and_recovery_does_not_double_count() {
let log = Arc::new(SwitchableLog::default());
let m = ExperimentMonitor::from_config(&config())
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
let fp_str = fp();
log.failing.store(true, std::sync::atomic::Ordering::SeqCst);
let tid = Uuid::now_v7();
let appended = m
.record_pair(
"t",
0,
&fp_str,
1.0,
0.0,
0.1,
0.05,
false,
0,
0,
tid,
Uuid::nil(),
)
.await;
assert!(appended, "the append itself succeeded");
assert_eq!(
m.reports()[0].report.pairs,
1,
"applied locally despite the failed read"
);
log.failing
.store(false, std::sync::atomic::Ordering::SeqCst);
m.sync_from_backend("t", 0, &fp_str).await.unwrap();
assert_eq!(
m.reports()[0].report.pairs,
1,
"recovery sync deduplicated by trace_id"
);
}
#[derive(Debug)]
struct HungAppendLog;
#[async_trait::async_trait]
impl ExperimentLogBackend for HungAppendLog {
async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
std::future::pending().await
}
async fn load_since(
&self,
_: &str,
_: usize,
_: &str,
_: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
Ok(vec![])
}
async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
false
}
}
#[tokio::test]
async fn hung_append_is_bounded_and_reported_as_not_in_log() {
tokio::time::pause();
let m = ExperimentMonitor::from_config(&config()).with_log(Arc::new(HungAppendLog));
let appended = m
.record_pair(
"t",
0,
&fp(),
1.0,
0.0,
0.1,
0.05,
false,
0,
0,
Uuid::now_v7(),
Uuid::nil(),
)
.await;
assert!(!appended, "a timed-out append is not in the log");
}
#[tokio::test]
async fn remote_pairs_carry_the_incumbent_cost() {
let shared = Arc::new(InMemoryExperimentLog::new());
let a = ExperimentMonitor::from_config(&config())
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let b = ExperimentMonitor::from_config(&config())
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
assert!(
a.record_pair(
"t",
0,
&fp(),
1.0,
1.0,
0.10,
0.02,
false,
0,
0,
Uuid::now_v7(),
Uuid::nil(),
)
.await
);
b.sync_from_backend("t", 0, &fp()).await.unwrap();
let inc = b.reports()[0].report.incumbent;
assert_eq!(inc.requests, 1, "{inc:?}");
assert!((inc.cost_usd - 0.10).abs() < 1e-12, "{inc:?}");
}
#[tokio::test]
async fn restart_counts_own_pre_restart_incumbent_cost_once() {
let shared = Arc::new(InMemoryExperimentLog::new());
let before = ExperimentMonitor::from_config(&config())
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
let served = trace("t", "s", Arm::Incumbent, 0.10);
assert!(
before
.record_pair(
"t",
0,
&fp(),
1.0,
1.0,
0.10,
0.02,
false,
0,
0,
Uuid::now_v7(),
served.trace_id,
)
.await
);
let after = monitor_from_traces(std::slice::from_ref(&served), &[], &[], &config())
.with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
after.sync_from_backend("t", 0, &fp()).await.unwrap();
let r = &after.reports()[0].report;
assert_eq!(r.pairs, 1, "the pair's evidence still arrives from the log");
assert_eq!(r.incumbent.requests, 1, "{:?}", r.incumbent);
assert!(
(r.incumbent.cost_usd - 0.10).abs() < 1e-12,
"{:?}",
r.incumbent
);
}
#[test]
fn startup_replay_counts_a_duplicated_pair_row_once() {
let tid = Uuid::now_v7();
let row = crate::store::ExperimentPairRow {
tenant: "t".to_owned(),
route_ix: 0,
fingerprint: fp(),
trace_id: tid,
x_inc: 1.0,
x_ch: 0.0,
challenger_cost_usd: 0.05,
ch_error: true,
ch_truncated: 0,
ts: jiff::Timestamp::now(),
in_log: crate::store::InLogStatus::NotInLog,
};
let m = monitor_from_traces(&[], &[], &[row.clone(), row], &config());
let r = &m.reports()[0];
assert_eq!(r.report.pairs, 1);
assert_eq!(r.health.challenger.errors, 1);
}
#[tokio::test]
async fn synced_state_expires_after_the_staleness_bound_when_the_log_is_down() {
tokio::time::pause();
let log = Arc::new(SwitchableLog::default());
let m = ExperimentMonitor::from_config(&config())
.with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
let fp_str = fp();
assert!(
m.ensure_synced("t", 0, &fp_str).await,
"first sync succeeds"
);
log.failing.store(true, std::sync::atomic::Ordering::SeqCst);
let reads = log.reads.load(std::sync::atomic::Ordering::SeqCst);
tokio::time::advance(SYNC_STALENESS_BOUND / 2).await;
assert!(m.ensure_synced("t", 0, &fp_str).await, "fresh: trusted");
assert_eq!(
log.reads.load(std::sync::atomic::Ordering::SeqCst),
reads,
"fresh state needs no read"
);
tokio::time::advance(SYNC_STALENESS_BOUND).await;
assert!(
!m.ensure_synced("t", 0, &fp_str).await,
"stale and the log is down: fail closed"
);
}
#[tokio::test]
async fn ensure_synced_times_out_on_hung_backend() {
tokio::time::pause();
#[derive(Debug)]
struct HangingLog;
#[async_trait::async_trait]
impl ExperimentLogBackend for HangingLog {
async fn append(
&self,
_: &str,
_: usize,
_: &str,
_: LogRecord,
) -> Result<u64, LogError> {
Err(LogError::Backend("down".to_owned()))
}
async fn load_since(
&self,
_: &str,
_: usize,
_: &str,
_: u64,
) -> Result<Vec<Option<LogRecord>>, LogError> {
std::future::pending::<Result<Vec<Option<LogRecord>>, LogError>>().await
}
async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
false
}
}
let m = Arc::new(
ExperimentMonitor::from_config(&config())
.with_log(Arc::new(HangingLog) as Arc<dyn ExperimentLogBackend>),
);
let fp_str = fp();
let result = tokio::select! {
r = m.ensure_synced("t", 0, &fp_str) => r,
_ = tokio::time::sleep(SYNC_TIMEOUT + std::time::Duration::from_millis(1)) => {
tokio::time::advance(SYNC_TIMEOUT + std::time::Duration::from_millis(10)).await;
m.ensure_synced("t", 0, &fp_str).await
}
};
assert!(
!result,
"ensure_synced must return false when load_since hangs (timeout fires)"
);
}
}