1use std::collections::{HashMap, HashSet};
41use std::fmt;
42use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
43
44use firstpass_core::{
45 Arm, Config, Experiment, ExperimentReport, ExperimentStatus, Trace, Verdict, select,
46};
47use serde::{Deserialize, Serialize};
48use uuid::Uuid;
49
50const MAX_TRACKED_SESSIONS: usize = 1_000_000;
55
56const SYNC_FAILURE_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(5);
62
63const SYNC_STALENESS_BOUND: std::time::Duration = std::time::Duration::from_secs(30);
68
69const SYNC_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
74
75const LOG_OP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
79
80async fn bounded<T>(
82 fut: impl std::future::Future<Output = Result<T, LogError>>,
83) -> Result<T, LogError> {
84 match tokio::time::timeout(LOG_OP_TIMEOUT, fut).await {
85 Ok(r) => r,
86 Err(_) => Err(LogError::Backend(format!(
87 "timed out after {} ms",
88 LOG_OP_TIMEOUT.as_millis()
89 ))),
90 }
91}
92
93#[derive(Debug, thiserror::Error)]
101pub enum LogError {
102 #[error("experiment log backend error: {0}")]
104 Backend(String),
105}
106
107#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
112#[serde(tag = "kind", rename_all = "snake_case")]
113pub enum LogRecord {
114 Outcome {
116 arm: Arm,
117 outcome: f64,
119 session_hash: u64,
121 cost_usd: f64,
123 },
124 Pair {
126 x_inc: f64,
127 x_ch: f64,
128 incumbent_cost_usd: f64,
130 challenger_cost_usd: f64,
132 #[serde(default)]
135 ch_error: bool,
136 #[serde(default)]
139 ch_truncated: u64,
140 #[serde(default)]
143 ts_unix: i64,
144 #[serde(default)]
149 replica_id: Uuid,
150 #[serde(default)]
155 trace_id: Uuid,
156 #[serde(default)]
161 incumbent_trace_id: Uuid,
162 },
163}
164
165#[async_trait::async_trait]
176pub trait ExperimentLogBackend: Send + Sync + fmt::Debug {
177 async fn append(
181 &self,
182 tenant: &str,
183 route_ix: usize,
184 fingerprint: &str,
185 record: LogRecord,
186 ) -> Result<u64, LogError>;
187
188 async fn load_since(
194 &self,
195 tenant: &str,
196 route_ix: usize,
197 fingerprint: &str,
198 offset: u64,
199 ) -> Result<Vec<Option<LogRecord>>, LogError>;
200
201 async fn claim_session(
207 &self,
208 tenant: &str,
209 route_ix: usize,
210 fingerprint: &str,
211 session_id: &str,
212 ) -> bool;
213}
214
215#[derive(Debug, Default)]
221pub struct InMemoryExperimentLog {
222 lists: Mutex<HashMap<(String, usize, String), Vec<LogRecord>>>,
223 sessions: Mutex<HashSet<(String, usize, String, String)>>,
224}
225
226impl InMemoryExperimentLog {
227 #[must_use]
229 pub fn new() -> Self {
230 Self::default()
231 }
232}
233
234#[async_trait::async_trait]
235impl ExperimentLogBackend for InMemoryExperimentLog {
236 async fn append(
237 &self,
238 tenant: &str,
239 route_ix: usize,
240 fingerprint: &str,
241 record: LogRecord,
242 ) -> Result<u64, LogError> {
243 let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
244 let mut lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
245 let list = lists.entry(key).or_default();
246 list.push(record);
247 Ok(list.len() as u64)
248 }
249
250 async fn load_since(
251 &self,
252 tenant: &str,
253 route_ix: usize,
254 fingerprint: &str,
255 offset: u64,
256 ) -> Result<Vec<Option<LogRecord>>, LogError> {
257 let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
258 let lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
259 match lists.get(&key) {
260 None => Ok(vec![]),
261 Some(list) => {
262 let start = offset as usize;
263 if start >= list.len() {
264 Ok(vec![])
265 } else {
266 Ok(list[start..].iter().map(|r| Some(r.clone())).collect())
268 }
269 }
270 }
271 }
272
273 async fn claim_session(
274 &self,
275 tenant: &str,
276 route_ix: usize,
277 fingerprint: &str,
278 session_id: &str,
279 ) -> bool {
280 let key = (
281 tenant.to_owned(),
282 route_ix,
283 fingerprint.to_owned(),
284 session_id.to_owned(),
285 );
286 let mut sessions = self.sessions.lock().unwrap_or_else(PoisonError::into_inner);
287 sessions.insert(key)
288 }
289}
290
291#[cfg(feature = "redis-cache")]
294pub use redis_log::RedisExperimentLog;
295
296#[cfg(feature = "redis-cache")]
297mod redis_log {
298 use super::{ExperimentLogBackend, LogRecord};
299 use std::fmt;
300
301 use tokio::sync::Mutex;
303
304 const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
305
306 fn redact(url: &str) -> String {
308 let Some((scheme, rest)) = url.split_once("://") else {
309 return "<malformed redis url>".to_owned();
310 };
311 let host = match rest.rfind('@') {
312 Some(i) => &rest[i + 1..],
313 None => rest,
314 };
315 format!("{scheme}://{host}")
316 }
317
318 pub struct RedisExperimentLog {
322 conn: Mutex<redis::aio::ConnectionManager>,
324 }
325
326 impl fmt::Debug for RedisExperimentLog {
327 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
328 f.debug_struct("RedisExperimentLog").finish_non_exhaustive()
329 }
330 }
331
332 impl RedisExperimentLog {
333 pub async fn connect(url: &str) -> Result<Self, String> {
339 let safe = redact(url);
340 let client = redis::Client::open(url).map_err(|e| format!("redis url {safe}: {e}"))?;
341 let conn =
342 tokio::time::timeout(CONNECT_TIMEOUT, redis::aio::ConnectionManager::new(client))
343 .await
344 .map_err(|_| format!("redis url {safe}: connect timed out"))?
345 .map_err(|e| format!("redis url {safe}: {e}"))?;
346 Ok(Self {
347 conn: Mutex::new(conn),
348 })
349 }
350
351 fn list_key(tenant: &str, route_ix: usize, fingerprint: &str) -> String {
352 format!("firstpass:exp:{tenant}:{route_ix}:{fingerprint}")
353 }
354
355 fn session_key(
356 tenant: &str,
357 route_ix: usize,
358 fingerprint: &str,
359 session_id: &str,
360 ) -> String {
361 format!("firstpass:exp:sess:{tenant}:{route_ix}:{fingerprint}:{session_id}")
362 }
363 }
364
365 #[async_trait::async_trait]
366 impl ExperimentLogBackend for RedisExperimentLog {
367 async fn append(
368 &self,
369 tenant: &str,
370 route_ix: usize,
371 fingerprint: &str,
372 record: LogRecord,
373 ) -> Result<u64, super::LogError> {
374 let key = Self::list_key(tenant, route_ix, fingerprint);
375 let json = serde_json::to_string(&record)
376 .map_err(|e| super::LogError::Backend(format!("serialize: {e}")))?;
377 let mut conn = self.conn.lock().await;
378 use redis::AsyncCommands as _;
379 conn.rpush::<_, _, u64>(&key, json)
380 .await
381 .map_err(|e| super::LogError::Backend(format!("RPUSH {key}: {e}")))
382 }
383
384 async fn load_since(
385 &self,
386 tenant: &str,
387 route_ix: usize,
388 fingerprint: &str,
389 offset: u64,
390 ) -> Result<Vec<Option<super::LogRecord>>, super::LogError> {
391 let key = Self::list_key(tenant, route_ix, fingerprint);
392 let mut conn = self.conn.lock().await;
393 use redis::AsyncCommands as _;
394 let start = isize::try_from(offset).unwrap_or(isize::MAX);
398 let raw: Vec<String> = conn
399 .lrange(&key, start, -1)
400 .await
401 .map_err(|e| super::LogError::Backend(format!("LRANGE {key}: {e}")))?;
402 Ok(raw
405 .iter()
406 .map(|s| match serde_json::from_str(s) {
407 Ok(r) => Some(r),
408 Err(e) => {
409 tracing::warn!(
410 %e,
411 "experiment log: malformed record; position consumed, evidence skipped"
412 );
413 None
414 }
415 })
416 .collect())
417 }
418
419 async fn claim_session(
420 &self,
421 tenant: &str,
422 route_ix: usize,
423 fingerprint: &str,
424 session_id: &str,
425 ) -> bool {
426 let key = Self::session_key(tenant, route_ix, fingerprint, session_id);
427 let mut conn = self.conn.lock().await;
428 use redis::AsyncCommands as _;
429 match conn
431 .set_options::<_, _, ()>(
432 &key,
433 "1",
434 redis::SetOptions::default()
435 .with_expiration(redis::SetExpiry::EX(86_400))
436 .conditional_set(redis::ExistenceCheck::NX),
437 )
438 .await
439 {
440 Ok(()) => true, Err(_) => false, }
443 }
444 }
445}
446
447#[derive(Debug)]
449struct Entry {
450 experiment: Experiment,
451 ignored_stale: u64,
453 health: Health,
455 sessions: HashSet<u64>,
458 saturated_logged: bool,
459 applied_pair_ids: HashSet<Uuid>,
476 locally_costed: HashSet<Uuid>,
481 remote_log_offset: u64,
483 sync_warn_logged: bool,
486 sync_failed_at: Option<tokio::time::Instant>,
492 last_synced_at: Option<tokio::time::Instant>,
497}
498
499fn session_hash(session_id: &str) -> u64 {
500 use sha2::{Digest, Sha256};
504 let d = Sha256::digest(session_id.as_bytes());
505 u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]])
506}
507
508fn status_code(s: ExperimentStatus) -> f64 {
510 match s {
511 ExperimentStatus::Undecided => 0.0,
512 ExperimentStatus::Certified => 1.0,
513 ExperimentStatus::Halted => 2.0,
514 }
515}
516
517#[derive(Debug)]
518struct Params {
519 margin: f64,
520 alpha: f64,
521 fingerprint: String,
522}
523
524type SyncGates = Mutex<HashMap<(String, usize), Arc<tokio::sync::Mutex<()>>>>;
526
527#[derive(Debug)]
529pub struct ExperimentMonitor {
530 params: HashMap<usize, Params>,
532 unavailable: bool,
535 entries: Mutex<HashMap<String, HashMap<usize, Entry>>>,
537 log: Option<Arc<dyn ExperimentLogBackend>>,
539 paired_ledger: Option<Arc<crate::shadow::ShadowLedger>>,
542 replica_id: Uuid,
546 sync_gates: SyncGates,
554}
555
556impl Default for ExperimentMonitor {
557 fn default() -> Self {
561 Self {
562 params: HashMap::new(),
563 unavailable: false,
564 entries: Mutex::new(HashMap::new()),
565 log: None,
566 paired_ledger: None,
567 replica_id: Uuid::now_v7(),
568 sync_gates: Mutex::new(HashMap::new()),
569 }
570 }
571}
572
573impl ExperimentMonitor {
574 #[must_use]
577 pub fn from_config(config: &Config) -> Self {
578 let params = config
579 .routes
580 .iter()
581 .enumerate()
582 .filter_map(|(i, r)| {
583 r.challenger.as_ref().map(|c| {
584 (
585 i,
586 Params {
587 margin: c.margin,
588 alpha: c.alpha,
589 fingerprint: c.fingerprint(&r.ladder),
590 },
591 )
592 })
593 })
594 .collect();
595 Self {
596 params,
597 unavailable: false,
598 entries: Mutex::default(),
599 log: None,
600 paired_ledger: None,
601 replica_id: Uuid::now_v7(),
602 sync_gates: Mutex::default(),
603 }
604 }
605
606 #[must_use]
608 pub fn unavailable(config: &Config) -> Self {
609 Self {
610 unavailable: true,
611 ..Self::from_config(config)
612 }
613 }
614
615 #[must_use]
619 pub fn with_log(mut self, log: Arc<dyn ExperimentLogBackend>) -> Self {
620 self.log = Some(log);
621 self
622 }
623
624 #[must_use]
628 pub fn with_paired_ledger(mut self, ledger: Arc<crate::shadow::ShadowLedger>) -> Self {
629 self.paired_ledger = Some(ledger);
630 self
631 }
632
633 #[must_use]
635 pub fn has_log(&self) -> bool {
636 self.log.is_some()
637 }
638
639 pub async fn ensure_synced(&self, tenant: &str, route_ix: usize, fingerprint: &str) -> bool {
658 if self.log.is_none() {
659 return true; }
661
662 {
665 let mut map = self.lock();
666 match self.entry(&mut map, tenant, route_ix) {
667 None => return true, Some(e) => {
669 if e.last_synced_at
672 .is_some_and(|t| t.elapsed() <= SYNC_STALENESS_BOUND)
673 {
674 return true;
675 }
676 if e.sync_failed_at
680 .is_some_and(|t| t.elapsed() < SYNC_FAILURE_COOLDOWN)
681 {
682 return false;
683 }
684 }
687 }
688 }
689
690 let gate = {
693 let mut gates = self
694 .sync_gates
695 .lock()
696 .unwrap_or_else(PoisonError::into_inner);
697 gates
698 .entry((tenant.to_owned(), route_ix))
699 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
700 .clone()
701 };
702
703 let _guard = gate.lock().await;
706
707 {
710 let map = self.lock();
711 if let Some(entry) = map.get(tenant).and_then(|r| r.get(&route_ix)) {
712 if entry
713 .last_synced_at
714 .is_some_and(|t| t.elapsed() <= SYNC_STALENESS_BOUND)
715 {
716 return true;
717 }
718 if entry
719 .sync_failed_at
720 .is_some_and(|t| t.elapsed() < SYNC_FAILURE_COOLDOWN)
721 {
722 return false;
723 }
724 }
725 }
726
727 let sync_result = tokio::time::timeout(
730 SYNC_TIMEOUT,
731 self.sync_from_backend(tenant, route_ix, fingerprint),
732 )
733 .await;
734
735 match sync_result {
736 Ok(Ok(())) => {
737 let mut gates = self
741 .sync_gates
742 .lock()
743 .unwrap_or_else(PoisonError::into_inner);
744 gates.remove(&(tenant.to_owned(), route_ix));
745 true
746 }
747 Ok(Err(e)) => {
748 let mut map = self.lock();
749 if let Some(entry) = self.entry(&mut map, tenant, route_ix) {
750 entry.sync_failed_at = Some(tokio::time::Instant::now());
751 if !entry.sync_warn_logged {
752 entry.sync_warn_logged = true;
753 tracing::warn!(
754 tenant,
755 route = route_ix,
756 error = %e,
757 "shared log unreachable; halt guard and spend cap may use stale \
758 state — serving incumbent only for this request"
759 );
760 }
761 }
762 false
763 }
764 Err(_elapsed) => {
765 let mut map = self.lock();
767 if let Some(entry) = self.entry(&mut map, tenant, route_ix) {
768 entry.sync_failed_at = Some(tokio::time::Instant::now());
769 if !entry.sync_warn_logged {
770 entry.sync_warn_logged = true;
771 tracing::warn!(
772 tenant,
773 route = route_ix,
774 "shared log sync timed out (>{} ms); serving incumbent only",
775 SYNC_TIMEOUT.as_millis()
776 );
777 }
778 }
779 false
780 }
781 }
782 }
783
784 #[must_use]
786 pub fn fingerprint(&self, route_ix: usize) -> Option<&str> {
787 self.params.get(&route_ix).map(|p| p.fingerprint.as_str())
788 }
789
790 #[must_use]
793 pub fn challenger_blocked(&self, tenant: &str, route_ix: usize) -> bool {
794 self.unavailable || self.status(tenant, route_ix) == ExperimentStatus::Halted
795 }
796
797 #[must_use]
799 pub fn is_active(&self) -> bool {
800 !self.params.is_empty()
801 }
802
803 fn lock(&self) -> MutexGuard<'_, HashMap<String, HashMap<usize, Entry>>> {
804 self.entries.lock().unwrap_or_else(PoisonError::into_inner)
807 }
808
809 fn entry<'a>(
810 &self,
811 map: &'a mut HashMap<String, HashMap<usize, Entry>>,
812 tenant: &str,
813 route_ix: usize,
814 ) -> Option<&'a mut Entry> {
815 let p = self.params.get(&route_ix)?;
816 let (margin, alpha) = (p.margin, p.alpha);
817 if !map.contains_key(tenant) {
818 map.insert(tenant.to_owned(), HashMap::new());
819 }
820 let routes = map.get_mut(tenant)?;
821 if let std::collections::hash_map::Entry::Vacant(v) = routes.entry(route_ix) {
822 match Experiment::new(margin, alpha) {
823 Ok(experiment) => {
824 v.insert(Entry {
825 experiment,
826 ignored_stale: 0,
827 health: Health::default(),
828 sessions: HashSet::new(),
829 saturated_logged: false,
830 applied_pair_ids: HashSet::new(),
831 locally_costed: HashSet::new(),
832 remote_log_offset: 0,
833 sync_warn_logged: false,
834 sync_failed_at: None,
835 last_synced_at: None,
836 });
837 }
838 Err(e) => {
839 tracing::warn!(route = route_ix, %e, "experiment parameters rejected");
841 return None;
842 }
843 }
844 }
845 routes.get_mut(&route_ix)
846 }
847
848 #[must_use]
852 pub fn status(&self, tenant: &str, route_ix: usize) -> ExperimentStatus {
853 self.lock()
854 .get(tenant)
855 .and_then(|r| r.get(&route_ix))
856 .map_or(ExperimentStatus::Undecided, |e| e.experiment.status())
857 }
858
859 pub fn observe_cost(
864 &self,
865 tenant: &str,
866 route_ix: usize,
867 fingerprint: &str,
868 arm: Arm,
869 usd: f64,
870 ) {
871 let current = self.fingerprint(route_ix) == Some(fingerprint);
872 let mut map = self.lock();
873 if let Some(e) = self.entry(&mut map, tenant, route_ix) {
874 if current {
875 e.experiment.observe_cost(arm, usd);
876 } else {
877 e.ignored_stale += 1;
878 }
879 }
880 }
881
882 pub fn observe_health(
886 &self,
887 tenant: &str,
888 route_ix: usize,
889 fingerprint: &str,
890 arm: Arm,
891 errored: bool,
892 truncated: u64,
893 ) {
894 if (!errored && truncated == 0) || self.fingerprint(route_ix) != Some(fingerprint) {
895 return;
896 }
897 let mut map = self.lock();
898 if let Some(e) = self.entry(&mut map, tenant, route_ix) {
899 let h = match arm {
900 Arm::Incumbent => &mut e.health.incumbent,
901 Arm::Challenger => &mut e.health.challenger,
902 };
903 h.errors += u64::from(errored);
904 h.truncated += truncated;
905 }
906 }
907
908 pub fn observe_outcome(
917 &self,
918 tenant: &str,
919 route_ix: usize,
920 fingerprint: &str,
921 arm: Arm,
922 session_id: &str,
923 verdict: Verdict,
924 ) -> bool {
925 if self.fingerprint(route_ix) != Some(fingerprint) {
927 return false;
928 }
929 let outcome = match verdict {
930 Verdict::Pass => 1.0,
931 Verdict::Fail => 0.0,
932 Verdict::Abstain => return false,
933 };
934 let mut map = self.lock();
935 let Some(e) = self.entry(&mut map, tenant, route_ix) else {
936 return false;
937 };
938 let key = session_hash(session_id);
939 if e.sessions.contains(&key) {
940 return false;
941 }
942 if e.sessions.len() >= MAX_TRACKED_SESSIONS {
943 if !e.saturated_logged {
944 e.saturated_logged = true;
945 tracing::warn!(
946 tenant,
947 route = route_ix,
948 "experiment session ledger is full; new sessions' outcomes are ignored"
949 );
950 }
951 return false;
952 }
953 let before = e.experiment.status();
954 if !e.experiment.observe_outcome(arm, outcome) {
955 return false;
956 }
957 e.sessions.insert(key);
958 let after = e.experiment.status();
959 if before != after {
960 match after {
961 ExperimentStatus::Halted => tracing::error!(
962 tenant,
963 route = route_ix,
964 "EXPERIMENT HALTED: the challenger ladder is worse than the incumbent by more \
965 than the margin — it is no longer served"
966 ),
967 ExperimentStatus::Certified => tracing::info!(
968 tenant,
969 route = route_ix,
970 "experiment certified: the challenger is within the margin of the incumbent; \
971 adopting it is a human decision (`firstpass experiment`)"
972 ),
973 ExperimentStatus::Undecided => {}
974 }
975 Self::publish(&map, route_ix);
976 }
977 true
978 }
979
980 fn publish(map: &HashMap<String, HashMap<usize, Entry>>, route_ix: usize) {
985 let worst = map
986 .values()
987 .filter_map(|r| r.get(&route_ix))
988 .map(|e| status_code(e.experiment.status()))
989 .fold(0.0, f64::max);
990 metrics::gauge!("firstpass_experiment_status", "route" => route_ix.to_string()).set(worst);
991 }
992
993 #[must_use]
995 pub fn reports(&self) -> Vec<Snapshot> {
996 let map = self.lock();
997 let mut out: Vec<_> = map
998 .iter()
999 .flat_map(|(t, routes)| {
1000 routes.iter().map(move |(ix, e)| Snapshot {
1001 tenant: t.clone(),
1002 route_ix: *ix,
1003 report: e.experiment.report(),
1004 ignored_stale: e.ignored_stale,
1005 health: e.health,
1006 })
1007 })
1008 .collect();
1009 out.sort_by(|a, b| (&a.tenant, a.route_ix).cmp(&(&b.tenant, b.route_ix)));
1010 out
1011 }
1012
1013 fn apply_log_record(
1021 &self,
1022 map: &mut HashMap<String, HashMap<usize, Entry>>,
1023 tenant: &str,
1024 route_ix: usize,
1025 fingerprint: &str,
1026 record: &LogRecord,
1027 ) -> bool {
1028 if self.fingerprint(route_ix) != Some(fingerprint) {
1029 if let Some(e) = self.entry(map, tenant, route_ix) {
1030 e.ignored_stale += 1;
1031 }
1032 return false;
1033 }
1034 match record {
1035 LogRecord::Outcome {
1036 arm,
1037 outcome,
1038 session_hash: hash,
1039 ..
1040 } => {
1041 let Some(e) = self.entry(map, tenant, route_ix) else {
1042 return false;
1043 };
1044 if e.sessions.contains(hash) {
1045 return false; }
1047 if e.sessions.len() >= MAX_TRACKED_SESSIONS {
1048 return false;
1049 }
1050 let before = e.experiment.status();
1051 if !e.experiment.observe_outcome(*arm, *outcome) {
1052 return false;
1053 }
1054 e.sessions.insert(*hash);
1055 before != e.experiment.status()
1056 }
1057 LogRecord::Pair {
1058 x_inc,
1059 x_ch,
1060 incumbent_cost_usd,
1061 challenger_cost_usd,
1062 ch_error,
1063 ch_truncated,
1064 ts_unix,
1065 replica_id: record_replica_id,
1066 trace_id: record_trace_id,
1067 incumbent_trace_id,
1068 } => {
1069 let Some(e) = self.entry(map, tenant, route_ix) else {
1070 return false;
1071 };
1072 let already_applied =
1077 !record_trace_id.is_nil() && e.applied_pair_ids.contains(record_trace_id);
1078 let before = e.experiment.status();
1079 if !already_applied {
1080 if e.applied_pair_ids.len() >= MAX_TRACKED_SESSIONS {
1081 if !e.saturated_logged {
1084 e.saturated_logged = true;
1085 tracing::warn!(
1086 tenant,
1087 route = route_ix,
1088 "experiment pair-id set is full; new pairs' evidence is skipped"
1089 );
1090 }
1091 } else {
1092 if e.experiment.observe_pair(*x_inc, *x_ch) {
1096 e.experiment
1099 .observe_cost(Arm::Challenger, *challenger_cost_usd);
1100 }
1101 if !record_trace_id.is_nil() {
1102 e.applied_pair_ids.insert(*record_trace_id);
1103 }
1104 }
1105 }
1106 let is_own = *record_replica_id == self.replica_id;
1115 if !already_applied && !is_own {
1116 let costed_here =
1117 !incumbent_trace_id.is_nil() && e.locally_costed.remove(incumbent_trace_id);
1118 if !costed_here {
1123 e.experiment
1124 .observe_cost(Arm::Incumbent, *incumbent_cost_usd);
1125 }
1126 if *ch_error || *ch_truncated > 0 {
1127 e.health.challenger.errors += u64::from(*ch_error);
1128 e.health.challenger.truncated += *ch_truncated;
1129 }
1130 if let Some(ledger) = &self.paired_ledger {
1131 let now = jiff::Timestamp::now();
1132 let pair_ts = jiff::Timestamp::from_second(*ts_unix)
1133 .unwrap_or(jiff::Timestamp::UNIX_EPOCH);
1134 ledger.seed(tenant, route_ix, *challenger_cost_usd, pair_ts, now);
1135 }
1136 }
1137 before != e.experiment.status()
1138 }
1139 }
1140 }
1141
1142 pub async fn sync_from_backend(
1162 &self,
1163 tenant: &str,
1164 route_ix: usize,
1165 fingerprint: &str,
1166 ) -> Result<(), LogError> {
1167 let Some(log) = &self.log else {
1168 return Ok(());
1169 };
1170 let offset = {
1172 let mut map = self.lock();
1173 self.entry(&mut map, tenant, route_ix)
1174 .map_or(0, |e| e.remote_log_offset)
1175 };
1176 let records = log
1178 .load_since(tenant, route_ix, fingerprint, offset)
1179 .await?;
1180 if records.is_empty() {
1181 let mut map = self.lock();
1184 if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
1185 e.last_synced_at = Some(tokio::time::Instant::now());
1186 e.sync_warn_logged = false;
1188 e.sync_failed_at = None;
1189 }
1190 return Ok(());
1191 }
1192 let mut map = self.lock();
1193 for (i, opt_rec) in records.iter().enumerate() {
1194 let abs_ix = offset + i as u64;
1195 let current_offset = map
1197 .get(tenant)
1198 .and_then(|r| r.get(&route_ix))
1199 .map_or(0, |e| e.remote_log_offset);
1200 if abs_ix < current_offset {
1201 continue;
1202 }
1203 let changed = if let Some(rec) = opt_rec {
1205 self.apply_log_record(&mut map, tenant, route_ix, fingerprint, rec)
1206 } else {
1207 false
1208 };
1209 if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
1213 e.remote_log_offset = e.remote_log_offset.max(abs_ix + 1);
1214 }
1215 if changed {
1216 let after = map
1217 .get(tenant)
1218 .and_then(|r| r.get(&route_ix))
1219 .map(|e| e.experiment.status());
1220 match after {
1221 Some(ExperimentStatus::Halted) => tracing::error!(
1222 tenant,
1223 route = route_ix,
1224 "EXPERIMENT HALTED (shared log): challenger worse than incumbent"
1225 ),
1226 Some(ExperimentStatus::Certified) => tracing::info!(
1227 tenant,
1228 route = route_ix,
1229 "experiment certified (shared log)"
1230 ),
1231 _ => {}
1232 }
1233 Self::publish(&map, route_ix);
1234 }
1235 }
1236 if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
1239 e.last_synced_at = Some(tokio::time::Instant::now());
1240 e.sync_warn_logged = false;
1242 e.sync_failed_at = None;
1243 }
1244 Ok(())
1245 }
1246
1247 #[allow(clippy::too_many_arguments)]
1253 pub async fn record_outcome(
1254 &self,
1255 tenant: &str,
1256 route_ix: usize,
1257 fingerprint: &str,
1258 arm: Arm,
1259 session_id: &str,
1260 verdict: Verdict,
1261 cost_usd: f64,
1262 ) -> bool {
1263 let outcome = match verdict {
1267 Verdict::Pass => 1.0,
1268 Verdict::Fail => 0.0,
1269 Verdict::Abstain => return false,
1270 };
1271 let Some(log) = self.log.as_ref().cloned() else {
1272 return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
1273 };
1274 if !log
1276 .claim_session(tenant, route_ix, fingerprint, session_id)
1277 .await
1278 {
1279 return false;
1280 }
1281 if let Err(e) = log
1282 .append(
1283 tenant,
1284 route_ix,
1285 fingerprint,
1286 LogRecord::Outcome {
1287 arm,
1288 outcome,
1289 session_hash: session_hash(session_id),
1290 cost_usd,
1291 },
1292 )
1293 .await
1294 {
1295 tracing::warn!(
1296 tenant,
1297 route = route_ix,
1298 error = %e,
1299 "outcome log append failed; evidence will not reach other replicas"
1300 );
1301 return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
1303 }
1304 let _ = self.sync_from_backend(tenant, route_ix, fingerprint).await;
1308 true
1309 }
1310
1311 #[allow(clippy::too_many_arguments)]
1324 pub fn apply_pair_locally(
1325 &self,
1326 tenant: &str,
1327 route_ix: usize,
1328 fingerprint: &str,
1329 x_inc: f64,
1330 x_ch: f64,
1331 challenger_cost_usd: f64,
1332 trace_id: Uuid,
1333 ) -> bool {
1334 if self.fingerprint(route_ix) != Some(fingerprint) {
1335 return false;
1336 }
1337 let mut map = self.lock();
1338 let Some(e) = self.entry(&mut map, tenant, route_ix) else {
1339 return false;
1340 };
1341 let already_applied = !trace_id.is_nil() && e.applied_pair_ids.contains(&trace_id);
1344 if already_applied {
1345 return false;
1346 }
1347 if e.applied_pair_ids.len() >= MAX_TRACKED_SESSIONS {
1348 if !e.saturated_logged {
1349 e.saturated_logged = true;
1350 tracing::warn!(
1351 tenant,
1352 route = route_ix,
1353 "experiment pair-id set is full; new pairs' evidence is skipped"
1354 );
1355 }
1356 return false;
1357 }
1358 let before = e.experiment.status();
1359 let ok = e.experiment.observe_pair(x_inc, x_ch);
1360 if ok {
1361 e.experiment
1364 .observe_cost(Arm::Challenger, challenger_cost_usd);
1365 }
1366 if !trace_id.is_nil() {
1367 e.applied_pair_ids.insert(trace_id);
1368 }
1369 if ok && before != e.experiment.status() {
1370 match e.experiment.status() {
1371 ExperimentStatus::Halted => tracing::error!(
1372 tenant,
1373 route = route_ix,
1374 "EXPERIMENT HALTED: paired challenger worse than incumbent (local evidence)"
1375 ),
1376 ExperimentStatus::Certified => tracing::info!(
1377 tenant,
1378 route = route_ix,
1379 "experiment certified (paired, local evidence): challenger within margin"
1380 ),
1381 ExperimentStatus::Undecided => {}
1382 }
1383 Self::publish(&map, route_ix);
1384 }
1385 ok
1386 }
1387
1388 #[allow(clippy::too_many_arguments)]
1410 pub async fn record_pair(
1411 &self,
1412 tenant: &str,
1413 route_ix: usize,
1414 fingerprint: &str,
1415 x_inc: f64,
1416 x_ch: f64,
1417 incumbent_cost_usd: f64,
1418 challenger_cost_usd: f64,
1419 ch_error: bool,
1420 ch_truncated: u64,
1421 ts_unix: i64,
1422 trace_id: Uuid,
1423 incumbent_trace_id: Uuid,
1424 ) -> bool {
1425 if self.fingerprint(route_ix) != Some(fingerprint) {
1426 return false;
1427 }
1428 let Some(log) = self.log.as_ref().cloned() else {
1429 return false;
1433 };
1434 if !x_inc.is_finite()
1436 || !(0.0..=1.0).contains(&x_inc)
1437 || !x_ch.is_finite()
1438 || !(0.0..=1.0).contains(&x_ch)
1439 {
1440 return false;
1441 }
1442 if let Err(e) = bounded(log.append(
1448 tenant,
1449 route_ix,
1450 fingerprint,
1451 LogRecord::Pair {
1452 x_inc,
1453 x_ch,
1454 incumbent_cost_usd,
1455 challenger_cost_usd,
1456 ch_error,
1457 ch_truncated,
1458 ts_unix,
1459 replica_id: self.replica_id,
1460 trace_id,
1461 incumbent_trace_id,
1462 },
1463 ))
1464 .await
1465 {
1466 tracing::warn!(
1467 tenant,
1468 route = route_ix,
1469 error = %e,
1470 "paired log append failed; evidence withheld until SQLite write succeeds \
1471 (local spend and health were already tallied)"
1472 );
1473 return false;
1475 }
1476 if bounded(self.sync_from_backend(tenant, route_ix, fingerprint))
1481 .await
1482 .is_err()
1483 {
1484 self.apply_pair_locally(
1485 tenant,
1486 route_ix,
1487 fingerprint,
1488 x_inc,
1489 x_ch,
1490 challenger_cost_usd,
1491 trace_id,
1492 );
1493 }
1494 true
1495 }
1496
1497 pub fn start_sync_task(self: &Arc<Self>, interval: std::time::Duration) {
1501 if self.log.is_none() {
1502 return;
1503 }
1504 let monitor = Arc::clone(self);
1505 tokio::spawn(async move {
1506 let mut ticker = tokio::time::interval(interval);
1507 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1508 loop {
1509 ticker.tick().await;
1510 let keys: Vec<(String, usize, String)> = {
1513 let map = monitor.lock();
1514 let mut v = Vec::new();
1515 for (tenant, routes) in map.iter() {
1516 for &route_ix in routes.keys() {
1517 if let Some(p) = monitor.params.get(&route_ix) {
1518 v.push((tenant.clone(), route_ix, p.fingerprint.clone()));
1519 }
1520 }
1521 }
1522 v
1523 };
1524 for (tenant, route_ix, fingerprint) in keys {
1525 if let Err(e) =
1526 bounded(monitor.sync_from_backend(&tenant, route_ix, &fingerprint)).await
1527 {
1528 let mut map = monitor.lock();
1535 if let Some(e_entry) =
1536 map.get_mut(&tenant).and_then(|r| r.get_mut(&route_ix))
1537 {
1538 e_entry.sync_failed_at = Some(tokio::time::Instant::now());
1539 }
1540 tracing::warn!(
1541 tenant,
1542 route = route_ix,
1543 error = %e,
1544 "background experiment sync failed"
1545 );
1546 }
1547 }
1548 }
1549 });
1550 }
1551}
1552
1553#[derive(Debug, Clone, PartialEq)]
1555pub struct Snapshot {
1556 pub tenant: String,
1558 pub route_ix: usize,
1560 pub report: ExperimentReport,
1562 pub ignored_stale: u64,
1564 pub health: Health,
1566}
1567
1568#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
1574pub struct ArmHealth {
1575 pub errors: u64,
1577 pub truncated: u64,
1579}
1580
1581#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
1585pub struct Health {
1586 pub incumbent: ArmHealth,
1588 pub challenger: ArmHealth,
1590}
1591
1592#[must_use]
1594pub fn truncated_attempts(t: &Trace) -> u64 {
1595 t.attempts.iter().filter(|a| a.truncated).count() as u64
1596}
1597
1598#[must_use]
1602pub fn is_incumbent_trace(t: &Trace) -> bool {
1603 t.experiment
1604 .as_ref()
1605 .is_none_or(|e| e.arm != Arm::Challenger)
1606}
1607
1608#[must_use]
1621pub fn monitor_from_traces(
1622 traces: &[Trace],
1623 deferred: &[(Uuid, Verdict)],
1624 pairs: &[crate::store::ExperimentPairRow],
1625 config: &Config,
1626) -> ExperimentMonitor {
1627 let monitor = ExperimentMonitor::from_config(config);
1628 if !monitor.is_active() {
1629 return monitor;
1630 }
1631 let mut by_id: HashMap<Uuid, &Trace> = HashMap::new();
1632 for t in traces {
1633 let Some(rec) = &t.experiment else { continue };
1634 let route_ix = t.route_ix.map_or(0, |r| r as usize);
1636 monitor.observe_cost(
1637 &t.tenant_id,
1638 route_ix,
1639 &rec.fingerprint,
1640 rec.arm,
1641 t.final_.total_cost_usd,
1642 );
1643 monitor.observe_health(
1644 &t.tenant_id,
1645 route_ix,
1646 &rec.fingerprint,
1647 rec.arm,
1648 t.final_.served_from == firstpass_core::ServedFrom::Error,
1649 truncated_attempts(t),
1650 );
1651 if rec.arm == Arm::Incumbent {
1652 let mut map = monitor.lock();
1653 if let Some(e) = monitor.entry(&mut map, &t.tenant_id, route_ix)
1654 && e.locally_costed.len() < MAX_TRACKED_SESSIONS
1655 {
1656 e.locally_costed.insert(t.trace_id);
1657 }
1658 }
1659 by_id.insert(t.trace_id, t);
1660 }
1661 for (id, verdict) in deferred {
1662 let Some(t) = by_id.get(id) else { continue };
1663 let Some(rec) = &t.experiment else { continue };
1664 let route_ix = t.route_ix.map_or(0, |r| r as usize);
1665 monitor.observe_outcome(
1666 &t.tenant_id,
1667 route_ix,
1668 &rec.fingerprint,
1669 rec.arm,
1670 &t.session_id,
1671 *verdict,
1672 );
1673 }
1674 for row in pairs {
1682 let mut map = monitor.lock();
1683 if let Some(e) = monitor.entry(&mut map, &row.tenant, row.route_ix) {
1684 if monitor.fingerprint(row.route_ix) == Some(row.fingerprint.as_str()) {
1685 if !row.trace_id.is_nil() && e.applied_pair_ids.contains(&row.trace_id) {
1687 continue;
1688 }
1689 if e.experiment.observe_pair(row.x_inc, row.x_ch) {
1690 e.experiment
1691 .observe_cost(Arm::Challenger, row.challenger_cost_usd);
1692 }
1693 if !row.trace_id.is_nil() {
1696 e.applied_pair_ids.insert(row.trace_id);
1697 }
1698 if row.ch_error || row.ch_truncated > 0 {
1700 e.health.challenger.errors += u64::from(row.ch_error);
1701 e.health.challenger.truncated += row.ch_truncated;
1702 }
1703 } else {
1704 e.ignored_stale += 1;
1705 }
1706 }
1707 }
1708 monitor
1709}
1710
1711#[must_use]
1718pub fn build_monitor(db_path: &str, config: &Config) -> ExperimentMonitor {
1719 build_monitor_with(db_path, config, false)
1720}
1721
1722#[must_use]
1730pub fn build_monitor_with(db_path: &str, config: &Config, shared_log: bool) -> ExperimentMonitor {
1731 let empty = ExperimentMonitor::from_config(config);
1732 if !empty.is_active() {
1733 return empty;
1734 }
1735 let loaded = crate::store::load_all_traces(db_path)
1736 .and_then(|t| crate::store::load_all_deferred(db_path).map(|d| (t, d)))
1737 .and_then(|(t, d)| crate::store::load_all_experiment_pairs(db_path).map(|p| (t, d, p)));
1738 match loaded {
1739 Ok((traces, deferred, pairs)) => {
1740 let offline_pairs: Vec<crate::store::ExperimentPairRow>;
1747 let replayed: &[crate::store::ExperimentPairRow] = if shared_log {
1748 offline_pairs = pairs
1749 .iter()
1750 .filter(|r| r.in_log == crate::store::InLogStatus::NotInLog)
1751 .cloned()
1752 .collect();
1753 &offline_pairs
1754 } else {
1755 &pairs
1756 };
1757 let m = monitor_from_traces(&traces, &deferred, replayed, config);
1758 if shared_log {
1759 let mut map = m.lock();
1760 for row in &pairs {
1761 let _ = m.entry(&mut map, &row.tenant, row.route_ix);
1762 }
1763 }
1764 for s in m.reports() {
1765 tracing::info!(
1766 tenant = %s.tenant,
1767 route = s.route_ix,
1768 status = ?s.report.status,
1769 pairs = s.report.pairs,
1770 ignored_stale = s.ignored_stale,
1771 "experiment restored from receipts"
1772 );
1773 ExperimentMonitor::publish(&m.lock(), s.route_ix);
1774 }
1775 m
1776 }
1777 Err(e) => {
1778 tracing::error!(
1779 %e,
1780 "could not read the trace store, so it is unknown whether a challenger was already \
1781 halted: challenger arms are NOT served (incumbent only) until restart with a \
1782 readable store"
1783 );
1784 ExperimentMonitor::unavailable(config)
1785 }
1786 }
1787}
1788
1789#[must_use]
1791pub fn experiments_from_traces(
1792 traces: &[Trace],
1793 deferred: &[(Uuid, Verdict)],
1794 pairs: &[crate::store::ExperimentPairRow],
1795 config: &Config,
1796) -> Vec<Snapshot> {
1797 monitor_from_traces(traces, deferred, pairs, config).reports()
1798}
1799
1800#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
1802#[serde(rename_all = "snake_case")]
1803pub enum Recommendation {
1804 Adopt,
1806 CertifiedNoSaving,
1808 ChallengerStopped,
1810 KeepRunning,
1812}
1813
1814#[must_use]
1818pub fn recommend(r: &ExperimentReport) -> Recommendation {
1819 match r.status {
1820 select::Status::Undecided => Recommendation::KeepRunning,
1821 select::Status::Halted => Recommendation::ChallengerStopped,
1822 select::Status::Certified => {
1823 match (
1824 r.challenger.cost_per_request(),
1825 r.incumbent.cost_per_request(),
1826 ) {
1827 (Some(c), Some(i)) if c < i => Recommendation::Adopt,
1828 _ => Recommendation::CertifiedNoSaving,
1829 }
1830 }
1831 }
1832}
1833
1834#[derive(Debug, Clone, Serialize)]
1836pub struct ExperimentView {
1837 pub tenant: String,
1839 pub route_ix: usize,
1841 pub report: ExperimentReport,
1843 pub ignored_stale: u64,
1845 pub health: Health,
1847 pub recommendation: Recommendation,
1849 pub challenger_ladder: Vec<String>,
1851}
1852
1853#[must_use]
1855pub fn views(reports: Vec<Snapshot>, config: &Config) -> Vec<ExperimentView> {
1856 reports
1857 .into_iter()
1858 .map(|s| ExperimentView {
1859 recommendation: recommend(&s.report),
1860 ignored_stale: s.ignored_stale,
1861 health: s.health,
1862 challenger_ladder: config
1863 .routes
1864 .get(s.route_ix)
1865 .and_then(|r| r.challenger.as_ref())
1866 .map(|c| c.ladder.clone())
1867 .unwrap_or_default(),
1868 tenant: s.tenant,
1869 route_ix: s.route_ix,
1870 report: s.report,
1871 })
1872 .collect()
1873}
1874
1875fn fmt_opt(v: Option<f64>, f: impl Fn(f64) -> String) -> String {
1876 v.map_or_else(|| "n/a".to_owned(), f)
1877}
1878
1879#[must_use]
1881pub fn render(views: &[ExperimentView]) -> String {
1882 use std::fmt::Write as _;
1883 if views.is_empty() {
1884 return "no experiment data yet — the config has no [route.challenger], or no experiment \
1885 traffic has been recorded\n"
1886 .to_owned();
1887 }
1888 let mut out = String::new();
1889 for v in views {
1890 let r = &v.report;
1891 let _ = writeln!(
1892 out,
1893 "tenant {} · route {} · status: {:?}",
1894 v.tenant, v.route_ix, r.status
1895 );
1896 let _ = writeln!(
1897 out,
1898 " pairs: {} (margin {}, alpha {})",
1899 r.pairs, r.margin, r.alpha
1900 );
1901 for (name, a) in [("incumbent ", &r.incumbent), ("challenger", &r.challenger)] {
1902 let _ = writeln!(
1903 out,
1904 " {name}: success {} over {} sessions · ${} per request over {} requests",
1905 fmt_opt(a.success_rate(), |s| format!("{:.1}%", s * 100.0)),
1906 a.outcomes,
1907 fmt_opt(a.cost_per_request(), |c| format!("{c:.6}")),
1908 a.requests,
1909 );
1910 }
1911 let _ = writeln!(
1912 out,
1913 " e-value to certify: {:.3} · e-value to halt: {:.3} (decide at {:.1})",
1914 r.e_certify,
1915 r.e_halt,
1916 1.0 / r.alpha
1917 );
1918 if v.ignored_stale > 0 {
1919 let _ = writeln!(
1920 out,
1921 " {} receipts from a previous experiment configuration ignored",
1922 v.ignored_stale
1923 );
1924 }
1925 for (name, h) in [
1926 ("incumbent", &v.health.incumbent),
1927 ("challenger", &v.health.challenger),
1928 ] {
1929 if h.errors > 0 || h.truncated > 0 {
1930 let _ = writeln!(
1931 out,
1932 " {name}: {} errors, {} answers cut off at max_tokens (neither is random: \
1933 check them before trusting the rates)",
1934 h.errors, h.truncated
1935 );
1936 }
1937 }
1938 match v.recommendation {
1939 Recommendation::Adopt => {
1940 let ladder = v
1941 .challenger_ladder
1942 .iter()
1943 .map(|m| format!("{m:?}"))
1944 .collect::<Vec<_>>()
1945 .join(", ");
1946 let _ = writeln!(
1947 out,
1948 " ADOPT: set this route's ladder to [{ladder}]\n\n ladder = [{ladder}]\n"
1949 );
1950 }
1951 Recommendation::CertifiedNoSaving => {
1952 let _ = writeln!(out, " certified, but no saving — keep incumbent");
1953 }
1954 Recommendation::ChallengerStopped => {
1955 let _ = writeln!(out, " challenger stopped: worse by more than the margin");
1956 }
1957 Recommendation::KeepRunning => {
1958 let _ = writeln!(out, " keep running");
1959 }
1960 }
1961 }
1962 out
1963}
1964
1965#[cfg(test)]
1966mod tests {
1967 use firstpass_core::trace::ExperimentRecord;
1968 use firstpass_core::{
1969 Features, FinalOutcome, GENESIS_HASH, Mode, PolicyRef, RequestInfo, ServedFrom, TaskKind,
1970 };
1971
1972 use super::*;
1973
1974 const CFG: &str = "[[route]]\nmatch = {}\nmode = \"enforce\"\n\
1975 ladder = [\"anthropic/claude-sonnet-5\"]\n\
1976 [route.challenger]\nladder = [\"anthropic/claude-haiku-4-5\"]\npercent = 50.0\n";
1977
1978 fn config() -> Config {
1979 Config::parse(CFG).expect("test config parses")
1980 }
1981
1982 fn fp() -> String {
1984 let c = config();
1985 c.routes[0]
1986 .challenger
1987 .as_ref()
1988 .expect("challenger")
1989 .fingerprint(&c.routes[0].ladder)
1990 }
1991
1992 fn trace(tenant: &str, session: &str, arm: Arm, cost: f64) -> Trace {
1993 Trace {
1994 trace_id: Uuid::now_v7(),
1995 prev_hash: GENESIS_HASH.to_owned(),
1996 tenant_id: tenant.to_owned(),
1997 session_id: session.to_owned(),
1998 ts: jiff::Timestamp::now(),
1999 mode: Mode::Enforce,
2000 policy: PolicyRef {
2001 id: "static-ladder@v0".to_owned(),
2002 explore: false,
2003 propensity: None,
2004 mode_profile: None,
2005 },
2006 request: RequestInfo {
2007 api: "anthropic.messages".to_owned(),
2008 prompt_hash: "h".to_owned(),
2009 features: Features::new(TaskKind::Other),
2010 },
2011 attempts: vec![],
2012 deferred: vec![],
2013 final_: FinalOutcome {
2014 served_rung: Some(0),
2015 served_from: ServedFrom::Attempt,
2016 total_cost_usd: cost,
2017 gate_cost_usd: 0.0,
2018 total_latency_ms: 1,
2019 escalations: 0,
2020 counterfactual_baseline_usd: cost,
2021 savings_usd: 0.0,
2022 cache_source: None,
2023 },
2024 probe: None,
2025 rollout: None,
2026 shadow: None,
2027 route_ix: Some(0),
2028 predicted_pass: None,
2029 decision_prior: None,
2030 elastic: None,
2031 risk: None,
2032 experiment: Some(ExperimentRecord {
2033 arm,
2034 percent: 50.0,
2035 bucket: 0,
2036 fingerprint: fp(),
2037 }),
2038 upstream_provider: String::new(),
2039 }
2040 }
2041
2042 #[test]
2045 fn arm_health_is_rebuilt_from_receipts() {
2046 let attempt = |truncated| firstpass_core::trace::Attempt {
2047 rung: 0,
2048 model: "anthropic/claude-sonnet-5".to_owned(),
2049 provider: "anthropic".to_owned(),
2050 in_tokens: 1,
2051 cache_write_tokens: 0,
2052 cache_read_tokens: 0,
2053 out_tokens: 1,
2054 cost_usd: 0.0,
2055 latency_ms: 1,
2056 gates: vec![],
2057 verdict: Verdict::Pass,
2058 truncated,
2059 };
2060 let mut errored = trace("t", "a", Arm::Incumbent, 0.01);
2061 errored.final_.served_from = ServedFrom::Error;
2062 let mut cut = trace("t", "b", Arm::Incumbent, 0.01);
2063 cut.attempts = vec![attempt(true), attempt(true), attempt(false)];
2064 let mut ch_cut = trace("t", "c", Arm::Challenger, 0.01);
2065 ch_cut.attempts = vec![attempt(true)];
2066 let mut stale = trace("t", "d", Arm::Incumbent, 0.01);
2067 stale.final_.served_from = ServedFrom::Error;
2068 if let Some(rec) = stale.experiment.as_mut() {
2069 rec.fingerprint = "an-older-config".to_owned();
2070 }
2071 let snaps = experiments_from_traces(&[errored, cut, ch_cut, stale], &[], &[], &config());
2072 let h = snaps[0].health;
2073 assert_eq!(
2074 h.incumbent,
2075 ArmHealth {
2076 errors: 1,
2077 truncated: 2
2078 },
2079 "{h:?}"
2080 );
2081 assert_eq!(
2082 h.challenger,
2083 ArmHealth {
2084 errors: 0,
2085 truncated: 1
2086 },
2087 "{h:?}"
2088 );
2089 assert_eq!(snaps[0].ignored_stale, 1);
2090 }
2091
2092 #[test]
2093 fn first_outcome_per_session_counts_and_abstain_is_ignored() {
2094 let m = ExperimentMonitor::from_config(&config());
2095 assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Abstain));
2096 assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Pass));
2097 assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Fail));
2099 assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s2", Verdict::Fail));
2100 let r = &m.reports()[0].report;
2101 assert_eq!(r.incumbent.outcomes, 2);
2102 assert!((r.incumbent.outcome_sum - 1.0).abs() < f64::EPSILON);
2103 }
2104
2105 #[test]
2106 fn routes_without_a_challenger_and_unknown_tenants_are_inert() {
2107 let m = ExperimentMonitor::from_config(&config());
2108 assert!(!m.observe_outcome("t", 7, &fp(), Arm::Incumbent, "s", Verdict::Pass));
2109 assert_eq!(m.status("nobody", 0), ExperimentStatus::Undecided);
2110 assert!(m.reports().is_empty());
2111 let off = ExperimentMonitor::default();
2112 assert!(!off.is_active());
2113 off.observe_cost("t", 0, &fp(), Arm::Incumbent, 1.0);
2114 assert!(off.reports().is_empty());
2115 }
2116
2117 #[test]
2118 fn tenants_never_share_an_experiment() {
2119 let m = ExperimentMonitor::from_config(&config());
2120 for i in 0..400 {
2121 m.observe_outcome(
2122 "a",
2123 0,
2124 &fp(),
2125 Arm::Incumbent,
2126 &format!("i{i}"),
2127 Verdict::Pass,
2128 );
2129 m.observe_outcome(
2130 "a",
2131 0,
2132 &fp(),
2133 Arm::Challenger,
2134 &format!("c{i}"),
2135 Verdict::Fail,
2136 );
2137 }
2138 assert_eq!(m.status("a", 0), ExperimentStatus::Halted);
2139 assert_eq!(m.status("b", 0), ExperimentStatus::Undecided);
2140 }
2141
2142 #[test]
2143 fn a_clearly_worse_challenger_halts() {
2144 let m = ExperimentMonitor::from_config(&config());
2145 let mut at = None;
2146 for i in 0..500 {
2147 m.observe_outcome(
2148 "t",
2149 0,
2150 &fp(),
2151 Arm::Incumbent,
2152 &format!("i{i}"),
2153 Verdict::Pass,
2154 );
2155 m.observe_outcome(
2156 "t",
2157 0,
2158 &fp(),
2159 Arm::Challenger,
2160 &format!("c{i}"),
2161 Verdict::Fail,
2162 );
2163 if at.is_none() && m.status("t", 0) == ExperimentStatus::Halted {
2164 at = Some(i);
2165 }
2166 }
2167 assert!(at.is_some(), "never halted");
2168 assert_eq!(m.status("t", 0), ExperimentStatus::Halted);
2169 }
2170
2171 #[test]
2172 fn replay_matches_live_with_session_dedup_and_halt() {
2173 let cfg = config();
2174 let mut traces = Vec::new();
2175 let mut deferred = Vec::new();
2176 for i in 0..400 {
2177 let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
2178 let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
2179 deferred.push((inc.trace_id, Verdict::Pass));
2180 deferred.push((chl.trace_id, Verdict::Fail));
2181 let again = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
2183 deferred.push((again.trace_id, Verdict::Pass));
2184 traces.extend([inc, chl, again]);
2185 }
2186 let reports = experiments_from_traces(&traces, &deferred, &[], &cfg);
2187 assert_eq!(reports.len(), 1);
2188 let snap = &reports[0];
2189 let r = &snap.report;
2190 assert_eq!((snap.tenant.as_str(), snap.route_ix), ("t", 0));
2191 assert_eq!(snap.ignored_stale, 0);
2192 assert_eq!(r.status, select::Status::Halted);
2193 assert_eq!(r.challenger.outcomes, 400);
2195 assert!(r.challenger.outcome_sum.abs() < f64::EPSILON);
2196 assert_eq!(r.challenger.requests, 800);
2198 assert!((r.challenger.cost_per_request().unwrap_or(0.0) - 0.02).abs() < 1e-12);
2199 assert!((r.incumbent.cost_per_request().unwrap_or(0.0) - 0.10).abs() < 1e-12);
2200 assert_eq!(recommend(r), Recommendation::ChallengerStopped);
2201
2202 let live = ExperimentMonitor::from_config(&cfg);
2204 for t in &traces {
2205 let rec = t.experiment.as_ref().expect("recorded");
2206 live.observe_cost(
2207 &t.tenant_id,
2208 0,
2209 &rec.fingerprint,
2210 rec.arm,
2211 t.final_.total_cost_usd,
2212 );
2213 }
2214 for (id, v) in &deferred {
2215 let t = traces.iter().find(|t| t.trace_id == *id).expect("trace");
2216 let rec = t.experiment.as_ref().expect("recorded");
2217 live.observe_outcome("t", 0, &rec.fingerprint, rec.arm, &t.session_id, *v);
2218 }
2219 assert_eq!(live.reports()[0], *snap);
2220 assert_eq!(
2222 experiments_from_traces(&traces, &deferred, &[], &cfg),
2223 reports
2224 );
2225 }
2226
2227 #[test]
2228 fn replay_ignores_unrecorded_traces_and_unknown_deferred() {
2229 let cfg = config();
2230 let mut plain = trace("t", "s", Arm::Incumbent, 0.1);
2231 plain.experiment = None;
2232 let deferred = [
2233 (plain.trace_id, Verdict::Pass),
2234 (Uuid::now_v7(), Verdict::Pass),
2235 ];
2236 assert!(experiments_from_traces(&[plain], &deferred, &[], &cfg).is_empty());
2237 let bare = Config::parse(
2239 "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
2240 )
2241 .expect("parses");
2242 let t = trace("t", "s", Arm::Incumbent, 0.1);
2243 assert!(experiments_from_traces(&[t], &[], &[], &bare).is_empty());
2244 }
2245
2246 fn view_with(status: select::Status, inc_cost: f64, ch_cost: f64) -> ExperimentView {
2247 let mut exp = Experiment::new(0.05, 0.05).expect("valid");
2248 exp.observe_cost(Arm::Incumbent, inc_cost);
2249 exp.observe_cost(Arm::Challenger, ch_cost);
2250 let mut report = exp.report();
2251 report.status = status;
2252 ExperimentView {
2253 tenant: "t".to_owned(),
2254 route_ix: 0,
2255 recommendation: recommend(&report),
2256 ignored_stale: 0,
2257 health: Health::default(),
2258 report,
2259 challenger_ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
2260 }
2261 }
2262
2263 #[test]
2264 fn recommendation_and_rendering_cover_every_state() {
2265 let adopt = view_with(select::Status::Certified, 0.10, 0.02);
2266 assert_eq!(adopt.recommendation, Recommendation::Adopt);
2267 let text = render(&[adopt]);
2268 assert!(
2269 text.contains("ADOPT: set this route's ladder to [\"anthropic/claude-haiku-4-5\"]"),
2270 "{text}"
2271 );
2272 assert!(text.contains("ladder = [\"anthropic/claude-haiku-4-5\"]"));
2273
2274 let dear = view_with(select::Status::Certified, 0.02, 0.10);
2275 assert_eq!(dear.recommendation, Recommendation::CertifiedNoSaving);
2276 assert!(render(&[dear]).contains("certified, but no saving — keep incumbent"));
2277
2278 let halted = view_with(select::Status::Halted, 0.10, 0.02);
2279 assert!(render(&[halted]).contains("challenger stopped: worse by more than the margin"));
2280
2281 let open = view_with(select::Status::Undecided, 0.10, 0.02);
2282 assert_eq!(open.recommendation, Recommendation::KeepRunning);
2283 assert!(render(&[open]).contains("keep running"));
2284 assert!(render(&[]).contains("no experiment data"));
2285 }
2286
2287 #[test]
2288 fn certified_without_cost_data_is_not_adopted() {
2289 let exp = Experiment::new(0.05, 0.05).expect("valid");
2290 let mut report = exp.report();
2291 report.status = select::Status::Certified;
2292 assert_eq!(recommend(&report), Recommendation::CertifiedNoSaving);
2293 }
2294
2295 #[test]
2298 fn changing_the_challenger_ladder_resets_the_experiment() {
2299 let old = config();
2300 let mut traces = Vec::new();
2301 let mut deferred = Vec::new();
2302 for i in 0..300 {
2303 let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
2304 let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
2305 deferred.push((inc.trace_id, Verdict::Pass));
2306 deferred.push((chl.trace_id, Verdict::Fail));
2307 traces.extend([inc, chl]);
2308 }
2309 let before = experiments_from_traces(&traces, &deferred, &[], &old);
2310 assert_eq!(before[0].report.status, select::Status::Halted);
2311 assert_eq!(before[0].ignored_stale, 0);
2312
2313 let new =
2314 Config::parse(&CFG.replace("claude-haiku-4-5", "claude-opus-4-8")).expect("parses");
2315 let after = experiments_from_traces(&traces, &deferred, &[], &new);
2316 assert_eq!(after[0].report.status, select::Status::Undecided);
2317 assert_eq!(after[0].report.pairs, 0);
2318 assert_eq!(after[0].report.challenger.requests, 0);
2319 assert_eq!(after[0].ignored_stale, 600);
2320 let text = render(&views(after, &new));
2321 assert!(
2322 text.contains("600 receipts from a previous experiment configuration ignored"),
2323 "{text}"
2324 );
2325
2326 let margin = Config::parse(&format!("{CFG}margin = 0.04\n")).expect("parses");
2328 assert_eq!(
2329 experiments_from_traces(&traces, &deferred, &[], &margin)[0].ignored_stale,
2330 600
2331 );
2332 let ramped = Config::parse(&CFG.replace("50.0", "30.0")).expect("parses");
2334 assert_eq!(
2335 experiments_from_traces(&traces, &deferred, &[], &ramped)[0].report,
2336 before[0].report
2337 );
2338 }
2339
2340 #[test]
2341 fn challenger_traces_are_not_incumbent_traces() {
2342 assert!(is_incumbent_trace(&trace("t", "s", Arm::Incumbent, 0.1)));
2343 assert!(!is_incumbent_trace(&trace("t", "s", Arm::Challenger, 0.1)));
2344 let mut plain = trace("t", "s", Arm::Challenger, 0.1);
2345 plain.experiment = None;
2346 assert!(is_incumbent_trace(&plain));
2347 }
2348
2349 #[tokio::test]
2351 async fn unreadable_store_fails_closed_on_the_challenger() {
2352 let cfg = config();
2353 let ok = ExperimentMonitor::from_config(&cfg);
2354 assert!(!ok.challenger_blocked("t", 0));
2355 let blocked = ExperimentMonitor::unavailable(&cfg);
2356 assert!(blocked.challenger_blocked("t", 0));
2357 assert!(blocked.challenger_blocked("any-other-tenant", 0));
2358
2359 let dir = std::env::temp_dir();
2361 let m = build_monitor(&dir.to_string_lossy(), &cfg);
2362 assert!(
2363 m.challenger_blocked("t", 0),
2364 "unreadable store served a challenger"
2365 );
2366
2367 let db = dir.join(format!("firstpass-exp-build-{}.db", Uuid::now_v7()));
2369 let (tx, handle) = crate::store::open(&db).expect("open");
2370 drop(tx);
2371 handle.await.expect("writer");
2372 let m = build_monitor(&db.to_string_lossy(), &cfg);
2373 assert!(!m.challenger_blocked("t", 0));
2374 let _ = std::fs::remove_file(&db);
2375
2376 let bare = Config::parse(
2378 "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
2379 )
2380 .expect("parses");
2381 assert!(!build_monitor(&dir.to_string_lossy(), &bare).challenger_blocked("t", 0));
2382 }
2383
2384 #[tokio::test]
2389 async fn paired_challenger_cost_reaches_the_report_on_every_path() {
2390 let cfg = config();
2391 let fp_str = fp();
2392 let challenger = |m: &ExperimentMonitor| {
2393 let r = m.reports();
2394 assert_eq!(r.len(), 1);
2395 let c = r[0].report.challenger;
2396 (c.requests, c.cost_usd, r[0].report.pairs)
2397 };
2398 let close = |(n, usd, pairs): (u64, f64, u64)| {
2399 assert_eq!((n, pairs), (3, 3));
2400 assert!((usd - 0.03).abs() < 1e-12, "{usd}");
2401 };
2402
2403 let direct = ExperimentMonitor::from_config(&cfg);
2406 for _ in 0..3 {
2407 let ok = direct
2408 .record_pair(
2409 "t",
2410 0,
2411 &fp_str,
2412 1.0,
2413 1.0,
2414 0.10,
2415 0.01,
2416 false,
2417 0,
2418 0,
2419 Uuid::nil(),
2420 Uuid::nil(),
2421 )
2422 .await;
2423 assert!(
2424 !ok,
2425 "no log → record_pair returns false (evidence via apply_pair_locally)"
2426 );
2427 direct.apply_pair_locally("t", 0, &fp_str, 1.0, 1.0, 0.01, Uuid::nil());
2428 }
2429 close(challenger(&direct));
2430
2431 let shared = Arc::new(InMemoryExperimentLog::new());
2432 let a = ExperimentMonitor::from_config(&cfg)
2433 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2434 let b = ExperimentMonitor::from_config(&cfg)
2435 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2436 for _ in 0..3 {
2437 a.record_pair(
2438 "t",
2439 0,
2440 &fp_str,
2441 1.0,
2442 1.0,
2443 0.10,
2444 0.01,
2445 false,
2446 0,
2447 0,
2448 Uuid::nil(),
2449 Uuid::nil(),
2450 )
2451 .await;
2452 }
2453 let _ = b.sync_from_backend("t", 0, &fp_str).await;
2454 close(challenger(&a));
2455 close(challenger(&b));
2456
2457 let db =
2458 std::env::temp_dir().join(format!("firstpass-paircost-{}.db", uuid::Uuid::now_v7()));
2459 let (tx, handle) = crate::store::open(&db).unwrap();
2460 drop(tx);
2461 handle.await.unwrap();
2462 for _ in 0..3 {
2463 crate::store::append_experiment_pair(
2464 &db,
2465 &crate::store::ExperimentPairRow {
2466 tenant: "t".to_owned(),
2467 route_ix: 0,
2468 fingerprint: fp_str.clone(),
2469 trace_id: uuid::Uuid::now_v7(),
2470 x_inc: 1.0,
2471 x_ch: 1.0,
2472 challenger_cost_usd: 0.01,
2473 ch_error: false,
2474 ch_truncated: 0,
2475 ts: jiff::Timestamp::now(),
2476 in_log: crate::store::InLogStatus::InLog,
2477 },
2478 )
2479 .unwrap();
2480 }
2481 close(challenger(&build_monitor(&db.to_string_lossy(), &cfg)));
2482
2483 let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
2486 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2487 let _ = restarted.sync_from_backend("t", 0, &fp_str).await;
2488 close(challenger(&restarted));
2489 let _ = std::fs::remove_file(&db);
2490 }
2491
2492 #[tokio::test]
2499 async fn two_replicas_sharing_one_log_converge_after_sync() {
2500 let shared = Arc::new(InMemoryExperimentLog::new());
2501 let cfg = config();
2502 let fp_str = fp();
2503
2504 let replica_a = ExperimentMonitor::from_config(&cfg)
2505 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2506 let replica_b = ExperimentMonitor::from_config(&cfg)
2507 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2508
2509 for i in 0..400 {
2511 replica_a
2512 .record_outcome(
2513 "t",
2514 0,
2515 &fp_str,
2516 Arm::Incumbent,
2517 &format!("i{i}"),
2518 Verdict::Pass,
2519 0.10,
2520 )
2521 .await;
2522 replica_a
2523 .record_outcome(
2524 "t",
2525 0,
2526 &fp_str,
2527 Arm::Challenger,
2528 &format!("c{i}"),
2529 Verdict::Fail,
2530 0.02,
2531 )
2532 .await;
2533 }
2534
2535 assert_eq!(
2538 replica_b.status("t", 0),
2539 ExperimentStatus::Undecided,
2540 "replica_b should not have any evidence before syncing"
2541 );
2542
2543 let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
2545 assert_eq!(
2546 replica_b.status("t", 0),
2547 ExperimentStatus::Halted,
2548 "replica_b must halt after syncing from shared log"
2549 );
2550 assert!(
2551 replica_b.challenger_blocked("t", 0),
2552 "halted experiment must block the challenger on the synced replica"
2553 );
2554
2555 let reps_a = replica_a.reports();
2557 let reps_b = replica_b.reports();
2558 assert_eq!(reps_a.len(), 1);
2559 assert_eq!(reps_b.len(), 1);
2560 assert_eq!(reps_a[0].report.status, reps_b[0].report.status);
2561 assert_eq!(reps_a[0].report.pairs, reps_b[0].report.pairs);
2562 }
2563
2564 #[tokio::test]
2573 async fn halt_on_one_replica_blocks_challenger_on_other_after_sync() {
2574 let shared = Arc::new(InMemoryExperimentLog::new());
2575 let cfg = config();
2576 let fp_str = fp();
2577
2578 let replica_a = ExperimentMonitor::from_config(&cfg)
2579 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2580 let replica_b = ExperimentMonitor::from_config(&cfg)
2581 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2582
2583 assert!(
2585 !replica_b.challenger_blocked("t", 0),
2586 "should be unblocked before any evidence"
2587 );
2588
2589 let mut halted = false;
2591 for i in 0..500 {
2592 replica_a
2593 .record_outcome(
2594 "t",
2595 0,
2596 &fp_str,
2597 Arm::Incumbent,
2598 &format!("i{i}"),
2599 Verdict::Pass,
2600 0.10,
2601 )
2602 .await;
2603 replica_a
2604 .record_outcome(
2605 "t",
2606 0,
2607 &fp_str,
2608 Arm::Challenger,
2609 &format!("c{i}"),
2610 Verdict::Fail,
2611 0.02,
2612 )
2613 .await;
2614 if replica_a.status("t", 0) == ExperimentStatus::Halted {
2615 halted = true;
2616 break;
2617 }
2618 }
2619 assert!(halted, "experiment never halted on replica A");
2620
2621 assert!(
2623 !replica_b.challenger_blocked("t", 0),
2624 "replica_b should not block before syncing"
2625 );
2626
2627 let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
2629 assert!(
2630 replica_b.challenger_blocked("t", 0),
2631 "replica_b must block after syncing the halt from the shared log"
2632 );
2633 }
2634
2635 #[tokio::test]
2641 async fn cross_replica_session_dedup_prevents_double_counting() {
2642 let shared = Arc::new(InMemoryExperimentLog::new());
2643 let cfg = config();
2644 let fp_str = fp();
2645
2646 let replica_a = ExperimentMonitor::from_config(&cfg)
2647 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2648 let replica_b = ExperimentMonitor::from_config(&cfg)
2649 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2650
2651 replica_a
2653 .record_outcome(
2654 "t",
2655 0,
2656 &fp_str,
2657 Arm::Incumbent,
2658 "session-1",
2659 Verdict::Pass,
2660 0.10,
2661 )
2662 .await;
2663 replica_b
2664 .record_outcome(
2665 "t",
2666 0,
2667 &fp_str,
2668 Arm::Incumbent,
2669 "session-1",
2670 Verdict::Pass,
2671 0.10,
2672 )
2673 .await;
2674
2675 let _ = replica_a.sync_from_backend("t", 0, &fp_str).await;
2677 let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
2678
2679 let reps_a = replica_a.reports();
2681 let reps_b = replica_b.reports();
2682 assert_eq!(
2683 reps_a[0].report.incumbent.outcomes, 1,
2684 "session should be counted exactly once"
2685 );
2686 assert_eq!(
2687 reps_b[0].report.incumbent.outcomes, 1,
2688 "both replicas should agree on outcome count"
2689 );
2690 }
2691
2692 #[tokio::test]
2699 async fn abstain_does_not_burn_session_slot_before_claim() {
2700 let shared = Arc::new(InMemoryExperimentLog::new());
2701 let cfg = config();
2702 let fp_str = fp();
2703
2704 let monitor = ExperimentMonitor::from_config(&cfg)
2705 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2706
2707 let counted = monitor
2709 .record_outcome(
2710 "t",
2711 0,
2712 &fp_str,
2713 Arm::Incumbent,
2714 "session-abstain",
2715 Verdict::Abstain,
2716 0.10,
2717 )
2718 .await;
2719 assert!(!counted, "Abstain must not be counted");
2720
2721 let counted = monitor
2723 .record_outcome(
2724 "t",
2725 0,
2726 &fp_str,
2727 Arm::Incumbent,
2728 "session-abstain",
2729 Verdict::Pass,
2730 0.10,
2731 )
2732 .await;
2733 assert!(counted, "Pass after Abstain must be counted");
2734
2735 let reps = monitor.reports();
2736 assert_eq!(
2737 reps[0].report.incumbent.outcomes, 1,
2738 "exactly one outcome must have been counted"
2739 );
2740 }
2741
2742 #[cfg(feature = "redis-cache")]
2747 async fn redis_cleanup(url: &str, unique_tenant: &str) {
2748 let client = redis::Client::open(url).expect("redis client");
2749 let mut conn = redis::aio::ConnectionManager::new(client)
2750 .await
2751 .expect("redis connection for cleanup");
2752 use redis::AsyncCommands as _;
2753 let pattern = format!("firstpass:exp*{unique_tenant}*");
2755 let keys: Vec<String> = {
2756 let mut out = Vec::new();
2757 let mut cursor: u64 = 0;
2758 loop {
2759 let (next, batch): (u64, Vec<String>) = redis::cmd("SCAN")
2760 .arg(cursor)
2761 .arg("MATCH")
2762 .arg(&pattern)
2763 .arg("COUNT")
2764 .arg(100u64)
2765 .query_async(&mut conn)
2766 .await
2767 .expect("SCAN");
2768 out.extend(batch);
2769 cursor = next;
2770 if cursor == 0 {
2771 break;
2772 }
2773 }
2774 out
2775 };
2776 if !keys.is_empty() {
2777 let _: () = conn.del(keys).await.expect("DEL cleanup");
2778 }
2779 }
2780
2781 #[cfg(feature = "redis-cache")]
2791 #[tokio::test]
2792 async fn redis_two_replicas_converge_to_identical_reports_after_sync() {
2793 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2794 panic!(
2795 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2796 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2797 is worse than a red one."
2798 )
2799 });
2800 let unique_tenant = format!("t-conv-{}", Uuid::now_v7());
2802 let log_a = Arc::new(
2803 RedisExperimentLog::connect(&url)
2804 .await
2805 .expect("RedisExperimentLog::connect"),
2806 );
2807 let log_b = Arc::new(
2808 RedisExperimentLog::connect(&url)
2809 .await
2810 .expect("RedisExperimentLog::connect"),
2811 );
2812 let cfg = config();
2813 let fp_str = fp();
2814
2815 let replica_a = ExperimentMonitor::from_config(&cfg)
2816 .with_log(Arc::clone(&log_a) as Arc<dyn ExperimentLogBackend>);
2817 let replica_b = ExperimentMonitor::from_config(&cfg)
2818 .with_log(Arc::clone(&log_b) as Arc<dyn ExperimentLogBackend>);
2819
2820 for i in 0..400 {
2822 replica_a
2823 .record_outcome(
2824 &unique_tenant,
2825 0,
2826 &fp_str,
2827 Arm::Incumbent,
2828 &format!("i{i}"),
2829 Verdict::Pass,
2830 0.10,
2831 )
2832 .await;
2833 replica_a
2834 .record_outcome(
2835 &unique_tenant,
2836 0,
2837 &fp_str,
2838 Arm::Challenger,
2839 &format!("c{i}"),
2840 Verdict::Fail,
2841 0.02,
2842 )
2843 .await;
2844 }
2845
2846 assert_eq!(
2848 replica_b.status(&unique_tenant, 0),
2849 ExperimentStatus::Undecided,
2850 "replica_b must be Undecided before syncing"
2851 );
2852
2853 let _ = replica_b
2855 .sync_from_backend(&unique_tenant, 0, &fp_str)
2856 .await;
2857 assert_eq!(
2858 replica_b.status(&unique_tenant, 0),
2859 ExperimentStatus::Halted,
2860 "replica_b must be Halted after syncing from Redis log"
2861 );
2862
2863 let reps_a = replica_a.reports();
2864 let reps_b = replica_b.reports();
2865 assert_eq!(reps_a.len(), 1);
2866 assert_eq!(reps_b.len(), 1);
2867 assert_eq!(
2868 reps_a[0].report.status, reps_b[0].report.status,
2869 "both replicas must agree on status"
2870 );
2871 assert_eq!(
2872 reps_a[0].report.pairs, reps_b[0].report.pairs,
2873 "both replicas must agree on pair counts"
2874 );
2875
2876 redis_cleanup(&url, &unique_tenant).await;
2877 }
2878
2879 #[cfg(feature = "redis-cache")]
2887 #[tokio::test]
2888 async fn redis_cross_replica_session_dedup_via_claim_session_holds() {
2889 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2890 panic!(
2891 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2892 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2893 is worse than a red one."
2894 )
2895 });
2896 let unique_tenant = format!("t-dedup-{}", Uuid::now_v7());
2897 let log = Arc::new(
2898 RedisExperimentLog::connect(&url)
2899 .await
2900 .expect("RedisExperimentLog::connect"),
2901 );
2902 let cfg = config();
2903 let fp_str = fp();
2904
2905 let replica_a = ExperimentMonitor::from_config(&cfg)
2906 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2907 let replica_b = ExperimentMonitor::from_config(&cfg)
2908 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2909
2910 replica_a
2912 .record_outcome(
2913 &unique_tenant,
2914 0,
2915 &fp_str,
2916 Arm::Incumbent,
2917 "session-x",
2918 Verdict::Pass,
2919 0.10,
2920 )
2921 .await;
2922 replica_b
2923 .record_outcome(
2924 &unique_tenant,
2925 0,
2926 &fp_str,
2927 Arm::Incumbent,
2928 "session-x",
2929 Verdict::Pass,
2930 0.10,
2931 )
2932 .await;
2933
2934 let _ = replica_a
2936 .sync_from_backend(&unique_tenant, 0, &fp_str)
2937 .await;
2938 let _ = replica_b
2939 .sync_from_backend(&unique_tenant, 0, &fp_str)
2940 .await;
2941
2942 let reps_a = replica_a.reports();
2943 let reps_b = replica_b.reports();
2944 assert_eq!(
2945 reps_a[0].report.incumbent.outcomes, 1,
2946 "session must be counted exactly once on replica A"
2947 );
2948 assert_eq!(
2949 reps_b[0].report.incumbent.outcomes, 1,
2950 "session must be counted exactly once on replica B"
2951 );
2952
2953 redis_cleanup(&url, &unique_tenant).await;
2954 }
2955
2956 #[cfg(feature = "redis-cache")]
2964 #[tokio::test]
2965 async fn redis_halt_on_one_replica_blocks_challenger_on_other() {
2966 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2967 panic!(
2968 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2969 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2970 is worse than a red one."
2971 )
2972 });
2973 let unique_tenant = format!("t-halt-{}", Uuid::now_v7());
2974 let log = Arc::new(
2975 RedisExperimentLog::connect(&url)
2976 .await
2977 .expect("RedisExperimentLog::connect"),
2978 );
2979 let cfg = config();
2980 let fp_str = fp();
2981
2982 let replica_a = ExperimentMonitor::from_config(&cfg)
2983 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2984 let replica_b = ExperimentMonitor::from_config(&cfg)
2985 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2986
2987 assert!(
2989 !replica_b.challenger_blocked(&unique_tenant, 0),
2990 "replica_b must be unblocked before any evidence"
2991 );
2992
2993 let mut halted = false;
2995 for i in 0..500 {
2996 replica_a
2997 .record_outcome(
2998 &unique_tenant,
2999 0,
3000 &fp_str,
3001 Arm::Incumbent,
3002 &format!("i{i}"),
3003 Verdict::Pass,
3004 0.10,
3005 )
3006 .await;
3007 replica_a
3008 .record_outcome(
3009 &unique_tenant,
3010 0,
3011 &fp_str,
3012 Arm::Challenger,
3013 &format!("c{i}"),
3014 Verdict::Fail,
3015 0.02,
3016 )
3017 .await;
3018 if replica_a.status(&unique_tenant, 0) == ExperimentStatus::Halted {
3019 halted = true;
3020 break;
3021 }
3022 }
3023 assert!(halted, "experiment never halted on replica A");
3024
3025 assert!(
3027 !replica_b.challenger_blocked(&unique_tenant, 0),
3028 "replica_b must not block before syncing"
3029 );
3030
3031 let _ = replica_b
3033 .sync_from_backend(&unique_tenant, 0, &fp_str)
3034 .await;
3035 assert!(
3036 replica_b.challenger_blocked(&unique_tenant, 0),
3037 "replica_b must block the challenger after syncing the halt from Redis"
3038 );
3039
3040 redis_cleanup(&url, &unique_tenant).await;
3041 }
3042
3043 #[cfg(feature = "redis-cache")]
3051 #[tokio::test]
3052 async fn redis_keys_are_isolated_per_tenant() {
3053 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
3054 panic!(
3055 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
3056 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
3057 is worse than a red one."
3058 )
3059 });
3060 let run_id = Uuid::now_v7();
3061 let tenant_a = format!("t-iso-a-{run_id}");
3062 let tenant_b = format!("t-iso-b-{run_id}");
3063 let log = Arc::new(
3064 RedisExperimentLog::connect(&url)
3065 .await
3066 .expect("RedisExperimentLog::connect"),
3067 );
3068 let cfg = config();
3069 let fp_str = fp();
3070
3071 let monitor_a = ExperimentMonitor::from_config(&cfg)
3072 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
3073 let monitor_b = ExperimentMonitor::from_config(&cfg)
3074 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
3075
3076 for i in 0..5 {
3078 monitor_a
3079 .record_outcome(
3080 &tenant_a,
3081 0,
3082 &fp_str,
3083 Arm::Incumbent,
3084 &format!("s{i}"),
3085 Verdict::Pass,
3086 0.10,
3087 )
3088 .await;
3089 }
3090
3091 let records = log
3093 .load_since(&tenant_b, 0, &fp_str, 0)
3094 .await
3095 .expect("load_since must not fail on a reachable Redis");
3096 assert!(
3097 records.is_empty(),
3098 "backend must return no records for tenant_b"
3099 );
3100
3101 assert_eq!(
3104 monitor_b.status(&tenant_b, 0),
3105 ExperimentStatus::Undecided,
3106 "evidence written under tenant_a must not appear under tenant_b"
3107 );
3108
3109 let reps = monitor_a.reports();
3111 assert_eq!(reps.len(), 1);
3112 assert_eq!(reps[0].report.incumbent.outcomes, 5);
3113
3114 redis_cleanup(&url, &tenant_a).await;
3116 redis_cleanup(&url, &tenant_b).await;
3117 }
3118
3119 #[test]
3127 fn old_log_record_pair_deserializes_without_new_fields() {
3128 let json = r#"{"kind":"pair","x_inc":1.0,"x_ch":0.5,"incumbent_cost_usd":0.1,"challenger_cost_usd":0.01}"#;
3129 let record: LogRecord = serde_json::from_str(json).unwrap();
3130 match record {
3131 LogRecord::Pair {
3132 ch_error,
3133 ch_truncated,
3134 ts_unix,
3135 x_inc,
3136 x_ch,
3137 replica_id,
3138 ..
3139 } => {
3140 assert!(!ch_error, "ch_error defaults to false");
3141 assert_eq!(ch_truncated, 0, "ch_truncated defaults to 0");
3142 assert_eq!(ts_unix, 0, "ts_unix defaults to 0");
3143 assert!((x_inc - 1.0).abs() < f64::EPSILON);
3144 assert!((x_ch - 0.5).abs() < f64::EPSILON);
3145 assert_eq!(
3148 replica_id,
3149 uuid::Uuid::nil(),
3150 "missing replica_id defaults to nil"
3151 );
3152 }
3153 _ => panic!("expected Pair variant"),
3154 }
3155 }
3156
3157 #[tokio::test]
3163 async fn pair_health_persists_and_rebuilds_from_store() {
3164 let db =
3165 std::env::temp_dir().join(format!("firstpass-health-persist-{}.db", Uuid::now_v7()));
3166 let (tx, handle) = crate::store::open(&db).unwrap();
3167 drop(tx);
3168 handle.await.unwrap();
3169 let cfg = config();
3170 let fp_str = fp();
3171 crate::store::append_experiment_pair(
3173 &db,
3174 &crate::store::ExperimentPairRow {
3175 tenant: "t".to_owned(),
3176 route_ix: 0,
3177 fingerprint: fp_str.clone(),
3178 trace_id: Uuid::now_v7(),
3179 x_inc: 1.0,
3180 x_ch: 1.0,
3181 challenger_cost_usd: 0.01,
3182 ch_error: true,
3183 ch_truncated: 2,
3184 ts: jiff::Timestamp::now(),
3185 in_log: crate::store::InLogStatus::InLog,
3186 },
3187 )
3188 .unwrap();
3189 let m = build_monitor(&db.to_string_lossy(), &cfg);
3191 let reps = m.reports();
3192 assert_eq!(reps.len(), 1, "one experiment entry");
3193 assert_eq!(reps[0].health.challenger.errors, 1, "ch_error rebuilt");
3194 assert_eq!(
3195 reps[0].health.challenger.truncated, 2,
3196 "ch_truncated rebuilt"
3197 );
3198 let _ = std::fs::remove_file(&db);
3199 }
3200
3201 #[derive(Debug)]
3213 struct DownLog;
3214
3215 #[async_trait::async_trait]
3216 impl ExperimentLogBackend for DownLog {
3217 async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
3218 Err(LogError::Backend("simulated outage".to_owned()))
3219 }
3220 async fn load_since(
3221 &self,
3222 _: &str,
3223 _: usize,
3224 _: &str,
3225 _: u64,
3226 ) -> Result<Vec<Option<LogRecord>>, LogError> {
3227 Err(LogError::Backend("simulated outage".to_owned()))
3228 }
3229 async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
3230 false
3231 }
3232 }
3233
3234 #[derive(Debug)]
3237 struct AppendOnlyLog(InMemoryExperimentLog);
3238
3239 #[async_trait::async_trait]
3240 impl ExperimentLogBackend for AppendOnlyLog {
3241 async fn append(
3242 &self,
3243 tenant: &str,
3244 route_ix: usize,
3245 fingerprint: &str,
3246 record: LogRecord,
3247 ) -> Result<u64, LogError> {
3248 self.0.append(tenant, route_ix, fingerprint, record).await
3249 }
3250 async fn load_since(
3251 &self,
3252 _: &str,
3253 _: usize,
3254 _: &str,
3255 _: u64,
3256 ) -> Result<Vec<Option<LogRecord>>, LogError> {
3257 Err(LogError::Backend("simulated read failure".to_owned()))
3258 }
3259 async fn claim_session(
3260 &self,
3261 tenant: &str,
3262 route_ix: usize,
3263 fingerprint: &str,
3264 session_id: &str,
3265 ) -> bool {
3266 self.0
3267 .claim_session(tenant, route_ix, fingerprint, session_id)
3268 .await
3269 }
3270 }
3271
3272 #[tokio::test]
3276 async fn paired_spend_counts_against_the_cap_when_the_shared_log_is_down() {
3277 let ledger = Arc::new(crate::shadow::ShadowLedger::new());
3278 let m = ExperimentMonitor::from_config(&config())
3279 .with_log(Arc::new(DownLog) as Arc<dyn ExperimentLogBackend>)
3280 .with_paired_ledger(Arc::clone(&ledger));
3281 let now = jiff::Timestamp::now();
3282
3283 ledger.debit("t", 0, 0.7, now);
3287 m.observe_health("t", 0, &fp(), Arm::Challenger, true, 2);
3288
3289 let accepted = m
3290 .record_pair(
3291 "t",
3292 0,
3293 &fp(),
3294 1.0,
3295 0.0,
3296 0.1,
3297 0.7,
3298 true,
3299 2,
3300 now.as_second(),
3301 Uuid::nil(),
3302 Uuid::nil(),
3303 )
3304 .await;
3305 assert!(!accepted, "nothing reached the log");
3306 assert!(
3307 (ledger.spent_today("t", 0, now) - 0.7).abs() < 1e-9,
3308 "spend debited locally"
3309 );
3310 assert!(
3311 !ledger.may_spend("t", 0, 0.5, now),
3312 "cap enforced during the outage"
3313 );
3314 let h = m.reports()[0].health.challenger;
3315 assert_eq!((h.errors, h.truncated), (1, 2), "{h:?}");
3316 }
3317
3318 #[tokio::test]
3322 async fn paired_spend_enforced_when_append_succeeds_but_read_fails() {
3323 let ledger = Arc::new(crate::shadow::ShadowLedger::new());
3324 let m = ExperimentMonitor::from_config(&config())
3325 .with_log(Arc::new(AppendOnlyLog(InMemoryExperimentLog::new()))
3326 as Arc<dyn ExperimentLogBackend>)
3327 .with_paired_ledger(Arc::clone(&ledger));
3328 let now = jiff::Timestamp::now();
3329
3330 ledger.debit("t", 0, 0.6, now);
3332 m.observe_health("t", 0, &fp(), Arm::Challenger, false, 1);
3333
3334 let accepted = m
3336 .record_pair(
3337 "t",
3338 0,
3339 &fp(),
3340 1.0,
3341 1.0,
3342 0.1,
3343 0.6,
3344 false,
3345 1,
3346 now.as_second(),
3347 Uuid::nil(),
3348 Uuid::nil(),
3349 )
3350 .await;
3351 assert!(accepted, "pair was appended to log");
3357
3358 assert!(
3360 (ledger.spent_today("t", 0, now) - 0.6).abs() < 1e-9,
3361 "local spend = 0.6"
3362 );
3363 assert!(
3364 !ledger.may_spend("t", 0, 0.5, now),
3365 "cap enforced by local debit even when read-back fails"
3366 );
3367 let h = m.reports()[0].health.challenger;
3368 assert_eq!((h.errors, h.truncated), (0, 1), "{h:?}");
3369 }
3370
3371 #[tokio::test]
3384 async fn two_replicas_health_and_spend_count_once() {
3385 let shared = Arc::new(InMemoryExperimentLog::new());
3386 let cfg = config();
3387 let fp_str = fp();
3388 let now_ts = jiff::Timestamp::now();
3389 let now_secs = now_ts.as_second();
3390
3391 let ledger_a = Arc::new(crate::shadow::ShadowLedger::new());
3392 let ledger_b = Arc::new(crate::shadow::ShadowLedger::new());
3393
3394 let replica_a = ExperimentMonitor::from_config(&cfg)
3395 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3396 .with_paired_ledger(Arc::clone(&ledger_a));
3397 let replica_b = ExperimentMonitor::from_config(&cfg)
3398 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3399 .with_paired_ledger(Arc::clone(&ledger_b));
3400
3401 assert!(
3402 replica_a.has_log(),
3403 "log must be attached for this test to be meaningful"
3404 );
3405
3406 ledger_a.debit("t", 0, 0.8, now_ts);
3410 replica_a.observe_health("t", 0, &fp_str, Arm::Challenger, true, 1);
3411
3412 let accepted = replica_a
3413 .record_pair(
3414 "t",
3415 0,
3416 &fp_str,
3417 1.0,
3418 0.5,
3419 0.1,
3420 0.8,
3421 true,
3422 1,
3423 now_secs,
3424 Uuid::nil(),
3425 Uuid::nil(),
3426 )
3427 .await;
3428 assert!(accepted, "pair must be accepted");
3429
3430 let reps_a = replica_a.reports();
3433 assert_eq!(
3434 reps_a[0].health.challenger.errors, 1,
3435 "A: challenger error counted once, not zero or doubled"
3436 );
3437 assert_eq!(
3438 reps_a[0].health.challenger.truncated, 1,
3439 "A: challenger truncated counted once"
3440 );
3441
3442 let a_spent = ledger_a.spent_today("t", 0, now_ts);
3444 assert!(
3445 (a_spent - 0.8).abs() < 1e-12,
3446 "A: ledger = 0.8 (not doubled or zero); got {a_spent}"
3447 );
3448
3449 assert_eq!(
3451 ledger_b.spent_today("t", 0, now_ts),
3452 0.0,
3453 "B: no spend before sync"
3454 );
3455
3456 let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
3459
3460 let reps_b = replica_b.reports();
3461 assert_eq!(
3462 reps_b[0].health.challenger.errors, 1,
3463 "B: challenger error visible after sync"
3464 );
3465 let b_spent = ledger_b.spent_today("t", 0, now_ts);
3466 assert!(
3467 (b_spent - 0.8).abs() < 1e-12,
3468 "B: ledger reflects A's spend after sync; got {b_spent}"
3469 );
3470
3471 ledger_a.debit("t", 0, 0.3, now_ts);
3473 replica_a.observe_health("t", 0, &fp_str, Arm::Challenger, false, 0);
3474 replica_a
3475 .record_pair(
3476 "t",
3477 0,
3478 &fp_str,
3479 1.0,
3480 0.5,
3481 0.1,
3482 0.3,
3483 false,
3484 0,
3485 now_secs,
3486 Uuid::nil(),
3487 Uuid::nil(),
3488 )
3489 .await;
3490
3491 let a_total = ledger_a.spent_today("t", 0, now_ts);
3492 assert!(
3493 (a_total - 1.1).abs() < 1e-12,
3494 "A: cluster total = 1.1; got {a_total}"
3495 );
3496 assert!(
3497 !ledger_a.may_spend("t", 0, 1.0, now_ts),
3498 "A: cap exhausted (1.1 > 1.0)"
3499 );
3500
3501 let _ = replica_b.sync_from_backend("t", 0, &fp_str).await;
3503 assert!(
3504 !ledger_b.may_spend("t", 0, 1.0, now_ts),
3505 "B: cap exhausted from cluster spend"
3506 );
3507 }
3508
3509 #[derive(Debug)]
3515 struct YieldingLog(InMemoryExperimentLog);
3516
3517 #[async_trait::async_trait]
3518 impl ExperimentLogBackend for YieldingLog {
3519 async fn append(
3520 &self,
3521 tenant: &str,
3522 route_ix: usize,
3523 fingerprint: &str,
3524 record: LogRecord,
3525 ) -> Result<u64, LogError> {
3526 self.0.append(tenant, route_ix, fingerprint, record).await
3527 }
3528
3529 async fn load_since(
3530 &self,
3531 tenant: &str,
3532 route_ix: usize,
3533 fingerprint: &str,
3534 offset: u64,
3535 ) -> Result<Vec<Option<LogRecord>>, LogError> {
3536 let records = self
3537 .0
3538 .load_since(tenant, route_ix, fingerprint, offset)
3539 .await?;
3540 tokio::task::yield_now().await;
3542 Ok(records)
3543 }
3544
3545 async fn claim_session(
3546 &self,
3547 tenant: &str,
3548 route_ix: usize,
3549 fingerprint: &str,
3550 session_id: &str,
3551 ) -> bool {
3552 self.0
3553 .claim_session(tenant, route_ix, fingerprint, session_id)
3554 .await
3555 }
3556 }
3557
3558 #[tokio::test]
3570 async fn concurrent_sync_does_not_double_count_pairs() {
3571 let shared = Arc::new(YieldingLog(InMemoryExperimentLog::new()));
3572 let cfg = config();
3573 let fp_str = fp();
3574
3575 for _ in 0..3 {
3577 let _ = shared
3578 .append(
3579 "t",
3580 0,
3581 &fp_str,
3582 LogRecord::Pair {
3583 x_inc: 1.0,
3584 x_ch: 1.0,
3585 incumbent_cost_usd: 0.10,
3586 challenger_cost_usd: 0.05,
3587 ch_error: true,
3588 ch_truncated: 1,
3589 ts_unix: 0,
3590 replica_id: uuid::Uuid::nil(),
3593 trace_id: uuid::Uuid::nil(),
3595 incumbent_trace_id: Uuid::nil(),
3596 },
3597 )
3598 .await;
3599 }
3600
3601 let monitor = Arc::new(
3602 ExperimentMonitor::from_config(&cfg)
3603 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>),
3604 );
3605
3606 let m1 = Arc::clone(&monitor);
3609 let m2 = Arc::clone(&monitor);
3610 let fp1 = fp_str.clone();
3611 let fp2 = fp_str.clone();
3612 tokio::join!(
3613 async move {
3614 let _ = m1.sync_from_backend("t", 0, &fp1).await;
3615 },
3616 async move {
3617 let _ = m2.sync_from_backend("t", 0, &fp2).await;
3618 },
3619 );
3620
3621 let snaps = monitor.reports();
3622 assert_eq!(snaps.len(), 1);
3623 let snap = &snaps[0];
3624 assert_eq!(
3625 snap.report.pairs, 3,
3626 "pairs must be 3, not double-counted: got {}",
3627 snap.report.pairs
3628 );
3629 assert_eq!(
3630 snap.health.challenger.errors, 3,
3631 "errors must be 3, not double-counted: got {}",
3632 snap.health.challenger.errors
3633 );
3634 assert_eq!(
3635 snap.health.challenger.truncated, 3,
3636 "truncated must be 3, not double-counted: got {}",
3637 snap.health.challenger.truncated
3638 );
3639 }
3640
3641 #[tokio::test]
3650 async fn ensure_synced_cold_start_respects_cluster_cap() {
3651 let shared = Arc::new(InMemoryExperimentLog::new());
3652 let cfg = config();
3653 let fp_str = fp();
3654 let now_ts = jiff::Timestamp::now();
3655 let cap = 0.5_f64;
3656
3657 let ledger_a = Arc::new(crate::shadow::ShadowLedger::new());
3658 let replica_a = ExperimentMonitor::from_config(&cfg)
3661 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3662 .with_paired_ledger(Arc::clone(&ledger_a));
3663 ledger_a.debit("t", 0, cap, now_ts);
3664 replica_a
3665 .record_pair(
3666 "t",
3667 0,
3668 &fp_str,
3669 1.0,
3670 1.0,
3671 0.10,
3672 cap,
3673 false,
3674 0,
3675 now_ts.as_second(),
3676 Uuid::nil(),
3677 Uuid::nil(),
3678 )
3679 .await;
3680 assert!(
3681 !ledger_a.may_spend("t", 0, cap, now_ts),
3682 "A: cap exhausted after recording one pair"
3683 );
3684
3685 let ledger_b = Arc::new(crate::shadow::ShadowLedger::new());
3687 let replica_b = ExperimentMonitor::from_config(&cfg)
3688 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>)
3689 .with_paired_ledger(Arc::clone(&ledger_b));
3690
3691 assert!(
3693 ledger_b.may_spend("t", 0, cap, now_ts),
3694 "B: before ensure_synced, empty ledger allows spend (expected)"
3695 );
3696
3697 let synced = replica_b.ensure_synced("t", 0, &fp_str).await;
3700 assert!(
3701 synced,
3702 "ensure_synced must return true when the log is healthy"
3703 );
3704 let may_spend_after_sync = ledger_b.may_spend("t", 0, cap, now_ts);
3705 assert!(
3706 !may_spend_after_sync,
3707 "B: after ensure_synced, cluster spend at cap — must refuse"
3708 );
3709 }
3710
3711 #[tokio::test]
3719 async fn ensure_synced_burst_waits_for_synced_state() {
3720 let shared = Arc::new(YieldingLog(InMemoryExperimentLog::new()));
3721 let cfg = config();
3722 let fp_str = fp();
3723 let replica_a = ExperimentMonitor::from_config(&cfg)
3724 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3725 for i in 0..500 {
3726 replica_a
3727 .record_outcome(
3728 "t",
3729 0,
3730 &fp_str,
3731 Arm::Incumbent,
3732 &format!("i{i}"),
3733 Verdict::Pass,
3734 0.10,
3735 )
3736 .await;
3737 replica_a
3738 .record_outcome(
3739 "t",
3740 0,
3741 &fp_str,
3742 Arm::Challenger,
3743 &format!("c{i}"),
3744 Verdict::Fail,
3745 0.02,
3746 )
3747 .await;
3748 if replica_a.status("t", 0) == ExperimentStatus::Halted {
3749 break;
3750 }
3751 }
3752 assert_eq!(replica_a.status("t", 0), ExperimentStatus::Halted);
3753
3754 let replica_b = ExperimentMonitor::from_config(&cfg)
3755 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3756 let (first, second) = tokio::join!(
3757 async {
3758 replica_b.ensure_synced("t", 0, &fp_str).await;
3759 replica_b.challenger_blocked("t", 0)
3760 },
3761 async {
3762 replica_b.ensure_synced("t", 0, &fp_str).await;
3763 replica_b.challenger_blocked("t", 0)
3764 },
3765 );
3766 assert!(
3767 first && second,
3768 "both burst requests see the halt: {first} {second}"
3769 );
3770 }
3771
3772 #[tokio::test]
3773 async fn ensure_synced_cold_start_respects_cluster_halt() {
3774 let shared = Arc::new(InMemoryExperimentLog::new());
3775 let cfg = config();
3776 let fp_str = fp();
3777
3778 let replica_a = ExperimentMonitor::from_config(&cfg)
3779 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3780
3781 let mut halted = false;
3783 for i in 0..500 {
3784 replica_a
3785 .record_outcome(
3786 "t",
3787 0,
3788 &fp_str,
3789 Arm::Incumbent,
3790 &format!("i{i}"),
3791 Verdict::Pass,
3792 0.10,
3793 )
3794 .await;
3795 replica_a
3796 .record_outcome(
3797 "t",
3798 0,
3799 &fp_str,
3800 Arm::Challenger,
3801 &format!("c{i}"),
3802 Verdict::Fail,
3803 0.02,
3804 )
3805 .await;
3806 if replica_a.status("t", 0) == ExperimentStatus::Halted {
3807 halted = true;
3808 break;
3809 }
3810 }
3811 assert!(halted, "experiment never halted on replica A");
3812
3813 let replica_b = ExperimentMonitor::from_config(&cfg)
3815 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
3816
3817 assert!(
3819 !replica_b.challenger_blocked("t", 0),
3820 "B: before ensure_synced, halt unknown (expected)"
3821 );
3822
3823 let synced = replica_b.ensure_synced("t", 0, &fp_str).await;
3825 assert!(synced, "ensure_synced must succeed with healthy log");
3826 assert!(
3827 replica_b.challenger_blocked("t", 0),
3828 "B: after ensure_synced, halt must block the challenger"
3829 );
3830 }
3831
3832 #[tokio::test]
3838 async fn restart_with_shared_log_replays_offline_pairs_and_skips_log_pairs() {
3839 let db =
3840 std::env::temp_dir().join(format!("firstpass-restart-{}.db", uuid::Uuid::now_v7()));
3841 let (tx, handle) = crate::store::open(&db).unwrap();
3843 drop(tx);
3844 handle.await.unwrap();
3845 let cfg = config();
3846 let fp_str = fp();
3847 let now_ts = jiff::Timestamp::now();
3848
3849 crate::store::append_experiment_pair(
3851 &db,
3852 &crate::store::ExperimentPairRow {
3853 tenant: "t".to_owned(),
3854 route_ix: 0,
3855 fingerprint: fp_str.clone(),
3856 trace_id: uuid::Uuid::now_v7(),
3857 x_inc: 1.0,
3858 x_ch: 0.0,
3859 challenger_cost_usd: 0.5,
3860 ch_error: false,
3861 ch_truncated: 0,
3862 ts: now_ts,
3863 in_log: crate::store::InLogStatus::NotInLog,
3864 },
3865 )
3866 .unwrap();
3867
3868 crate::store::append_experiment_pair(
3870 &db,
3871 &crate::store::ExperimentPairRow {
3872 tenant: "t".to_owned(),
3873 route_ix: 0,
3874 fingerprint: fp_str.clone(),
3875 trace_id: uuid::Uuid::now_v7(),
3876 x_inc: 1.0,
3877 x_ch: 0.0,
3878 challenger_cost_usd: 0.3,
3879 ch_error: false,
3880 ch_truncated: 0,
3881 ts: now_ts,
3882 in_log: crate::store::InLogStatus::InLog,
3883 },
3884 )
3885 .unwrap();
3886
3887 let m = build_monitor_with(&db.to_string_lossy(), &cfg, true);
3889 let reps = m.reports();
3890 assert_eq!(
3891 reps[0].report.pairs, 1,
3892 "only in_log=false row replayed; got {}",
3893 reps[0].report.pairs
3894 );
3895
3896 let m2 = build_monitor_with(&db.to_string_lossy(), &cfg, false);
3898 let reps2 = m2.reports();
3899 assert_eq!(
3900 reps2[0].report.pairs, 2,
3901 "shared_log=false replays all rows; got {}",
3902 reps2[0].report.pairs
3903 );
3904
3905 let _ = std::fs::remove_file(&db);
3906 }
3907
3908 #[derive(Debug)]
3912 struct CountingFailLog {
3913 load_count: Arc<std::sync::atomic::AtomicUsize>,
3914 }
3915
3916 #[async_trait::async_trait]
3917 impl ExperimentLogBackend for CountingFailLog {
3918 async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
3919 Err(LogError::Backend("down".to_owned()))
3920 }
3921 async fn load_since(
3922 &self,
3923 _: &str,
3924 _: usize,
3925 _: &str,
3926 _: u64,
3927 ) -> Result<Vec<Option<LogRecord>>, LogError> {
3928 self.load_count
3929 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
3930 Err(LogError::Backend("down".to_owned()))
3931 }
3932 async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
3933 false
3934 }
3935 }
3936
3937 #[tokio::test]
3941 async fn ensure_synced_failure_burst_issues_exactly_one_read() {
3942 let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3943 let m = Arc::new(ExperimentMonitor::from_config(&config()).with_log(Arc::new(
3944 CountingFailLog {
3945 load_count: Arc::clone(&load_count),
3946 },
3947 )
3948 as Arc<dyn ExperimentLogBackend>));
3949 let fp_str = fp();
3950
3951 let handles: Vec<_> = (0..10)
3953 .map(|_| {
3954 let m = Arc::clone(&m);
3955 let fp_str = fp_str.clone();
3956 tokio::spawn(async move { m.ensure_synced("t", 0, &fp_str).await })
3957 })
3958 .collect();
3959 for h in handles {
3960 let result = h.await.unwrap();
3961 assert!(!result, "every caller gets false on backend failure");
3962 }
3963
3964 assert_eq!(
3965 load_count.load(std::sync::atomic::Ordering::SeqCst),
3966 1,
3967 "exactly one backend read issued during a failure burst"
3968 );
3969 }
3970
3971 #[tokio::test]
3973 async fn ensure_synced_cooldown_skips_retry_within_window() {
3974 let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3975 let m = ExperimentMonitor::from_config(&config())
3976 .with_log(Arc::new(CountingFailLog {
3977 load_count: Arc::clone(&load_count),
3978 }) as Arc<dyn ExperimentLogBackend>);
3979 let fp_str = fp();
3980
3981 assert!(!m.ensure_synced("t", 0, &fp_str).await);
3983 assert_eq!(load_count.load(std::sync::atomic::Ordering::SeqCst), 1);
3984
3985 assert!(!m.ensure_synced("t", 0, &fp_str).await);
3987 assert_eq!(
3988 load_count.load(std::sync::atomic::Ordering::SeqCst),
3989 1,
3990 "no additional read within the cooldown window"
3991 );
3992 }
3993
3994 #[tokio::test]
3996 async fn ensure_synced_cooldown_allows_retry_after_expiry() {
3997 tokio::time::pause();
3998 let load_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3999 let m = ExperimentMonitor::from_config(&config())
4000 .with_log(Arc::new(CountingFailLog {
4001 load_count: Arc::clone(&load_count),
4002 }) as Arc<dyn ExperimentLogBackend>);
4003 let fp_str = fp();
4004
4005 assert!(!m.ensure_synced("t", 0, &fp_str).await);
4007 assert_eq!(load_count.load(std::sync::atomic::Ordering::SeqCst), 1);
4008
4009 tokio::time::advance(SYNC_FAILURE_COOLDOWN + std::time::Duration::from_millis(1)).await;
4011
4012 assert!(!m.ensure_synced("t", 0, &fp_str).await);
4014 assert_eq!(
4015 load_count.load(std::sync::atomic::Ordering::SeqCst),
4016 2,
4017 "one more read after the cooldown expires"
4018 );
4019 }
4020
4021 #[tokio::test]
4028 async fn record_pair_does_not_apply_evidence_on_append_failure_caller_does() {
4029 let m = ExperimentMonitor::from_config(&config())
4030 .with_log(Arc::new(DownLog) as Arc<dyn ExperimentLogBackend>);
4031 let fp_str = fp();
4032
4033 let mut pairs_applied = 0u32;
4034 for _ in 0..500 {
4035 if m.challenger_blocked("t", 0) {
4036 break;
4037 }
4038 let accepted = m
4040 .record_pair(
4041 "t",
4042 0,
4043 &fp_str,
4044 1.0,
4045 0.0,
4046 0.1,
4047 0.01,
4048 false,
4049 0,
4050 0,
4051 Uuid::nil(),
4052 Uuid::nil(),
4053 )
4054 .await;
4055 assert!(!accepted, "append failing log returns false");
4056 m.apply_pair_locally("t", 0, &fp_str, 1.0, 0.0, 0.01, Uuid::nil());
4058 pairs_applied += 1;
4059 }
4060
4061 assert!(
4063 m.challenger_blocked("t", 0),
4064 "experiment must halt locally after {pairs_applied} pairs via apply_pair_locally"
4065 );
4066 assert!(
4067 m.reports()[0].report.pairs > 0,
4068 "pairs counted locally via apply_pair_locally"
4069 );
4070 }
4071
4072 #[tokio::test]
4076 async fn record_pair_no_log_does_not_apply_evidence() {
4077 let m = ExperimentMonitor::from_config(&config()); let fp_str = fp();
4079
4080 for _ in 0..10 {
4082 let ok = m
4083 .record_pair(
4084 "t",
4085 0,
4086 &fp_str,
4087 1.0,
4088 0.0,
4089 0.1,
4090 0.01,
4091 false,
4092 0,
4093 0,
4094 Uuid::nil(),
4095 Uuid::nil(),
4096 )
4097 .await;
4098 assert!(!ok, "no log → record_pair returns false");
4099 }
4100 assert_eq!(
4102 m.reports().len(),
4103 0,
4104 "record_pair in no-log mode must not create an entry or advance evidence"
4105 );
4106
4107 m.apply_pair_locally("t", 0, &fp_str, 1.0, 0.0, 0.01, Uuid::nil());
4109 assert_eq!(
4110 m.reports()[0].report.pairs,
4111 1,
4112 "apply_pair_locally counted the pair"
4113 );
4114 }
4115
4116 #[tokio::test]
4122 async fn unknown_rows_contribute_spend_but_not_evidence_with_shared_log() {
4123 use crate::store::InLogStatus;
4124 let db =
4125 std::env::temp_dir().join(format!("firstpass-unknown-{}.db", uuid::Uuid::now_v7()));
4126 let (tx, handle) = crate::store::open(&db).unwrap();
4127 drop(tx);
4128 handle.await.unwrap();
4129 let cfg = config();
4130 let fp_str = fp();
4131 let now_ts = jiff::Timestamp::now();
4132
4133 for (in_log, x_ch) in [
4135 (InLogStatus::Unknown, 0.8_f64),
4136 (InLogStatus::NotInLog, 0.6),
4137 ] {
4138 crate::store::append_experiment_pair(
4139 &db,
4140 &crate::store::ExperimentPairRow {
4141 tenant: "t".to_owned(),
4142 route_ix: 0,
4143 fingerprint: fp_str.clone(),
4144 trace_id: uuid::Uuid::now_v7(),
4145 x_inc: 1.0,
4146 x_ch,
4147 challenger_cost_usd: 0.1,
4148 ch_error: false,
4149 ch_truncated: 0,
4150 ts: now_ts,
4151 in_log,
4152 },
4153 )
4154 .unwrap();
4155 }
4156
4157 let m = build_monitor_with(&db.to_string_lossy(), &cfg, true);
4159 let reps = m.reports();
4160 assert_eq!(
4161 reps[0].report.pairs, 1,
4162 "shared_log=true: only NotInLog evidence replayed; got {}",
4163 reps[0].report.pairs
4164 );
4165
4166 let m2 = build_monitor_with(&db.to_string_lossy(), &cfg, false);
4168 let reps2 = m2.reports();
4169 assert_eq!(
4170 reps2[0].report.pairs, 2,
4171 "shared_log=false: all rows replayed; got {}",
4172 reps2[0].report.pairs
4173 );
4174
4175 let _ = std::fs::remove_file(&db);
4176 }
4177
4178 #[derive(Debug)]
4183 struct MalformedMiddleLog {
4184 inner: InMemoryExperimentLog,
4185 }
4186
4187 #[async_trait::async_trait]
4188 impl ExperimentLogBackend for MalformedMiddleLog {
4189 async fn append(
4190 &self,
4191 tenant: &str,
4192 route_ix: usize,
4193 fingerprint: &str,
4194 record: LogRecord,
4195 ) -> Result<u64, LogError> {
4196 self.inner
4197 .append(tenant, route_ix, fingerprint, record)
4198 .await
4199 }
4200
4201 async fn load_since(
4202 &self,
4203 tenant: &str,
4204 route_ix: usize,
4205 fingerprint: &str,
4206 offset: u64,
4207 ) -> Result<Vec<Option<LogRecord>>, LogError> {
4208 let mut records = self
4210 .inner
4211 .load_since(tenant, route_ix, fingerprint, offset)
4212 .await?;
4213 if records.len() >= 2 {
4216 records[1] = None;
4217 }
4218 Ok(records)
4219 }
4220
4221 async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4222 false
4223 }
4224 }
4225
4226 #[tokio::test]
4231 async fn malformed_log_entry_advances_offset_and_valid_entries_applied_once() {
4232 let log = Arc::new(MalformedMiddleLog {
4233 inner: InMemoryExperimentLog::new(),
4234 });
4235 let cfg = config();
4236 let fp_str = fp();
4237
4238 for _ in 0..3 {
4240 let _ = log
4241 .inner
4242 .append(
4243 "t",
4244 0,
4245 &fp_str,
4246 LogRecord::Pair {
4247 x_inc: 1.0,
4248 x_ch: 1.0,
4249 incumbent_cost_usd: 0.1,
4250 challenger_cost_usd: 0.05,
4251 ch_error: false,
4252 ch_truncated: 0,
4253 ts_unix: 0,
4254 replica_id: uuid::Uuid::nil(),
4255 trace_id: uuid::Uuid::nil(),
4256 incumbent_trace_id: Uuid::nil(),
4257 },
4258 )
4259 .await;
4260 }
4261 let m = ExperimentMonitor::from_config(&cfg)
4264 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4265
4266 let _ = m.sync_from_backend("t", 0, &fp_str).await;
4268 let pairs_after_first = m.reports()[0].report.pairs;
4269 assert_eq!(
4271 pairs_after_first, 2,
4272 "first sync: 2 valid entries applied, got {pairs_after_first}"
4273 );
4274
4275 let _ = m.sync_from_backend("t", 0, &fp_str).await;
4277 let pairs_after_second = m.reports()[0].report.pairs;
4278 assert_eq!(
4279 pairs_after_second, 2,
4280 "second sync: no re-application, pairs still 2, got {pairs_after_second}"
4281 );
4282
4283 let offset = {
4285 let map = m.lock();
4286 map.get("t")
4287 .and_then(|r| r.get(&0))
4288 .map_or(0, |e| e.remote_log_offset)
4289 };
4290 assert_eq!(
4291 offset, 3,
4292 "offset must equal list length after consuming all positions"
4293 );
4294 }
4295
4296 #[derive(Debug)]
4301 struct PushThenErrorLog {
4302 inner: InMemoryExperimentLog,
4303 }
4304
4305 #[async_trait::async_trait]
4306 impl ExperimentLogBackend for PushThenErrorLog {
4307 async fn append(
4308 &self,
4309 tenant: &str,
4310 route_ix: usize,
4311 fingerprint: &str,
4312 record: LogRecord,
4313 ) -> Result<u64, LogError> {
4314 let _ = self
4316 .inner
4317 .append(tenant, route_ix, fingerprint, record)
4318 .await;
4319 Err(LogError::Backend("timeout after write".to_owned()))
4321 }
4322
4323 async fn load_since(
4324 &self,
4325 tenant: &str,
4326 route_ix: usize,
4327 fingerprint: &str,
4328 offset: u64,
4329 ) -> Result<Vec<Option<LogRecord>>, LogError> {
4330 self.inner
4331 .load_since(tenant, route_ix, fingerprint, offset)
4332 .await
4333 }
4334
4335 async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4336 false
4337 }
4338 }
4339
4340 #[tokio::test]
4347 async fn trace_id_dedup_prevents_double_count_on_append_timeout() {
4348 let log = Arc::new(PushThenErrorLog {
4349 inner: InMemoryExperimentLog::new(),
4350 });
4351 let cfg = config();
4352 let fp_str = fp();
4353 let tid = Uuid::now_v7();
4354 let now_ts = jiff::Timestamp::now();
4355
4356 let ledger = Arc::new(crate::shadow::ShadowLedger::new());
4357 let m = ExperimentMonitor::from_config(&cfg)
4358 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>)
4359 .with_paired_ledger(Arc::clone(&ledger));
4360
4361 let appended = m
4365 .record_pair(
4366 "t",
4367 0,
4368 &fp_str,
4369 1.0,
4370 1.0,
4371 0.1,
4372 0.05,
4373 false,
4374 1,
4375 now_ts.as_second(),
4376 tid,
4377 Uuid::nil(),
4378 )
4379 .await;
4380 assert!(!appended, "append returned Err → record_pair returns false");
4381 assert_eq!(
4384 m.reports().len(),
4385 0,
4386 "no entry created yet by record_pair alone"
4387 );
4388
4389 ledger.debit("t", 0, 0.05, now_ts);
4392 m.observe_health("t", 0, &fp_str, Arm::Challenger, false, 1);
4393 m.apply_pair_locally("t", 0, &fp_str, 1.0, 1.0, 0.05, tid);
4394 assert_eq!(
4395 m.reports()[0].report.pairs,
4396 1,
4397 "evidence applied via apply_pair_locally"
4398 );
4399
4400 let _ = m.sync_from_backend("t", 0, &fp_str).await;
4405 assert_eq!(
4406 m.reports()[0].report.pairs,
4407 1,
4408 "after sync: pairs still 1 (trace_id dedup prevented double-count)"
4409 );
4410 let h = m.reports()[0].health.challenger;
4411 assert_eq!(
4412 (h.errors, h.truncated),
4413 (0, 1),
4414 "health counted exactly once (not re-applied by apply_log_record)"
4415 );
4416 assert!(
4417 (ledger.spent_today("t", 0, now_ts) - 0.05).abs() < 1e-9,
4418 "ledger spend counted exactly once (not re-seeded by apply_log_record)"
4419 );
4420 }
4421
4422 #[tokio::test]
4429 async fn ensure_synced_stale_after_bound_exceeded() {
4430 tokio::time::pause();
4431 let shared = Arc::new(InMemoryExperimentLog::new());
4432 let m = Arc::new(
4433 ExperimentMonitor::from_config(&config())
4434 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>),
4435 );
4436 let fp_str = fp();
4437
4438 assert!(
4440 m.ensure_synced("t", 0, &fp_str).await,
4441 "first sync must succeed"
4442 );
4443
4444 tokio::time::advance(SYNC_STALENESS_BOUND / 2).await;
4446 assert!(
4447 m.ensure_synced("t", 0, &fp_str).await,
4448 "still within staleness bound → true"
4449 );
4450
4451 tokio::time::advance(SYNC_STALENESS_BOUND).await;
4453
4454 assert!(
4456 m.ensure_synced("t", 0, &fp_str).await,
4457 "stale but log reachable → sync succeeds → true"
4458 );
4459
4460 let m_down = Arc::new(ExperimentMonitor::from_config(&config()).with_log(Arc::new(
4463 CountingFailLog {
4464 load_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
4465 },
4466 )
4467 as Arc<dyn ExperimentLogBackend>));
4468 assert!(
4470 !m_down.ensure_synced("t", 0, &fp_str).await,
4471 "log down on first sync → false"
4472 );
4473 assert!(
4475 !m_down.ensure_synced("t", 0, &fp_str).await,
4476 "cooldown active → false immediately"
4477 );
4478 }
4479
4480 #[tokio::test]
4485 async fn restart_counts_a_timed_out_pair_once_across_store_and_log() {
4486 let db =
4487 std::env::temp_dir().join(format!("firstpass-restart-dedup-{}.db", Uuid::now_v7()));
4488 crate::store::ensure_migrated(&db).unwrap();
4489 let log = Arc::new(InMemoryExperimentLog::new());
4490 let cfg = config();
4491 let fp_str = fp();
4492 let tid = Uuid::now_v7();
4493 let now = jiff::Timestamp::now();
4494 let before_restart = ExperimentMonitor::from_config(&cfg)
4496 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4497 log.append(
4498 "t",
4499 0,
4500 &fp_str,
4501 LogRecord::Pair {
4502 x_inc: 1.0,
4503 x_ch: 1.0,
4504 incumbent_cost_usd: 0.1,
4505 challenger_cost_usd: 0.05,
4506 ch_error: false,
4507 ch_truncated: 1,
4508 ts_unix: now.as_second(),
4509 replica_id: before_restart.replica_id,
4510 trace_id: tid,
4511 incumbent_trace_id: Uuid::nil(),
4512 },
4513 )
4514 .await
4515 .unwrap();
4516 crate::store::append_experiment_pair(
4518 &db,
4519 &crate::store::ExperimentPairRow {
4520 tenant: "t".to_owned(),
4521 route_ix: 0,
4522 fingerprint: fp_str.clone(),
4523 trace_id: tid,
4524 x_inc: 1.0,
4525 x_ch: 1.0,
4526 challenger_cost_usd: 0.05,
4527 ch_error: false,
4528 ch_truncated: 1,
4529 ts: now,
4530 in_log: crate::store::InLogStatus::NotInLog,
4531 },
4532 )
4533 .unwrap();
4534
4535 let ledger = Arc::new(crate::shadow::ShadowLedger::new());
4537 ledger.debit("t", 0, 0.05, now); let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
4539 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>)
4540 .with_paired_ledger(Arc::clone(&ledger));
4541 assert_ne!(restarted.replica_id, before_restart.replica_id);
4542 restarted.sync_from_backend("t", 0, &fp_str).await.unwrap();
4543
4544 let r = &restarted.reports()[0];
4545 assert_eq!(r.report.pairs, 1, "evidence once");
4546 assert_eq!(r.health.challenger.truncated, 1, "health once");
4547 assert!(
4548 (ledger.spent_today("t", 0, now) - 0.05).abs() < 1e-9,
4549 "spend once: {}",
4550 ledger.spent_today("t", 0, now)
4551 );
4552 let _ = std::fs::remove_file(&db);
4553 }
4554
4555 #[derive(Debug, Default)]
4557 struct SwitchableLog {
4558 inner: InMemoryExperimentLog,
4559 failing: std::sync::atomic::AtomicBool,
4560 reads: std::sync::atomic::AtomicUsize,
4561 }
4562
4563 #[async_trait::async_trait]
4564 impl ExperimentLogBackend for SwitchableLog {
4565 async fn append(
4566 &self,
4567 t: &str,
4568 r: usize,
4569 f: &str,
4570 rec: LogRecord,
4571 ) -> Result<u64, LogError> {
4572 self.inner.append(t, r, f, rec).await
4573 }
4574 async fn load_since(
4575 &self,
4576 t: &str,
4577 r: usize,
4578 f: &str,
4579 o: u64,
4580 ) -> Result<Vec<Option<LogRecord>>, LogError> {
4581 self.reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
4582 if self.failing.load(std::sync::atomic::Ordering::SeqCst) {
4583 return Err(LogError::Backend("down".to_owned()));
4584 }
4585 self.inner.load_since(t, r, f, o).await
4586 }
4587 async fn claim_session(&self, t: &str, r: usize, f: &str, s: &str) -> bool {
4588 self.inner.claim_session(t, r, f, s).await
4589 }
4590 }
4591
4592 #[tokio::test]
4596 async fn read_back_failure_applies_the_pair_locally_and_recovery_does_not_double_count() {
4597 let log = Arc::new(SwitchableLog::default());
4598 let m = ExperimentMonitor::from_config(&config())
4599 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4600 let fp_str = fp();
4601 log.failing.store(true, std::sync::atomic::Ordering::SeqCst);
4602 let tid = Uuid::now_v7();
4603 let appended = m
4604 .record_pair(
4605 "t",
4606 0,
4607 &fp_str,
4608 1.0,
4609 0.0,
4610 0.1,
4611 0.05,
4612 false,
4613 0,
4614 0,
4615 tid,
4616 Uuid::nil(),
4617 )
4618 .await;
4619 assert!(appended, "the append itself succeeded");
4620 assert_eq!(
4621 m.reports()[0].report.pairs,
4622 1,
4623 "applied locally despite the failed read"
4624 );
4625 log.failing
4626 .store(false, std::sync::atomic::Ordering::SeqCst);
4627 m.sync_from_backend("t", 0, &fp_str).await.unwrap();
4628 assert_eq!(
4629 m.reports()[0].report.pairs,
4630 1,
4631 "recovery sync deduplicated by trace_id"
4632 );
4633 }
4634
4635 #[derive(Debug)]
4637 struct HungAppendLog;
4638
4639 #[async_trait::async_trait]
4640 impl ExperimentLogBackend for HungAppendLog {
4641 async fn append(&self, _: &str, _: usize, _: &str, _: LogRecord) -> Result<u64, LogError> {
4642 std::future::pending().await
4643 }
4644 async fn load_since(
4645 &self,
4646 _: &str,
4647 _: usize,
4648 _: &str,
4649 _: u64,
4650 ) -> Result<Vec<Option<LogRecord>>, LogError> {
4651 Ok(vec![])
4652 }
4653 async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4654 false
4655 }
4656 }
4657
4658 #[tokio::test]
4661 async fn hung_append_is_bounded_and_reported_as_not_in_log() {
4662 tokio::time::pause();
4663 let m = ExperimentMonitor::from_config(&config()).with_log(Arc::new(HungAppendLog));
4664 let appended = m
4665 .record_pair(
4666 "t",
4667 0,
4668 &fp(),
4669 1.0,
4670 0.0,
4671 0.1,
4672 0.05,
4673 false,
4674 0,
4675 0,
4676 Uuid::now_v7(),
4677 Uuid::nil(),
4678 )
4679 .await;
4680 assert!(!appended, "a timed-out append is not in the log");
4681 }
4682
4683 #[tokio::test]
4686 async fn remote_pairs_carry_the_incumbent_cost() {
4687 let shared = Arc::new(InMemoryExperimentLog::new());
4688 let a = ExperimentMonitor::from_config(&config())
4689 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4690 let b = ExperimentMonitor::from_config(&config())
4691 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4692 assert!(
4693 a.record_pair(
4694 "t",
4695 0,
4696 &fp(),
4697 1.0,
4698 1.0,
4699 0.10,
4700 0.02,
4701 false,
4702 0,
4703 0,
4704 Uuid::now_v7(),
4705 Uuid::nil(),
4706 )
4707 .await
4708 );
4709 b.sync_from_backend("t", 0, &fp()).await.unwrap();
4710 let inc = b.reports()[0].report.incumbent;
4711 assert_eq!(inc.requests, 1, "{inc:?}");
4712 assert!((inc.cost_usd - 0.10).abs() < 1e-12, "{inc:?}");
4713 }
4714
4715 #[tokio::test]
4719 async fn restart_counts_own_pre_restart_incumbent_cost_once() {
4720 let shared = Arc::new(InMemoryExperimentLog::new());
4721 let before = ExperimentMonitor::from_config(&config())
4722 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4723 let served = trace("t", "s", Arm::Incumbent, 0.10);
4724 assert!(
4725 before
4726 .record_pair(
4727 "t",
4728 0,
4729 &fp(),
4730 1.0,
4731 1.0,
4732 0.10,
4733 0.02,
4734 false,
4735 0,
4736 0,
4737 Uuid::now_v7(),
4738 served.trace_id,
4739 )
4740 .await
4741 );
4742 let after = monitor_from_traces(std::slice::from_ref(&served), &[], &[], &config())
4743 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
4744 after.sync_from_backend("t", 0, &fp()).await.unwrap();
4745 let r = &after.reports()[0].report;
4746 assert_eq!(r.pairs, 1, "the pair's evidence still arrives from the log");
4747 assert_eq!(r.incumbent.requests, 1, "{:?}", r.incumbent);
4748 assert!(
4749 (r.incumbent.cost_usd - 0.10).abs() < 1e-12,
4750 "{:?}",
4751 r.incumbent
4752 );
4753 }
4754
4755 #[test]
4757 fn startup_replay_counts_a_duplicated_pair_row_once() {
4758 let tid = Uuid::now_v7();
4759 let row = crate::store::ExperimentPairRow {
4760 tenant: "t".to_owned(),
4761 route_ix: 0,
4762 fingerprint: fp(),
4763 trace_id: tid,
4764 x_inc: 1.0,
4765 x_ch: 0.0,
4766 challenger_cost_usd: 0.05,
4767 ch_error: true,
4768 ch_truncated: 0,
4769 ts: jiff::Timestamp::now(),
4770 in_log: crate::store::InLogStatus::NotInLog,
4771 };
4772 let m = monitor_from_traces(&[], &[], &[row.clone(), row], &config());
4773 let r = &m.reports()[0];
4774 assert_eq!(r.report.pairs, 1);
4775 assert_eq!(r.health.challenger.errors, 1);
4776 }
4777
4778 #[tokio::test]
4782 async fn synced_state_expires_after_the_staleness_bound_when_the_log_is_down() {
4783 tokio::time::pause();
4784 let log = Arc::new(SwitchableLog::default());
4785 let m = ExperimentMonitor::from_config(&config())
4786 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
4787 let fp_str = fp();
4788 assert!(
4789 m.ensure_synced("t", 0, &fp_str).await,
4790 "first sync succeeds"
4791 );
4792 log.failing.store(true, std::sync::atomic::Ordering::SeqCst);
4793 let reads = log.reads.load(std::sync::atomic::Ordering::SeqCst);
4794 tokio::time::advance(SYNC_STALENESS_BOUND / 2).await;
4795 assert!(m.ensure_synced("t", 0, &fp_str).await, "fresh: trusted");
4796 assert_eq!(
4797 log.reads.load(std::sync::atomic::Ordering::SeqCst),
4798 reads,
4799 "fresh state needs no read"
4800 );
4801 tokio::time::advance(SYNC_STALENESS_BOUND).await;
4802 assert!(
4803 !m.ensure_synced("t", 0, &fp_str).await,
4804 "stale and the log is down: fail closed"
4805 );
4806 }
4807
4808 #[tokio::test]
4813 async fn ensure_synced_times_out_on_hung_backend() {
4814 tokio::time::pause();
4815
4816 #[derive(Debug)]
4818 struct HangingLog;
4819
4820 #[async_trait::async_trait]
4821 impl ExperimentLogBackend for HangingLog {
4822 async fn append(
4823 &self,
4824 _: &str,
4825 _: usize,
4826 _: &str,
4827 _: LogRecord,
4828 ) -> Result<u64, LogError> {
4829 Err(LogError::Backend("down".to_owned()))
4830 }
4831 async fn load_since(
4832 &self,
4833 _: &str,
4834 _: usize,
4835 _: &str,
4836 _: u64,
4837 ) -> Result<Vec<Option<LogRecord>>, LogError> {
4838 std::future::pending::<Result<Vec<Option<LogRecord>>, LogError>>().await
4840 }
4841 async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
4842 false
4843 }
4844 }
4845
4846 let m = Arc::new(
4847 ExperimentMonitor::from_config(&config())
4848 .with_log(Arc::new(HangingLog) as Arc<dyn ExperimentLogBackend>),
4849 );
4850 let fp_str = fp();
4851
4852 let result = tokio::select! {
4855 r = m.ensure_synced("t", 0, &fp_str) => r,
4856 _ = tokio::time::sleep(SYNC_TIMEOUT + std::time::Duration::from_millis(1)) => {
4857 tokio::time::advance(SYNC_TIMEOUT + std::time::Duration::from_millis(10)).await;
4859 m.ensure_synced("t", 0, &fp_str).await
4860 }
4861 };
4862 assert!(
4863 !result,
4864 "ensure_synced must return false when load_since hangs (timeout fires)"
4865 );
4866 }
4867}