1use std::collections::{HashMap, HashSet};
41use std::fmt;
42use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
43
44use firstpass_core::{
45 Arm, Config, Experiment, ExperimentReport, ExperimentStatus, Trace, Verdict, select,
46};
47use serde::{Deserialize, Serialize};
48use uuid::Uuid;
49
50const MAX_TRACKED_SESSIONS: usize = 1_000_000;
55
56#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
63#[serde(tag = "kind", rename_all = "snake_case")]
64pub enum LogRecord {
65 Outcome {
67 arm: Arm,
68 outcome: f64,
70 session_hash: u64,
72 cost_usd: f64,
74 },
75 Pair {
77 x_inc: f64,
78 x_ch: f64,
79 incumbent_cost_usd: f64,
81 challenger_cost_usd: f64,
83 },
84}
85
86#[async_trait::async_trait]
97pub trait ExperimentLogBackend: Send + Sync + fmt::Debug {
98 async fn append(
100 &self,
101 tenant: &str,
102 route_ix: usize,
103 fingerprint: &str,
104 record: LogRecord,
105 ) -> u64;
106
107 async fn load_since(
109 &self,
110 tenant: &str,
111 route_ix: usize,
112 fingerprint: &str,
113 offset: u64,
114 ) -> Vec<LogRecord>;
115
116 async fn claim_session(
122 &self,
123 tenant: &str,
124 route_ix: usize,
125 fingerprint: &str,
126 session_id: &str,
127 ) -> bool;
128}
129
130#[derive(Debug, Default)]
136pub struct InMemoryExperimentLog {
137 lists: Mutex<HashMap<(String, usize, String), Vec<LogRecord>>>,
138 sessions: Mutex<HashSet<(String, usize, String, String)>>,
139}
140
141impl InMemoryExperimentLog {
142 #[must_use]
144 pub fn new() -> Self {
145 Self::default()
146 }
147}
148
149#[async_trait::async_trait]
150impl ExperimentLogBackend for InMemoryExperimentLog {
151 async fn append(
152 &self,
153 tenant: &str,
154 route_ix: usize,
155 fingerprint: &str,
156 record: LogRecord,
157 ) -> u64 {
158 let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
159 let mut lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
160 let list = lists.entry(key).or_default();
161 list.push(record);
162 list.len() as u64
163 }
164
165 async fn load_since(
166 &self,
167 tenant: &str,
168 route_ix: usize,
169 fingerprint: &str,
170 offset: u64,
171 ) -> Vec<LogRecord> {
172 let key = (tenant.to_owned(), route_ix, fingerprint.to_owned());
173 let lists = self.lists.lock().unwrap_or_else(PoisonError::into_inner);
174 match lists.get(&key) {
175 None => vec![],
176 Some(list) => {
177 let start = offset as usize;
178 if start >= list.len() {
179 vec![]
180 } else {
181 list[start..].to_vec()
182 }
183 }
184 }
185 }
186
187 async fn claim_session(
188 &self,
189 tenant: &str,
190 route_ix: usize,
191 fingerprint: &str,
192 session_id: &str,
193 ) -> bool {
194 let key = (
195 tenant.to_owned(),
196 route_ix,
197 fingerprint.to_owned(),
198 session_id.to_owned(),
199 );
200 let mut sessions = self.sessions.lock().unwrap_or_else(PoisonError::into_inner);
201 sessions.insert(key)
202 }
203}
204
205#[cfg(feature = "redis-cache")]
208pub use redis_log::RedisExperimentLog;
209
210#[cfg(feature = "redis-cache")]
211mod redis_log {
212 use super::{ExperimentLogBackend, LogRecord};
213 use std::fmt;
214
215 use tokio::sync::Mutex;
217
218 const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
219
220 fn redact(url: &str) -> String {
222 let Some((scheme, rest)) = url.split_once("://") else {
223 return "<malformed redis url>".to_owned();
224 };
225 let host = match rest.rfind('@') {
226 Some(i) => &rest[i + 1..],
227 None => rest,
228 };
229 format!("{scheme}://{host}")
230 }
231
232 pub struct RedisExperimentLog {
236 conn: Mutex<redis::aio::ConnectionManager>,
238 }
239
240 impl fmt::Debug for RedisExperimentLog {
241 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
242 f.debug_struct("RedisExperimentLog").finish_non_exhaustive()
243 }
244 }
245
246 impl RedisExperimentLog {
247 pub async fn connect(url: &str) -> Result<Self, String> {
253 let safe = redact(url);
254 let client = redis::Client::open(url).map_err(|e| format!("redis url {safe}: {e}"))?;
255 let conn =
256 tokio::time::timeout(CONNECT_TIMEOUT, redis::aio::ConnectionManager::new(client))
257 .await
258 .map_err(|_| format!("redis url {safe}: connect timed out"))?
259 .map_err(|e| format!("redis url {safe}: {e}"))?;
260 Ok(Self {
261 conn: Mutex::new(conn),
262 })
263 }
264
265 fn list_key(tenant: &str, route_ix: usize, fingerprint: &str) -> String {
266 format!("firstpass:exp:{tenant}:{route_ix}:{fingerprint}")
267 }
268
269 fn session_key(
270 tenant: &str,
271 route_ix: usize,
272 fingerprint: &str,
273 session_id: &str,
274 ) -> String {
275 format!("firstpass:exp:sess:{tenant}:{route_ix}:{fingerprint}:{session_id}")
276 }
277 }
278
279 #[async_trait::async_trait]
280 impl ExperimentLogBackend for RedisExperimentLog {
281 async fn append(
282 &self,
283 tenant: &str,
284 route_ix: usize,
285 fingerprint: &str,
286 record: LogRecord,
287 ) -> u64 {
288 let key = Self::list_key(tenant, route_ix, fingerprint);
289 let Ok(json) = serde_json::to_string(&record) else {
290 return 0;
291 };
292 let mut conn = self.conn.lock().await;
293 use redis::AsyncCommands as _;
294 match conn.rpush::<_, _, u64>(&key, json).await {
295 Ok(n) => n,
296 Err(e) => {
297 tracing::warn!(key, %e, "experiment log: redis RPUSH failed");
298 0
299 }
300 }
301 }
302
303 async fn load_since(
304 &self,
305 tenant: &str,
306 route_ix: usize,
307 fingerprint: &str,
308 offset: u64,
309 ) -> Vec<LogRecord> {
310 let key = Self::list_key(tenant, route_ix, fingerprint);
311 let mut conn = self.conn.lock().await;
312 use redis::AsyncCommands as _;
313 let raw: Vec<String> = match conn.lrange(&key, offset as isize, -1).await {
314 Ok(v) => v,
315 Err(e) => {
316 tracing::warn!(key, %e, "experiment log: redis LRANGE failed");
317 return vec![];
318 }
319 };
320 raw.iter()
321 .filter_map(|s| match serde_json::from_str(s) {
322 Ok(r) => Some(r),
323 Err(e) => {
324 tracing::warn!(%e, "experiment log: skipping malformed log record");
325 None
326 }
327 })
328 .collect()
329 }
330
331 async fn claim_session(
332 &self,
333 tenant: &str,
334 route_ix: usize,
335 fingerprint: &str,
336 session_id: &str,
337 ) -> bool {
338 let key = Self::session_key(tenant, route_ix, fingerprint, session_id);
339 let mut conn = self.conn.lock().await;
340 use redis::AsyncCommands as _;
341 match conn
343 .set_options::<_, _, ()>(
344 &key,
345 "1",
346 redis::SetOptions::default()
347 .with_expiration(redis::SetExpiry::EX(86_400))
348 .conditional_set(redis::ExistenceCheck::NX),
349 )
350 .await
351 {
352 Ok(()) => true, Err(_) => false, }
355 }
356 }
357}
358
359#[derive(Debug)]
361struct Entry {
362 experiment: Experiment,
363 ignored_stale: u64,
365 health: Health,
367 sessions: HashSet<u64>,
370 saturated_logged: bool,
371 remote_log_offset: u64,
373}
374
375fn session_hash(session_id: &str) -> u64 {
376 use sha2::{Digest, Sha256};
380 let d = Sha256::digest(session_id.as_bytes());
381 u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]])
382}
383
384fn status_code(s: ExperimentStatus) -> f64 {
386 match s {
387 ExperimentStatus::Undecided => 0.0,
388 ExperimentStatus::Certified => 1.0,
389 ExperimentStatus::Halted => 2.0,
390 }
391}
392
393#[derive(Debug)]
394struct Params {
395 margin: f64,
396 alpha: f64,
397 fingerprint: String,
398}
399
400#[derive(Debug, Default)]
402pub struct ExperimentMonitor {
403 params: HashMap<usize, Params>,
405 unavailable: bool,
408 entries: Mutex<HashMap<String, HashMap<usize, Entry>>>,
410 log: Option<Arc<dyn ExperimentLogBackend>>,
412}
413
414impl ExperimentMonitor {
415 #[must_use]
418 pub fn from_config(config: &Config) -> Self {
419 let params = config
420 .routes
421 .iter()
422 .enumerate()
423 .filter_map(|(i, r)| {
424 r.challenger.as_ref().map(|c| {
425 (
426 i,
427 Params {
428 margin: c.margin,
429 alpha: c.alpha,
430 fingerprint: c.fingerprint(&r.ladder),
431 },
432 )
433 })
434 })
435 .collect();
436 Self {
437 params,
438 unavailable: false,
439 entries: Mutex::default(),
440 log: None,
441 }
442 }
443
444 #[must_use]
446 pub fn unavailable(config: &Config) -> Self {
447 Self {
448 unavailable: true,
449 ..Self::from_config(config)
450 }
451 }
452
453 #[must_use]
457 pub fn with_log(mut self, log: Arc<dyn ExperimentLogBackend>) -> Self {
458 self.log = Some(log);
459 self
460 }
461
462 #[must_use]
464 pub fn fingerprint(&self, route_ix: usize) -> Option<&str> {
465 self.params.get(&route_ix).map(|p| p.fingerprint.as_str())
466 }
467
468 #[must_use]
471 pub fn challenger_blocked(&self, tenant: &str, route_ix: usize) -> bool {
472 self.unavailable || self.status(tenant, route_ix) == ExperimentStatus::Halted
473 }
474
475 #[must_use]
477 pub fn is_active(&self) -> bool {
478 !self.params.is_empty()
479 }
480
481 fn lock(&self) -> MutexGuard<'_, HashMap<String, HashMap<usize, Entry>>> {
482 self.entries.lock().unwrap_or_else(PoisonError::into_inner)
485 }
486
487 fn entry<'a>(
488 &self,
489 map: &'a mut HashMap<String, HashMap<usize, Entry>>,
490 tenant: &str,
491 route_ix: usize,
492 ) -> Option<&'a mut Entry> {
493 let p = self.params.get(&route_ix)?;
494 let (margin, alpha) = (p.margin, p.alpha);
495 if !map.contains_key(tenant) {
496 map.insert(tenant.to_owned(), HashMap::new());
497 }
498 let routes = map.get_mut(tenant)?;
499 if let std::collections::hash_map::Entry::Vacant(v) = routes.entry(route_ix) {
500 match Experiment::new(margin, alpha) {
501 Ok(experiment) => {
502 v.insert(Entry {
503 experiment,
504 ignored_stale: 0,
505 health: Health::default(),
506 sessions: HashSet::new(),
507 saturated_logged: false,
508 remote_log_offset: 0,
509 });
510 }
511 Err(e) => {
512 tracing::warn!(route = route_ix, %e, "experiment parameters rejected");
514 return None;
515 }
516 }
517 }
518 routes.get_mut(&route_ix)
519 }
520
521 #[must_use]
525 pub fn status(&self, tenant: &str, route_ix: usize) -> ExperimentStatus {
526 self.lock()
527 .get(tenant)
528 .and_then(|r| r.get(&route_ix))
529 .map_or(ExperimentStatus::Undecided, |e| e.experiment.status())
530 }
531
532 pub fn observe_cost(
537 &self,
538 tenant: &str,
539 route_ix: usize,
540 fingerprint: &str,
541 arm: Arm,
542 usd: f64,
543 ) {
544 let current = self.fingerprint(route_ix) == Some(fingerprint);
545 let mut map = self.lock();
546 if let Some(e) = self.entry(&mut map, tenant, route_ix) {
547 if current {
548 e.experiment.observe_cost(arm, usd);
549 } else {
550 e.ignored_stale += 1;
551 }
552 }
553 }
554
555 pub fn observe_health(
559 &self,
560 tenant: &str,
561 route_ix: usize,
562 fingerprint: &str,
563 arm: Arm,
564 errored: bool,
565 truncated: u64,
566 ) {
567 if (!errored && truncated == 0) || self.fingerprint(route_ix) != Some(fingerprint) {
568 return;
569 }
570 let mut map = self.lock();
571 if let Some(e) = self.entry(&mut map, tenant, route_ix) {
572 let h = match arm {
573 Arm::Incumbent => &mut e.health.incumbent,
574 Arm::Challenger => &mut e.health.challenger,
575 };
576 h.errors += u64::from(errored);
577 h.truncated += truncated;
578 }
579 }
580
581 pub fn observe_outcome(
590 &self,
591 tenant: &str,
592 route_ix: usize,
593 fingerprint: &str,
594 arm: Arm,
595 session_id: &str,
596 verdict: Verdict,
597 ) -> bool {
598 if self.fingerprint(route_ix) != Some(fingerprint) {
600 return false;
601 }
602 let outcome = match verdict {
603 Verdict::Pass => 1.0,
604 Verdict::Fail => 0.0,
605 Verdict::Abstain => return false,
606 };
607 let mut map = self.lock();
608 let Some(e) = self.entry(&mut map, tenant, route_ix) else {
609 return false;
610 };
611 let key = session_hash(session_id);
612 if e.sessions.contains(&key) {
613 return false;
614 }
615 if e.sessions.len() >= MAX_TRACKED_SESSIONS {
616 if !e.saturated_logged {
617 e.saturated_logged = true;
618 tracing::warn!(
619 tenant,
620 route = route_ix,
621 "experiment session ledger is full; new sessions' outcomes are ignored"
622 );
623 }
624 return false;
625 }
626 let before = e.experiment.status();
627 if !e.experiment.observe_outcome(arm, outcome) {
628 return false;
629 }
630 e.sessions.insert(key);
631 let after = e.experiment.status();
632 if before != after {
633 match after {
634 ExperimentStatus::Halted => tracing::error!(
635 tenant,
636 route = route_ix,
637 "EXPERIMENT HALTED: the challenger ladder is worse than the incumbent by more \
638 than the margin — it is no longer served"
639 ),
640 ExperimentStatus::Certified => tracing::info!(
641 tenant,
642 route = route_ix,
643 "experiment certified: the challenger is within the margin of the incumbent; \
644 adopting it is a human decision (`firstpass experiment`)"
645 ),
646 ExperimentStatus::Undecided => {}
647 }
648 Self::publish(&map, route_ix);
649 }
650 true
651 }
652
653 fn publish(map: &HashMap<String, HashMap<usize, Entry>>, route_ix: usize) {
658 let worst = map
659 .values()
660 .filter_map(|r| r.get(&route_ix))
661 .map(|e| status_code(e.experiment.status()))
662 .fold(0.0, f64::max);
663 metrics::gauge!("firstpass_experiment_status", "route" => route_ix.to_string()).set(worst);
664 }
665
666 #[must_use]
668 pub fn reports(&self) -> Vec<Snapshot> {
669 let map = self.lock();
670 let mut out: Vec<_> = map
671 .iter()
672 .flat_map(|(t, routes)| {
673 routes.iter().map(move |(ix, e)| Snapshot {
674 tenant: t.clone(),
675 route_ix: *ix,
676 report: e.experiment.report(),
677 ignored_stale: e.ignored_stale,
678 health: e.health,
679 })
680 })
681 .collect();
682 out.sort_by(|a, b| (&a.tenant, a.route_ix).cmp(&(&b.tenant, b.route_ix)));
683 out
684 }
685
686 fn apply_log_record(
694 &self,
695 map: &mut HashMap<String, HashMap<usize, Entry>>,
696 tenant: &str,
697 route_ix: usize,
698 fingerprint: &str,
699 record: &LogRecord,
700 ) -> bool {
701 if self.fingerprint(route_ix) != Some(fingerprint) {
702 if let Some(e) = self.entry(map, tenant, route_ix) {
703 e.ignored_stale += 1;
704 }
705 return false;
706 }
707 match record {
708 LogRecord::Outcome {
709 arm,
710 outcome,
711 session_hash: hash,
712 ..
713 } => {
714 let Some(e) = self.entry(map, tenant, route_ix) else {
715 return false;
716 };
717 if e.sessions.contains(hash) {
718 return false; }
720 if e.sessions.len() >= MAX_TRACKED_SESSIONS {
721 return false;
722 }
723 let before = e.experiment.status();
724 if !e.experiment.observe_outcome(*arm, *outcome) {
725 return false;
726 }
727 e.sessions.insert(*hash);
728 before != e.experiment.status()
729 }
730 LogRecord::Pair {
731 x_inc,
732 x_ch,
733 challenger_cost_usd,
734 ..
735 } => {
736 let Some(e) = self.entry(map, tenant, route_ix) else {
737 return false;
738 };
739 let before = e.experiment.status();
740 if e.experiment.observe_pair(*x_inc, *x_ch) {
741 e.experiment
744 .observe_cost(Arm::Challenger, *challenger_cost_usd);
745 }
746 before != e.experiment.status()
747 }
748 }
749 }
750
751 pub async fn sync_from_backend(&self, tenant: &str, route_ix: usize, fingerprint: &str) {
759 let Some(log) = &self.log else {
760 return;
761 };
762 let offset = {
763 let mut map = self.lock();
764 self.entry(&mut map, tenant, route_ix)
765 .map_or(0, |e| e.remote_log_offset)
766 };
767 let records = log.load_since(tenant, route_ix, fingerprint, offset).await;
768 if records.is_empty() {
769 return;
770 }
771 let mut map = self.lock();
772 for (i, rec) in records.iter().enumerate() {
773 let changed = self.apply_log_record(&mut map, tenant, route_ix, fingerprint, rec);
774 if let Some(e) = map.get_mut(tenant).and_then(|r| r.get_mut(&route_ix)) {
777 e.remote_log_offset = offset + i as u64 + 1;
778 }
779 if changed {
780 let after = map
781 .get(tenant)
782 .and_then(|r| r.get(&route_ix))
783 .map(|e| e.experiment.status());
784 match after {
785 Some(ExperimentStatus::Halted) => tracing::error!(
786 tenant,
787 route = route_ix,
788 "EXPERIMENT HALTED (shared log): challenger worse than incumbent"
789 ),
790 Some(ExperimentStatus::Certified) => tracing::info!(
791 tenant,
792 route = route_ix,
793 "experiment certified (shared log)"
794 ),
795 _ => {}
796 }
797 Self::publish(&map, route_ix);
798 }
799 }
800 }
801
802 #[allow(clippy::too_many_arguments)]
808 pub async fn record_outcome(
809 &self,
810 tenant: &str,
811 route_ix: usize,
812 fingerprint: &str,
813 arm: Arm,
814 session_id: &str,
815 verdict: Verdict,
816 cost_usd: f64,
817 ) -> bool {
818 let outcome = match verdict {
822 Verdict::Pass => 1.0,
823 Verdict::Fail => 0.0,
824 Verdict::Abstain => return false,
825 };
826 let Some(log) = self.log.as_ref().cloned() else {
827 return self.observe_outcome(tenant, route_ix, fingerprint, arm, session_id, verdict);
828 };
829 if !log
831 .claim_session(tenant, route_ix, fingerprint, session_id)
832 .await
833 {
834 return false;
835 }
836 log.append(
837 tenant,
838 route_ix,
839 fingerprint,
840 LogRecord::Outcome {
841 arm,
842 outcome,
843 session_hash: session_hash(session_id),
844 cost_usd,
845 },
846 )
847 .await;
848 self.sync_from_backend(tenant, route_ix, fingerprint).await;
851 true
852 }
853
854 #[allow(clippy::too_many_arguments)]
860 pub async fn record_pair(
861 &self,
862 tenant: &str,
863 route_ix: usize,
864 fingerprint: &str,
865 x_inc: f64,
866 x_ch: f64,
867 incumbent_cost_usd: f64,
868 challenger_cost_usd: f64,
869 ) -> bool {
870 if self.fingerprint(route_ix) != Some(fingerprint) {
871 return false;
872 }
873 let Some(log) = self.log.as_ref().cloned() else {
874 let mut map = self.lock();
876 let Some(e) = self.entry(&mut map, tenant, route_ix) else {
877 return false;
878 };
879 let before = e.experiment.status();
880 let ok = e.experiment.observe_pair(x_inc, x_ch);
881 if ok {
882 e.experiment
885 .observe_cost(Arm::Challenger, challenger_cost_usd);
886 }
887 if ok && before != e.experiment.status() {
888 match e.experiment.status() {
889 ExperimentStatus::Halted => tracing::error!(
890 tenant,
891 route = route_ix,
892 "EXPERIMENT HALTED: paired challenger worse than incumbent"
893 ),
894 ExperimentStatus::Certified => tracing::info!(
895 tenant,
896 route = route_ix,
897 "experiment certified (paired): challenger within margin"
898 ),
899 ExperimentStatus::Undecided => {}
900 }
901 Self::publish(&map, route_ix);
902 }
903 return ok;
904 };
905 if !x_inc.is_finite()
907 || !(0.0..=1.0).contains(&x_inc)
908 || !x_ch.is_finite()
909 || !(0.0..=1.0).contains(&x_ch)
910 {
911 return false;
912 }
913 log.append(
914 tenant,
915 route_ix,
916 fingerprint,
917 LogRecord::Pair {
918 x_inc,
919 x_ch,
920 incumbent_cost_usd,
921 challenger_cost_usd,
922 },
923 )
924 .await;
925 self.sync_from_backend(tenant, route_ix, fingerprint).await;
926 true
927 }
928
929 pub fn start_sync_task(self: &Arc<Self>, interval: std::time::Duration) {
933 if self.log.is_none() {
934 return;
935 }
936 let monitor = Arc::clone(self);
937 tokio::spawn(async move {
938 let mut ticker = tokio::time::interval(interval);
939 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
940 loop {
941 ticker.tick().await;
942 let keys: Vec<(String, usize, String)> = {
945 let map = monitor.lock();
946 let mut v = Vec::new();
947 for (tenant, routes) in map.iter() {
948 for &route_ix in routes.keys() {
949 if let Some(p) = monitor.params.get(&route_ix) {
950 v.push((tenant.clone(), route_ix, p.fingerprint.clone()));
951 }
952 }
953 }
954 v
955 };
956 for (tenant, route_ix, fingerprint) in keys {
957 monitor
958 .sync_from_backend(&tenant, route_ix, &fingerprint)
959 .await;
960 }
961 }
962 });
963 }
964}
965
966#[derive(Debug, Clone, PartialEq)]
968pub struct Snapshot {
969 pub tenant: String,
971 pub route_ix: usize,
973 pub report: ExperimentReport,
975 pub ignored_stale: u64,
977 pub health: Health,
979}
980
981#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
987pub struct ArmHealth {
988 pub errors: u64,
990 pub truncated: u64,
992}
993
994#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
997pub struct Health {
998 pub incumbent: ArmHealth,
1000 pub challenger: ArmHealth,
1002}
1003
1004#[must_use]
1006pub fn truncated_attempts(t: &Trace) -> u64 {
1007 t.attempts.iter().filter(|a| a.truncated).count() as u64
1008}
1009
1010#[must_use]
1014pub fn is_incumbent_trace(t: &Trace) -> bool {
1015 t.experiment
1016 .as_ref()
1017 .is_none_or(|e| e.arm != Arm::Challenger)
1018}
1019
1020#[must_use]
1033pub fn monitor_from_traces(
1034 traces: &[Trace],
1035 deferred: &[(Uuid, Verdict)],
1036 pairs: &[crate::store::ExperimentPairRow],
1037 config: &Config,
1038) -> ExperimentMonitor {
1039 let monitor = ExperimentMonitor::from_config(config);
1040 if !monitor.is_active() {
1041 return monitor;
1042 }
1043 let mut by_id: HashMap<Uuid, &Trace> = HashMap::new();
1044 for t in traces {
1045 let Some(rec) = &t.experiment else { continue };
1046 let route_ix = t.route_ix.map_or(0, |r| r as usize);
1048 monitor.observe_cost(
1049 &t.tenant_id,
1050 route_ix,
1051 &rec.fingerprint,
1052 rec.arm,
1053 t.final_.total_cost_usd,
1054 );
1055 monitor.observe_health(
1056 &t.tenant_id,
1057 route_ix,
1058 &rec.fingerprint,
1059 rec.arm,
1060 t.final_.served_from == firstpass_core::ServedFrom::Error,
1061 truncated_attempts(t),
1062 );
1063 by_id.insert(t.trace_id, t);
1064 }
1065 for (id, verdict) in deferred {
1066 let Some(t) = by_id.get(id) else { continue };
1067 let Some(rec) = &t.experiment else { continue };
1068 let route_ix = t.route_ix.map_or(0, |r| r as usize);
1069 monitor.observe_outcome(
1070 &t.tenant_id,
1071 route_ix,
1072 &rec.fingerprint,
1073 rec.arm,
1074 &t.session_id,
1075 *verdict,
1076 );
1077 }
1078 for row in pairs {
1082 let mut map = monitor.lock();
1083 if let Some(e) = monitor.entry(&mut map, &row.tenant, row.route_ix) {
1084 if monitor.fingerprint(row.route_ix) == Some(row.fingerprint.as_str()) {
1085 if e.experiment.observe_pair(row.x_inc, row.x_ch) {
1086 e.experiment
1087 .observe_cost(Arm::Challenger, row.challenger_cost_usd);
1088 }
1089 } else {
1090 e.ignored_stale += 1;
1091 }
1092 }
1093 }
1094 monitor
1095}
1096
1097#[must_use]
1104pub fn build_monitor(db_path: &str, config: &Config) -> ExperimentMonitor {
1105 build_monitor_with(db_path, config, false)
1106}
1107
1108#[must_use]
1116pub fn build_monitor_with(db_path: &str, config: &Config, shared_log: bool) -> ExperimentMonitor {
1117 let empty = ExperimentMonitor::from_config(config);
1118 if !empty.is_active() {
1119 return empty;
1120 }
1121 let loaded = crate::store::load_all_traces(db_path)
1122 .and_then(|t| crate::store::load_all_deferred(db_path).map(|d| (t, d)))
1123 .and_then(|(t, d)| crate::store::load_all_experiment_pairs(db_path).map(|p| (t, d, p)));
1124 match loaded {
1125 Ok((traces, deferred, pairs)) => {
1126 let replayed: &[crate::store::ExperimentPairRow] =
1127 if shared_log { &[] } else { &pairs };
1128 let m = monitor_from_traces(&traces, &deferred, replayed, config);
1129 if shared_log {
1130 let mut map = m.lock();
1131 for row in &pairs {
1132 let _ = m.entry(&mut map, &row.tenant, row.route_ix);
1133 }
1134 }
1135 for s in m.reports() {
1136 tracing::info!(
1137 tenant = %s.tenant,
1138 route = s.route_ix,
1139 status = ?s.report.status,
1140 pairs = s.report.pairs,
1141 ignored_stale = s.ignored_stale,
1142 "experiment restored from receipts"
1143 );
1144 ExperimentMonitor::publish(&m.lock(), s.route_ix);
1145 }
1146 m
1147 }
1148 Err(e) => {
1149 tracing::error!(
1150 %e,
1151 "could not read the trace store, so it is unknown whether a challenger was already \
1152 halted: challenger arms are NOT served (incumbent only) until restart with a \
1153 readable store"
1154 );
1155 ExperimentMonitor::unavailable(config)
1156 }
1157 }
1158}
1159
1160#[must_use]
1162pub fn experiments_from_traces(
1163 traces: &[Trace],
1164 deferred: &[(Uuid, Verdict)],
1165 pairs: &[crate::store::ExperimentPairRow],
1166 config: &Config,
1167) -> Vec<Snapshot> {
1168 monitor_from_traces(traces, deferred, pairs, config).reports()
1169}
1170
1171#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
1173#[serde(rename_all = "snake_case")]
1174pub enum Recommendation {
1175 Adopt,
1177 CertifiedNoSaving,
1179 ChallengerStopped,
1181 KeepRunning,
1183}
1184
1185#[must_use]
1189pub fn recommend(r: &ExperimentReport) -> Recommendation {
1190 match r.status {
1191 select::Status::Undecided => Recommendation::KeepRunning,
1192 select::Status::Halted => Recommendation::ChallengerStopped,
1193 select::Status::Certified => {
1194 match (
1195 r.challenger.cost_per_request(),
1196 r.incumbent.cost_per_request(),
1197 ) {
1198 (Some(c), Some(i)) if c < i => Recommendation::Adopt,
1199 _ => Recommendation::CertifiedNoSaving,
1200 }
1201 }
1202 }
1203}
1204
1205#[derive(Debug, Clone, Serialize)]
1207pub struct ExperimentView {
1208 pub tenant: String,
1210 pub route_ix: usize,
1212 pub report: ExperimentReport,
1214 pub ignored_stale: u64,
1216 pub health: Health,
1218 pub recommendation: Recommendation,
1220 pub challenger_ladder: Vec<String>,
1222}
1223
1224#[must_use]
1226pub fn views(reports: Vec<Snapshot>, config: &Config) -> Vec<ExperimentView> {
1227 reports
1228 .into_iter()
1229 .map(|s| ExperimentView {
1230 recommendation: recommend(&s.report),
1231 ignored_stale: s.ignored_stale,
1232 health: s.health,
1233 challenger_ladder: config
1234 .routes
1235 .get(s.route_ix)
1236 .and_then(|r| r.challenger.as_ref())
1237 .map(|c| c.ladder.clone())
1238 .unwrap_or_default(),
1239 tenant: s.tenant,
1240 route_ix: s.route_ix,
1241 report: s.report,
1242 })
1243 .collect()
1244}
1245
1246fn fmt_opt(v: Option<f64>, f: impl Fn(f64) -> String) -> String {
1247 v.map_or_else(|| "n/a".to_owned(), f)
1248}
1249
1250#[must_use]
1252pub fn render(views: &[ExperimentView]) -> String {
1253 use std::fmt::Write as _;
1254 if views.is_empty() {
1255 return "no experiment data yet — the config has no [route.challenger], or no experiment \
1256 traffic has been recorded\n"
1257 .to_owned();
1258 }
1259 let mut out = String::new();
1260 for v in views {
1261 let r = &v.report;
1262 let _ = writeln!(
1263 out,
1264 "tenant {} · route {} · status: {:?}",
1265 v.tenant, v.route_ix, r.status
1266 );
1267 let _ = writeln!(
1268 out,
1269 " pairs: {} (margin {}, alpha {})",
1270 r.pairs, r.margin, r.alpha
1271 );
1272 for (name, a) in [("incumbent ", &r.incumbent), ("challenger", &r.challenger)] {
1273 let _ = writeln!(
1274 out,
1275 " {name}: success {} over {} sessions · ${} per request over {} requests",
1276 fmt_opt(a.success_rate(), |s| format!("{:.1}%", s * 100.0)),
1277 a.outcomes,
1278 fmt_opt(a.cost_per_request(), |c| format!("{c:.6}")),
1279 a.requests,
1280 );
1281 }
1282 let _ = writeln!(
1283 out,
1284 " e-value to certify: {:.3} · e-value to halt: {:.3} (decide at {:.1})",
1285 r.e_certify,
1286 r.e_halt,
1287 1.0 / r.alpha
1288 );
1289 if v.ignored_stale > 0 {
1290 let _ = writeln!(
1291 out,
1292 " {} receipts from a previous experiment configuration ignored",
1293 v.ignored_stale
1294 );
1295 }
1296 for (name, h) in [
1297 ("incumbent", &v.health.incumbent),
1298 ("challenger", &v.health.challenger),
1299 ] {
1300 if h.errors > 0 || h.truncated > 0 {
1301 let _ = writeln!(
1302 out,
1303 " {name}: {} errors, {} answers cut off at max_tokens (neither is random: \
1304 check them before trusting the rates)",
1305 h.errors, h.truncated
1306 );
1307 }
1308 }
1309 match v.recommendation {
1310 Recommendation::Adopt => {
1311 let ladder = v
1312 .challenger_ladder
1313 .iter()
1314 .map(|m| format!("{m:?}"))
1315 .collect::<Vec<_>>()
1316 .join(", ");
1317 let _ = writeln!(
1318 out,
1319 " ADOPT: set this route's ladder to [{ladder}]\n\n ladder = [{ladder}]\n"
1320 );
1321 }
1322 Recommendation::CertifiedNoSaving => {
1323 let _ = writeln!(out, " certified, but no saving — keep incumbent");
1324 }
1325 Recommendation::ChallengerStopped => {
1326 let _ = writeln!(out, " challenger stopped: worse by more than the margin");
1327 }
1328 Recommendation::KeepRunning => {
1329 let _ = writeln!(out, " keep running");
1330 }
1331 }
1332 }
1333 out
1334}
1335
1336#[cfg(test)]
1337mod tests {
1338 use firstpass_core::trace::ExperimentRecord;
1339 use firstpass_core::{
1340 Features, FinalOutcome, GENESIS_HASH, Mode, PolicyRef, RequestInfo, ServedFrom, TaskKind,
1341 };
1342
1343 use super::*;
1344
1345 const CFG: &str = "[[route]]\nmatch = {}\nmode = \"enforce\"\n\
1346 ladder = [\"anthropic/claude-sonnet-5\"]\n\
1347 [route.challenger]\nladder = [\"anthropic/claude-haiku-4-5\"]\npercent = 50.0\n";
1348
1349 fn config() -> Config {
1350 Config::parse(CFG).expect("test config parses")
1351 }
1352
1353 fn fp() -> String {
1355 let c = config();
1356 c.routes[0]
1357 .challenger
1358 .as_ref()
1359 .expect("challenger")
1360 .fingerprint(&c.routes[0].ladder)
1361 }
1362
1363 fn trace(tenant: &str, session: &str, arm: Arm, cost: f64) -> Trace {
1364 Trace {
1365 trace_id: Uuid::now_v7(),
1366 prev_hash: GENESIS_HASH.to_owned(),
1367 tenant_id: tenant.to_owned(),
1368 session_id: session.to_owned(),
1369 ts: jiff::Timestamp::now(),
1370 mode: Mode::Enforce,
1371 policy: PolicyRef {
1372 id: "static-ladder@v0".to_owned(),
1373 explore: false,
1374 propensity: None,
1375 mode_profile: None,
1376 },
1377 request: RequestInfo {
1378 api: "anthropic.messages".to_owned(),
1379 prompt_hash: "h".to_owned(),
1380 features: Features::new(TaskKind::Other),
1381 },
1382 attempts: vec![],
1383 deferred: vec![],
1384 final_: FinalOutcome {
1385 served_rung: Some(0),
1386 served_from: ServedFrom::Attempt,
1387 total_cost_usd: cost,
1388 gate_cost_usd: 0.0,
1389 total_latency_ms: 1,
1390 escalations: 0,
1391 counterfactual_baseline_usd: cost,
1392 savings_usd: 0.0,
1393 cache_source: None,
1394 },
1395 probe: None,
1396 rollout: None,
1397 shadow: None,
1398 route_ix: Some(0),
1399 predicted_pass: None,
1400 decision_prior: None,
1401 elastic: None,
1402 risk: None,
1403 experiment: Some(ExperimentRecord {
1404 arm,
1405 percent: 50.0,
1406 bucket: 0,
1407 fingerprint: fp(),
1408 }),
1409 }
1410 }
1411
1412 #[test]
1415 fn arm_health_is_rebuilt_from_receipts() {
1416 let attempt = |truncated| firstpass_core::trace::Attempt {
1417 rung: 0,
1418 model: "anthropic/claude-sonnet-5".to_owned(),
1419 provider: "anthropic".to_owned(),
1420 in_tokens: 1,
1421 cache_write_tokens: 0,
1422 cache_read_tokens: 0,
1423 out_tokens: 1,
1424 cost_usd: 0.0,
1425 latency_ms: 1,
1426 gates: vec![],
1427 verdict: Verdict::Pass,
1428 truncated,
1429 };
1430 let mut errored = trace("t", "a", Arm::Incumbent, 0.01);
1431 errored.final_.served_from = ServedFrom::Error;
1432 let mut cut = trace("t", "b", Arm::Incumbent, 0.01);
1433 cut.attempts = vec![attempt(true), attempt(true), attempt(false)];
1434 let mut ch_cut = trace("t", "c", Arm::Challenger, 0.01);
1435 ch_cut.attempts = vec![attempt(true)];
1436 let mut stale = trace("t", "d", Arm::Incumbent, 0.01);
1437 stale.final_.served_from = ServedFrom::Error;
1438 if let Some(rec) = stale.experiment.as_mut() {
1439 rec.fingerprint = "an-older-config".to_owned();
1440 }
1441 let snaps = experiments_from_traces(&[errored, cut, ch_cut, stale], &[], &[], &config());
1442 let h = snaps[0].health;
1443 assert_eq!(
1444 h.incumbent,
1445 ArmHealth {
1446 errors: 1,
1447 truncated: 2
1448 },
1449 "{h:?}"
1450 );
1451 assert_eq!(
1452 h.challenger,
1453 ArmHealth {
1454 errors: 0,
1455 truncated: 1
1456 },
1457 "{h:?}"
1458 );
1459 assert_eq!(snaps[0].ignored_stale, 1);
1460 }
1461
1462 #[test]
1463 fn first_outcome_per_session_counts_and_abstain_is_ignored() {
1464 let m = ExperimentMonitor::from_config(&config());
1465 assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Abstain));
1466 assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Pass));
1467 assert!(!m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s1", Verdict::Fail));
1469 assert!(m.observe_outcome("t", 0, &fp(), Arm::Incumbent, "s2", Verdict::Fail));
1470 let r = &m.reports()[0].report;
1471 assert_eq!(r.incumbent.outcomes, 2);
1472 assert!((r.incumbent.outcome_sum - 1.0).abs() < f64::EPSILON);
1473 }
1474
1475 #[test]
1476 fn routes_without_a_challenger_and_unknown_tenants_are_inert() {
1477 let m = ExperimentMonitor::from_config(&config());
1478 assert!(!m.observe_outcome("t", 7, &fp(), Arm::Incumbent, "s", Verdict::Pass));
1479 assert_eq!(m.status("nobody", 0), ExperimentStatus::Undecided);
1480 assert!(m.reports().is_empty());
1481 let off = ExperimentMonitor::default();
1482 assert!(!off.is_active());
1483 off.observe_cost("t", 0, &fp(), Arm::Incumbent, 1.0);
1484 assert!(off.reports().is_empty());
1485 }
1486
1487 #[test]
1488 fn tenants_never_share_an_experiment() {
1489 let m = ExperimentMonitor::from_config(&config());
1490 for i in 0..400 {
1491 m.observe_outcome(
1492 "a",
1493 0,
1494 &fp(),
1495 Arm::Incumbent,
1496 &format!("i{i}"),
1497 Verdict::Pass,
1498 );
1499 m.observe_outcome(
1500 "a",
1501 0,
1502 &fp(),
1503 Arm::Challenger,
1504 &format!("c{i}"),
1505 Verdict::Fail,
1506 );
1507 }
1508 assert_eq!(m.status("a", 0), ExperimentStatus::Halted);
1509 assert_eq!(m.status("b", 0), ExperimentStatus::Undecided);
1510 }
1511
1512 #[test]
1513 fn a_clearly_worse_challenger_halts() {
1514 let m = ExperimentMonitor::from_config(&config());
1515 let mut at = None;
1516 for i in 0..500 {
1517 m.observe_outcome(
1518 "t",
1519 0,
1520 &fp(),
1521 Arm::Incumbent,
1522 &format!("i{i}"),
1523 Verdict::Pass,
1524 );
1525 m.observe_outcome(
1526 "t",
1527 0,
1528 &fp(),
1529 Arm::Challenger,
1530 &format!("c{i}"),
1531 Verdict::Fail,
1532 );
1533 if at.is_none() && m.status("t", 0) == ExperimentStatus::Halted {
1534 at = Some(i);
1535 }
1536 }
1537 assert!(at.is_some(), "never halted");
1538 assert_eq!(m.status("t", 0), ExperimentStatus::Halted);
1539 }
1540
1541 #[test]
1542 fn replay_matches_live_with_session_dedup_and_halt() {
1543 let cfg = config();
1544 let mut traces = Vec::new();
1545 let mut deferred = Vec::new();
1546 for i in 0..400 {
1547 let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
1548 let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
1549 deferred.push((inc.trace_id, Verdict::Pass));
1550 deferred.push((chl.trace_id, Verdict::Fail));
1551 let again = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
1553 deferred.push((again.trace_id, Verdict::Pass));
1554 traces.extend([inc, chl, again]);
1555 }
1556 let reports = experiments_from_traces(&traces, &deferred, &[], &cfg);
1557 assert_eq!(reports.len(), 1);
1558 let snap = &reports[0];
1559 let r = &snap.report;
1560 assert_eq!((snap.tenant.as_str(), snap.route_ix), ("t", 0));
1561 assert_eq!(snap.ignored_stale, 0);
1562 assert_eq!(r.status, select::Status::Halted);
1563 assert_eq!(r.challenger.outcomes, 400);
1565 assert!(r.challenger.outcome_sum.abs() < f64::EPSILON);
1566 assert_eq!(r.challenger.requests, 800);
1568 assert!((r.challenger.cost_per_request().unwrap_or(0.0) - 0.02).abs() < 1e-12);
1569 assert!((r.incumbent.cost_per_request().unwrap_or(0.0) - 0.10).abs() < 1e-12);
1570 assert_eq!(recommend(r), Recommendation::ChallengerStopped);
1571
1572 let live = ExperimentMonitor::from_config(&cfg);
1574 for t in &traces {
1575 let rec = t.experiment.as_ref().expect("recorded");
1576 live.observe_cost(
1577 &t.tenant_id,
1578 0,
1579 &rec.fingerprint,
1580 rec.arm,
1581 t.final_.total_cost_usd,
1582 );
1583 }
1584 for (id, v) in &deferred {
1585 let t = traces.iter().find(|t| t.trace_id == *id).expect("trace");
1586 let rec = t.experiment.as_ref().expect("recorded");
1587 live.observe_outcome("t", 0, &rec.fingerprint, rec.arm, &t.session_id, *v);
1588 }
1589 assert_eq!(live.reports()[0], *snap);
1590 assert_eq!(
1592 experiments_from_traces(&traces, &deferred, &[], &cfg),
1593 reports
1594 );
1595 }
1596
1597 #[test]
1598 fn replay_ignores_unrecorded_traces_and_unknown_deferred() {
1599 let cfg = config();
1600 let mut plain = trace("t", "s", Arm::Incumbent, 0.1);
1601 plain.experiment = None;
1602 let deferred = [
1603 (plain.trace_id, Verdict::Pass),
1604 (Uuid::now_v7(), Verdict::Pass),
1605 ];
1606 assert!(experiments_from_traces(&[plain], &deferred, &[], &cfg).is_empty());
1607 let bare = Config::parse(
1609 "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
1610 )
1611 .expect("parses");
1612 let t = trace("t", "s", Arm::Incumbent, 0.1);
1613 assert!(experiments_from_traces(&[t], &[], &[], &bare).is_empty());
1614 }
1615
1616 fn view_with(status: select::Status, inc_cost: f64, ch_cost: f64) -> ExperimentView {
1617 let mut exp = Experiment::new(0.05, 0.05).expect("valid");
1618 exp.observe_cost(Arm::Incumbent, inc_cost);
1619 exp.observe_cost(Arm::Challenger, ch_cost);
1620 let mut report = exp.report();
1621 report.status = status;
1622 ExperimentView {
1623 tenant: "t".to_owned(),
1624 route_ix: 0,
1625 recommendation: recommend(&report),
1626 ignored_stale: 0,
1627 health: Health::default(),
1628 report,
1629 challenger_ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
1630 }
1631 }
1632
1633 #[test]
1634 fn recommendation_and_rendering_cover_every_state() {
1635 let adopt = view_with(select::Status::Certified, 0.10, 0.02);
1636 assert_eq!(adopt.recommendation, Recommendation::Adopt);
1637 let text = render(&[adopt]);
1638 assert!(
1639 text.contains("ADOPT: set this route's ladder to [\"anthropic/claude-haiku-4-5\"]"),
1640 "{text}"
1641 );
1642 assert!(text.contains("ladder = [\"anthropic/claude-haiku-4-5\"]"));
1643
1644 let dear = view_with(select::Status::Certified, 0.02, 0.10);
1645 assert_eq!(dear.recommendation, Recommendation::CertifiedNoSaving);
1646 assert!(render(&[dear]).contains("certified, but no saving — keep incumbent"));
1647
1648 let halted = view_with(select::Status::Halted, 0.10, 0.02);
1649 assert!(render(&[halted]).contains("challenger stopped: worse by more than the margin"));
1650
1651 let open = view_with(select::Status::Undecided, 0.10, 0.02);
1652 assert_eq!(open.recommendation, Recommendation::KeepRunning);
1653 assert!(render(&[open]).contains("keep running"));
1654 assert!(render(&[]).contains("no experiment data"));
1655 }
1656
1657 #[test]
1658 fn certified_without_cost_data_is_not_adopted() {
1659 let exp = Experiment::new(0.05, 0.05).expect("valid");
1660 let mut report = exp.report();
1661 report.status = select::Status::Certified;
1662 assert_eq!(recommend(&report), Recommendation::CertifiedNoSaving);
1663 }
1664
1665 #[test]
1668 fn changing_the_challenger_ladder_resets_the_experiment() {
1669 let old = config();
1670 let mut traces = Vec::new();
1671 let mut deferred = Vec::new();
1672 for i in 0..300 {
1673 let inc = trace("t", &format!("inc{i}"), Arm::Incumbent, 0.10);
1674 let chl = trace("t", &format!("chl{i}"), Arm::Challenger, 0.02);
1675 deferred.push((inc.trace_id, Verdict::Pass));
1676 deferred.push((chl.trace_id, Verdict::Fail));
1677 traces.extend([inc, chl]);
1678 }
1679 let before = experiments_from_traces(&traces, &deferred, &[], &old);
1680 assert_eq!(before[0].report.status, select::Status::Halted);
1681 assert_eq!(before[0].ignored_stale, 0);
1682
1683 let new =
1684 Config::parse(&CFG.replace("claude-haiku-4-5", "claude-opus-4-8")).expect("parses");
1685 let after = experiments_from_traces(&traces, &deferred, &[], &new);
1686 assert_eq!(after[0].report.status, select::Status::Undecided);
1687 assert_eq!(after[0].report.pairs, 0);
1688 assert_eq!(after[0].report.challenger.requests, 0);
1689 assert_eq!(after[0].ignored_stale, 600);
1690 let text = render(&views(after, &new));
1691 assert!(
1692 text.contains("600 receipts from a previous experiment configuration ignored"),
1693 "{text}"
1694 );
1695
1696 let margin = Config::parse(&format!("{CFG}margin = 0.04\n")).expect("parses");
1698 assert_eq!(
1699 experiments_from_traces(&traces, &deferred, &[], &margin)[0].ignored_stale,
1700 600
1701 );
1702 let ramped = Config::parse(&CFG.replace("50.0", "30.0")).expect("parses");
1704 assert_eq!(
1705 experiments_from_traces(&traces, &deferred, &[], &ramped)[0].report,
1706 before[0].report
1707 );
1708 }
1709
1710 #[test]
1711 fn challenger_traces_are_not_incumbent_traces() {
1712 assert!(is_incumbent_trace(&trace("t", "s", Arm::Incumbent, 0.1)));
1713 assert!(!is_incumbent_trace(&trace("t", "s", Arm::Challenger, 0.1)));
1714 let mut plain = trace("t", "s", Arm::Challenger, 0.1);
1715 plain.experiment = None;
1716 assert!(is_incumbent_trace(&plain));
1717 }
1718
1719 #[tokio::test]
1721 async fn unreadable_store_fails_closed_on_the_challenger() {
1722 let cfg = config();
1723 let ok = ExperimentMonitor::from_config(&cfg);
1724 assert!(!ok.challenger_blocked("t", 0));
1725 let blocked = ExperimentMonitor::unavailable(&cfg);
1726 assert!(blocked.challenger_blocked("t", 0));
1727 assert!(blocked.challenger_blocked("any-other-tenant", 0));
1728
1729 let dir = std::env::temp_dir();
1731 let m = build_monitor(&dir.to_string_lossy(), &cfg);
1732 assert!(
1733 m.challenger_blocked("t", 0),
1734 "unreadable store served a challenger"
1735 );
1736
1737 let db = dir.join(format!("firstpass-exp-build-{}.db", Uuid::now_v7()));
1739 let (tx, handle) = crate::store::open(&db).expect("open");
1740 drop(tx);
1741 handle.await.expect("writer");
1742 let m = build_monitor(&db.to_string_lossy(), &cfg);
1743 assert!(!m.challenger_blocked("t", 0));
1744 let _ = std::fs::remove_file(&db);
1745
1746 let bare = Config::parse(
1748 "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-sonnet-5\"]\n",
1749 )
1750 .expect("parses");
1751 assert!(!build_monitor(&dir.to_string_lossy(), &bare).challenger_blocked("t", 0));
1752 }
1753
1754 #[tokio::test]
1759 async fn paired_challenger_cost_reaches_the_report_on_every_path() {
1760 let cfg = config();
1761 let fp_str = fp();
1762 let challenger = |m: &ExperimentMonitor| {
1763 let r = m.reports();
1764 assert_eq!(r.len(), 1);
1765 let c = r[0].report.challenger;
1766 (c.requests, c.cost_usd, r[0].report.pairs)
1767 };
1768 let close = |(n, usd, pairs): (u64, f64, u64)| {
1769 assert_eq!((n, pairs), (3, 3));
1770 assert!((usd - 0.03).abs() < 1e-12, "{usd}");
1771 };
1772
1773 let direct = ExperimentMonitor::from_config(&cfg);
1774 for _ in 0..3 {
1775 assert!(
1776 direct
1777 .record_pair("t", 0, &fp_str, 1.0, 1.0, 0.10, 0.01)
1778 .await
1779 );
1780 }
1781 close(challenger(&direct));
1782
1783 let shared = Arc::new(InMemoryExperimentLog::new());
1784 let a = ExperimentMonitor::from_config(&cfg)
1785 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1786 let b = ExperimentMonitor::from_config(&cfg)
1787 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1788 for _ in 0..3 {
1789 a.record_pair("t", 0, &fp_str, 1.0, 1.0, 0.10, 0.01).await;
1790 }
1791 b.sync_from_backend("t", 0, &fp_str).await;
1792 close(challenger(&a));
1793 close(challenger(&b));
1794
1795 let db =
1796 std::env::temp_dir().join(format!("firstpass-paircost-{}.db", uuid::Uuid::now_v7()));
1797 let (tx, handle) = crate::store::open(&db).unwrap();
1798 drop(tx);
1799 handle.await.unwrap();
1800 for _ in 0..3 {
1801 crate::store::append_experiment_pair(
1802 &db,
1803 &crate::store::ExperimentPairRow {
1804 tenant: "t".to_owned(),
1805 route_ix: 0,
1806 fingerprint: fp_str.clone(),
1807 trace_id: uuid::Uuid::now_v7(),
1808 x_inc: 1.0,
1809 x_ch: 1.0,
1810 challenger_cost_usd: 0.01,
1811 ts: jiff::Timestamp::now(),
1812 },
1813 )
1814 .unwrap();
1815 }
1816 close(challenger(&build_monitor(&db.to_string_lossy(), &cfg)));
1817
1818 let restarted = build_monitor_with(&db.to_string_lossy(), &cfg, true)
1821 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1822 restarted.sync_from_backend("t", 0, &fp_str).await;
1823 close(challenger(&restarted));
1824 let _ = std::fs::remove_file(&db);
1825 }
1826
1827 #[tokio::test]
1834 async fn two_replicas_sharing_one_log_converge_after_sync() {
1835 let shared = Arc::new(InMemoryExperimentLog::new());
1836 let cfg = config();
1837 let fp_str = fp();
1838
1839 let replica_a = ExperimentMonitor::from_config(&cfg)
1840 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1841 let replica_b = ExperimentMonitor::from_config(&cfg)
1842 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1843
1844 for i in 0..400 {
1846 replica_a
1847 .record_outcome(
1848 "t",
1849 0,
1850 &fp_str,
1851 Arm::Incumbent,
1852 &format!("i{i}"),
1853 Verdict::Pass,
1854 0.10,
1855 )
1856 .await;
1857 replica_a
1858 .record_outcome(
1859 "t",
1860 0,
1861 &fp_str,
1862 Arm::Challenger,
1863 &format!("c{i}"),
1864 Verdict::Fail,
1865 0.02,
1866 )
1867 .await;
1868 }
1869
1870 assert_eq!(
1873 replica_b.status("t", 0),
1874 ExperimentStatus::Undecided,
1875 "replica_b should not have any evidence before syncing"
1876 );
1877
1878 replica_b.sync_from_backend("t", 0, &fp_str).await;
1880 assert_eq!(
1881 replica_b.status("t", 0),
1882 ExperimentStatus::Halted,
1883 "replica_b must halt after syncing from shared log"
1884 );
1885 assert!(
1886 replica_b.challenger_blocked("t", 0),
1887 "halted experiment must block the challenger on the synced replica"
1888 );
1889
1890 let reps_a = replica_a.reports();
1892 let reps_b = replica_b.reports();
1893 assert_eq!(reps_a.len(), 1);
1894 assert_eq!(reps_b.len(), 1);
1895 assert_eq!(reps_a[0].report.status, reps_b[0].report.status);
1896 assert_eq!(reps_a[0].report.pairs, reps_b[0].report.pairs);
1897 }
1898
1899 #[tokio::test]
1908 async fn halt_on_one_replica_blocks_challenger_on_other_after_sync() {
1909 let shared = Arc::new(InMemoryExperimentLog::new());
1910 let cfg = config();
1911 let fp_str = fp();
1912
1913 let replica_a = ExperimentMonitor::from_config(&cfg)
1914 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1915 let replica_b = ExperimentMonitor::from_config(&cfg)
1916 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1917
1918 assert!(
1920 !replica_b.challenger_blocked("t", 0),
1921 "should be unblocked before any evidence"
1922 );
1923
1924 let mut halted = false;
1926 for i in 0..500 {
1927 replica_a
1928 .record_outcome(
1929 "t",
1930 0,
1931 &fp_str,
1932 Arm::Incumbent,
1933 &format!("i{i}"),
1934 Verdict::Pass,
1935 0.10,
1936 )
1937 .await;
1938 replica_a
1939 .record_outcome(
1940 "t",
1941 0,
1942 &fp_str,
1943 Arm::Challenger,
1944 &format!("c{i}"),
1945 Verdict::Fail,
1946 0.02,
1947 )
1948 .await;
1949 if replica_a.status("t", 0) == ExperimentStatus::Halted {
1950 halted = true;
1951 break;
1952 }
1953 }
1954 assert!(halted, "experiment never halted on replica A");
1955
1956 assert!(
1958 !replica_b.challenger_blocked("t", 0),
1959 "replica_b should not block before syncing"
1960 );
1961
1962 replica_b.sync_from_backend("t", 0, &fp_str).await;
1964 assert!(
1965 replica_b.challenger_blocked("t", 0),
1966 "replica_b must block after syncing the halt from the shared log"
1967 );
1968 }
1969
1970 #[tokio::test]
1976 async fn cross_replica_session_dedup_prevents_double_counting() {
1977 let shared = Arc::new(InMemoryExperimentLog::new());
1978 let cfg = config();
1979 let fp_str = fp();
1980
1981 let replica_a = ExperimentMonitor::from_config(&cfg)
1982 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1983 let replica_b = ExperimentMonitor::from_config(&cfg)
1984 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
1985
1986 replica_a
1988 .record_outcome(
1989 "t",
1990 0,
1991 &fp_str,
1992 Arm::Incumbent,
1993 "session-1",
1994 Verdict::Pass,
1995 0.10,
1996 )
1997 .await;
1998 replica_b
1999 .record_outcome(
2000 "t",
2001 0,
2002 &fp_str,
2003 Arm::Incumbent,
2004 "session-1",
2005 Verdict::Pass,
2006 0.10,
2007 )
2008 .await;
2009
2010 replica_a.sync_from_backend("t", 0, &fp_str).await;
2012 replica_b.sync_from_backend("t", 0, &fp_str).await;
2013
2014 let reps_a = replica_a.reports();
2016 let reps_b = replica_b.reports();
2017 assert_eq!(
2018 reps_a[0].report.incumbent.outcomes, 1,
2019 "session should be counted exactly once"
2020 );
2021 assert_eq!(
2022 reps_b[0].report.incumbent.outcomes, 1,
2023 "both replicas should agree on outcome count"
2024 );
2025 }
2026
2027 #[tokio::test]
2034 async fn abstain_does_not_burn_session_slot_before_claim() {
2035 let shared = Arc::new(InMemoryExperimentLog::new());
2036 let cfg = config();
2037 let fp_str = fp();
2038
2039 let monitor = ExperimentMonitor::from_config(&cfg)
2040 .with_log(Arc::clone(&shared) as Arc<dyn ExperimentLogBackend>);
2041
2042 let counted = monitor
2044 .record_outcome(
2045 "t",
2046 0,
2047 &fp_str,
2048 Arm::Incumbent,
2049 "session-abstain",
2050 Verdict::Abstain,
2051 0.10,
2052 )
2053 .await;
2054 assert!(!counted, "Abstain must not be counted");
2055
2056 let counted = monitor
2058 .record_outcome(
2059 "t",
2060 0,
2061 &fp_str,
2062 Arm::Incumbent,
2063 "session-abstain",
2064 Verdict::Pass,
2065 0.10,
2066 )
2067 .await;
2068 assert!(counted, "Pass after Abstain must be counted");
2069
2070 let reps = monitor.reports();
2071 assert_eq!(
2072 reps[0].report.incumbent.outcomes, 1,
2073 "exactly one outcome must have been counted"
2074 );
2075 }
2076
2077 #[cfg(feature = "redis-cache")]
2082 async fn redis_cleanup(url: &str, unique_tenant: &str) {
2083 let client = redis::Client::open(url).expect("redis client");
2084 let mut conn = redis::aio::ConnectionManager::new(client)
2085 .await
2086 .expect("redis connection for cleanup");
2087 use redis::AsyncCommands as _;
2088 let pattern = format!("firstpass:exp*{unique_tenant}*");
2090 let keys: Vec<String> = {
2091 let mut out = Vec::new();
2092 let mut cursor: u64 = 0;
2093 loop {
2094 let (next, batch): (u64, Vec<String>) = redis::cmd("SCAN")
2095 .arg(cursor)
2096 .arg("MATCH")
2097 .arg(&pattern)
2098 .arg("COUNT")
2099 .arg(100u64)
2100 .query_async(&mut conn)
2101 .await
2102 .expect("SCAN");
2103 out.extend(batch);
2104 cursor = next;
2105 if cursor == 0 {
2106 break;
2107 }
2108 }
2109 out
2110 };
2111 if !keys.is_empty() {
2112 let _: () = conn.del(keys).await.expect("DEL cleanup");
2113 }
2114 }
2115
2116 #[cfg(feature = "redis-cache")]
2126 #[tokio::test]
2127 async fn redis_two_replicas_converge_to_identical_reports_after_sync() {
2128 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2129 panic!(
2130 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2131 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2132 is worse than a red one."
2133 )
2134 });
2135 let unique_tenant = format!("t-conv-{}", Uuid::now_v7());
2137 let log_a = Arc::new(
2138 RedisExperimentLog::connect(&url)
2139 .await
2140 .expect("RedisExperimentLog::connect"),
2141 );
2142 let log_b = Arc::new(
2143 RedisExperimentLog::connect(&url)
2144 .await
2145 .expect("RedisExperimentLog::connect"),
2146 );
2147 let cfg = config();
2148 let fp_str = fp();
2149
2150 let replica_a = ExperimentMonitor::from_config(&cfg)
2151 .with_log(Arc::clone(&log_a) as Arc<dyn ExperimentLogBackend>);
2152 let replica_b = ExperimentMonitor::from_config(&cfg)
2153 .with_log(Arc::clone(&log_b) as Arc<dyn ExperimentLogBackend>);
2154
2155 for i in 0..400 {
2157 replica_a
2158 .record_outcome(
2159 &unique_tenant,
2160 0,
2161 &fp_str,
2162 Arm::Incumbent,
2163 &format!("i{i}"),
2164 Verdict::Pass,
2165 0.10,
2166 )
2167 .await;
2168 replica_a
2169 .record_outcome(
2170 &unique_tenant,
2171 0,
2172 &fp_str,
2173 Arm::Challenger,
2174 &format!("c{i}"),
2175 Verdict::Fail,
2176 0.02,
2177 )
2178 .await;
2179 }
2180
2181 assert_eq!(
2183 replica_b.status(&unique_tenant, 0),
2184 ExperimentStatus::Undecided,
2185 "replica_b must be Undecided before syncing"
2186 );
2187
2188 replica_b
2190 .sync_from_backend(&unique_tenant, 0, &fp_str)
2191 .await;
2192 assert_eq!(
2193 replica_b.status(&unique_tenant, 0),
2194 ExperimentStatus::Halted,
2195 "replica_b must be Halted after syncing from Redis log"
2196 );
2197
2198 let reps_a = replica_a.reports();
2199 let reps_b = replica_b.reports();
2200 assert_eq!(reps_a.len(), 1);
2201 assert_eq!(reps_b.len(), 1);
2202 assert_eq!(
2203 reps_a[0].report.status, reps_b[0].report.status,
2204 "both replicas must agree on status"
2205 );
2206 assert_eq!(
2207 reps_a[0].report.pairs, reps_b[0].report.pairs,
2208 "both replicas must agree on pair counts"
2209 );
2210
2211 redis_cleanup(&url, &unique_tenant).await;
2212 }
2213
2214 #[cfg(feature = "redis-cache")]
2222 #[tokio::test]
2223 async fn redis_cross_replica_session_dedup_via_claim_session_holds() {
2224 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2225 panic!(
2226 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2227 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2228 is worse than a red one."
2229 )
2230 });
2231 let unique_tenant = format!("t-dedup-{}", Uuid::now_v7());
2232 let log = Arc::new(
2233 RedisExperimentLog::connect(&url)
2234 .await
2235 .expect("RedisExperimentLog::connect"),
2236 );
2237 let cfg = config();
2238 let fp_str = fp();
2239
2240 let replica_a = ExperimentMonitor::from_config(&cfg)
2241 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2242 let replica_b = ExperimentMonitor::from_config(&cfg)
2243 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2244
2245 replica_a
2247 .record_outcome(
2248 &unique_tenant,
2249 0,
2250 &fp_str,
2251 Arm::Incumbent,
2252 "session-x",
2253 Verdict::Pass,
2254 0.10,
2255 )
2256 .await;
2257 replica_b
2258 .record_outcome(
2259 &unique_tenant,
2260 0,
2261 &fp_str,
2262 Arm::Incumbent,
2263 "session-x",
2264 Verdict::Pass,
2265 0.10,
2266 )
2267 .await;
2268
2269 replica_a
2271 .sync_from_backend(&unique_tenant, 0, &fp_str)
2272 .await;
2273 replica_b
2274 .sync_from_backend(&unique_tenant, 0, &fp_str)
2275 .await;
2276
2277 let reps_a = replica_a.reports();
2278 let reps_b = replica_b.reports();
2279 assert_eq!(
2280 reps_a[0].report.incumbent.outcomes, 1,
2281 "session must be counted exactly once on replica A"
2282 );
2283 assert_eq!(
2284 reps_b[0].report.incumbent.outcomes, 1,
2285 "session must be counted exactly once on replica B"
2286 );
2287
2288 redis_cleanup(&url, &unique_tenant).await;
2289 }
2290
2291 #[cfg(feature = "redis-cache")]
2299 #[tokio::test]
2300 async fn redis_halt_on_one_replica_blocks_challenger_on_other() {
2301 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2302 panic!(
2303 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2304 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2305 is worse than a red one."
2306 )
2307 });
2308 let unique_tenant = format!("t-halt-{}", Uuid::now_v7());
2309 let log = Arc::new(
2310 RedisExperimentLog::connect(&url)
2311 .await
2312 .expect("RedisExperimentLog::connect"),
2313 );
2314 let cfg = config();
2315 let fp_str = fp();
2316
2317 let replica_a = ExperimentMonitor::from_config(&cfg)
2318 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2319 let replica_b = ExperimentMonitor::from_config(&cfg)
2320 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2321
2322 assert!(
2324 !replica_b.challenger_blocked(&unique_tenant, 0),
2325 "replica_b must be unblocked before any evidence"
2326 );
2327
2328 let mut halted = false;
2330 for i in 0..500 {
2331 replica_a
2332 .record_outcome(
2333 &unique_tenant,
2334 0,
2335 &fp_str,
2336 Arm::Incumbent,
2337 &format!("i{i}"),
2338 Verdict::Pass,
2339 0.10,
2340 )
2341 .await;
2342 replica_a
2343 .record_outcome(
2344 &unique_tenant,
2345 0,
2346 &fp_str,
2347 Arm::Challenger,
2348 &format!("c{i}"),
2349 Verdict::Fail,
2350 0.02,
2351 )
2352 .await;
2353 if replica_a.status(&unique_tenant, 0) == ExperimentStatus::Halted {
2354 halted = true;
2355 break;
2356 }
2357 }
2358 assert!(halted, "experiment never halted on replica A");
2359
2360 assert!(
2362 !replica_b.challenger_blocked(&unique_tenant, 0),
2363 "replica_b must not block before syncing"
2364 );
2365
2366 replica_b
2368 .sync_from_backend(&unique_tenant, 0, &fp_str)
2369 .await;
2370 assert!(
2371 replica_b.challenger_blocked(&unique_tenant, 0),
2372 "replica_b must block the challenger after syncing the halt from Redis"
2373 );
2374
2375 redis_cleanup(&url, &unique_tenant).await;
2376 }
2377
2378 #[cfg(feature = "redis-cache")]
2386 #[tokio::test]
2387 async fn redis_keys_are_isolated_per_tenant() {
2388 let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
2389 panic!(
2390 "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:56411/15) to run the \
2391 redis-cache tests. Not skipped on purpose: a green suite that tested nothing \
2392 is worse than a red one."
2393 )
2394 });
2395 let run_id = Uuid::now_v7();
2396 let tenant_a = format!("t-iso-a-{run_id}");
2397 let tenant_b = format!("t-iso-b-{run_id}");
2398 let log = Arc::new(
2399 RedisExperimentLog::connect(&url)
2400 .await
2401 .expect("RedisExperimentLog::connect"),
2402 );
2403 let cfg = config();
2404 let fp_str = fp();
2405
2406 let monitor_a = ExperimentMonitor::from_config(&cfg)
2407 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2408 let monitor_b = ExperimentMonitor::from_config(&cfg)
2409 .with_log(Arc::clone(&log) as Arc<dyn ExperimentLogBackend>);
2410
2411 for i in 0..5 {
2413 monitor_a
2414 .record_outcome(
2415 &tenant_a,
2416 0,
2417 &fp_str,
2418 Arm::Incumbent,
2419 &format!("s{i}"),
2420 Verdict::Pass,
2421 0.10,
2422 )
2423 .await;
2424 }
2425
2426 let records = log.load_since(&tenant_b, 0, &fp_str, 0).await;
2428 assert!(
2429 records.is_empty(),
2430 "backend must return no records for tenant_b"
2431 );
2432
2433 assert_eq!(
2436 monitor_b.status(&tenant_b, 0),
2437 ExperimentStatus::Undecided,
2438 "evidence written under tenant_a must not appear under tenant_b"
2439 );
2440
2441 let reps = monitor_a.reports();
2443 assert_eq!(reps.len(), 1);
2444 assert_eq!(reps[0].report.incumbent.outcomes, 5);
2445
2446 redis_cleanup(&url, &tenant_a).await;
2448 redis_cleanup(&url, &tenant_b).await;
2449 }
2450}