lean_ctx/core/ocla/
routing_quality.rs

1//! In-memory quality tracking for model-routing decisions.
2
3use std::collections::VecDeque;
4
5const MAX_DECISIONS: usize = 1_000;
6const QUALITY_THRESHOLD: f64 = 0.8;
7
8/// The model route selected for one request.
9#[derive(Clone, Debug, Eq, PartialEq)]
10pub struct RoutingDecision {
11    pub original_model: String,
12    pub routed_model: String,
13    pub reason: String,
14    pub timestamp: String,
15}
16
17/// Measured result for a model route.
18#[derive(Clone, Debug, PartialEq)]
19pub struct RoutingOutcome {
20    pub decision: RoutingDecision,
21    pub quality_score: Option<f64>,
22    pub tokens_saved: u64,
23    pub latency_delta_ms: i64,
24}
25
26/// Bounded tracker for recent model-routing outcomes.
27#[derive(Debug, Default)]
28pub struct RoutingQualityTracker {
29    outcomes: VecDeque<RoutingOutcome>,
30}
31
32impl RoutingQualityTracker {
33    /// Creates an empty tracker.
34    pub fn new() -> Self {
35        Self::default()
36    }
37
38    /// Records an outcome, retaining only the most recent 1000 outcomes.
39    pub fn record(&mut self, outcome: RoutingOutcome) {
40        if self.outcomes.len() == MAX_DECISIONS {
41            self.outcomes.pop_front();
42        }
43        self.outcomes.push_back(outcome);
44
45        if self.should_fallback() {
46            tracing::warn!(
47                success_rate = self.success_rate(),
48                "routing quality below threshold; suggest fallback"
49            );
50        }
51    }
52
53    /// Returns the fraction of recorded outcomes meeting the quality threshold.
54    pub fn success_rate(&self) -> f64 {
55        if self.outcomes.is_empty() {
56            return 0.0;
57        }
58
59        self.outcomes
60            .iter()
61            .filter(|outcome| outcome.quality_score.unwrap_or(0.0) >= QUALITY_THRESHOLD)
62            .count() as f64
63            / self.outcomes.len() as f64
64    }
65
66    /// Returns the average token savings across recorded outcomes.
67    pub fn average_savings(&self) -> f64 {
68        if self.outcomes.is_empty() {
69            return 0.0;
70        }
71
72        self.outcomes
73            .iter()
74            .map(|outcome| outcome.tokens_saved as f64)
75            .sum::<f64>()
76            / self.outcomes.len() as f64
77    }
78
79    /// Returns whether recent route quality should trigger a fallback.
80    pub fn should_fallback(&self) -> bool {
81        !self.outcomes.is_empty() && self.success_rate() < QUALITY_THRESHOLD
82    }
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88
89    fn outcome(score: Option<f64>, tokens_saved: u64) -> RoutingOutcome {
90        RoutingOutcome {
91            decision: RoutingDecision {
92                original_model: "expensive".into(),
93                routed_model: "fast".into(),
94                reason: "quality test".into(),
95                timestamp: "2026-01-01T00:00:00Z".into(),
96            },
97            quality_score: score,
98            tokens_saved,
99            latency_delta_ms: -5,
100        }
101    }
102
103    #[test]
104    fn new_tracker_starts_empty() {
105        let tracker = RoutingQualityTracker::new();
106
107        assert_eq!(tracker.success_rate(), 0.0);
108        assert_eq!(tracker.average_savings(), 0.0);
109        assert!(!tracker.should_fallback());
110    }
111
112    #[test]
113    fn mixed_outcomes_update_quality_and_savings() {
114        let mut tracker = RoutingQualityTracker::new();
115        tracker.record(outcome(Some(0.95), 100));
116        tracker.record(outcome(Some(0.4), 20));
117        tracker.record(outcome(None, 0));
118        tracker.record(outcome(Some(0.8), 40));
119
120        assert!((tracker.success_rate() - 0.5).abs() < f64::EPSILON);
121        assert!((tracker.average_savings() - 40.0).abs() < f64::EPSILON);
122        assert!(tracker.should_fallback());
123    }
124
125    #[test]
126    fn ring_buffer_discards_oldest_outcome() {
127        let mut tracker = RoutingQualityTracker::new();
128        tracker.record(outcome(Some(0.0), 0));
129        for _ in 0..MAX_DECISIONS {
130            tracker.record(outcome(Some(1.0), 100));
131        }
132
133        assert!((tracker.success_rate() - 1.0).abs() < f64::EPSILON);
134        assert!((tracker.average_savings() - 100.0).abs() < f64::EPSILON);
135        assert!(!tracker.should_fallback());
136    }
137}
lean_ctx/core/ocla/routing_quality.rs

lean_ctx/core/ocla/
routing_quality.rs
