lean_ctx/core/ocla/
routing_quality.rs

1//! In-memory quality tracking for model-routing decisions.
2
3use std::collections::VecDeque;
4
5const MAX_DECISIONS: usize = 1_000;
6const QUALITY_THRESHOLD: f64 = 0.8;
7const MIN_SCORED_OUTCOMES: usize = 20;
8
9/// The model route selected for one request.
10#[derive(Clone, Debug, Eq, PartialEq)]
11pub struct RoutingDecision {
12    pub decision_id: String,
13    pub original_model: String,
14    pub routed_model: String,
15    pub reason: String,
16    pub timestamp: String,
17}
18
19/// Measured result for a model route.
20#[derive(Clone, Debug, PartialEq)]
21pub struct RoutingOutcome {
22    pub decision: RoutingDecision,
23    pub quality_score: Option<f64>,
24    pub tokens_saved: u64,
25    pub latency_delta_ms: i64,
26}
27
28/// Bounded tracker for recent model-routing outcomes.
29#[derive(Debug, Default)]
30pub struct RoutingQualityTracker {
31    outcomes: VecDeque<RoutingOutcome>,
32}
33
34impl RoutingQualityTracker {
35    /// Creates an empty tracker.
36    pub fn new() -> Self {
37        Self::default()
38    }
39
40    /// Records an outcome, retaining only the most recent 1000 outcomes.
41    pub fn record(&mut self, outcome: RoutingOutcome) {
42        if self.outcomes.len() == MAX_DECISIONS {
43            self.outcomes.pop_front();
44        }
45        self.outcomes.push_back(RoutingOutcome {
46            quality_score: sanitize_quality(outcome.quality_score),
47            ..outcome
48        });
49
50        if self.should_fallback() {
51            tracing::warn!(
52                success_rate = self.success_rate(),
53                "routing quality below threshold; suggest fallback"
54            );
55        }
56    }
57
58    /// Returns the fraction of recorded outcomes meeting the quality threshold.
59    pub fn success_rate(&self) -> f64 {
60        let scored = self.scored_outcomes();
61        if scored == 0 {
62            return 0.0;
63        }
64
65        self.outcomes
66            .iter()
67            .filter(|outcome| {
68                outcome
69                    .quality_score
70                    .is_some_and(|score| score >= QUALITY_THRESHOLD)
71            })
72            .count() as f64
73            / scored as f64
74    }
75
76    /// Returns the average token savings across recorded outcomes.
77    pub fn average_savings(&self) -> f64 {
78        if self.outcomes.is_empty() {
79            return 0.0;
80        }
81
82        self.outcomes
83            .iter()
84            .map(|outcome| outcome.tokens_saved as f64)
85            .sum::<f64>()
86            / self.outcomes.len() as f64
87    }
88
89    /// Returns whether recent route quality should trigger a fallback.
90    pub fn should_fallback(&self) -> bool {
91        self.scored_outcomes() >= MIN_SCORED_OUTCOMES && self.success_rate() < QUALITY_THRESHOLD
92    }
93
94    fn scored_outcomes(&self) -> usize {
95        self.outcomes
96            .iter()
97            .filter(|outcome| outcome.quality_score.is_some())
98            .count()
99    }
100}
101
102fn sanitize_quality(score: Option<f64>) -> Option<f64> {
103    score.filter(|value| value.is_finite() && (0.0..=1.0).contains(value))
104}
105
106#[cfg(test)]
107mod tests {
108    use super::*;
109
110    fn outcome(score: Option<f64>, tokens_saved: u64) -> RoutingOutcome {
111        RoutingOutcome {
112            decision: RoutingDecision {
113                decision_id: "decision-test".into(),
114                original_model: "expensive".into(),
115                routed_model: "fast".into(),
116                reason: "quality test".into(),
117                timestamp: "2026-01-01T00:00:00Z".into(),
118            },
119            quality_score: score,
120            tokens_saved,
121            latency_delta_ms: -5,
122        }
123    }
124
125    #[test]
126    fn new_tracker_starts_empty() {
127        let tracker = RoutingQualityTracker::new();
128
129        assert_eq!(tracker.success_rate(), 0.0);
130        assert_eq!(tracker.average_savings(), 0.0);
131        assert!(!tracker.should_fallback());
132    }
133
134    #[test]
135    fn mixed_outcomes_update_quality_and_savings() {
136        let mut tracker = RoutingQualityTracker::new();
137        tracker.record(outcome(Some(0.95), 100));
138        tracker.record(outcome(Some(0.4), 20));
139        tracker.record(outcome(None, 0));
140        tracker.record(outcome(Some(0.8), 40));
141
142        assert!((tracker.success_rate() - (2.0 / 3.0)).abs() < f64::EPSILON);
143        assert!((tracker.average_savings() - 40.0).abs() < f64::EPSILON);
144        assert!(!tracker.should_fallback());
145    }
146
147    #[test]
148    fn ring_buffer_discards_oldest_outcome() {
149        let mut tracker = RoutingQualityTracker::new();
150        tracker.record(outcome(Some(0.0), 0));
151        for _ in 0..MAX_DECISIONS {
152            tracker.record(outcome(Some(1.0), 100));
153        }
154
155        assert!((tracker.success_rate() - 1.0).abs() < f64::EPSILON);
156        assert!((tracker.average_savings() - 100.0).abs() < f64::EPSILON);
157        assert!(!tracker.should_fallback());
158    }
159
160    #[test]
161    fn fallback_requires_minimum_scored_samples() {
162        let mut tracker = RoutingQualityTracker::new();
163        for _ in 0..(MIN_SCORED_OUTCOMES - 1) {
164            tracker.record(outcome(Some(0.0), 0));
165        }
166        assert!(!tracker.should_fallback());
167
168        tracker.record(outcome(Some(0.0), 0));
169        assert!(tracker.should_fallback());
170    }
171
172    #[test]
173    fn invalid_and_absent_quality_do_not_count_as_failure() {
174        let mut tracker = RoutingQualityTracker::new();
175        tracker.record(outcome(None, 10));
176        tracker.record(outcome(Some(f64::NAN), 10));
177        tracker.record(outcome(Some(1.5), 10));
178
179        assert_eq!(tracker.success_rate(), 0.0);
180        assert!(!tracker.should_fallback());
181    }
182}
lean_ctx/core/ocla/routing_quality.rs

lean_ctx/core/ocla/
routing_quality.rs
