lean_ctx/core/context_kernel/
outcome_signal.rs

1//! Outcome quality signals inferred from observable LLM behavior.
2
3use std::time::{SystemTime, UNIX_EPOCH};
4
5use super::activation::{connect_feedback, record_real_outcome};
6use super::types::{ContextReceiptV1, ReceiptOutcome};
7
8/// The signal that determined the outcome classification.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
10pub enum OutcomeSignal {
11    /// LLM used the context on first try.
12    FirstPass,
13    /// LLM retried after receiving this context (indicates rejection).
14    Retry,
15    /// LLM produced zero response tokens (likely ignored the context).
16    Ignored,
17    /// Ambiguous — not enough signal to determine.
18    Ambiguous,
19}
20
21/// An outcome inferred from LLM behavior heuristics.
22#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
23pub struct InferredOutcome {
24    /// Accept/reject classification inferred from the observed behavior.
25    pub outcome: ReceiptOutcome,
26    /// Confidence in the inferred classification, from zero to one.
27    pub confidence: f64,
28    /// Behavioral signal used to determine the classification.
29    pub signal: OutcomeSignal,
30}
31
32/// Infers a context outcome from request and response behavior.
33pub fn infer_outcome(
34    request_count: usize,
35    was_retry: bool,
36    response_tokens: usize,
37) -> InferredOutcome {
38    if was_retry && request_count > 1 {
39        InferredOutcome {
40            outcome: ReceiptOutcome::Rejected,
41            confidence: 0.8,
42            signal: OutcomeSignal::Retry,
43        }
44    } else if response_tokens == 0 {
45        InferredOutcome {
46            outcome: ReceiptOutcome::Rejected,
47            confidence: 0.6,
48            signal: OutcomeSignal::Ignored,
49        }
50    } else if request_count == 1 {
51        InferredOutcome {
52            outcome: ReceiptOutcome::Accepted,
53            confidence: 0.9,
54            signal: OutcomeSignal::FirstPass,
55        }
56    } else {
57        InferredOutcome {
58            outcome: ReceiptOutcome::Accepted,
59            confidence: 0.5,
60            signal: OutcomeSignal::Ambiguous,
61        }
62    }
63}
64
65/// Records an inferred outcome and feeds it into provider learning.
66///
67/// Feedback failures are contained so outcome tracking cannot disrupt delivery.
68pub fn record_and_learn(outcome: &InferredOutcome, receipt: &ContextReceiptV1, project_root: &str) {
69    let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
70        let recorded = record_real_outcome(receipt, outcome.outcome == ReceiptOutcome::Accepted);
71        connect_feedback(&recorded, project_root);
72    }));
73}
74
75/// Tracks recent inferred outcomes for aggregate quality monitoring.
76#[derive(Debug, Clone, Default)]
77pub struct OutcomeTracker {
78    outcomes: Vec<(OutcomeSignal, f64)>,
79}
80
81impl OutcomeTracker {
82    /// Appends an inferred outcome with its observation timestamp.
83    pub fn record(&mut self, outcome: &InferredOutcome) {
84        let timestamp = SystemTime::now()
85            .duration_since(UNIX_EPOCH)
86            .map_or(0.0, |duration| duration.as_secs_f64());
87        self.outcomes.push((outcome.signal, timestamp));
88    }
89
90    /// Returns the fraction of tracked outcomes classified as accepted.
91    pub fn acceptance_rate(&self) -> f64 {
92        if self.outcomes.is_empty() {
93            return 0.0;
94        }
95
96        let accepted = self
97            .outcomes
98            .iter()
99            .filter(|(signal, _)| is_accepted(*signal))
100            .count();
101        accepted as f64 / self.outcomes.len() as f64
102    }
103
104    /// Returns whether a complete recent window has below 50% acceptance.
105    pub fn is_degrading(&self, window: usize) -> bool {
106        if window == 0 || self.outcomes.len() < window {
107            return false;
108        }
109
110        let accepted = self
111            .outcomes
112            .iter()
113            .rev()
114            .take(window)
115            .filter(|(signal, _)| is_accepted(*signal))
116            .count();
117        accepted * 2 < window
118    }
119
120    /// Returns the number of tracked outcomes.
121    pub fn len(&self) -> usize {
122        self.outcomes.len()
123    }
124
125    /// Returns whether the tracker contains no outcomes.
126    pub fn is_empty(&self) -> bool {
127        self.outcomes.is_empty()
128    }
129}
130
131fn is_accepted(signal: OutcomeSignal) -> bool {
132    matches!(signal, OutcomeSignal::FirstPass | OutcomeSignal::Ambiguous)
133}
134
135#[cfg(test)]
136mod tests {
137    use std::collections::HashMap;
138
139    use super::{InferredOutcome, OutcomeSignal, OutcomeTracker, infer_outcome, record_and_learn};
140    use crate::core::context_kernel::types::{ContextReceiptV1, ReceiptOutcome};
141
142    fn receipt() -> ContextReceiptV1 {
143        ContextReceiptV1 {
144            receipt_id: "receipt-1".to_owned(),
145            plan_id: "plan-1".to_owned(),
146            delivered_tokens: 10,
147            cache_hits: 0,
148            cache_misses: 0,
149            outcome: ReceiptOutcome::Unknown,
150            quality_signals: Vec::new(),
151            feedback_attribution: HashMap::new(),
152        }
153    }
154
155    fn tracked(outcome: ReceiptOutcome, signal: OutcomeSignal) -> InferredOutcome {
156        InferredOutcome {
157            outcome,
158            confidence: 1.0,
159            signal,
160        }
161    }
162
163    #[test]
164    fn first_pass_accepted() {
165        let inferred = infer_outcome(1, false, 20);
166        assert_eq!(inferred.outcome, ReceiptOutcome::Accepted);
167        assert_eq!(inferred.confidence, 0.9);
168        assert_eq!(inferred.signal, OutcomeSignal::FirstPass);
169    }
170
171    #[test]
172    fn retry_rejected() {
173        let inferred = infer_outcome(2, true, 20);
174        assert_eq!(inferred.outcome, ReceiptOutcome::Rejected);
175        assert_eq!(inferred.confidence, 0.8);
176        assert_eq!(inferred.signal, OutcomeSignal::Retry);
177    }
178
179    #[test]
180    fn zero_tokens_ignored() {
181        let inferred = infer_outcome(1, false, 0);
182        assert_eq!(inferred.outcome, ReceiptOutcome::Rejected);
183        assert_eq!(inferred.confidence, 0.6);
184        assert_eq!(inferred.signal, OutcomeSignal::Ignored);
185    }
186
187    #[test]
188    fn ambiguous_default_accepted() {
189        let inferred = infer_outcome(2, false, 20);
190        assert_eq!(inferred.outcome, ReceiptOutcome::Accepted);
191        assert_eq!(inferred.confidence, 0.5);
192        assert_eq!(inferred.signal, OutcomeSignal::Ambiguous);
193    }
194
195    #[test]
196    fn tracker_acceptance_rate() {
197        let mut tracker = OutcomeTracker::default();
198        for _ in 0..7 {
199            tracker.record(&tracked(ReceiptOutcome::Accepted, OutcomeSignal::FirstPass));
200        }
201        for _ in 0..3 {
202            tracker.record(&tracked(ReceiptOutcome::Rejected, OutcomeSignal::Retry));
203        }
204
205        assert!((tracker.acceptance_rate() - 0.7).abs() < f64::EPSILON);
206        assert_eq!(tracker.len(), 10);
207    }
208
209    #[test]
210    fn tracker_detects_degradation() {
211        let mut tracker = OutcomeTracker::default();
212        for _ in 0..5 {
213            tracker.record(&tracked(ReceiptOutcome::Rejected, OutcomeSignal::Retry));
214        }
215
216        assert!(tracker.is_degrading(5));
217    }
218
219    #[test]
220    fn record_and_learn_no_panic() {
221        let inferred = infer_outcome(2, true, 20);
222        record_and_learn(&inferred, &receipt(), "/path/that/does/not/exist");
223    }
224}
lean_ctx/core/context_kernel/outcome_signal.rs

lean_ctx/core/context_kernel/
outcome_signal.rs
