lean_ctx/core/context_kernel/
etpao.rs

1//! Effective tokens per accepted outcome metrics.
2
3use std::collections::HashMap;
4
5use serde::{Deserialize, Serialize};
6
7use super::types::{ContextReceiptV1, ReceiptOutcome};
8
9/// Aggregated token efficiency metrics for a single scope (project/agent/model).
10#[derive(Debug, Clone, Default, Serialize, Deserialize)]
11pub struct EtpaoMetrics {
12    pub scope: String,
13    pub tokens_input: u64,
14    pub tokens_output: u64,
15    pub tokens_reasoning: u64,
16    pub tokens_schema: u64,
17    pub tokens_retry: u64,
18    pub tokens_cache_write: u64,
19    pub tokens_handoff: u64,
20    pub accepted_outcomes: u64,
21    pub first_pass_successes: u64,
22    pub total_requests: u64,
23    pub cache_hits: u64,
24    pub cache_misses: u64,
25}
26
27impl EtpaoMetrics {
28    /// Returns total accounted tokens per accepted outcome.
29    pub fn etpao(&self) -> f64 {
30        if self.accepted_outcomes == 0 {
31            return f64::INFINITY;
32        }
33
34        let total_tokens = self
35            .tokens_input
36            .saturating_add(self.tokens_output)
37            .saturating_add(self.tokens_reasoning)
38            .saturating_add(self.tokens_schema)
39            .saturating_add(self.tokens_retry)
40            .saturating_add(self.tokens_cache_write)
41            .saturating_add(self.tokens_handoff);
42        total_tokens as f64 / self.accepted_outcomes as f64
43    }
44
45    /// Returns the fraction of requests accepted on the first pass.
46    pub fn first_pass_success_rate(&self) -> f64 {
47        ratio(self.first_pass_successes, self.total_requests)
48    }
49
50    /// Returns cache hits as a fraction of total requests.
51    pub fn cache_hit_rate(&self) -> f64 {
52        ratio(
53            self.cache_hits,
54            self.cache_hits.saturating_add(self.cache_misses),
55        )
56    }
57
58    /// Adds another scope's counters to this metric set.
59    pub fn merge(&mut self, other: &EtpaoMetrics) {
60        self.tokens_input = self.tokens_input.saturating_add(other.tokens_input);
61        self.tokens_output = self.tokens_output.saturating_add(other.tokens_output);
62        self.tokens_reasoning = self.tokens_reasoning.saturating_add(other.tokens_reasoning);
63        self.tokens_schema = self.tokens_schema.saturating_add(other.tokens_schema);
64        self.tokens_retry = self.tokens_retry.saturating_add(other.tokens_retry);
65        self.tokens_cache_write = self
66            .tokens_cache_write
67            .saturating_add(other.tokens_cache_write);
68        self.tokens_handoff = self.tokens_handoff.saturating_add(other.tokens_handoff);
69        self.accepted_outcomes = self
70            .accepted_outcomes
71            .saturating_add(other.accepted_outcomes);
72        self.first_pass_successes = self
73            .first_pass_successes
74            .saturating_add(other.first_pass_successes);
75        self.total_requests = self.total_requests.saturating_add(other.total_requests);
76        self.cache_hits = self.cache_hits.saturating_add(other.cache_hits);
77        self.cache_misses = self.cache_misses.saturating_add(other.cache_misses);
78    }
79}
80
81/// Tracks ETPAO metrics per scope, updated from context receipt outcomes.
82#[derive(Debug, Default)]
83pub struct EtpaoTracker {
84    scopes: HashMap<String, EtpaoMetrics>,
85}
86
87impl EtpaoTracker {
88    /// Records one delivered receipt and its evaluated outcome for a scope.
89    pub fn record_receipt(
90        &mut self,
91        scope: &str,
92        receipt: &ContextReceiptV1,
93        outcome: ReceiptOutcome,
94    ) {
95        let metrics = self
96            .scopes
97            .entry(scope.to_owned())
98            .or_insert_with(|| EtpaoMetrics {
99                scope: scope.to_owned(),
100                ..EtpaoMetrics::default()
101            });
102        metrics.tokens_input = metrics
103            .tokens_input
104            .saturating_add(receipt.delivered_tokens as u64);
105        metrics.total_requests = metrics.total_requests.saturating_add(1);
106
107        metrics.cache_hits = metrics.cache_hits.saturating_add(receipt.cache_hits as u64);
108        metrics.cache_misses = metrics
109            .cache_misses
110            .saturating_add(receipt.cache_misses as u64);
111
112        if outcome == ReceiptOutcome::Accepted {
113            metrics.accepted_outcomes = metrics.accepted_outcomes.saturating_add(1);
114            // Only count first-pass if no cache misses (proxy for no retries)
115            if receipt.cache_misses == 0 {
116                metrics.first_pass_successes = metrics.first_pass_successes.saturating_add(1);
117            }
118        }
119    }
120
121    /// Returns metrics for a tracked scope.
122    pub fn get(&self, scope: &str) -> Option<&EtpaoMetrics> {
123        self.scopes.get(scope)
124    }
125
126    /// Combines every tracked scope into a single aggregate.
127    pub fn aggregate(&self) -> EtpaoMetrics {
128        let mut aggregate = EtpaoMetrics {
129            scope: "aggregate".to_owned(),
130            ..EtpaoMetrics::default()
131        };
132        for metrics in self.scopes.values() {
133            aggregate.merge(metrics);
134        }
135        aggregate
136    }
137
138    /// Lists tracked scope names in stable lexical order.
139    pub fn all_scopes(&self) -> Vec<&str> {
140        let mut scopes: Vec<_> = self.scopes.keys().map(String::as_str).collect();
141        scopes.sort_unstable();
142        scopes
143    }
144
145    /// Returns serializable scope metrics in stable lexical order.
146    pub fn to_wire(&self) -> Vec<EtpaoMetrics> {
147        let mut metrics: Vec<_> = self.scopes.values().cloned().collect();
148        metrics.sort_unstable_by_key(|item| item.scope.clone());
149        metrics
150    }
151}
152
153/// Formats a privacy-safe numeric ETPAO summary.
154pub fn format_etpao_summary(metrics: &EtpaoMetrics) -> String {
155    format!(
156        "ETPAO: {:.2}, first-pass rate: {:.2}%, requests: {}",
157        metrics.etpao(),
158        metrics.first_pass_success_rate() * 100.0,
159        metrics.total_requests
160    )
161}
162
163fn ratio(numerator: u64, denominator: u64) -> f64 {
164    if denominator == 0 {
165        0.0
166    } else {
167        numerator as f64 / denominator as f64
168    }
169}
170
171#[cfg(test)]
172mod tests {
173    use super::{EtpaoMetrics, EtpaoTracker, format_etpao_summary};
174    use crate::core::context_kernel::types::{ContextReceiptV1, ReceiptOutcome};
175    use std::collections::HashMap;
176
177    fn receipt(tokens: usize) -> ContextReceiptV1 {
178        ContextReceiptV1 {
179            receipt_id: "receipt:test".to_owned(),
180            plan_id: "plan:test".to_owned(),
181            delivered_tokens: tokens,
182            cache_hits: 0,
183            cache_misses: 0,
184            outcome: ReceiptOutcome::Unknown,
185            quality_signals: Vec::new(),
186            feedback_attribution: HashMap::new(),
187        }
188    }
189
190    #[test]
191    fn empty_tracker_returns_infinity_etpao() {
192        assert!(EtpaoTracker::default().aggregate().etpao().is_infinite());
193    }
194
195    #[test]
196    fn single_receipt_updates_metrics() {
197        let mut tracker = EtpaoTracker::default();
198        tracker.record_receipt("agent-1", &receipt(120), ReceiptOutcome::Accepted);
199
200        let metrics = tracker.get("agent-1").expect("scope metrics");
201        assert_eq!(metrics.tokens_input, 120);
202        assert_eq!(metrics.total_requests, 1);
203        assert_eq!(metrics.accepted_outcomes, 1);
204        assert_eq!(metrics.first_pass_successes, 1);
205        assert_eq!(metrics.etpao(), 120.0);
206    }
207
208    #[test]
209    fn merge_sums_all_fields() {
210        let mut left = EtpaoMetrics {
211            tokens_input: 1,
212            tokens_output: 2,
213            tokens_reasoning: 3,
214            tokens_schema: 4,
215            tokens_retry: 5,
216            tokens_cache_write: 6,
217            tokens_handoff: 7,
218            accepted_outcomes: 8,
219            first_pass_successes: 9,
220            total_requests: 10,
221            ..EtpaoMetrics::default()
222        };
223        let right = left.clone();
224        left.merge(&right);
225
226        assert_eq!(left.tokens_input, 2);
227        assert_eq!(left.tokens_output, 4);
228        assert_eq!(left.tokens_reasoning, 6);
229        assert_eq!(left.tokens_schema, 8);
230        assert_eq!(left.tokens_retry, 10);
231        assert_eq!(left.tokens_cache_write, 12);
232        assert_eq!(left.tokens_handoff, 14);
233        assert_eq!(left.accepted_outcomes, 16);
234        assert_eq!(left.first_pass_successes, 18);
235        assert_eq!(left.total_requests, 20);
236    }
237
238    #[test]
239    fn aggregate_combines_scopes() {
240        let mut tracker = EtpaoTracker::default();
241        tracker.record_receipt("a", &receipt(10), ReceiptOutcome::Accepted);
242        tracker.record_receipt("b", &receipt(20), ReceiptOutcome::Rejected);
243        tracker.record_receipt("c", &receipt(30), ReceiptOutcome::Partial);
244
245        let aggregate = tracker.aggregate();
246        assert_eq!(aggregate.scope, "aggregate");
247        assert_eq!(aggregate.tokens_input, 60);
248        assert_eq!(aggregate.total_requests, 3);
249        assert_eq!(aggregate.accepted_outcomes, 1);
250    }
251
252    #[test]
253    fn first_pass_rate_calculation() {
254        let metrics = EtpaoMetrics {
255            first_pass_successes: 3,
256            total_requests: 4,
257            cache_hits: 2,
258            cache_misses: 2,
259            ..EtpaoMetrics::default()
260        };
261
262        assert_eq!(metrics.first_pass_success_rate(), 0.75);
263        assert_eq!(metrics.cache_hit_rate(), 0.5);
264    }
265
266    #[test]
267    fn format_summary_no_content_leak() {
268        let metrics = EtpaoMetrics {
269            scope: "/secret/project/private.rs".to_owned(),
270            tokens_input: 25,
271            accepted_outcomes: 1,
272            first_pass_successes: 1,
273            total_requests: 1,
274            ..EtpaoMetrics::default()
275        };
276        let summary = format_etpao_summary(&metrics);
277
278        assert!(summary.contains("25.00"));
279        assert!(summary.contains("100.00%"));
280        assert!(!summary.contains("secret"));
281        assert!(!summary.contains("private.rs"));
282    }
283
284    #[test]
285    fn scope_exports_are_sorted() {
286        let mut tracker = EtpaoTracker::default();
287        tracker.record_receipt("z", &receipt(1), ReceiptOutcome::Unknown);
288        tracker.record_receipt("a", &receipt(1), ReceiptOutcome::Unknown);
289
290        assert_eq!(tracker.all_scopes(), vec!["a", "z"]);
291        assert_eq!(tracker.to_wire()[0].scope, "a");
292    }
293}
lean_ctx/core/context_kernel/etpao.rs

lean_ctx/core/context_kernel/
etpao.rs
