lc_evaluation/
price.rs

1//! E1 — eval cost as a first-class citizen (v0.22.1 §S7).
2//!
3//! Evaluation runs burn tokens through the predictor (and any LLM-as-judge). This module gives
4//! the eval report a real cost ledger: a per-`TokenUsage` meter plus a configurable USD `PriceBook`
5//! whose [`PriceBook::estimate_cost`] is a pure function of token counts and a model name.
6//!
7//! Design notes:
8//! - Rates are a static snapshot of list prices (per 1M tokens), deliberately approximate.
9//! - `PriceBook` matches model names **exactly** (not substring) — an eval run configures the
10//!   book it expects, so an unknown model means "I don't know this price", returning `None`
11//!   rather than a guess.
12//! - `TokenUsage` is additive: a predictor that has no token metering reports `None` and the
13//!   report simply carries a zero/None cost ledger (zero behavior change).
14
15use std::collections::HashMap;
16
17use serde::{Deserialize, Serialize};
18
19/// USD price of a model, per 1M tokens.
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub struct Price {
22    /// USD per 1M input (prompt) tokens.
23    pub input_per_1m: u64,
24    /// USD per 1M output (completion) tokens.
25    pub output_per_1m: u64,
26}
27
28/// A token-usage report from one predictor step (an eval input).
29#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
30pub struct TokenUsage {
31    /// Prompt tokens consumed.
32    pub prompt_tokens: usize,
33    /// Completion tokens consumed.
34    pub completion_tokens: usize,
35    /// Model that produced the prediction, for price lookup. `None` when unknown.
36    #[serde(default, skip_serializing_if = "Option::is_none")]
37    pub model: Option<String>,
38}
39
40impl TokenUsage {
41    /// Total tokens consumed.
42    pub fn total(&self) -> usize {
43        self.prompt_tokens.saturating_add(self.completion_tokens)
44    }
45}
46
47/// Configurable USD price book for eval-run cost estimation.
48#[derive(Debug, Clone, Default)]
49pub struct PriceBook {
50    rates: HashMap<String, Price>,
51}
52
53impl PriceBook {
54    /// A price book with a few common models preloaded (approximate list prices).
55    pub fn default_set() -> Self {
56        let mut book = Self::default();
57        book.set(
58            "claude-3-5-sonnet",
59            Price {
60                input_per_1m: 3,
61                output_per_1m: 15,
62            },
63        );
64        book.set(
65            "claude-3-5-haiku",
66            Price {
67                input_per_1m: 80,
68                output_per_1m: 400,
69            },
70        );
71        book.set(
72            "gpt-4o-mini",
73            Price {
74                input_per_1m: 15,
75                output_per_1m: 60,
76            },
77        );
78        book.set(
79            "gpt-4o",
80            Price {
81                input_per_1m: 250,
82                output_per_1m: 1000,
83            },
84        );
85        book.set(
86            "deepseek-chat",
87            Price {
88                input_per_1m: 27,
89                output_per_1m: 110,
90            },
91        );
92        book
93    }
94
95    /// Sets (or overrides) the price for a model. Prices are in "units per 1M": a price of `3`
96    /// means $3.00 per 1M input tokens.
97    pub fn set(&mut self, model: impl Into<String>, price: Price) {
98        self.rates.insert(model.into(), price);
99    }
100
101    /// Looks up an exact model price.
102    pub fn get(&self, model: &str) -> Option<Price> {
103        self.rates.get(model).copied()
104    }
105
106    /// Estimates the USD cost of a usage report; `None` when the model isn't in the book.
107    ///
108    /// Pure function over the book + usage, so it is trivially unit-testable.
109    pub fn estimate(&self, usage: &TokenUsage) -> Option<f64> {
110        let model = usage.model.as_deref()?;
111        self.estimate_cost(usage.prompt_tokens, usage.completion_tokens, model)
112    }
113
114    /// Estimates the USD cost of token counts for a named model.
115    ///
116    /// Integer cents-per-1M math: `prompt / 1e6 * input_price/100`. Returns `None` when the
117    /// model is not in the book.
118    pub fn estimate_cost(
119        &self,
120        prompt_tokens: usize,
121        completion_tokens: usize,
122        model: &str,
123    ) -> Option<f64> {
124        let p = self.rates.get(model)?;
125        let usd = prompt_tokens as f64 / 1e6 * (p.input_per_1m as f64 / 100.0)
126            + completion_tokens as f64 / 1e6 * (p.output_per_1m as f64 / 100.0);
127        Some(usd)
128    }
129}
130
131/// Cost ledger carried on an eval `Report` (E1).
132///
133/// Token buckets accumulate every reported predictor usage; `cost_usd` is `Some` only when at
134/// least one usage had a model that the configured price book knows. Default is all-zero + `None`
135/// so old reports serialize compatibly under `#[serde(default)]`.
136#[derive(Debug, Clone, Default, Serialize, Deserialize)]
137pub struct OverallCost {
138    /// Total prompt tokens across the run.
139    pub prompt_tokens: usize,
140    /// Total completion tokens across the run.
141    pub completion_tokens: usize,
142    /// Total tokens (prompt + completion).
143    pub total_tokens: usize,
144    /// Estimated USD cost of the run; `None` when no priced usage was reported.
145    pub cost_usd: Option<f64>,
146}
147
148impl OverallCost {
149    /// Adds a usage report, accumulating token totals and (when priced) the USD estimate.
150    pub fn accumulate(&mut self, usage: &TokenUsage, book: &PriceBook) {
151        self.prompt_tokens = self.prompt_tokens.saturating_add(usage.prompt_tokens);
152        self.completion_tokens = self
153            .completion_tokens
154            .saturating_add(usage.completion_tokens);
155        self.total_tokens = self.total_tokens.saturating_add(usage.total());
156        if let Some(usd) = book.estimate(usage) {
157            self.cost_usd = Some(self.cost_usd.unwrap_or(0.0) + usd);
158        }
159    }
160}
161
162#[cfg(test)]
163mod tests {
164    use super::*;
165
166    fn cheap_book() -> PriceBook {
167        let mut b = PriceBook::default();
168        b.set(
169            "test-model",
170            Price {
171                input_per_1m: 300,
172                output_per_1m: 1500,
173            },
174        ); // $3.00 / $15.00
175        b
176    }
177
178    #[test]
179    fn estimate_is_pure_and_per_1m_scaled() {
180        let b = cheap_book();
181        // 1M in + 1M out on the $3/$15 model = $18.00
182        let usd = b.estimate_cost(1_000_000, 1_000_000, "test-model").unwrap();
183        assert!((usd - 18.0).abs() < 1e-9, "got {usd}");
184        // 1k each = $0.018
185        let small = b.estimate_cost(1000, 1000, "test-model").unwrap();
186        assert!((small - 0.018).abs() < 1e-9, "got {small}");
187    }
188
189    #[test]
190    fn unknown_model_yields_none_not_a_guess() {
191        let b = cheap_book();
192        assert!(b.estimate_cost(1, 1, "mystery-model").is_none());
193        assert!(b.get("nonexistent").is_none());
194    }
195
196    #[test]
197    fn exact_match_not_substring() {
198        let mut b = cheap_book();
199        b.set(
200            "gpt-4o",
201            Price {
202                input_per_1m: 250,
203                output_per_1m: 1000,
204            },
205        );
206        // "gpt-4o" and "gpt-4o-mini" are distinct entries; no substring matching
207        assert!(b.get("gpt-4o-mini").is_none());
208        assert!(b.get("gpt-4o").is_some());
209    }
210
211    #[test]
212    fn accumulate_totals_and_priced_usd() {
213        let b = cheap_book();
214        let mut cost = OverallCost::default();
215        cost.accumulate(
216            &TokenUsage {
217                prompt_tokens: 1000,
218                completion_tokens: 1000,
219                model: Some("test-model".into()),
220            },
221            &b,
222        );
223        cost.accumulate(
224            &TokenUsage {
225                prompt_tokens: 500,
226                completion_tokens: 0,
227                model: None,
228            },
229            &b,
230        );
231        assert_eq!(cost.prompt_tokens, 1500);
232        assert_eq!(cost.completion_tokens, 1000);
233        assert_eq!(cost.total_tokens, 2500);
234        // only the priced usage contributes USD
235        assert!(
236            (cost.cost_usd.unwrap() - 0.018).abs() < 1e-9,
237            "got {:?}",
238            cost.cost_usd
239        );
240    }
241
242    #[test]
243    fn accumulate_without_any_priced_usage_keeps_cost_none() {
244        let b = cheap_book();
245        let mut cost = OverallCost::default();
246        cost.accumulate(
247            &TokenUsage {
248                prompt_tokens: 10,
249                completion_tokens: 10,
250                model: None,
251            },
252            &b,
253        );
254        assert_eq!(cost.total_tokens, 20);
255        assert!(cost.cost_usd.is_none());
256    }
257
258    #[test]
259    fn default_set_has_common_models() {
260        let b = PriceBook::default_set();
261        assert!(b.get("gpt-4o-mini").is_some());
262        assert!(b.get("deepseek-chat").is_some());
263        // 1M in + 1M out on gpt-4o-mini: $0.15 + $0.60 = $0.75
264        let usd = b
265            .estimate_cost(1_000_000, 1_000_000, "gpt-4o-mini")
266            .unwrap();
267        assert!((usd - 0.75).abs() < 1e-9, "got {usd}");
268    }
269}
lc_evaluation/price.rs

lc_evaluation/
price.rs
