llm_browser_testkit/
pricing.rs

1//! Automatic pricing lookup from providers that expose exact, public
2//! per-token prices.
3//!
4//! Today only `OpenRouter` qualifies: its `GET /api/v1/models` response
5//! carries exact per-token `prompt`, `completion`, `input_cache_read` and
6//! `input_cache_write` prices with no authentication. Other providers either
7//! expose no public price API (`OpenAI`, Groq, xAI, `DeepSeek`, Google) or
8//! expose prices that cannot be mapped to a model exactly without heuristics
9//! (AWS Bedrock Price List, Azure Retail Prices).
10
11use std::collections::HashMap;
12
13use crate::scenario::EndpointConfig;
14
15/// Exact per-1M-token prices for a model.
16#[derive(Debug, Clone, Copy, PartialEq)]
17pub struct ModelPricing {
18    /// Input (prompt) price per 1M tokens.
19    pub input_per_1m: f64,
20    /// Output (completion) price per 1M tokens.
21    pub output_per_1m: f64,
22    /// Cached-input (cache read) price per 1M tokens, when offered.
23    pub cached_input_per_1m: Option<f64>,
24    /// Cache-write (cache creation) price per 1M tokens, when offered.
25    pub cache_write_per_1m: Option<f64>,
26}
27
28/// Parses an `OpenRouter` `/api/v1/models` response for `model`.
29///
30/// Matches the full model id case-insensitively first, then falls back to the
31/// leaf after the last `/` (so `claude-3.5-sonnet` matches
32/// `anthropic/claude-3.5-sonnet`). Returns `None` when the model is absent or
33/// its pricing is not a usable positive number.
34#[must_use]
35pub fn parse_openrouter_models(json: &serde_json::Value, model: &str) -> Option<ModelPricing> {
36    let wanted = model.to_ascii_lowercase();
37    let leaf = wanted.rsplit('/').next().unwrap_or(wanted.as_str());
38    let data = json["data"].as_array()?;
39    let entry = data.iter().find(|m| {
40        let id = m["id"].as_str().unwrap_or_default().to_ascii_lowercase();
41        id == wanted || id.rsplit('/').next().unwrap_or_default() == leaf
42    })?;
43    let pricing = &entry["pricing"];
44    Some(ModelPricing {
45        input_per_1m: per_million(pricing, "prompt")?,
46        output_per_1m: per_million(pricing, "completion")?,
47        cached_input_per_1m: per_million(pricing, "input_cache_read"),
48        cache_write_per_1m: per_million(pricing, "input_cache_write"),
49    })
50}
51
52/// Reads a per-token price (string or number) and converts it to per-1M
53/// tokens. Negative values (`OpenRouter` uses `-1` for dynamic/auto prices)
54/// and missing keys yield `None`.
55fn per_million(pricing: &serde_json::Value, key: &str) -> Option<f64> {
56    let value = match &pricing[key] {
57        serde_json::Value::String(s) => s.parse::<f64>().ok()?,
58        serde_json::Value::Number(n) => n.as_f64()?,
59        _ => return None,
60    };
61    (value >= 0.0).then_some(value * 1_000_000.0)
62}
63
64/// Fetches exact `OpenRouter` pricing for `model`.
65///
66/// # Errors
67///
68/// Returns a description when the request fails, the response is not JSON, or
69/// the model is not present in the catalog.
70pub async fn fetch_openrouter_pricing(
71    client: &reqwest::Client,
72    model: &str,
73) -> Result<ModelPricing, String> {
74    let resp = client
75        .get("https://openrouter.ai/api/v1/models")
76        .send()
77        .await
78        .map_err(|e| format!("OpenRouter models request failed: {e}"))?;
79    if !resp.status().is_success() {
80        return Err(format!(
81            "OpenRouter models request returned HTTP {}",
82            resp.status()
83        ));
84    }
85    let json: serde_json::Value = resp
86        .json()
87        .await
88        .map_err(|e| format!("OpenRouter models response was not JSON: {e}"))?;
89    parse_openrouter_models(&json, model)
90        .ok_or_else(|| format!("model `{model}` not found in OpenRouter pricing"))
91}
92
93/// Whether an endpoint opts into automatic pricing, and from where.
94#[must_use]
95fn wants_openrouter_pricing(ec: &EndpointConfig) -> bool {
96    match ec.pricing_source.as_deref() {
97        Some("openrouter") => true,
98        Some("auto") => ec
99            .url
100            .as_deref()
101            .is_some_and(|url| url.to_ascii_lowercase().contains("openrouter.ai")),
102        _ => false,
103    }
104}
105
106/// Fills unset pricing fields on `ec` from fetched `pricing`; explicit values
107/// win.
108pub fn apply_pricing(ec: &mut EndpointConfig, pricing: &ModelPricing) {
109    let p = ec.pricing.get_or_insert_with(Default::default);
110    if p.input_per_1m_tokens == 0.0 {
111        p.input_per_1m_tokens = pricing.input_per_1m;
112    }
113    if p.output_per_1m_tokens == 0.0 {
114        p.output_per_1m_tokens = pricing.output_per_1m;
115    }
116    if p.cached_input_per_1m_tokens.is_none() {
117        p.cached_input_per_1m_tokens = pricing.cached_input_per_1m;
118    }
119    if p.cache_write_per_1m_tokens.is_none() {
120        p.cache_write_per_1m_tokens = pricing.cache_write_per_1m;
121    }
122}
123
124/// Applies automatic pricing to every endpoint that opts in.
125///
126/// Returns the number of endpoints priced. Endpoints without a model, without
127/// a supported source, or whose lookup fails are left untouched. The first
128/// failure is returned as an error string while the remaining endpoints are
129/// still processed, so a single unknown model never blocks a run.
130///
131/// # Errors
132///
133/// Returns the first lookup error when **no** endpoint could be priced; a
134/// partial success returns `Ok` with the number priced.
135#[allow(clippy::implicit_hasher)]
136pub async fn apply_auto_pricing(
137    endpoints: &mut HashMap<String, EndpointConfig>,
138    client: &reqwest::Client,
139) -> Result<usize, String> {
140    let mut priced = 0;
141    let mut first_error: Option<String> = None;
142    for ec in endpoints.values_mut() {
143        if !wants_openrouter_pricing(ec) {
144            continue;
145        }
146        let Some(model) = ec.model.clone() else {
147            continue;
148        };
149        match fetch_openrouter_pricing(client, &model).await {
150            Ok(pricing) => {
151                apply_pricing(ec, &pricing);
152                priced += 1;
153            }
154            Err(e) => {
155                first_error.get_or_insert(e);
156            }
157        };
158    }
159    match first_error {
160        Some(e) if priced == 0 => Err(e),
161        _ => Ok(priced),
162    }
163}
164
165#[cfg(test)]
166mod tests {
167    use super::{apply_pricing, parse_openrouter_models};
168    use crate::scenario::{EndpointConfig, PricingConfig};
169
170    fn catalog() -> serde_json::Value {
171        serde_json::json!({
172            "data": [
173                {"id": "anthropic/claude-3.5-sonnet", "pricing": {
174                    "prompt": "0.000003", "completion": "0.000015",
175                    "input_cache_read": "0.0000003", "input_cache_write": "0.00000375"
176                }},
177                {"id": "openrouter/auto", "pricing": {"prompt": "-1", "completion": "-1"}}
178            ]
179        })
180    }
181
182    #[test]
183    fn parses_exact_match_with_cache_prices() {
184        let p = parse_openrouter_models(&catalog(), "anthropic/claude-3.5-sonnet").unwrap();
185        assert!((p.input_per_1m - 3.0).abs() < 1e-9);
186        assert!((p.output_per_1m - 15.0).abs() < 1e-9);
187        assert!((p.cached_input_per_1m.unwrap() - 0.3).abs() < 1e-9);
188        assert!((p.cache_write_per_1m.unwrap() - 3.75).abs() < 1e-9);
189    }
190
191    #[test]
192    fn parses_by_leaf_when_namespaced_omitted() {
193        let p = parse_openrouter_models(&catalog(), "claude-3.5-sonnet").unwrap();
194        assert!((p.input_per_1m - 3.0).abs() < 1e-9);
195    }
196
197    #[test]
198    fn dynamic_negative_prices_are_not_usable() {
199        assert!(parse_openrouter_models(&catalog(), "openrouter/auto").is_none());
200    }
201
202    #[test]
203    fn unknown_model_is_none() {
204        assert!(parse_openrouter_models(&catalog(), "does/not-exist").is_none());
205    }
206
207    #[test]
208    fn apply_pricing_keeps_explicit_values() {
209        let mut ec = EndpointConfig {
210            model: Some("anthropic/claude-3.5-sonnet".to_owned()),
211            pricing: Some(PricingConfig {
212                input_per_1m_tokens: 9.0,
213                output_per_1m_tokens: 0.0,
214                ..PricingConfig::default()
215            }),
216            ..EndpointConfig::default()
217        };
218        let p = parse_openrouter_models(&catalog(), "anthropic/claude-3.5-sonnet").unwrap();
219        apply_pricing(&mut ec, &p);
220        let pricing = ec.pricing.unwrap();
221        assert!(
222            (pricing.input_per_1m_tokens - 9.0).abs() < 1e-9,
223            "explicit wins"
224        );
225        assert!(
226            (pricing.output_per_1m_tokens - 15.0).abs() < 1e-9,
227            "fetched fills gap"
228        );
229        assert!(pricing.cached_input_per_1m_tokens.is_some());
230        assert!(pricing.cache_write_per_1m_tokens.is_some());
231    }
232}
llm_browser_testkit/pricing.rs

llm_browser_testkit/
pricing.rs
