use std::collections::HashMap;
use std::io::Read as _;
use std::path::{Path, PathBuf};
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
use std::time::Duration;
use serde::{Deserialize, Serialize};
use crate::profile::{atomic_write, clauth_dir};
use crate::tokens::ModelTokens;
use crate::usage::now_ms;
const FEED_URL: &str =
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
const REFRESH_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
const RECV_TIMEOUT: Duration = Duration::from_secs(15);
const MAX_BODY_BYTES: u64 = 8 * 1024 * 1024;
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub(crate) struct ModelRate {
pub(crate) input: f64,
pub(crate) output: f64,
pub(crate) cache_read: f64,
pub(crate) cache_write: f64,
}
#[derive(Debug, Clone)]
pub(crate) struct PriceTable {
rates: HashMap<String, ModelRate>,
pub(crate) fetched_at_ms: u64,
}
impl PriceTable {
pub(crate) fn rate(&self, model: &str) -> Option<ModelRate> {
if let Some(r) = self.rates.get(model) {
return Some(*r);
}
let mut cur = model;
while let Some((head, _)) = cur.rsplit_once('-') {
if head.contains('-')
&& let Some(r) = self.rates.get(head)
{
return Some(*r);
}
cur = head;
}
None
}
pub(crate) fn cost(&self, m: &ModelTokens) -> Option<f64> {
let r = self.rate(&m.model)?;
Some(
m.input as f64 * r.input
+ m.output as f64 * r.output
+ m.cache_read as f64 * r.cache_read
+ m.cache_create as f64 * r.cache_write,
)
}
pub(crate) fn total_cost(&self, models: &[ModelTokens]) -> (f64, usize) {
let mut total = 0.0;
let mut unpriced = 0usize;
for m in models {
match self.cost(m) {
Some(c) => total += c,
None if m.total() > 0 => unpriced += 1,
None => {}
}
}
(total, unpriced)
}
}
pub(crate) enum PricingEvent {
Loaded(Box<PriceTable>),
Failed,
}
pub(crate) fn spawn(tx: Sender<PricingEvent>, refresh_rx: Receiver<()>) {
let Some(cache_file) = cache_path() else {
return;
};
std::thread::spawn(move || {
if let Some(table) = load_cache(&cache_file) {
let _ = tx.send(PricingEvent::Loaded(Box::new(table)));
}
loop {
run_fetch(&tx, &cache_file);
match refresh_rx.recv_timeout(REFRESH_INTERVAL) {
Ok(()) => while refresh_rx.try_recv().is_ok() {},
Err(RecvTimeoutError::Timeout) => {}
Err(RecvTimeoutError::Disconnected) => return,
}
}
});
}
fn run_fetch(tx: &Sender<PricingEvent>, cache_file: &Path) {
match fetch_table() {
Ok(mut table) => {
table.fetched_at_ms = now_ms();
save_cache(cache_file, &table);
let _ = tx.send(PricingEvent::Loaded(Box::new(table)));
}
Err(_) => match load_cache(cache_file) {
Some(table) => {
let _ = tx.send(PricingEvent::Loaded(Box::new(table)));
}
None => {
let _ = tx.send(PricingEvent::Failed);
}
},
}
}
fn fetch_table() -> anyhow::Result<PriceTable> {
let agent: ureq::Agent = ureq::Agent::config_builder()
.timeout_connect(Some(CONNECT_TIMEOUT))
.timeout_recv_response(Some(RECV_TIMEOUT))
.build()
.into();
let reader = agent
.get(FEED_URL)
.header("User-Agent", "clauth-pricing")
.call()
.map_err(anyhow::Error::from)?
.into_body()
.into_reader();
let mut capped = reader.take(MAX_BODY_BYTES + 1);
let mut bytes = Vec::new();
capped
.read_to_end(&mut bytes)
.map_err(anyhow::Error::from)?;
if bytes.len() as u64 > MAX_BODY_BYTES {
anyhow::bail!("price feed exceeded {MAX_BODY_BYTES} byte cap");
}
let json = String::from_utf8(bytes).map_err(anyhow::Error::from)?;
let rates = distill(&json)?;
if rates.is_empty() {
anyhow::bail!("price feed parsed but contained no priced models");
}
Ok(PriceTable {
rates,
fetched_at_ms: 0, })
}
fn distill(json: &str) -> anyhow::Result<HashMap<String, ModelRate>> {
let root: serde_json::Value = serde_json::from_str(json).map_err(anyhow::Error::from)?;
let obj = root
.as_object()
.ok_or_else(|| anyhow::anyhow!("price feed root is not a JSON object"))?;
let mut rates = HashMap::new();
for (id, v) in obj {
if id.contains('/') {
continue;
}
let Some(input) = v
.get("input_cost_per_token")
.and_then(serde_json::Value::as_f64)
else {
continue;
};
let f = |key: &str| {
v.get(key)
.and_then(serde_json::Value::as_f64)
.unwrap_or(0.0)
};
rates.insert(
id.clone(),
ModelRate {
input,
output: f("output_cost_per_token"),
cache_read: f("cache_read_input_token_cost"),
cache_write: f("cache_creation_input_token_cost"),
},
);
}
Ok(rates)
}
#[derive(Debug, Serialize, Deserialize)]
struct CacheFile {
fetched_at_ms: u64,
rates: HashMap<String, ModelRate>,
}
fn cache_path() -> Option<PathBuf> {
clauth_dir().ok().map(|d| d.join("price_cache.json"))
}
fn load_cache(path: &Path) -> Option<PriceTable> {
let bytes = std::fs::read_to_string(path).ok()?;
let cache: CacheFile = serde_json::from_str(&bytes).ok()?;
Some(PriceTable {
rates: cache.rates,
fetched_at_ms: cache.fetched_at_ms,
})
}
fn save_cache(path: &Path, table: &PriceTable) {
let cache = CacheFile {
fetched_at_ms: table.fetched_at_ms,
rates: table.rates.clone(),
};
if let Ok(json) = serde_json::to_string(&cache) {
let _ = atomic_write(path, json);
}
}
#[cfg(test)]
#[path = "../tests/inline/pricing.rs"]
mod tests;