rig_core/completion/
cache_cost.rs

1//! What prompt caching cost and saved, priced the same way on every provider.
2
3use std::iter::Sum;
4use std::ops::{Add, AddAssign};
5
6use serde::{Deserialize, Serialize};
7
8use super::Usage;
9
10/// Prices of one model and tier, in USD per 1M tokens (storage: per 1M
11/// token-hours). The built-in catalog's [`Pricing`](crate::catalog::Pricing)
12/// holds the input, cached-read and cache-write prices of the models it
13/// lists, but no storage price. A provider without a separate write price
14/// bills writes at `input`; one without storage leaves `storage_per_hour`
15/// at zero.
16///
17/// ```no_run
18/// use rig_core::completion::CacheRates;
19///
20/// // A 5-minute cache write at 1.25 times the input price, reads at a tenth.
21/// let rates = CacheRates { input: 3.0, cached_read: 0.3, cache_write: 3.75, storage_per_hour: 0.0 };
22/// # let _ = rates;
23/// ```
24#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
25pub struct CacheRates {
26    /// Uncached input.
27    pub input: f64,
28    /// Input read from a cache.
29    pub cached_read: f64,
30    /// Input written to a cache, or put into a cache resource.
31    pub cache_write: f64,
32    /// Storing one token for one hour, for providers that bill cache storage.
33    pub storage_per_hour: f64,
34}
35
36/// A request's or a run's input tokens by how they are billed, plus the
37/// token-hours its cache resources were stored. Sum the costs of a run's
38/// calls (and of any cache resources beside them) and price the total with
39/// [`CacheCost::usd`].
40///
41/// A run priced the same way on any provider, with the caller's rates from
42/// the provider's price list:
43///
44/// ```no_run
45/// use rig_core::completion::{CacheCost, CacheRates, Usage};
46/// use rig_core::providers::gemini::{AutoCache, CacheBook};
47///
48/// # let calls: Vec<Usage> = Vec::new();
49/// # let book = CacheBook::new(AutoCache::default());
50/// let run: CacheCost = calls.iter().map(CacheCost::from_usage).sum();
51///
52/// let opus = CacheRates { input: 4.0, cached_read: 0.2, cache_write: 5.0, storage_per_hour: 0.0 };
53/// println!("${:.3}, {:.1}% saved", run.usd(&opus), run.saving(&opus) * 100.0);
54///
55/// // Gemini: the calls, plus the cache book's creation and storage.
56/// let gemini = run + CacheCost::from(&book.report());
57/// let flash = CacheRates { input: 0.75, cached_read: 0.075, cache_write: 0.75, storage_per_hour: 0.5 };
58/// println!("${:.3}", gemini.usd(&flash));
59/// ```
60#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
61pub struct CacheCost {
62    /// Input billed at the uncached price.
63    pub uncached_input: u64,
64    /// Input read from a cache.
65    pub cache_reads: u64,
66    /// Input written to a cache.
67    pub cache_writes: u64,
68    /// Σ tokens × hours of stored cache resources.
69    pub storage_token_hours: f64,
70}
71
72impl CacheCost {
73    /// The billed input of one call: the input neither read from nor written
74    /// to a cache is `input_tokens - cached_input_tokens -
75    /// cache_creation_input_tokens`. An unreported counter counts as zero;
76    /// reads and writes above `input_tokens` leave no uncached input rather
77    /// than a negative one.
78    pub fn from_usage(usage: &Usage) -> Self {
79        let input = usage.input_tokens.unwrap_or(0);
80        let cache_reads = usage.cached_input_tokens.unwrap_or(0);
81        let cache_writes = usage.cache_creation_input_tokens.unwrap_or(0);
82        Self {
83            uncached_input: input.saturating_sub(cache_reads + cache_writes),
84            cache_reads,
85            cache_writes,
86            storage_token_hours: 0.0,
87        }
88    }
89
90    /// Every input token, however it was billed.
91    pub fn prompt_tokens(&self) -> u64 {
92        self.uncached_input + self.cache_reads + self.cache_writes
93    }
94
95    /// Input cost in USD at `rates`: uncached, read, written and stored.
96    pub fn usd(&self, rates: &CacheRates) -> f64 {
97        (self.uncached_input as f64 * rates.input
98            + self.cache_reads as f64 * rates.cached_read
99            + self.cache_writes as f64 * rates.cache_write
100            + self.storage_token_hours * rates.storage_per_hour)
101            / 1e6
102    }
103
104    /// Input cost in USD at `rates` had every prompt token been uncached.
105    pub fn uncached_usd(&self, rates: &CacheRates) -> f64 {
106        self.prompt_tokens() as f64 * rates.input / 1e6
107    }
108
109    /// The share of the uncached input cost caching saved: `1 - usd /
110    /// uncached_usd`. Negative when caching cost more; zero without input.
111    pub fn saving(&self, rates: &CacheRates) -> f64 {
112        let uncached = self.uncached_usd(rates);
113        if uncached > 0.0 {
114            1.0 - self.usd(rates) / uncached
115        } else {
116            0.0
117        }
118    }
119}
120
121impl Add for CacheCost {
122    type Output = Self;
123
124    fn add(mut self, other: Self) -> Self {
125        self += other;
126        self
127    }
128}
129
130impl AddAssign for CacheCost {
131    fn add_assign(&mut self, other: Self) {
132        self.uncached_input += other.uncached_input;
133        self.cache_reads += other.cache_reads;
134        self.cache_writes += other.cache_writes;
135        self.storage_token_hours += other.storage_token_hours;
136    }
137}
138
139impl Sum for CacheCost {
140    fn sum<I: Iterator<Item = Self>>(costs: I) -> Self {
141        costs.fold(Self::default(), Add::add)
142    }
143}
144
145#[cfg(test)]
146mod tests;
rig_core/completion/cache_cost.rs

rig_core/completion/
cache_cost.rs
