Skip to main content

rigspark_runtime/
usage.rs

1use serde::Serialize;
2use serde_json::Value;
3use std::sync::{Arc, Mutex};
4
5#[derive(Default, Debug, Clone, PartialEq, Serialize)]
6#[serde(rename_all = "camelCase")]
7pub struct InferenceUsage {
8    pub input_tokens: Option<u64>,
9    pub output_tokens: Option<u64>,
10    pub cache_hit_tokens: Option<u64>,
11    pub cache_miss_tokens: Option<u64>,
12}
13#[derive(Clone, Copy)]
14pub enum Provider {
15    OpenAi,
16    Claude,
17    Ollama,
18}
19fn count(value: &Value) -> Option<u64> {
20    value.as_u64().filter(|count| *count <= 9007199254740991)
21}
22pub fn parse(value: &Value, provider: Provider) -> InferenceUsage {
23    let mut result = InferenceUsage::default();
24    if matches!(provider, Provider::Ollama) {
25        result.input_tokens = count(&value["prompt_eval_count"]);
26        result.output_tokens = count(&value["eval_count"]);
27        return result;
28    }
29    let usage = value
30        .get("usage")
31        .filter(|usage| !usage.is_null())
32        .unwrap_or(&value["message"]["usage"]);
33    if matches!(provider, Provider::Claude) {
34        let input = count(&usage["input_tokens"]);
35        let hits = count(&usage["cache_read_input_tokens"]);
36        let writes = count(&usage["cache_creation_input_tokens"]);
37        result.output_tokens = count(&usage["output_tokens"]);
38        if let (Some(input), Some(hits), Some(writes)) = (input, hits, writes) {
39            if input + hits + writes <= 9007199254740991 {
40                result.input_tokens = Some(input + hits + writes);
41                result.cache_hit_tokens = Some(hits);
42                result.cache_miss_tokens = Some(input + writes);
43            }
44        } else if usage.get("cache_read_input_tokens").is_none()
45            && usage.get("cache_creation_input_tokens").is_none()
46        {
47            result.input_tokens = input;
48        }
49    } else {
50        result.input_tokens = count(&usage["prompt_tokens"]);
51        result.output_tokens = count(&usage["completion_tokens"]);
52        if let (Some(input), Some(hits)) = (
53            result.input_tokens,
54            count(&usage["prompt_tokens_details"]["cached_tokens"]),
55        ) && hits <= input
56        {
57            result.cache_hit_tokens = Some(hits);
58            result.cache_miss_tokens = Some(input - hits);
59        }
60    }
61    result
62}
63tokio::task_local! { static CURRENT: Arc<Mutex<InferenceUsage>>; }
64pub async fn scope<ResultValue>(
65    usage: Arc<Mutex<InferenceUsage>>,
66    operation: impl std::future::Future<Output = ResultValue>,
67) -> ResultValue {
68    CURRENT.scope(usage, operation).await
69}
70pub fn begin() {
71    let _ = CURRENT.try_with(|usage| {
72        if let Ok(mut usage) = usage.lock() {
73            *usage = InferenceUsage::default();
74        }
75    });
76}
77pub fn record(value: &Value, provider: Provider) {
78    let parsed = parse(value, provider);
79    let _ = CURRENT.try_with(|usage| {
80        if let Ok(mut usage) = usage.lock() {
81            if parsed.input_tokens.is_some() {
82                usage.input_tokens = parsed.input_tokens;
83            }
84            if parsed.output_tokens.is_some() {
85                usage.output_tokens = parsed.output_tokens;
86            }
87            if parsed.cache_hit_tokens.is_some() {
88                usage.cache_hit_tokens = parsed.cache_hit_tokens;
89            }
90            if parsed.cache_miss_tokens.is_some() {
91                usage.cache_miss_tokens = parsed.cache_miss_tokens;
92            }
93        }
94    });
95}
96#[cfg(test)]
97mod tests {
98    use super::*;
99    use serde_json::json;
100    #[tokio::test]
101    async fn provider_counters_merge_without_double_counting() {
102        let usage = Arc::new(Mutex::new(InferenceUsage::default()));
103        scope(usage.clone(), async {
104            record(&json!({"message":{"usage":{"input_tokens":10,"cache_read_input_tokens":60,"cache_creation_input_tokens":30,"output_tokens":0}}}), Provider::Claude);
105            record(&json!({"usage":{"output_tokens":20}}), Provider::Claude);
106        }).await;
107        assert_eq!(
108            *usage.lock().unwrap(),
109            InferenceUsage {
110                input_tokens: Some(100),
111                output_tokens: Some(20),
112                cache_hit_tokens: Some(60),
113                cache_miss_tokens: Some(40)
114            }
115        );
116        assert_eq!(
117            parse(
118                &json!({"prompt_eval_count":100,"eval_count":20}),
119                Provider::Ollama
120            )
121            .cache_hit_tokens,
122            None
123        );
124        assert_eq!(
125            parse(
126                &json!({"usage":{"prompt_tokens":10,"prompt_tokens_details":{"cached_tokens":20}}}),
127                Provider::OpenAi
128            )
129            .cache_hit_tokens,
130            None
131        );
132    }
133}