rigspark_runtime/
usage.rs1use serde::Serialize;
2use serde_json::Value;
3use std::sync::{Arc, Mutex};
4
5#[derive(Default, Debug, Clone, PartialEq, Serialize)]
6#[serde(rename_all = "camelCase")]
7pub struct InferenceUsage {
8 pub input_tokens: Option<u64>,
9 pub output_tokens: Option<u64>,
10 pub cache_hit_tokens: Option<u64>,
11 pub cache_miss_tokens: Option<u64>,
12}
13#[derive(Clone, Copy)]
14pub enum Provider {
15 OpenAi,
16 Claude,
17 Ollama,
18}
19fn count(value: &Value) -> Option<u64> {
20 value.as_u64().filter(|count| *count <= 9007199254740991)
21}
22pub fn parse(value: &Value, provider: Provider) -> InferenceUsage {
23 let mut result = InferenceUsage::default();
24 if matches!(provider, Provider::Ollama) {
25 result.input_tokens = count(&value["prompt_eval_count"]);
26 result.output_tokens = count(&value["eval_count"]);
27 return result;
28 }
29 let usage = value
30 .get("usage")
31 .filter(|usage| !usage.is_null())
32 .unwrap_or(&value["message"]["usage"]);
33 if matches!(provider, Provider::Claude) {
34 let input = count(&usage["input_tokens"]);
35 let hits = count(&usage["cache_read_input_tokens"]);
36 let writes = count(&usage["cache_creation_input_tokens"]);
37 result.output_tokens = count(&usage["output_tokens"]);
38 if let (Some(input), Some(hits), Some(writes)) = (input, hits, writes) {
39 if input + hits + writes <= 9007199254740991 {
40 result.input_tokens = Some(input + hits + writes);
41 result.cache_hit_tokens = Some(hits);
42 result.cache_miss_tokens = Some(input + writes);
43 }
44 } else if usage.get("cache_read_input_tokens").is_none()
45 && usage.get("cache_creation_input_tokens").is_none()
46 {
47 result.input_tokens = input;
48 }
49 } else {
50 result.input_tokens = count(&usage["prompt_tokens"]);
51 result.output_tokens = count(&usage["completion_tokens"]);
52 if let (Some(input), Some(hits)) = (
53 result.input_tokens,
54 count(&usage["prompt_tokens_details"]["cached_tokens"]),
55 ) && hits <= input
56 {
57 result.cache_hit_tokens = Some(hits);
58 result.cache_miss_tokens = Some(input - hits);
59 }
60 }
61 result
62}
63tokio::task_local! { static CURRENT: Arc<Mutex<InferenceUsage>>; }
64pub async fn scope<ResultValue>(
65 usage: Arc<Mutex<InferenceUsage>>,
66 operation: impl std::future::Future<Output = ResultValue>,
67) -> ResultValue {
68 CURRENT.scope(usage, operation).await
69}
70pub fn begin() {
71 let _ = CURRENT.try_with(|usage| {
72 if let Ok(mut usage) = usage.lock() {
73 *usage = InferenceUsage::default();
74 }
75 });
76}
77pub fn record(value: &Value, provider: Provider) {
78 let parsed = parse(value, provider);
79 let _ = CURRENT.try_with(|usage| {
80 if let Ok(mut usage) = usage.lock() {
81 if parsed.input_tokens.is_some() {
82 usage.input_tokens = parsed.input_tokens;
83 }
84 if parsed.output_tokens.is_some() {
85 usage.output_tokens = parsed.output_tokens;
86 }
87 if parsed.cache_hit_tokens.is_some() {
88 usage.cache_hit_tokens = parsed.cache_hit_tokens;
89 }
90 if parsed.cache_miss_tokens.is_some() {
91 usage.cache_miss_tokens = parsed.cache_miss_tokens;
92 }
93 }
94 });
95}
96#[cfg(test)]
97mod tests {
98 use super::*;
99 use serde_json::json;
100 #[tokio::test]
101 async fn provider_counters_merge_without_double_counting() {
102 let usage = Arc::new(Mutex::new(InferenceUsage::default()));
103 scope(usage.clone(), async {
104 record(&json!({"message":{"usage":{"input_tokens":10,"cache_read_input_tokens":60,"cache_creation_input_tokens":30,"output_tokens":0}}}), Provider::Claude);
105 record(&json!({"usage":{"output_tokens":20}}), Provider::Claude);
106 }).await;
107 assert_eq!(
108 *usage.lock().unwrap(),
109 InferenceUsage {
110 input_tokens: Some(100),
111 output_tokens: Some(20),
112 cache_hit_tokens: Some(60),
113 cache_miss_tokens: Some(40)
114 }
115 );
116 assert_eq!(
117 parse(
118 &json!({"prompt_eval_count":100,"eval_count":20}),
119 Provider::Ollama
120 )
121 .cache_hit_tokens,
122 None
123 );
124 assert_eq!(
125 parse(
126 &json!({"usage":{"prompt_tokens":10,"prompt_tokens_details":{"cached_tokens":20}}}),
127 Provider::OpenAi
128 )
129 .cache_hit_tokens,
130 None
131 );
132 }
133}