use super::AtomicTokenCounter;
impl AtomicTokenCounter {
pub fn record_model_usage(&self, model: &str, prompt: u64, completion: u64) {
self.record_model_usage_with_cache(model, prompt, completion, 0, 0);
}
pub fn record_model_usage_with_cache(
&self,
model: &str,
prompt: u64,
completion: u64,
cache_read: u64,
cache_write: u64,
) {
tracing::debug!(
model,
prompt,
completion,
cache_read,
cache_write,
"Recording model usage"
);
self.record(prompt, completion);
if let Ok(mut usage) = self.model_usage.try_lock() {
let entry = usage.entry(model.to_string()).or_insert((0, 0));
entry.0 += prompt;
entry.1 += completion;
}
if let Ok(mut last) = self.model_last_prompt_tokens.try_lock() {
last.insert(model.to_string(), prompt + cache_read + cache_write);
}
if (cache_read > 0 || cache_write > 0)
&& let Ok(mut cache) = self.model_cache_usage.try_lock()
{
let entry = cache.entry(model.to_string()).or_insert((0, 0));
entry.0 += cache_read;
entry.1 += cache_write;
}
}
pub fn cache_usage_for(&self, model: &str) -> (u64, u64) {
self.model_cache_usage
.try_lock()
.ok()
.and_then(|m| m.get(model).copied())
.unwrap_or((0, 0))
}
pub fn last_prompt_tokens_for(&self, model: &str) -> Option<u64> {
self.model_last_prompt_tokens
.try_lock()
.ok()
.and_then(|m| m.get(model).copied())
}
}