vv_agent/memory/
token_utils.rs

1use serde_json::Value;
2use std::path::Path;
3
4use crate::types::Message;
5
6pub fn resolve_model_token_limits(
7    settings: &Value,
8    backend: &str,
9    model: &str,
10) -> (Option<u64>, Option<u64>) {
11    let backend = backend.trim();
12    let model = model.trim();
13    if backend.is_empty() || model.is_empty() {
14        return (None, None);
15    }
16    let Ok(resolved) = crate::config::resolve_model_endpoint(settings, backend, model) else {
17        return (None, None);
18    };
19    (resolved.context_length, resolved.max_output_tokens)
20}
21
22pub fn resolve_model_token_limits_from_file(
23    path: impl AsRef<Path>,
24    backend: &str,
25    model: &str,
26) -> (Option<u64>, Option<u64>) {
27    let Ok(settings) = crate::config::load_llm_settings_from_file(path) else {
28        return (None, None);
29    };
30    resolve_model_token_limits(&settings, backend, model)
31}
32
33pub fn compute_compaction_threshold(
34    configured_threshold: u64,
35    model_context_window: u64,
36    reserved_output_tokens: u64,
37    autocompact_buffer_tokens: u64,
38) -> u64 {
39    let derived = if model_context_window > 0 {
40        model_context_window
41            .saturating_sub(reserved_output_tokens)
42            .saturating_sub(autocompact_buffer_tokens)
43    } else {
44        0
45    };
46    match (configured_threshold, derived) {
47        (configured, derived) if configured > 0 && derived > 0 => configured.min(derived),
48        (configured, _) if configured > 0 => configured,
49        (_, derived) => derived,
50    }
51}
52
53pub fn count_messages_tokens(messages: &[Message], model: &str) -> u64 {
54    if messages.is_empty() {
55        return 0;
56    }
57    let payload = messages
58        .iter()
59        .map(|message| message.to_openai_message(true))
60        .collect::<Vec<_>>();
61    count_tokens(&serde_json::to_string(&payload).unwrap_or_default(), model)
62}
63
64pub trait TokenCountPayload {
65    fn to_token_count_text(&self) -> String;
66}
67
68impl TokenCountPayload for str {
69    fn to_token_count_text(&self) -> String {
70        self.to_string()
71    }
72}
73
74impl TokenCountPayload for String {
75    fn to_token_count_text(&self) -> String {
76        self.clone()
77    }
78}
79
80impl TokenCountPayload for Value {
81    fn to_token_count_text(&self) -> String {
82        serde_json::to_string(self).unwrap_or_default()
83    }
84}
85
86pub fn count_tokens<T: TokenCountPayload + ?Sized>(payload: &T, model: &str) -> u64 {
87    let text = payload.to_token_count_text();
88    if text.is_empty() {
89        return 0;
90    }
91    if vv_llm_tokenizer_supported(model) {
92        if let Ok(count) = vv_llm::utilities::count_tokens(&text, model) {
93            if count > 0 {
94                return count as u64;
95            }
96        }
97    }
98    estimate_tokens(&text, model)
99}
100
101pub fn estimate_tokens(text: &str, _model: &str) -> u64 {
102    if text.is_empty() {
103        return 0;
104    }
105    let mut cjk_chars = 0_u64;
106    let mut other_chars = 0_u64;
107    for ch in text.chars() {
108        if is_cjk(ch) {
109            cjk_chars += 1;
110        } else {
111            other_chars += 1;
112        }
113    }
114    ((cjk_chars as f64 * 1.5) + (other_chars as f64 * 0.25))
115        .floor()
116        .max(1.0) as u64
117}
118
119fn vv_llm_tokenizer_supported(model: &str) -> bool {
120    model == "gpt-3.5-turbo"
121        || model.starts_with("gpt-4o")
122        || model.starts_with("o1-")
123        || model.starts_with("o3-")
124}
125
126fn is_cjk(ch: char) -> bool {
127    matches!(
128        ch as u32,
129        0x4E00..=0x9FFF | 0x3000..=0x303F | 0xFF00..=0xFFEF
130    )
131}
vv_agent/memory/token_utils.rs

vv_agent/memory/
token_utils.rs
