kmp-domain 0.18.5

Domain model of the Kernel Memory Protocol: aggregates, value objects, repositories and projections, with no IO
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

/// Estimates token counts for text content.
///
/// Implementations may range from simple heuristics (chars / 4) to
/// model-specific tokenizers. The kernel uses this trait for budget
/// enforcement during context rendering.
pub trait TokenEstimator: Send + Sync {
    fn estimate_tokens(&self, text: &str) -> u32;

    /// Estimate a sequence of records without requiring callers to join it
    /// into one large temporary string. The default preserves the exact
    /// concatenated-text semantics required by arbitrary estimators.
    fn estimate_token_records(&self, records: &mut dyn Iterator<Item = String>) -> u32 {
        let mut text = String::new();
        for record in records {
            text.push_str(&record);
        }
        self.estimate_tokens(&text)
    }

    fn name(&self) -> &str;
}