token_count/output/
basic.rs

1//! Basic formatter - outputs model info and token count without percentage
2
3use crate::output::OutputFormatter;
4use crate::tokenizers::TokenizationResult;
5
6/// Basic formatter that outputs model info and token count (no percentage)
7pub struct BasicFormatter;
8
9impl OutputFormatter for BasicFormatter {
10    fn format(&self, result: &TokenizationResult) -> String {
11        format!(
12            "Model: {} ({})\nTokens: {}",
13            result.model_info.name, result.model_info.encoding, result.token_count
14        )
15    }
16}
17
18#[cfg(test)]
19mod tests {
20    use super::*;
21    use crate::tokenizers::ModelInfo;
22
23    #[test]
24    fn test_basic_formatter() {
25        let formatter = BasicFormatter;
26        let result = TokenizationResult {
27            token_count: 2,
28            model_info: ModelInfo {
29                name: "gpt-4".to_string(),
30                encoding: "cl100k_base".to_string(),
31                context_window: 128000,
32                description: "GPT-4".to_string(),
33            },
34            token_details: None,
35        };
36
37        let output = formatter.format(&result);
38        assert!(output.contains("Model: gpt-4"));
39        assert!(output.contains("Tokens: 2"));
40        assert!(!output.contains("%"), "Should not contain percentage");
41        assert!(!output.contains("Context window"), "Should not contain context window");
42    }
43}
token_count/output/basic.rs

token_count/output/
basic.rs
