token_count/output/
debug.rs

1//! Debug formatter - outputs token IDs and sample decoded tokens
2
3use crate::output::OutputFormatter;
4use crate::tokenizers::TokenizationResult;
5
6/// Debug formatter that outputs token IDs and sample decoded tokens
7pub struct DebugFormatter;
8
9impl OutputFormatter for DebugFormatter {
10    fn format(&self, result: &TokenizationResult) -> String {
11        format!(
12            "Model: {} ({})\nTokens: {}\nContext window: {} tokens\n\nNote: Detailed token ID output will be available in v0.2.0",
13            result.model_info.name,
14            result.model_info.encoding,
15            result.token_count,
16            result.model_info.context_window
17        )
18    }
19}
20
21#[cfg(test)]
22mod tests {
23    use super::*;
24    use crate::tokenizers::ModelInfo;
25
26    #[test]
27    fn test_debug_formatter() {
28        let formatter = DebugFormatter;
29        let result = TokenizationResult {
30            token_count: 2,
31            model_info: ModelInfo {
32                name: "gpt-4".to_string(),
33                encoding: "cl100k_base".to_string(),
34                context_window: 128000,
35                description: "GPT-4".to_string(),
36            },
37        };
38
39        let output = formatter.format(&result);
40        assert!(output.contains("Model: gpt-4"));
41        assert!(output.contains("Tokens: 2"));
42        assert!(output.contains("v0.2.0"));
43    }
44}
token_count/output/debug.rs

token_count/output/
debug.rs
