llm_token_visualizer/
lib.rs

1pub mod data;
2pub mod detect;
3#[cfg(feature = "live")]
4pub mod live;
5pub mod renderer;
6pub mod report;
7pub mod utils;
8
9pub use data::{
10    ConfidenceLevel, FlagType, TokenAnalysis, TokenFlag, TokenInfo, VisualizationConfig,
11};
12pub use renderer::{HtmlRenderer, MarkdownRenderer, Renderer, TerminalRenderer};
13pub use utils::{create_mock_analysis, detect_issues, simple_tokenize, AnalysisMetrics};
14
15use anyhow::Result;
16
17/// Main visualization function that can be used by other applications
18pub fn visualize_tokens(
19    text: &str,
20    analysis: &TokenAnalysis,
21    format: &str,
22    config: Option<VisualizationConfig>,
23) -> Result<String> {
24    let config = config.unwrap_or_default();
25
26    match format {
27        "terminal" => {
28            let renderer = TerminalRenderer::new();
29            renderer.render(text, analysis, &config)
30        }
31        "html" => {
32            let renderer = HtmlRenderer::new();
33            renderer.render(text, analysis, &config)
34        }
35        "markdown" => {
36            let renderer = MarkdownRenderer::new();
37            renderer.render(text, analysis, &config)
38        }
39        _ => Err(anyhow::anyhow!("Unsupported format: {}", format)),
40    }
41}
42
43/// Quick analysis function for testing - creates mock data and visualizes
44pub fn quick_analyze(text: &str, format: &str) -> Result<String> {
45    let analysis = utils::create_mock_analysis(text);
46    let config = VisualizationConfig::default();
47    visualize_tokens(text, &analysis, format, Some(config))
48}
49
50/// Comprehensive analysis with issue detection
51pub fn analyze_with_issues(analysis: &TokenAnalysis) -> (utils::AnalysisMetrics, Vec<String>) {
52    let metrics = utils::AnalysisMetrics::from_analysis(analysis);
53    let issues = utils::detect_issues(analysis);
54    (metrics, issues)
55}
56
57#[cfg(test)]
58mod tests {
59    use super::*;
60
61    #[test]
62    fn test_visualize_tokens() {
63        let analysis = TokenAnalysis {
64            tokens: vec![
65                TokenInfo {
66                    text: "Hello".to_string(),
67                    confidence: 0.9,
68                },
69                TokenInfo {
70                    text: " world".to_string(),
71                    confidence: 0.8,
72                },
73            ],
74            flags: vec![],
75        };
76
77        let result = visualize_tokens("Hello world", &analysis, "markdown", None);
78        assert!(result.is_ok());
79        assert!(result.unwrap().contains("Hello"));
80    }
81
82    #[test]
83    fn test_quick_analyze() {
84        let result = quick_analyze("This is a test", "markdown");
85        assert!(result.is_ok());
86        assert!(result.unwrap().contains("Token Analysis"));
87    }
88
89    #[test]
90    fn test_analyze_with_issues() {
91        let analysis = TokenAnalysis {
92            tokens: vec![
93                TokenInfo {
94                    text: "good".to_string(),
95                    confidence: 0.9,
96                },
97                TokenInfo {
98                    text: "bad".to_string(),
99                    confidence: 0.1,
100                },
101            ],
102            flags: vec![],
103        };
104
105        let (metrics, issues) = analyze_with_issues(&analysis);
106        assert_eq!(metrics.total_tokens, 2);
107        assert!(!issues.is_empty());
108    }
109}
llm_token_visualizer/lib.rs

llm_token_visualizer/
lib.rs
