llm-token-visualizer 0.3.0

Flag low-confidence spans in LLM output from token logprobs, and render per-token confidence as terminal, HTML or Markdown
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

pub mod data;
pub mod detect;
#[cfg(feature = "live")]
pub mod live;
pub mod renderer;
pub mod report;
pub mod utils;

pub use data::{
    ConfidenceLevel, FlagType, TokenAnalysis, TokenFlag, TokenInfo, VisualizationConfig,
};
pub use renderer::{HtmlRenderer, MarkdownRenderer, Renderer, TerminalRenderer};
pub use utils::{create_mock_analysis, detect_issues, simple_tokenize, AnalysisMetrics};

use anyhow::Result;

/// Main visualization function that can be used by other applications
pub fn visualize_tokens(
    text: &str,
    analysis: &TokenAnalysis,
    format: &str,
    config: Option<VisualizationConfig>,
) -> Result<String> {
    let config = config.unwrap_or_default();

    match format {
        "terminal" => {
            let renderer = TerminalRenderer::new();
            renderer.render(text, analysis, &config)
        }
        "html" => {
            let renderer = HtmlRenderer::new();
            renderer.render(text, analysis, &config)
        }
        "markdown" => {
            let renderer = MarkdownRenderer::new();
            renderer.render(text, analysis, &config)
        }
        _ => Err(anyhow::anyhow!("Unsupported format: {}", format)),
    }
}

/// Quick analysis function for testing - creates mock data and visualizes
pub fn quick_analyze(text: &str, format: &str) -> Result<String> {
    let analysis = utils::create_mock_analysis(text);
    let config = VisualizationConfig::default();
    visualize_tokens(text, &analysis, format, Some(config))
}

/// Comprehensive analysis with issue detection
pub fn analyze_with_issues(analysis: &TokenAnalysis) -> (utils::AnalysisMetrics, Vec<String>) {
    let metrics = utils::AnalysisMetrics::from_analysis(analysis);
    let issues = utils::detect_issues(analysis);
    (metrics, issues)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_visualize_tokens() {
        let analysis = TokenAnalysis {
            tokens: vec![
                TokenInfo {
                    text: "Hello".to_string(),
                    confidence: 0.9,
                },
                TokenInfo {
                    text: " world".to_string(),
                    confidence: 0.8,
                },
            ],
            flags: vec![],
        };

        let result = visualize_tokens("Hello world", &analysis, "markdown", None);
        assert!(result.is_ok());
        assert!(result.unwrap().contains("Hello"));
    }

    #[test]
    fn test_quick_analyze() {
        let result = quick_analyze("This is a test", "markdown");
        assert!(result.is_ok());
        assert!(result.unwrap().contains("Token Analysis"));
    }

    #[test]
    fn test_analyze_with_issues() {
        let analysis = TokenAnalysis {
            tokens: vec![
                TokenInfo {
                    text: "good".to_string(),
                    confidence: 0.9,
                },
                TokenInfo {
                    text: "bad".to_string(),
                    confidence: 0.1,
                },
            ],
            flags: vec![],
        };

        let (metrics, issues) = analyze_with_issues(&analysis);
        assert_eq!(metrics.total_tokens, 2);
        assert!(!issues.is_empty());
    }
}