Struct TokenVisualizer 

pub struct TokenVisualizer { /* private fields */ }

Token visualizer implementation

Implementations

impl TokenVisualizer

pub fn new(config: VisualizationConfig) -> Self

Create a new token visualizer

pub fn default() -> Self

Create visualizer with default configuration

pub fn with_special_tokens(self, special_tokens: HashMap<String, u32>) -> Self

Add special tokens for recognition

pub fn visualize<T: Tokenizer>( &self, tokenizer: &T, text: &str, ) -> Result<TokenVisualization>

Visualize tokenization from any tokenizer

pub fn compare_tokenizers<T: Tokenizer>( &self, tokenizers: HashMap<String, &T>, text: &str, ) -> Result<TokenizerComparison>

Compare multiple tokenizers

pub fn to_html(&self, visualization: &TokenVisualization) -> String

Generate HTML visualization

pub fn to_text(&self, visualization: &TokenVisualization) -> String

Generate plain text visualization

pub fn to_json(&self, visualization: &TokenVisualization) -> Result<String>

Export visualization to JSON

pub fn comparison_report(&self, comparison: &TokenizerComparison) -> String

Generate comparison report