Struct TokenizationDebugger 

pub struct TokenizationDebugger { /* private fields */ }

Comprehensive tokenization debugger for analyzing tokenization behavior

Implementations

impl TokenizationDebugger

pub fn new() -> Self

Create a new tokenization debugger

pub fn with_config(config: DebuggerConfig) -> Self

Create debugger with custom configuration

pub fn add_tokenizer(&mut self, name: String, tokenizer: TokenizerWrapper)

Add a tokenizer to the debugger

pub fn remove_tokenizer(&mut self, name: &str) -> Option<TokenizerWrapper>

Remove a tokenizer from the debugger

pub fn list_tokenizers(&self) -> Vec<String>

List all available tokenizers

pub fn debug_text(&mut self, text: &str) -> Result<DebugSession>

Debug tokenization of input text with all registered tokenizers

pub fn compare_texts(&mut self, texts: &[String]) -> Result<Vec<DebugSession>>

Compare tokenization across multiple texts

pub fn get_history(&self) -> &[DebugSession]

Get debugging history

pub fn get_session(&self, session_id: &str) -> Option<&DebugSession>

Get a specific session by ID

pub fn generate_report(&self, session_id: &str) -> Result<String>

Generate a detailed debug report

pub fn generate_html_visualization(&self, session_id: &str) -> Result<String>

Generate HTML visualization of tokenization

Trait Implementations

impl Default for TokenizationDebugger

fn default() -> Self

Returns the “default value” for a type.