Trait TokenizerRegistry 

pub trait TokenizerRegistry: Send + Sync {
    // Required methods
    fn register(
        &mut self,
        name: &str,
        tokenizer: Box<dyn Tokenizer>,
    ) -> Result<()>;
    fn get(&self, name: &str) -> Option<&dyn Tokenizer>;
    fn remove(&mut self, name: &str) -> Option<Box<dyn Tokenizer>>;
    fn list_names(&self) -> Vec<String>;
    fn contains(&self, name: &str) -> bool;
}

Tokenizer registry for managing multiple tokenizers

Required Methods

fn register(&mut self, name: &str, tokenizer: Box<dyn Tokenizer>) -> Result<()>

Register a tokenizer with a name

fn get(&self, name: &str) -> Option<&dyn Tokenizer>

Get tokenizer by name

fn remove(&mut self, name: &str) -> Option<Box<dyn Tokenizer>>

Remove tokenizer by name

fn list_names(&self) -> Vec<String>

List all registered tokenizer names

fn contains(&self, name: &str) -> bool

Check if tokenizer exists

Implementors