Struct MathTokenizer 

pub struct MathTokenizer { /* private fields */ }

Mathematical formula tokenizer

Implementations

impl MathTokenizer

pub fn new() -> Result<Self>

Create a new math tokenizer with default configuration

pub fn with_config(config: MathTokenizerConfig) -> Result<Self>

Create a new math tokenizer with custom configuration

pub fn tokenize_math(&mut self, text: &str) -> Result<Vec<MathToken>>

Tokenize mathematical text into MathTokens

pub fn math_tokens_to_input(&mut self, tokens: Vec<MathToken>) -> TokenizedInput

Convert MathTokens to standard TokenizedInput

pub fn analyze_math(&self, tokens: &[MathToken]) -> MathAnalysis

Get mathematical analysis of tokenized text

pub fn vocab_stats(&self) -> HashMap<String, usize>

Get vocabulary statistics

Trait Implementations

impl Clone for MathTokenizer

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for MathTokenizer

fn default() -> Self

Returns the “default value” for a type.

impl Tokenizer for MathTokenizer

fn encode(&self, text: &str) -> Result<TokenizedInput>

Encodes a single text string into tokens.

fn decode(&self, token_ids: &[u32]) -> Result<String>

Decodes token IDs back into text.

fn get_vocab(&self) -> HashMap<String, u32>

Returns a copy of the vocabulary as a mapping from tokens to IDs.

fn token_to_id(&self, token: &str) -> Option<u32>

Converts a token string to its corresponding ID.

fn id_to_token(&self, id: u32) -> Option<String>

Converts a token ID to its corresponding token string.

fn encode_pair(&self, text_a: &str, text_b: &str) -> Result<TokenizedInput>

Encodes a pair of texts for sequence-pair tasks.

fn vocab_size(&self) -> usize

Returns the size of the tokenizer’s vocabulary.