Struct ZeroCopyTokenizer 

pub struct ZeroCopyTokenizer { /* private fields */ }

Zero-copy tokenizer implementation

Implementations

impl ZeroCopyTokenizer

pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self>

Load a tokenizer from a zero-copy file

pub fn header(&self) -> &ZeroCopyHeader

Get the header information

pub fn vocab_size(&self) -> usize

Get vocabulary size

pub fn get_token_unchecked(&self, id: u32) -> Option<&str>

Get a token by ID without copying

pub fn get_id_unchecked(&self, token: &str) -> Option<u32>

Get token ID without copying

pub fn get_vocab_entry(&self, index: usize) -> Option<&ZeroCopyVocabEntry>

Get vocabulary entry by index

pub fn vocab_entries(&self) -> impl Iterator<Item = &ZeroCopyVocabEntry>

Iterate over all vocabulary entries

pub fn metadata_bytes(&self) -> &[u8]

Get metadata section as bytes

pub fn special_tokens_bytes(&self) -> &[u8]

Get special tokens section as bytes

pub fn memory_stats(&self) -> ZeroCopyMemoryStats

Get memory usage statistics

pub fn verify_integrity(&self) -> Result<bool>

Verify file integrity using checksum

Trait Implementations

impl Tokenizer for ZeroCopyTokenizer

fn encode(&self, text: &str) -> Result<TokenizedInput>

Encodes a single text string into tokens.

fn decode(&self, token_ids: &[u32]) -> Result<String>

Decodes token IDs back into text.

fn get_vocab(&self) -> HashMap<String, u32>

Returns a copy of the vocabulary as a mapping from tokens to IDs.

fn token_to_id(&self, token: &str) -> Option<u32>

Converts a token string to its corresponding ID.

fn id_to_token(&self, id: u32) -> Option<String>

Converts a token ID to its corresponding token string.

fn encode_pair(&self, text_a: &str, text_b: &str) -> Result<TokenizedInput>

Encodes a pair of texts for sequence-pair tasks.

fn vocab_size(&self) -> usize

Returns the size of the tokenizer’s vocabulary.