Struct CustomFormatTokenizer 

pub struct CustomFormatTokenizer { /* private fields */ }

Custom format tokenizer implementation

Implementations

impl CustomFormatTokenizer

pub fn from_format(format: CustomTokenizerFormat) -> Result<Self>

Create a new tokenizer from a custom format

pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self>

Load tokenizer from custom format file

pub fn save_to_file<P: AsRef<Path>>(&self, path: P) -> Result<()>

Save tokenizer to custom format file

pub fn with_max_length(self, max_length: Option<usize>) -> Self

Set maximum sequence length

pub fn vocab_size(&self) -> usize

Get vocabulary size

pub fn token_to_id(&self, token: &str) -> Option<u32>

Get token ID

pub fn id_to_token(&self, id: u32) -> Option<String>

Get token from ID

pub fn get_vocab(&self) -> &HashMap<String, u32>

Get vocabulary

Trait Implementations

impl Clone for CustomFormatTokenizer

fn clone(&self) -> CustomFormatTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CustomFormatTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Tokenizer for CustomFormatTokenizer

fn encode(&self, text: &str) -> Result<TokenizedInput>

Encodes a single text string into tokens.

fn decode(&self, ids: &[u32]) -> Result<String>

Decodes token IDs back into text.

fn encode_pair(&self, text_a: &str, text_b: &str) -> Result<TokenizedInput>

Encodes a pair of texts for sequence-pair tasks.

fn vocab_size(&self) -> usize

Returns the size of the tokenizer’s vocabulary.

fn get_vocab(&self) -> HashMap<String, u32>

Returns a copy of the vocabulary as a mapping from tokens to IDs.

fn token_to_id(&self, token: &str) -> Option<u32>

Converts a token string to its corresponding ID.

fn id_to_token(&self, id: u32) -> Option<String>

Converts a token ID to its corresponding token string.