Struct edge_transformers::tokenizer::AutoTokenizer

pub struct AutoTokenizer {
    pub tokenizer: Tokenizer,
    pub eos_token: String,
    pub pad_token: String,
    pub eos_token_id: u32,
    pub pad_token_id: u32,
}

A simple tokenizer wrapper that reads special token map for further use in the pipeline.

Fields

tokenizer: Tokenizer

eos_token: String

pad_token: String

eos_token_id: u32

pad_token_id: u32

Implementations

impl AutoTokenizer

pub fn new_from_memory( tokenizer_config: String, special_tokens_map: String ) -> Result<AutoTokenizer, Error>

pub fn new( tokenizer_config_path: PathBuf, special_tokens_map_path: PathBuf ) -> Result<AutoTokenizer, Error>