Struct FastTokenizer 

pub struct FastTokenizer { /* private fields */ }

Hybrid tokenizer: fast BPE encoding via fastokens, decoding via HuggingFace.

Both backends are loaded from the same tokenizer.json file.

Implementations

impl FastTokenizer

pub fn from_file(path: &str) -> Result<Self>

Trait Implementations

impl Decoder for FastTokenizer

fn decode( &self, token_ids: &[TokenIdType], skip_special_tokens: bool, ) -> Result<DecodeResult>

impl Encoder for FastTokenizer

fn encode(&self, input: &str) -> Result<Encoding>

fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>

impl Tokenizer for FastTokenizer

fn with_options(self, options: TokenizerOptions) -> Self

where Self: Sized,

Apply construction-time TokenizerOptions.