Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Tokenizer wrapper for counting tokens

Implementations

impl Tokenizer

pub fn new() -> MemoryResult<Self>

pub fn count_tokens(&self, text: &str) -> usize

Count tokens in text

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text to tokens

pub fn decode(&self, tokens: &[u32]) -> String

Decode tokens to text

Trait Implementations

impl Default for Tokenizer

fn default() -> Self

Returns the “default value” for a type.