Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Implementations

impl Tokenizer

Auto-generated by derive_getters::Getters.

pub fn first_bytes_to_lengths(&self) -> &Vec<Box<[u16]>>

Get field first_bytes_to_lengths from instance of Tokenizer.

pub fn bytes_to_token_index(&self) -> &HashMap<Vec<u8>, u32>

Get field bytes_to_token_index from instance of Tokenizer.

pub fn token_index_to_bytes(&self) -> &Vec<Vec<u8>>

Get field token_index_to_bytes from instance of Tokenizer.

impl Tokenizer

pub fn new(vocab: &str) -> Result<Self, TokenizerError>

pub fn encode(&self, input: &[u8]) -> Result<Vec<u32>, TokenizerError>

pub fn decode(&self, tokens: &[u32]) -> Result<Vec<u8>, TokenizerError>

impl Tokenizer

pub fn encode_into( &self, input: &[u8], output: &mut Vec<u32>, ) -> Result<(), TokenizerError>

pub fn decode_into( &self, tokens: &[u32], output: &mut Vec<u8>, ) -> Result<(), TokenizerError>

Trait Implementations

impl Clone for Tokenizer

fn clone(&self) -> Tokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.