Struct CharTokenizer 

pub struct CharTokenizer {
    pub lowercase: bool,
    /* private fields */
}

Character-level tokenizer

Fields

lowercase: bool

Implementations

impl CharTokenizer

pub fn new() -> Self

pub fn lowercase(self, lowercase: bool) -> Self

pub fn fit(&mut self, texts: &[String])

pub fn texts_to_sequences(&self, texts: &[String]) -> Vec<Vec<usize>>

pub fn sequences_to_texts(&self, sequences: &[Vec<usize>]) -> Vec<String>

pub fn vocab_size(&self) -> usize

Trait Implementations

impl Default for CharTokenizer

fn default() -> Self

Returns the “default value” for a type.