Struct WordTokenizer 

pub struct WordTokenizer {
    pub lowercase: bool,
    pub remove_punctuation: bool,
    pub min_word_length: usize,
    /* private fields */
}

Word-level tokenizer

Fields

lowercase: bool

remove_punctuation: bool

min_word_length: usize

Implementations

impl WordTokenizer

pub fn new() -> Self

pub fn lowercase(self, lowercase: bool) -> Self

pub fn remove_punctuation(self, remove: bool) -> Self

pub fn min_word_length(self, length: usize) -> Self

pub fn tokenize(&self, text: &str) -> Vec<String>

pub fn fit(&mut self, texts: &[String])

pub fn texts_to_sequences(&self, texts: &[String]) -> Vec<Vec<usize>>

pub fn sequences_to_texts(&self, sequences: &[Vec<usize>]) -> Vec<String>

pub fn vocab_size(&self) -> usize

pub fn vocab(&self) -> Option<&HashMap<String, usize>>

Trait Implementations

impl Default for WordTokenizer

fn default() -> Self

Returns the “default value” for a type.