Struct Tokenizer

pub struct Tokenizer { /* private fields */ }

Implementations

impl Tokenizer

pub fn new( text: &'_ str, stopwords: &'_ [String], punctuation: Option<&'_ [String]>, ) -> Self

Create a new Tokenizer instance.

pub fn split_into_words(&self) -> Vec<String>

Split text into words by splitting on word bounds.

pub fn sync_split_into_words(&self) -> Vec<String>

Split text into words by splitting on word bounds (always synchronous even with parallel flag).

pub fn split_into_sentences(&self) -> Vec<String>

Split text into unicode sentences by splitting on punctuation.

pub fn sync_split_into_sentences(&self) -> Vec<String>

Split text into unicode sentences (always synchronous even with parallel flag).

pub fn split_into_phrases(&self, length: Option<usize>) -> Vec<String>

Split text into phrases by splitting on stopwords.

pub fn sync_split_into_phrases(&self, length: Option<usize>) -> Vec<String>

Split text into words by splitting on word bounds (always synchronous even with parallel flag).

pub fn split_into_paragraphs(&self) -> Vec<String>

Split text into paragraphs by splitting on newlines.

pub fn sync_split_into_paragraphs(&self) -> Vec<String>

Split text into paragraphs (always synchronous even with parallel flag).