Struct TextPreprocessor 

pub struct TextPreprocessor {
    pub lowercase: bool,
    pub remove_stopwords: bool,
    pub lemmatize: bool,
    pub stem: bool,
    pub stopwords: Vec<String>,
}

Text preprocessing for NLP

Fields

lowercase: bool

Whether to convert to lowercase

remove_stopwords: bool

Whether to remove stopwords

lemmatize: bool

Whether to lemmatize

stem: bool

Whether to stem

stopwords: Vec<String>

Custom stopwords

Implementations

impl TextPreprocessor

pub fn new() -> Self

Creates a new text preprocessor with default settings

pub fn with_lowercase(self, lowercase: bool) -> Self

Sets whether to convert to lowercase

pub fn with_remove_stopwords(self, remove_stopwords: bool) -> Self

Sets whether to remove stopwords

pub fn with_lemmatize(self, lemmatize: bool) -> Self

Sets whether to lemmatize

pub fn with_stem(self, stem: bool) -> Self

Sets whether to stem

pub fn with_stopwords(self, stopwords: Vec<String>) -> Self

Sets custom stopwords

pub fn preprocess(&self, text: &str) -> Result<String>

Preprocesses text

pub fn tokenize(&self, text: &str) -> Result<Vec<String>>

Tokenizes text

Trait Implementations

impl Clone for TextPreprocessor

fn clone(&self) -> TextPreprocessor

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TextPreprocessor

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.