Struct TextPreprocessor 

pub struct TextPreprocessor { /* private fields */ }

Text preprocessor that combines tokenization, stop word removal, and stemming

Examples

use sklears_feature_extraction::text::preprocessing::TextPreprocessor;

let preprocessor = TextPreprocessor::new()
    .enable_stemming(true)
    .enable_stop_word_removal(true);

let processed = preprocessor.preprocess("The running dogs are playing");
// Result will be stemmed and stop words removed

Implementations

impl TextPreprocessor

pub fn new() -> Self

Create a new text preprocessor

pub fn enable_stemming(self, enable: bool) -> Self

Enable or disable stemming

pub fn enable_stop_word_removal(self, enable: bool) -> Self

Enable or disable stop word removal

pub fn stop_words(self, stop_words: HashSet<String>) -> Self

Set custom stop words

pub fn tokenizer(self, tokenizer: SimpleTokenizer) -> Self

Configure the tokenizer

pub fn preprocess(&self, text: &str) -> Vec<String>

Preprocess text through the configured pipeline

Trait Implementations

impl Clone for TextPreprocessor

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TextPreprocessor

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TextPreprocessor

fn default() -> Self

Returns the “default value” for a type.