Struct BasicTextCleaner 

pub struct BasicTextCleaner { /* private fields */ }

Text cleaner for removing special characters, extra whitespace, and stopwords

Implementations

impl BasicTextCleaner

pub fn new( remove_special_chars: bool, remove_stopwords: bool, normalize_whitespace: bool, ) -> Self

Create a new text cleaner

Examples found in repository

examples/text_processing_demo.rs (line 35)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== SciRS2 Text Processing Demo ===\n");

    let documents = [
        "The quick brown fox jumps over the lazy dog.",
        "A fast red fox leaped over the sleeping canine.",
        "Machine learning algorithms process textual data efficiently.",
        "Text processing and natural language understanding are important.",
    ];

    // 1. Text Normalization
    println!("1. Text Normalization");
    let normalizer = BasicNormalizer::new(true, true);
    for (i, doc) in documents.iter().enumerate() {
        let normalized = normalizer.normalize(doc)?;
        println!("Doc {}: {}", i + 1, normalized);
    }
    println!();

    // 2. Text Cleaning
    println!("2. Text Cleaning");
    let cleaner = BasicTextCleaner::new(true, true, true);
    for (i, doc) in documents.iter().enumerate() {
        let cleaned = cleaner.clean(doc)?;
        println!("Doc {}: {}", i + 1, cleaned);
    }
    println!();

    // 3. Tokenization Examples
    println!("3. Tokenization Examples");

    // Word tokenization
    let word_tokenizer = WordTokenizer::new(true);
    let tokens = word_tokenizer.tokenize(documents[0])?;
    println!("Word tokens: {tokens:?}");

    // N-gram tokenization
    let ngram_tokenizer = NgramTokenizer::new(2)?;
    let ngrams = ngram_tokenizer.tokenize(documents[0])?;
    println!("2-grams: {ngrams:?}");

    // Regex tokenization
    let regex_tokenizer = RegexTokenizer::new(r"\b\w+\b", false)?;
    let regex_tokens = regex_tokenizer.tokenize(documents[0])?;
    println!("Regex tokens: {regex_tokens:?}");
    println!();

    // 4. Stemming and Lemmatization
    println!("4. Stemming and Lemmatization");
    let porter_stemmer = PorterStemmer::new();
    let lemmatizer = SimpleLemmatizer::new();

    let test_words = vec!["running", "jumped", "better", "processing"];
    for word in test_words {
        let stemmed = porter_stemmer.stem(word)?;
        let lemmatized = lemmatizer.stem(word)?;
        println!("{word}: stemmed={stemmed}, lemmatized={lemmatized}");
    }
    println!();

    // 5. Count Vectorization
    println!("5. Count Vectorization");
    let mut count_vectorizer = CountVectorizer::new(false);

    let doc_refs = documents.to_vec();
    count_vectorizer.fit(&doc_refs)?;

    // Transform individual documents
    let count_matrix = count_vectorizer.transform_batch(&doc_refs)?;
    println!("Count vector shape: {:?}", count_matrix.shape());
    println!("Vocabulary size: {}", count_vectorizer.vocabulary().len());

    println!();

    // 6. TF-IDF Vectorization
    println!("6. TF-IDF Vectorization");
    let mut tfidf_vectorizer = TfidfVectorizer::new(false, true, Some("l2".to_string()));

    tfidf_vectorizer.fit(&doc_refs)?;
    let tfidf_matrix = tfidf_vectorizer.transform_batch(&doc_refs)?;

    println!("TF-IDF vector shape: {:?}", tfidf_matrix.shape());
    println!("Sample TF-IDF values:");
    for i in 0..3.min(tfidf_matrix.nrows()) {
        for j in 0..5.min(tfidf_matrix.ncols()) {
            print!("{:.3} ", tfidf_matrix[[i, j]]);
        }
        println!();
    }
    println!();

    // 7. Complete Pipeline Example
    println!("7. Complete Text Processing Pipeline");
    let testtext = "The cats were running quickly through the gardens.";

    // Normalize
    let normalized = normalizer.normalize(testtext)?;
    println!("Normalized: {normalized}");

    // Clean
    let cleaned = cleaner.clean(&normalized)?;
    println!("Cleaned: {cleaned}");

    // Tokenize
    let tokens = word_tokenizer.tokenize(&cleaned)?;
    println!("Tokens: {tokens:?}");

    // Stem
    let stemmed_tokens: Result<Vec<_>, _> = tokens
        .iter()
        .map(|token| porter_stemmer.stem(token))
        .collect();
    let stemmed_tokens = stemmed_tokens?;
    println!("Stemmed: {stemmed_tokens:?}");

    Ok(())
}

examples/enhanced_vectorization_demo.rs (line 117)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== Enhanced Text Vectorization Demo ===\n");

    let documents = vec![
        "The quick brown fox jumps over the lazy dog.",
        "A fast red fox leaped over the sleeping canine.",
        "Machine learning algorithms process textual data efficiently.",
        "Text processing and natural language understanding are important.",
        "Natural language processing is a field of artificial intelligence.",
        "Deep learning models can understand complex text patterns.",
    ];

    // 1. Enhanced Count Vectorizer with Unigrams
    println!("1. Enhanced Count Vectorizer (Unigrams only)");
    let mut count_vec_unigram = EnhancedCountVectorizer::new()
        .set_binary(false)
        .set_max_features(Some(20));

    count_vec_unigram.fit(&documents)?;
    let count_matrix = count_vec_unigram.transform_batch(&documents)?;

    println!("Vocabulary size: {}", count_vec_unigram.vocabulary().len());
    println!("Count matrix shape: {:?}", count_matrix.shape());
    println!();

    // 2. Enhanced Count Vectorizer with N-grams
    println!("2. Enhanced Count Vectorizer (Unigrams + Bigrams)");
    let mut count_vec_ngram = EnhancedCountVectorizer::new()
        .set_ngram_range((1, 2))?
        .set_max_features(Some(30));

    count_vec_ngram.fit(&documents)?;
    let ngram_matrix = count_vec_ngram.transform_batch(&documents)?;

    println!(
        "Vocabulary size with n-grams: {}",
        count_vec_ngram.vocabulary().len()
    );
    println!("N-gram count matrix shape: {:?}", ngram_matrix.shape());

    // Show some n-gram tokens
    let vocab = count_vec_ngram.vocabulary();
    let mut ngram_tokens: Vec<String> = Vec::new();
    for i in 0..vocab.len().min(10) {
        if let Some(token) = vocab.get_token(i) {
            if token.contains(' ') {
                // This is a bigram
                ngram_tokens.push(token.to_string());
            }
        }
    }
    println!("Sample bigrams: {ngram_tokens:?}");
    println!();

    // 3. Enhanced Count Vectorizer with Document Frequency Filtering
    println!("3. Count Vectorizer with Document Frequency Filtering");
    let mut count_vec_filtered = EnhancedCountVectorizer::new()
        .set_min_df(0.3)?  // Token must appear in at least 30% of documents
        .set_max_df(0.8)?; // Token must appear in at most 80% of documents

    count_vec_filtered.fit(&documents)?;

    println!(
        "Vocabulary size after DF filtering: {}",
        count_vec_filtered.vocabulary().len()
    );
    println!();

    // 4. Enhanced TF-IDF Vectorizer with N-grams
    println!("4. Enhanced TF-IDF Vectorizer with N-grams");
    let mut tfidf_vec = EnhancedTfidfVectorizer::new()
        .set_ngram_range((1, 2))?
        .set_max_features(Some(50))
        .set_smooth_idf(true)
        .set_sublinear_tf(true)
        .set_norm(Some("l2".to_string()))?;

    tfidf_vec.fit(&documents)?;
    let tfidf_matrix = tfidf_vec.transform_batch(&documents)?;

    println!("TF-IDF matrix shape: {:?}", tfidf_matrix.shape());
    println!("TF-IDF with smoothing and sublinear TF applied");

    // Show TF-IDF values for first document
    let first_doc_tfidf = tfidf_matrix.row(0);
    let mut top_features: Vec<(String, f64)> = Vec::new();

    for (idx, &value) in first_doc_tfidf.iter().enumerate() {
        if value > 0.0 {
            if let Some(token) = tfidf_vec.vocabulary().get_token(idx) {
                top_features.push((token.to_string(), value));
            }
        }
    }

    top_features.sort_by(|a, b| b.1.partial_cmp(&a.1).expect("Operation failed"));
    println!("\nTop TF-IDF features for first document:");
    for (token, score) in top_features.iter().take(5) {
        println!("  {token}: {score:.3}");
    }
    println!();

    // 5. Processing with Text Cleaning
    println!("5. Vectorization with Text Preprocessing");
    let cleaner = BasicTextCleaner::new(true, true, true);

    // Clean documents first
    let cleaned_docs: Result<Vec<_>, _> = documents.iter().map(|doc| cleaner.clean(doc)).collect();
    let cleaned_docs = cleaned_docs?;
    let cleaned_refs: Vec<&str> = cleaned_docs.iter().map(|s| s.as_str()).collect();

    let mut tfidf_cleaned = EnhancedTfidfVectorizer::new()
        .set_ngram_range((1, 2))?
        .set_max_features(Some(30));

    tfidf_cleaned.fit(&cleaned_refs)?;
    let cleaned_matrix = tfidf_cleaned.transform_batch(&cleaned_refs)?;

    println!("TF-IDF shape after cleaning: {:?}", cleaned_matrix.shape());
    println!("Processing pipeline: Clean -> Tokenize -> Vectorize");

    Ok(())
}

pub fn with_stopwords( remove_special_chars: bool, remove_stopwords: bool, normalize_whitespace: bool, stopwords: HashSet<String>, ) -> Self

Create a text cleaner with custom stopwords

pub fn add_stopwords(&mut self, words: &[&str])

Add stopwords to the cleaner

pub fn is_stopword(&self, word: &str) -> bool

Check if a word is a stopword

Trait Implementations

impl Clone for BasicTextCleaner

fn clone(&self) -> BasicTextCleaner

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BasicTextCleaner

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for BasicTextCleaner

fn default() -> Self

Returns the “default value” for a type.

impl TextCleaner for BasicTextCleaner

fn clean(&self, text: &str) -> Result<String>

Clean the input text

fn clean_batch(&self, texts: &[&str]) -> Result<Vec<String>>

Clean a batch of texts