Struct WordTokenizer 

pub struct WordTokenizer { /* private fields */ }

Tokenizer for splitting text into words

Implementations

impl WordTokenizer

pub fn new(lowercase: bool) -> Self

Create a new word tokenizer

Examples found in repository

examples/text_processing_demo.rs (line 46)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== SciRS2 Text Processing Demo ===\n");

    let documents = [
        "The quick brown fox jumps over the lazy dog.",
        "A fast red fox leaped over the sleeping canine.",
        "Machine learning algorithms process textual data efficiently.",
        "Text processing and natural language understanding are important.",
    ];

    // 1. Text Normalization
    println!("1. Text Normalization");
    let normalizer = BasicNormalizer::new(true, true);
    for (i, doc) in documents.iter().enumerate() {
        let normalized = normalizer.normalize(doc)?;
        println!("Doc {}: {}", i + 1, normalized);
    }
    println!();

    // 2. Text Cleaning
    println!("2. Text Cleaning");
    let cleaner = BasicTextCleaner::new(true, true, true);
    for (i, doc) in documents.iter().enumerate() {
        let cleaned = cleaner.clean(doc)?;
        println!("Doc {}: {}", i + 1, cleaned);
    }
    println!();

    // 3. Tokenization Examples
    println!("3. Tokenization Examples");

    // Word tokenization
    let word_tokenizer = WordTokenizer::new(true);
    let tokens = word_tokenizer.tokenize(documents[0])?;
    println!("Word tokens: {tokens:?}");

    // N-gram tokenization
    let ngram_tokenizer = NgramTokenizer::new(2)?;
    let ngrams = ngram_tokenizer.tokenize(documents[0])?;
    println!("2-grams: {ngrams:?}");

    // Regex tokenization
    let regex_tokenizer = RegexTokenizer::new(r"\b\w+\b", false)?;
    let regex_tokens = regex_tokenizer.tokenize(documents[0])?;
    println!("Regex tokens: {regex_tokens:?}");
    println!();

    // 4. Stemming and Lemmatization
    println!("4. Stemming and Lemmatization");
    let porter_stemmer = PorterStemmer::new();
    let lemmatizer = SimpleLemmatizer::new();

    let test_words = vec!["running", "jumped", "better", "processing"];
    for word in test_words {
        let stemmed = porter_stemmer.stem(word)?;
        let lemmatized = lemmatizer.stem(word)?;
        println!("{word}: stemmed={stemmed}, lemmatized={lemmatized}");
    }
    println!();

    // 5. Count Vectorization
    println!("5. Count Vectorization");
    let mut count_vectorizer = CountVectorizer::new(false);

    let doc_refs = documents.to_vec();
    count_vectorizer.fit(&doc_refs)?;

    // Transform individual documents
    let count_matrix = count_vectorizer.transform_batch(&doc_refs)?;
    println!("Count vector shape: {:?}", count_matrix.shape());
    println!("Vocabulary size: {}", count_vectorizer.vocabulary().len());

    println!();

    // 6. TF-IDF Vectorization
    println!("6. TF-IDF Vectorization");
    let mut tfidf_vectorizer = TfidfVectorizer::new(false, true, Some("l2".to_string()));

    tfidf_vectorizer.fit(&doc_refs)?;
    let tfidf_matrix = tfidf_vectorizer.transform_batch(&doc_refs)?;

    println!("TF-IDF vector shape: {:?}", tfidf_matrix.shape());
    println!("Sample TF-IDF values:");
    for i in 0..3.min(tfidf_matrix.nrows()) {
        for j in 0..5.min(tfidf_matrix.ncols()) {
            print!("{:.3} ", tfidf_matrix[[i, j]]);
        }
        println!();
    }
    println!();

    // 7. Complete Pipeline Example
    println!("7. Complete Text Processing Pipeline");
    let testtext = "The cats were running quickly through the gardens.";

    // Normalize
    let normalized = normalizer.normalize(testtext)?;
    println!("Normalized: {normalized}");

    // Clean
    let cleaned = cleaner.clean(&normalized)?;
    println!("Cleaned: {cleaned}");

    // Tokenize
    let tokens = word_tokenizer.tokenize(&cleaned)?;
    println!("Tokens: {tokens:?}");

    // Stem
    let stemmed_tokens: Result<Vec<_>, _> = tokens
        .iter()
        .map(|token| porter_stemmer.stem(token))
        .collect();
    let stemmed_tokens = stemmed_tokens?;
    println!("Stemmed: {stemmed_tokens:?}");

    Ok(())
}

examples/parallel_processing_demo.rs (line 61)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Parallel Text Processing Demo");
    println!("============================\n");

    // Create test data with larger size to demonstrate parallelism
    println!("Creating test data...");
    let texts = create_testtexts(1000);

    // Create references to handle &[&str] requirements
    let text_refs: Vec<&str> = texts.iter().map(|s| s.as_str()).collect();

    println!("Total documents: {}", texts.len());
    println!("Example document: {}", texts[0]);

    // 1. Simple Parallel Text Processing
    println!("\n1. Basic Parallel Processing");
    println!("---------------------------");

    let processor = ParallelTextProcessor::new();

    let start = Instant::now();
    let word_counts = processor.process(&text_refs, |text| {
        // Count words in each document
        text.split_whitespace().count()
    });
    let duration = start.elapsed();

    println!("Processed {} documents in {:.2?}", texts.len(), duration);
    println!(
        "Average word count: {:.2}",
        word_counts.iter().sum::<usize>() as f64 / word_counts.len() as f64
    );

    // Sequential comparison
    let start = Instant::now();
    let _seq_word_counts: Vec<_> = texts
        .iter()
        .map(|text| text.split_whitespace().count())
        .collect();
    let seq_duration = start.elapsed();

    println!("Sequential processing took {seq_duration:.2?}");
    println!(
        "Speedup factor: {:.2}x",
        seq_duration.as_secs_f64() / duration.as_secs_f64()
    );

    // 2. Parallel Tokenization
    println!("\n2. Parallel Tokenization");
    println!("----------------------");

    let tokenizer = ParallelTokenizer::new(WordTokenizer::new(true)); // Pass 'lowercase' parameter

    let start = Instant::now();
    let tokens = tokenizer.tokenize(&text_refs)?;
    let duration = start.elapsed();

    println!("Tokenized {} documents in {:.2?}", texts.len(), duration);
    println!(
        "Total tokens: {}",
        tokens.iter().map(|t| t.len()).sum::<usize>()
    );
    println!(
        "Sample tokens from first document: {:?}",
        tokens[0].iter().take(5).collect::<Vec<_>>()
    );

    // Custom token processing
    println!("\nCustom token processing...");
    let start = Instant::now();
    let token_stats = tokenizer.tokenize_and_map(&text_refs, |tokens| {
        // Calculate token statistics
        let count = tokens.len();
        let avg_len = if count > 0 {
            tokens.iter().map(|t| t.len()).sum::<usize>() as f64 / count as f64
        } else {
            0.0
        };
        (count, avg_len)
    })?;
    let duration = start.elapsed();

    println!("Processed token statistics in {duration:.2?}");
    println!(
        "Average tokens per document: {:.2}",
        token_stats.iter().map(|(count_, _)| *count_).sum::<usize>() as f64
            / token_stats.len() as f64
    );
    println!(
        "Average token length: {:.2}",
        token_stats.iter().map(|(_, avg_len)| *avg_len).sum::<f64>() / token_stats.len() as f64
    );

    // 3. Parallel Vectorization
    println!("\n3. Parallel Vectorization");
    println!("------------------------");

    // First fit the vectorizer
    let mut vectorizer = TfidfVectorizer::default();
    let start = Instant::now();

    // Import the Vectorizer trait to use its methods
    use scirs2_text::Vectorizer;
    vectorizer.fit(&text_refs)?;
    let fit_duration = start.elapsed();

    println!("Fitted vectorizer in {fit_duration:.2?}");

    // Now transform in parallel
    let parallel_vectorizer = ParallelVectorizer::new(vectorizer).with_chunk_size(100);

    let start = Instant::now();
    let vectors = parallel_vectorizer.transform(&text_refs)?;
    let transform_duration = start.elapsed();

    println!(
        "Transformed {} documents in {:.2?}",
        texts.len(),
        transform_duration
    );
    println!("Vector shape: {:?}", vectors.shape());
    println!(
        "Non-zero elements: {}",
        vectors.iter().filter(|&&x| x > 0.0).count()
    );

    // 4. Batch Processing with Progress
    println!("\n4. Batch Processing with Progress");
    println!("--------------------------------");

    let processor = ParallelCorpusProcessor::new(100).with_threads(num_cpus::get());

    println!("Processing with {} threads...", num_cpus::get());
    let start = Instant::now();

    let last_progress = std::sync::Mutex::new(0);
    let result = processor.process_with_progress(
        &text_refs,
        |batch| {
            // Analyze batch of documents
            let mut word_counts = Vec::new();
            let mut char_counts = Vec::new();

            for &text in batch {
                word_counts.push(text.split_whitespace().count());
                char_counts.push(text.chars().count());
            }

            Ok(word_counts.into_iter().zip(char_counts).collect::<Vec<_>>())
        },
        |current, total| {
            // Only print progress updates at 10% intervals
            let percent = current * 100 / total;
            let mut last = last_progress.lock().unwrap();
            if percent / 10 > *last / 10 {
                println!("  Progress: {current}/{total}  ({percent}%)");
                *last = percent;
            }
        },
    )?;

    let duration = start.elapsed();

    println!("Processed {} documents in {:.2?}", texts.len(), duration);
    println!(
        "Average words per document: {:.2}",
        result.iter().map(|(words_, _)| *words_).sum::<usize>() as f64 / result.len() as f64
    );
    println!(
        "Average characters per document: {:.2}",
        result.iter().map(|(_, chars)| chars).sum::<usize>() as f64 / result.len() as f64
    );

    // 5. Memory-efficient processing
    println!("\n5. Memory-Efficient Large Corpus Processing");
    println!("------------------------------------------");

    println!("Simulating processing of a large corpus...");
    let largetexts: Vec<&str> = text_refs.iter().cycle().take(5000).copied().collect();
    println!("Large corpus size: {} documents", largetexts.len());

    let processor = ParallelCorpusProcessor::new(250).with_max_memory(1024 * 1024 * 1024); // 1 GB limit

    let start = Instant::now();
    let summary = processor.process(&largetexts, |batch| {
        // Compute simple statistics for the batch
        let batch_size = batch.len();
        let total_words: usize = batch
            .iter()
            .map(|&text| text.split_whitespace().count())
            .sum();
        let total_chars: usize = batch.iter().map(|&text| text.chars().count()).sum();

        Ok(vec![(batch_size, total_words, total_chars)])
    })?;
    let duration = start.elapsed();

    let total_words: usize = summary.iter().map(|(_, words_, _)| *words_).sum();
    let total_chars: usize = summary.iter().map(|(_, _, chars)| *chars).sum();

    println!("Processed large corpus in {duration:.2?}");
    println!("Total words: {total_words}");
    println!("Total chars: {total_chars}");
    println!(
        "Average processing speed: {:.2} documents/second",
        largetexts.len() as f64 / duration.as_secs_f64()
    );

    Ok(())
}

pub fn withpattern(lowercase: bool, pattern: &str) -> Result<Self>

Create a new word tokenizer with a custom pattern

Trait Implementations

impl Clone for WordTokenizer

fn clone(&self) -> WordTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for WordTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for WordTokenizer

fn default() -> Self

Returns the “default value” for a type.

impl Tokenizer for WordTokenizer

fn tokenize(&self, text: &str) -> Result<Vec<String>>

Tokenize the input text into tokens

fn clone_box(&self) -> Box<dyn Tokenizer + Send + Sync>

Clone the tokenizer (for use in parallel processing)

fn tokenize_batch(&self, texts: &[&str]) -> Result<Vec<Vec<String>>>

Tokenize batch of text