Struct CountVectorizer 

pub struct CountVectorizer { /* private fields */ }

Count vectorizer that uses a bag-of-words representation

Implementations

impl CountVectorizer

pub fn new(binary: bool) -> Self

Create a new count vectorizer

Examples found in repository

examples/text_processing_demo.rs (line 76)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== SciRS2 Text Processing Demo ===\n");

    let documents = [
        "The quick brown fox jumps over the lazy dog.",
        "A fast red fox leaped over the sleeping canine.",
        "Machine learning algorithms process textual data efficiently.",
        "Text processing and natural language understanding are important.",
    ];

    // 1. Text Normalization
    println!("1. Text Normalization");
    let normalizer = BasicNormalizer::new(true, true);
    for (i, doc) in documents.iter().enumerate() {
        let normalized = normalizer.normalize(doc)?;
        println!("Doc {}: {}", i + 1, normalized);
    }
    println!();

    // 2. Text Cleaning
    println!("2. Text Cleaning");
    let cleaner = BasicTextCleaner::new(true, true, true);
    for (i, doc) in documents.iter().enumerate() {
        let cleaned = cleaner.clean(doc)?;
        println!("Doc {}: {}", i + 1, cleaned);
    }
    println!();

    // 3. Tokenization Examples
    println!("3. Tokenization Examples");

    // Word tokenization
    let word_tokenizer = WordTokenizer::new(true);
    let tokens = word_tokenizer.tokenize(documents[0])?;
    println!("Word tokens: {tokens:?}");

    // N-gram tokenization
    let ngram_tokenizer = NgramTokenizer::new(2)?;
    let ngrams = ngram_tokenizer.tokenize(documents[0])?;
    println!("2-grams: {ngrams:?}");

    // Regex tokenization
    let regex_tokenizer = RegexTokenizer::new(r"\b\w+\b", false)?;
    let regex_tokens = regex_tokenizer.tokenize(documents[0])?;
    println!("Regex tokens: {regex_tokens:?}");
    println!();

    // 4. Stemming and Lemmatization
    println!("4. Stemming and Lemmatization");
    let porter_stemmer = PorterStemmer::new();
    let lemmatizer = SimpleLemmatizer::new();

    let test_words = vec!["running", "jumped", "better", "processing"];
    for word in test_words {
        let stemmed = porter_stemmer.stem(word)?;
        let lemmatized = lemmatizer.stem(word)?;
        println!("{word}: stemmed={stemmed}, lemmatized={lemmatized}");
    }
    println!();

    // 5. Count Vectorization
    println!("5. Count Vectorization");
    let mut count_vectorizer = CountVectorizer::new(false);

    let doc_refs = documents.to_vec();
    count_vectorizer.fit(&doc_refs)?;

    // Transform individual documents
    let count_matrix = count_vectorizer.transform_batch(&doc_refs)?;
    println!("Count vector shape: {:?}", count_matrix.shape());
    println!("Vocabulary size: {}", count_vectorizer.vocabulary().len());

    println!();

    // 6. TF-IDF Vectorization
    println!("6. TF-IDF Vectorization");
    let mut tfidf_vectorizer = TfidfVectorizer::new(false, true, Some("l2".to_string()));

    tfidf_vectorizer.fit(&doc_refs)?;
    let tfidf_matrix = tfidf_vectorizer.transform_batch(&doc_refs)?;

    println!("TF-IDF vector shape: {:?}", tfidf_matrix.shape());
    println!("Sample TF-IDF values:");
    for i in 0..3.min(tfidf_matrix.nrows()) {
        for j in 0..5.min(tfidf_matrix.ncols()) {
            print!("{:.3} ", tfidf_matrix[[i, j]]);
        }
        println!();
    }
    println!();

    // 7. Complete Pipeline Example
    println!("7. Complete Text Processing Pipeline");
    let testtext = "The cats were running quickly through the gardens.";

    // Normalize
    let normalized = normalizer.normalize(testtext)?;
    println!("Normalized: {normalized}");

    // Clean
    let cleaned = cleaner.clean(&normalized)?;
    println!("Cleaned: {cleaned}");

    // Tokenize
    let tokens = word_tokenizer.tokenize(&cleaned)?;
    println!("Tokens: {tokens:?}");

    // Stem
    let stemmed_tokens: Result<Vec<_>, _> = tokens
        .iter()
        .map(|token| porter_stemmer.stem(token))
        .collect();
    let stemmed_tokens = stemmed_tokens?;
    println!("Stemmed: {stemmed_tokens:?}");

    Ok(())
}

pub fn with_tokenizer( tokenizer: Box<dyn Tokenizer + Send + Sync>, binary: bool, ) -> Self

Create a count vectorizer with a custom tokenizer

pub fn vocabulary(&self) -> &Vocabulary

Get a reference to the vocabulary

Examples found in repository

examples/text_processing_demo.rs (line 84)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== SciRS2 Text Processing Demo ===\n");

    let documents = [
        "The quick brown fox jumps over the lazy dog.",
        "A fast red fox leaped over the sleeping canine.",
        "Machine learning algorithms process textual data efficiently.",
        "Text processing and natural language understanding are important.",
    ];

    // 1. Text Normalization
    println!("1. Text Normalization");
    let normalizer = BasicNormalizer::new(true, true);
    for (i, doc) in documents.iter().enumerate() {
        let normalized = normalizer.normalize(doc)?;
        println!("Doc {}: {}", i + 1, normalized);
    }
    println!();

    // 2. Text Cleaning
    println!("2. Text Cleaning");
    let cleaner = BasicTextCleaner::new(true, true, true);
    for (i, doc) in documents.iter().enumerate() {
        let cleaned = cleaner.clean(doc)?;
        println!("Doc {}: {}", i + 1, cleaned);
    }
    println!();

    // 3. Tokenization Examples
    println!("3. Tokenization Examples");

    // Word tokenization
    let word_tokenizer = WordTokenizer::new(true);
    let tokens = word_tokenizer.tokenize(documents[0])?;
    println!("Word tokens: {tokens:?}");

    // N-gram tokenization
    let ngram_tokenizer = NgramTokenizer::new(2)?;
    let ngrams = ngram_tokenizer.tokenize(documents[0])?;
    println!("2-grams: {ngrams:?}");

    // Regex tokenization
    let regex_tokenizer = RegexTokenizer::new(r"\b\w+\b", false)?;
    let regex_tokens = regex_tokenizer.tokenize(documents[0])?;
    println!("Regex tokens: {regex_tokens:?}");
    println!();

    // 4. Stemming and Lemmatization
    println!("4. Stemming and Lemmatization");
    let porter_stemmer = PorterStemmer::new();
    let lemmatizer = SimpleLemmatizer::new();

    let test_words = vec!["running", "jumped", "better", "processing"];
    for word in test_words {
        let stemmed = porter_stemmer.stem(word)?;
        let lemmatized = lemmatizer.stem(word)?;
        println!("{word}: stemmed={stemmed}, lemmatized={lemmatized}");
    }
    println!();

    // 5. Count Vectorization
    println!("5. Count Vectorization");
    let mut count_vectorizer = CountVectorizer::new(false);

    let doc_refs = documents.to_vec();
    count_vectorizer.fit(&doc_refs)?;

    // Transform individual documents
    let count_matrix = count_vectorizer.transform_batch(&doc_refs)?;
    println!("Count vector shape: {:?}", count_matrix.shape());
    println!("Vocabulary size: {}", count_vectorizer.vocabulary().len());

    println!();

    // 6. TF-IDF Vectorization
    println!("6. TF-IDF Vectorization");
    let mut tfidf_vectorizer = TfidfVectorizer::new(false, true, Some("l2".to_string()));

    tfidf_vectorizer.fit(&doc_refs)?;
    let tfidf_matrix = tfidf_vectorizer.transform_batch(&doc_refs)?;

    println!("TF-IDF vector shape: {:?}", tfidf_matrix.shape());
    println!("Sample TF-IDF values:");
    for i in 0..3.min(tfidf_matrix.nrows()) {
        for j in 0..5.min(tfidf_matrix.ncols()) {
            print!("{:.3} ", tfidf_matrix[[i, j]]);
        }
        println!();
    }
    println!();

    // 7. Complete Pipeline Example
    println!("7. Complete Text Processing Pipeline");
    let testtext = "The cats were running quickly through the gardens.";

    // Normalize
    let normalized = normalizer.normalize(testtext)?;
    println!("Normalized: {normalized}");

    // Clean
    let cleaned = cleaner.clean(&normalized)?;
    println!("Cleaned: {cleaned}");

    // Tokenize
    let tokens = word_tokenizer.tokenize(&cleaned)?;
    println!("Tokens: {tokens:?}");

    // Stem
    let stemmed_tokens: Result<Vec<_>, _> = tokens
        .iter()
        .map(|token| porter_stemmer.stem(token))
        .collect();
    let stemmed_tokens = stemmed_tokens?;
    println!("Stemmed: {stemmed_tokens:?}");

    Ok(())
}

examples/topic_modeling_demo.rs (line 45)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Topic Modeling with LDA Demo");
    println!("===========================\n");

    // Sample documents about different topics
    let documents = vec![
        // Technology documents
        "Artificial intelligence and machine learning are transforming the tech industry",
        "Deep learning neural networks require powerful GPUs for training",
        "Computer vision algorithms can now recognize objects in real time",
        "Natural language processing helps computers understand human language",
        // Sports documents
        "The basketball team won the championship after a thrilling final game",
        "Football players need excellent physical conditioning and teamwork",
        "Tennis requires both physical fitness and mental concentration",
        "Swimming is an excellent full-body workout and competitive sport",
        // Science documents
        "Climate change is affecting global weather patterns and ecosystems",
        "Quantum physics explores the behavior of matter at atomic scales",
        "Genetic research is unlocking the secrets of human DNA",
        "Space exploration continues to reveal mysteries of the universe",
    ];

    // Convert documents to document-term matrix
    let mut vectorizer = CountVectorizer::default();
    let doc_term_matrix = vectorizer.fit_transform(&documents)?;

    println!("Document-Term Matrix:");
    println!(
        "  Shape: ({}, {})",
        doc_term_matrix.nrows(),
        doc_term_matrix.ncols()
    );
    println!("  Vocabulary size: {}\n", vectorizer.vocabulary_size());

    // Create vocabulary mapping
    let vocabulary = vectorizer.vocabulary();
    let mut word_index_map = HashMap::new();
    for (word, &idx) in vocabulary.token_to_index().iter() {
        word_index_map.insert(idx, word.clone());
    }

    // Train LDA model
    let mut lda = LdaBuilder::new()
        .ntopics(3)
        .maxiter(100)
        .random_seed(42)
        .doc_topic_prior(0.1)
        .topic_word_prior(0.01)
        .learning_method(LdaLearningMethod::Batch)
        .build();

    println!("Training LDA model with 3 topics...");
    let doc_topics = lda.fit_transform(&doc_term_matrix)?;
    println!("Training completed!\n");

    // Display document-topic assignments
    println!("Document-Topic Assignments:");
    for (doc_idx, topic_dist) in doc_topics.outer_iter().enumerate() {
        let max_topic = topic_dist
            .iter()
            .enumerate()
            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap())
            .map(|(idx_, _)| idx_)
            .unwrap();

        println!(
            "Document {}: Topic {} (probabilities: {:.3}, {:.3}, {:.3})",
            doc_idx + 1,
            max_topic,
            topic_dist[0],
            topic_dist[1],
            topic_dist[2]
        );
    }
    println!();

    // Get topics with top words
    let topics = lda.get_topics(10, &word_index_map)?;

    println!("Discovered Topics:");
    for topic in &topics {
        println!("\nTopic {}:", topic.id);
        println!("Top words:");
        for (word, weight) in &topic.top_words {
            println!("  {word} ({weight:.4})");
        }
    }

    // Analyze a new document
    println!("\n\nAnalyzing a new document:");
    let new_doc = "Machine learning algorithms are revolutionizing artificial intelligence";
    let new_doc_vec = vectorizer.transform(new_doc)?;
    let new_doc_topics = lda.transform(&new_doc_vec.insert_axis(scirs2_core::ndarray::Axis(0)))?;

    println!("Document: \"{new_doc}\"");
    println!("Topic distribution:");
    for (topic_idx, &prob) in new_doc_topics.row(0).iter().enumerate() {
        println!("  Topic {topic_idx}: {prob:.3}");
    }

    // Create another LDA model with different configuration
    println!("\n\nTrying different LDA configuration:");
    let mut lda2 = LatentDirichletAllocation::with_ntopics(4);
    lda2.fit(&doc_term_matrix)?;

    let topics2 = lda2.get_topics(5, &word_index_map)?;
    println!("Discovered {} topics with top 5 words each:", topics2.len());
    for topic in &topics2 {
        let words: Vec<String> = topic
            .top_words
            .iter()
            .map(|(word_, _)| word_.clone())
            .collect();
        println!("Topic {}: {}", topic.id, words.join(", "));
    }

    Ok(())
}

pub fn vocabulary_size(&self) -> usize

Get the vocabulary size

Examples found in repository

examples/topic_modeling_demo.rs (line 42)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Topic Modeling with LDA Demo");
    println!("===========================\n");

    // Sample documents about different topics
    let documents = vec![
        // Technology documents
        "Artificial intelligence and machine learning are transforming the tech industry",
        "Deep learning neural networks require powerful GPUs for training",
        "Computer vision algorithms can now recognize objects in real time",
        "Natural language processing helps computers understand human language",
        // Sports documents
        "The basketball team won the championship after a thrilling final game",
        "Football players need excellent physical conditioning and teamwork",
        "Tennis requires both physical fitness and mental concentration",
        "Swimming is an excellent full-body workout and competitive sport",
        // Science documents
        "Climate change is affecting global weather patterns and ecosystems",
        "Quantum physics explores the behavior of matter at atomic scales",
        "Genetic research is unlocking the secrets of human DNA",
        "Space exploration continues to reveal mysteries of the universe",
    ];

    // Convert documents to document-term matrix
    let mut vectorizer = CountVectorizer::default();
    let doc_term_matrix = vectorizer.fit_transform(&documents)?;

    println!("Document-Term Matrix:");
    println!(
        "  Shape: ({}, {})",
        doc_term_matrix.nrows(),
        doc_term_matrix.ncols()
    );
    println!("  Vocabulary size: {}\n", vectorizer.vocabulary_size());

    // Create vocabulary mapping
    let vocabulary = vectorizer.vocabulary();
    let mut word_index_map = HashMap::new();
    for (word, &idx) in vocabulary.token_to_index().iter() {
        word_index_map.insert(idx, word.clone());
    }

    // Train LDA model
    let mut lda = LdaBuilder::new()
        .ntopics(3)
        .maxiter(100)
        .random_seed(42)
        .doc_topic_prior(0.1)
        .topic_word_prior(0.01)
        .learning_method(LdaLearningMethod::Batch)
        .build();

    println!("Training LDA model with 3 topics...");
    let doc_topics = lda.fit_transform(&doc_term_matrix)?;
    println!("Training completed!\n");

    // Display document-topic assignments
    println!("Document-Topic Assignments:");
    for (doc_idx, topic_dist) in doc_topics.outer_iter().enumerate() {
        let max_topic = topic_dist
            .iter()
            .enumerate()
            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap())
            .map(|(idx_, _)| idx_)
            .unwrap();

        println!(
            "Document {}: Topic {} (probabilities: {:.3}, {:.3}, {:.3})",
            doc_idx + 1,
            max_topic,
            topic_dist[0],
            topic_dist[1],
            topic_dist[2]
        );
    }
    println!();

    // Get topics with top words
    let topics = lda.get_topics(10, &word_index_map)?;

    println!("Discovered Topics:");
    for topic in &topics {
        println!("\nTopic {}:", topic.id);
        println!("Top words:");
        for (word, weight) in &topic.top_words {
            println!("  {word} ({weight:.4})");
        }
    }

    // Analyze a new document
    println!("\n\nAnalyzing a new document:");
    let new_doc = "Machine learning algorithms are revolutionizing artificial intelligence";
    let new_doc_vec = vectorizer.transform(new_doc)?;
    let new_doc_topics = lda.transform(&new_doc_vec.insert_axis(scirs2_core::ndarray::Axis(0)))?;

    println!("Document: \"{new_doc}\"");
    println!("Topic distribution:");
    for (topic_idx, &prob) in new_doc_topics.row(0).iter().enumerate() {
        println!("  Topic {topic_idx}: {prob:.3}");
    }

    // Create another LDA model with different configuration
    println!("\n\nTrying different LDA configuration:");
    let mut lda2 = LatentDirichletAllocation::with_ntopics(4);
    lda2.fit(&doc_term_matrix)?;

    let topics2 = lda2.get_topics(5, &word_index_map)?;
    println!("Discovered {} topics with top 5 words each:", topics2.len());
    for topic in &topics2 {
        let words: Vec<String> = topic
            .top_words
            .iter()
            .map(|(word_, _)| word_.clone())
            .collect();
        println!("Topic {}: {}", topic.id, words.join(", "));
    }

    Ok(())
}

pub fn get_feature_count( &self, matrix: &Array2<f64>, document_index: usize, feature_index: usize, ) -> Option<f64>

Get feature count for a specific document and feature index from a matrix

pub fn vocabulary_map(&self) -> HashMap<String, usize>

Get vocabulary as HashMap for compatibility with visualization

Trait Implementations

impl Clone for CountVectorizer

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for CountVectorizer

fn default() -> Self

Returns the “default value” for a type.

impl Vectorizer for CountVectorizer

fn fit(&mut self, texts: &[&str]) -> Result<()>

Fit the vectorizer on a corpus of texts

fn transform(&self, text: &str) -> Result<Array1<f64>>

Transform a text into a vector

fn transform_batch(&self, texts: &[&str]) -> Result<Array2<f64>>

Transform a batch of texts into a matrix where each row is a document vector

fn fit_transform(&mut self, texts: &[&str]) -> Result<Array2<f64>>

Fit on a corpus and then transform a batch of texts