Struct SimpleLemmatizer 

pub struct SimpleLemmatizer { /* private fields */ }

Simple lemmatizer using a dictionary-based approach

Implementations

impl SimpleLemmatizer

pub fn new() -> Self

Create a new lemmatizer

Examples found in repository

examples/stemming_comparison_demo.rs (line 13)

fn main() -> Result<(), Box<dyn Error>> {
    println!("Stemming Algorithms Comparison Demo");
    println!("-----------------------------------");

    // Create instances of different stemmers
    let porter_stemmer = PorterStemmer::new();
    let snowball_stemmer = SnowballStemmer::new("english")?;
    let lancaster_stemmer = LancasterStemmer::new();
    let lemmatizer = SimpleLemmatizer::new();

    // Test words to compare stemming results
    let test_words = vec![
        "running",
        "ran",
        "runs",
        "easily",
        "fishing",
        "fished",
        "troubled",
        "troubling",
        "troubles",
        "production",
        "productive",
        "argument",
        "arguing",
        "university",
        "universities",
        "maximizing",
        "maximum",
        "presumably",
        "multiply",
        "opposition",
        "computational",
    ];

    // Print results in a table format
    println!(
        "{:<15} {:<15} {:<15} {:<15} {:<15}",
        "Original", "Porter", "Snowball", "Lancaster", "Lemmatizer"
    );
    println!("{}", "-".repeat(75));

    for word in test_words {
        let porter_result = porter_stemmer.stem(word)?;
        let snowball_result = snowball_stemmer.stem(word)?;
        let lancaster_result = lancaster_stemmer.stem(word)?;
        let lemma_result = lemmatizer.stem(word)?;

        println!(
            "{word:<15} {porter_result:<15} {snowball_result:<15} {lancaster_result:<15} {lemma_result:<15}"
        );
    }

    // Demonstrate configurability of the Lancaster stemmer
    println!("\nLancaster Stemmer Configuration Options");
    println!("------------------------------------");

    let default_lancaster = LancasterStemmer::new();
    let custom_lancaster = LancasterStemmer::new()
        .with_min_stemmed_length(3)
        .with_acceptable_check(false);

    println!(
        "{:<15} {:<20} {:<20}",
        "Original", "Default Lancaster", "Custom Lancaster"
    );
    println!("{}", "-".repeat(55));

    let custom_test_words = vec!["provision", "ear", "me", "fishing", "multiply"];

    for word in custom_test_words {
        let default_result = default_lancaster.stem(word)?;
        let custom_result = custom_lancaster.stem(word)?;

        println!("{word:<15} {default_result:<20} {custom_result:<20}");
    }

    println!("\nNotes:");
    println!("- Porter stemmer: Established algorithm, medium aggressiveness");
    println!("- Snowball stemmer: Improved Porter algorithm with language-specific rules");
    println!("- Lancaster stemmer: Most aggressive stemming, can be configured");
    println!("- Lemmatizer: Dictionary-based approach, produces actual words");

    Ok(())
}

examples/text_processing_demo.rs (line 64)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== SciRS2 Text Processing Demo ===\n");

    let documents = [
        "The quick brown fox jumps over the lazy dog.",
        "A fast red fox leaped over the sleeping canine.",
        "Machine learning algorithms process textual data efficiently.",
        "Text processing and natural language understanding are important.",
    ];

    // 1. Text Normalization
    println!("1. Text Normalization");
    let normalizer = BasicNormalizer::new(true, true);
    for (i, doc) in documents.iter().enumerate() {
        let normalized = normalizer.normalize(doc)?;
        println!("Doc {}: {}", i + 1, normalized);
    }
    println!();

    // 2. Text Cleaning
    println!("2. Text Cleaning");
    let cleaner = BasicTextCleaner::new(true, true, true);
    for (i, doc) in documents.iter().enumerate() {
        let cleaned = cleaner.clean(doc)?;
        println!("Doc {}: {}", i + 1, cleaned);
    }
    println!();

    // 3. Tokenization Examples
    println!("3. Tokenization Examples");

    // Word tokenization
    let word_tokenizer = WordTokenizer::new(true);
    let tokens = word_tokenizer.tokenize(documents[0])?;
    println!("Word tokens: {tokens:?}");

    // N-gram tokenization
    let ngram_tokenizer = NgramTokenizer::new(2)?;
    let ngrams = ngram_tokenizer.tokenize(documents[0])?;
    println!("2-grams: {ngrams:?}");

    // Regex tokenization
    let regex_tokenizer = RegexTokenizer::new(r"\b\w+\b", false)?;
    let regex_tokens = regex_tokenizer.tokenize(documents[0])?;
    println!("Regex tokens: {regex_tokens:?}");
    println!();

    // 4. Stemming and Lemmatization
    println!("4. Stemming and Lemmatization");
    let porter_stemmer = PorterStemmer::new();
    let lemmatizer = SimpleLemmatizer::new();

    let test_words = vec!["running", "jumped", "better", "processing"];
    for word in test_words {
        let stemmed = porter_stemmer.stem(word)?;
        let lemmatized = lemmatizer.stem(word)?;
        println!("{word}: stemmed={stemmed}, lemmatized={lemmatized}");
    }
    println!();

    // 5. Count Vectorization
    println!("5. Count Vectorization");
    let mut count_vectorizer = CountVectorizer::new(false);

    let doc_refs = documents.to_vec();
    count_vectorizer.fit(&doc_refs)?;

    // Transform individual documents
    let count_matrix = count_vectorizer.transform_batch(&doc_refs)?;
    println!("Count vector shape: {:?}", count_matrix.shape());
    println!("Vocabulary size: {}", count_vectorizer.vocabulary().len());

    println!();

    // 6. TF-IDF Vectorization
    println!("6. TF-IDF Vectorization");
    let mut tfidf_vectorizer = TfidfVectorizer::new(false, true, Some("l2".to_string()));

    tfidf_vectorizer.fit(&doc_refs)?;
    let tfidf_matrix = tfidf_vectorizer.transform_batch(&doc_refs)?;

    println!("TF-IDF vector shape: {:?}", tfidf_matrix.shape());
    println!("Sample TF-IDF values:");
    for i in 0..3.min(tfidf_matrix.nrows()) {
        for j in 0..5.min(tfidf_matrix.ncols()) {
            print!("{:.3} ", tfidf_matrix[[i, j]]);
        }
        println!();
    }
    println!();

    // 7. Complete Pipeline Example
    println!("7. Complete Text Processing Pipeline");
    let testtext = "The cats were running quickly through the gardens.";

    // Normalize
    let normalized = normalizer.normalize(testtext)?;
    println!("Normalized: {normalized}");

    // Clean
    let cleaned = cleaner.clean(&normalized)?;
    println!("Cleaned: {cleaned}");

    // Tokenize
    let tokens = word_tokenizer.tokenize(&cleaned)?;
    println!("Tokens: {tokens:?}");

    // Stem
    let stemmed_tokens: Result<Vec<_>, _> = tokens
        .iter()
        .map(|token| porter_stemmer.stem(token))
        .collect();
    let stemmed_tokens = stemmed_tokens?;
    println!("Stemmed: {stemmed_tokens:?}");

    Ok(())
}

examples/rule_lemmatizer_demo.rs (line 30)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Rule-based Lemmatization Demo\n");

    // Create lemmatizers and stemmers
    let simple_lemmatizer = SimpleLemmatizer::new();
    let rule_lemmatizer = RuleLemmatizer::new();
    let porter_stemmer = PorterStemmer::new();

    // Create a POS-aware lemmatizer using the builder pattern
    let pos_aware_lemmatizer = RuleLemmatizerBuilder::new()
        .use_pos_tagging(true)
        .apply_case_restoration(true)
        .check_vowels(true)
        .build();

    // Simple demo comparing lemmatization results
    println!("\n=== Lemmatization Comparison ===\n");
    let test_words = vec![
        ("running", Some(PosTag::Verb)),
        ("ran", Some(PosTag::Verb)),
        ("better", Some(PosTag::Adjective)),
        ("best", Some(PosTag::Adjective)),
        ("feet", Some(PosTag::Noun)),
        ("children", Some(PosTag::Noun)),
        ("went", Some(PosTag::Verb)),
        ("mice", Some(PosTag::Noun)),
        ("quickly", Some(PosTag::Adverb)),
        ("universities", Some(PosTag::Noun)),
        ("studying", Some(PosTag::Verb)),
        ("studied", Some(PosTag::Verb)),
        ("studies", Some(PosTag::Verb)),
    ];

    println!(
        "{:<15} {:<15} {:<15} {:<15}",
        "Word", "Simple", "Rule-based", "Porter"
    );
    println!("{:-<60}", "");

    for (word, pos) in &test_words {
        let simple_result = simple_lemmatizer.stem(word)?;
        let rule_result = if let Some(pos_tag) = pos {
            rule_lemmatizer.lemmatize(word, Some(pos_tag.clone()))
        } else {
            rule_lemmatizer.stem(word)?
        };
        let porter_result = porter_stemmer.stem(word)?;

        println!("{word:<15} {simple_result:<15} {rule_result:<15} {porter_result:<15}");
    }

    // POS tagging demonstration
    println!("\n=== Part-of-Speech Aware Lemmatization ===\n");
    println!("Demonstrating how the same word can lemmatize differently based on POS tag:\n");

    let ambiguous_words = vec![
        ("left", vec![PosTag::Verb, PosTag::Adjective, PosTag::Noun]),
        ("close", vec![PosTag::Verb, PosTag::Adjective, PosTag::Noun]),
        ("flies", vec![PosTag::Verb, PosTag::Noun]),
        ("saw", vec![PosTag::Verb, PosTag::Noun]),
        ("light", vec![PosTag::Noun, PosTag::Verb, PosTag::Adjective]),
    ];

    for (word, pos_tags) in &ambiguous_words {
        println!("Word: \"{word}\"");

        for pos in pos_tags {
            let pos_name = match pos {
                PosTag::Verb => "Verb",
                PosTag::Noun => "Noun",
                PosTag::Adjective => "Adjective",
                PosTag::Adverb => "Adverb",
                PosTag::Other => "Other",
            };
            println!(
                "  as {:<10}: {}",
                pos_name,
                pos_aware_lemmatizer.lemmatize(word, Some(pos.clone()))
            );
        }
        println!();
    }

    // Performance comparison
    println!("\n=== Performance Comparison ===\n");

    // Preprocess text into tokens for benchmarking
    let mut all_tokens = Vec::new();
    for text in TEXTS {
        all_tokens.extend(
            text.split_whitespace()
                .map(|s| s.trim_matches(|c: char| !c.is_alphabetic()))
                .filter(|s| !s.is_empty())
                .collect::<Vec<_>>(),
        );
    }

    // Assign random POS tags for the benchmark
    let mut tokens_with_pos = Vec::new();
    let pos_tags = [
        PosTag::Verb,
        PosTag::Noun,
        PosTag::Adjective,
        PosTag::Adverb,
        PosTag::Other,
    ];
    for (i, token) in all_tokens.iter().enumerate() {
        tokens_with_pos.push((token.to_string(), pos_tags[i % pos_tags.len()].clone()));
    }

    // Benchmark simple lemmatizer
    let start = Instant::now();
    for token in &all_tokens {
        let _ = simple_lemmatizer.stem(token)?;
    }
    let simple_time = start.elapsed();

    // Benchmark rule-based lemmatizer without POS
    let start = Instant::now();
    for token in &all_tokens {
        let _ = rule_lemmatizer.stem(token)?;
    }
    let rule_time = start.elapsed();

    // Benchmark rule-based lemmatizer with POS
    let start = Instant::now();
    for (token, pos) in &tokens_with_pos {
        let _ = pos_aware_lemmatizer.lemmatize(token, Some(pos.clone()));
    }
    let pos_rule_time = start.elapsed();

    // Benchmark Porter stemmer
    let start = Instant::now();
    for token in &all_tokens {
        let _ = porter_stemmer.stem(token)?;
    }
    let porter_time = start.elapsed();

    println!("Processing {} tokens:\n", all_tokens.len());
    println!("- SimpleLemmatizer: {simple_time:.2?}");
    println!("- RuleLemmatizer (without POS): {rule_time:.2?}");
    println!("- RuleLemmatizer (with POS): {pos_rule_time:.2?}");
    println!("- PorterStemmer: {porter_time:.2?}");

    // Example using RuleLemmatizer on real text
    println!("\n=== Text Processing Example ===\n");
    let text = "The scientists were running experiments to test their hypotheses. \
                The children went to the museum, where they saw the fossils of prehistoric animals. \
                Universities are studying better methods to address these issues quickly.";

    println!("Original text:\n{text}\n");

    // Simple tokenization and lemmatization with POS tags
    let tokens: Vec<&str> = text
        .split_whitespace()
        .map(|s| s.trim_matches(|c: char| !c.is_alphabetic()))
        .filter(|s| !s.is_empty())
        .collect();

    // Simulate a very basic POS tagger (in a real application, you would use a proper tagger)
    let mut pos_map: HashMap<&str, PosTag> = HashMap::new();
    pos_map.insert("scientists", PosTag::Noun);
    pos_map.insert("were", PosTag::Verb);
    pos_map.insert("running", PosTag::Verb);
    pos_map.insert("experiments", PosTag::Noun);
    pos_map.insert("test", PosTag::Verb);
    pos_map.insert("their", PosTag::Other);
    pos_map.insert("hypotheses", PosTag::Noun);
    pos_map.insert("children", PosTag::Noun);
    pos_map.insert("went", PosTag::Verb);
    pos_map.insert("museum", PosTag::Noun);
    pos_map.insert("saw", PosTag::Verb);
    pos_map.insert("fossils", PosTag::Noun);
    pos_map.insert("prehistoric", PosTag::Adjective);
    pos_map.insert("animals", PosTag::Noun);
    pos_map.insert("universities", PosTag::Noun);
    pos_map.insert("studying", PosTag::Verb);
    pos_map.insert("better", PosTag::Adjective);
    pos_map.insert("methods", PosTag::Noun);
    pos_map.insert("address", PosTag::Verb);
    pos_map.insert("issues", PosTag::Noun);
    pos_map.insert("quickly", PosTag::Adverb);

    // Process text
    println!("Word-by-word lemmatization results:");
    println!(
        "{:<15} {:<15} {:<15} {:<15}",
        "Word", "RuleLemmatizer", "With POS", "Porter"
    );
    println!("{:-<60}", "");

    for token in tokens {
        let pos = pos_map.get(token.to_lowercase().as_str()).cloned();

        let rule_result = rule_lemmatizer.stem(token)?;
        let pos_result = if let Some(pos_tag) = &pos {
            pos_aware_lemmatizer.lemmatize(token, Some(pos_tag.clone()))
        } else {
            pos_aware_lemmatizer.stem(token)?
        };
        let porter_result = porter_stemmer.stem(token)?;

        println!("{token:<15} {rule_result:<15} {pos_result:<15} {porter_result:<15}");
    }

    // Custom rules example
    println!("\n=== Custom Rules and Exceptions ===\n");

    // Create a custom lemmatizer with additional rules and exceptions
    let custom_lemmatizer = RuleLemmatizerBuilder::new()
        .add_exception("dataset", "data")
        .add_exception("corpora", "corpus")
        .add_dict_entry("nlp", "natural language processing")
        .build();

    let custom_words = vec!["dataset", "corpora", "nlp", "datasets"];

    println!("Custom lemmatizer results:");
    for word in custom_words {
        println!(
            "{:<15} -> {}",
            word,
            custom_lemmatizer.lemmatize(word, None)
        );
    }

    Ok(())
}

pub fn from_dict_file(path: &str) -> Result<Self>

Load lemmatization dictionary from a file

pub fn add_lemma(&mut self, word: &str, lemma: &str)

Add a lemma mapping

Trait Implementations

impl Clone for SimpleLemmatizer

fn clone(&self) -> SimpleLemmatizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for SimpleLemmatizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for SimpleLemmatizer

fn default() -> Self

Returns the “default value” for a type.

impl Stemmer for SimpleLemmatizer

fn stem(&self, word: &str) -> Result<String>

Stem a single word

fn stem_batch(&self, words: &[&str]) -> Result<Vec<String>>

Stem multiple words