Struct StopwordsFilter 

pub struct StopwordsFilter { /* private fields */ }

Filter tokens using a predefined stopword list

Implementations

impl StopwordsFilter

pub fn new(_stopwords: Vec<String>, removestopwords: bool) -> Self

Create a new stopwords filter

Examples found in repository

examples/token_filtering_demo.rs (line 81)

fn main() -> Result<()> {
    println!("Token Filtering Demo");
    println!("===================\n");

    // Create a sample text
    let text = "The quick brown fox jumps over the lazy dog. The fox is quick and brown.";
    println!("Original text: {text}\n");

    // Create a tokenizer
    let tokenizer = WordTokenizer::default();
    let tokens = tokenizer.tokenize(text)?;
    println!("Tokenized: {tokens:?}\n");

    // 1. Filter by length
    println!("1. Length Filtering");
    println!("------------------");
    let length_filter = LengthFilter::new(4, 6);
    let filtered = length_filter.apply(&tokens);
    println!("Tokens with length 4-6: {filtered:?}");

    let filteredtext = length_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 2. Filter by frequency
    println!("2. Frequency Filtering");
    println!("---------------------");

    // Create token counts
    let mut counts = HashMap::new();
    for token in &tokens {
        *counts.entry(token.clone()).or_insert(0) += 1;
    }

    // Print counts
    println!("Token counts:");
    for (token, count) in &counts {
        println!("  {token} : {count}");
    }

    // Filter tokens that appear more than once
    let freq_filter = FrequencyFilter::from_counts(counts.clone(), 2);
    let filtered = freq_filter.apply(&tokens);
    println!("\nTokens that appear 2+ times: {filtered:?}");

    let filteredtext = freq_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 3. Filter by regex pattern
    println!("3. Regex Filtering");
    println!("-----------------");

    // Keep only tokens that contain a vowel followed by 'w' or 'r'
    let regex_filter = RegexFilter::new("[aeiou][wr]", true)?;
    let filtered = regex_filter.apply(&tokens);
    println!("Tokens containing a vowel followed by 'w' or 'r': {filtered:?}");

    let filteredtext = regex_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 4. Stopwords filtering
    println!("4. Stopwords Filtering");
    println!("---------------------");

    // Define common stopwords
    let stopwords = vec![
        "the".to_string(),
        "is".to_string(),
        "and".to_string(),
        "over".to_string(),
        "a".to_string(),
        "an".to_string(),
    ];

    let stopwords_filter = StopwordsFilter::new(stopwords, true);
    let filtered = stopwords_filter.apply(&tokens);
    println!("Tokens with stopwords removed: {filtered:?}");

    let filteredtext = stopwords_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 5. Composite filtering
    println!("5. Composite Filtering");
    println!("---------------------");

    // Create separate filters
    let length_filter = LengthFilter::new(3, 5);
    let regex_filter = RegexFilter::new("^[a-z]", true)?;

    // Apply filters sequentially
    let filtered_by_length = length_filter.apply(&tokens);
    let filtered = regex_filter.apply(&filtered_by_length);
    println!("Tokens with length 3-5 AND starting with lowercase letter: {filtered:?}");

    // First filter by length
    let text_with_length = length_filter.filtertext(text, &tokenizer)?;

    // Then apply regex filter to the already filtered text
    let filteredtext = regex_filter.filtertext(&text_with_length, &tokenizer)?;

    // We should see only words that are 3-5 chars AND start with lowercase
    println!("Filtered text: {filteredtext}\n");

    Ok(())
}

pub fn from_file(path: &str) -> Result<Self>

Create a stopwords filter from a file

pub fn remove_stopwords(self, remove: bool) -> Self

Set whether to remove stopwords

pub fn add_stopwords(&mut self, words: &[String])

Add stopwords to the filter

pub fn get_stopwords(&self) -> Vec<String>

Get the current stopwords

Trait Implementations

impl Clone for StopwordsFilter

fn clone(&self) -> StopwordsFilter

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for StopwordsFilter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl TokenFilter for StopwordsFilter

fn apply(&self, tokens: &[String]) -> Vec<String>

Filter tokens based on the strategy

fn filtertext(&self, text: &str, tokenizer: &dyn Tokenizer) -> Result<String>

Apply the filter directly to text