Struct FrequencyFilter 

pub struct FrequencyFilter {
    pub min_count: usize,
    pub max_count: Option<usize>,
    pub max_freq: Option<f64>,
    /* private fields */
}

Filter tokens by frequency in a corpus

Fields

min_count: usize

Minimum token frequency

max_count: Option<usize>

Maximum token frequency (absolute count)

max_freq: Option<f64>

Maximum token frequency (as a fraction of total)

Implementations

impl FrequencyFilter

pub fn from_tokens_with_vocabulary( tokens: &[String], vocabulary: &Vocabulary, min_count: usize, ) -> Self

Create a new frequency filter from tokens with a vocabulary for reference

pub fn from_counts( _token_counts: HashMap<String, usize>, mincount: usize, ) -> Self

Create a new frequency filter from token counts

Examples found in repository

examples/token_filtering_demo.rs (line 48)

fn main() -> Result<()> {
    println!("Token Filtering Demo");
    println!("===================\n");

    // Create a sample text
    let text = "The quick brown fox jumps over the lazy dog. The fox is quick and brown.";
    println!("Original text: {text}\n");

    // Create a tokenizer
    let tokenizer = WordTokenizer::default();
    let tokens = tokenizer.tokenize(text)?;
    println!("Tokenized: {tokens:?}\n");

    // 1. Filter by length
    println!("1. Length Filtering");
    println!("------------------");
    let length_filter = LengthFilter::new(4, 6);
    let filtered = length_filter.apply(&tokens);
    println!("Tokens with length 4-6: {filtered:?}");

    let filteredtext = length_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 2. Filter by frequency
    println!("2. Frequency Filtering");
    println!("---------------------");

    // Create token counts
    let mut counts = HashMap::new();
    for token in &tokens {
        *counts.entry(token.clone()).or_insert(0) += 1;
    }

    // Print counts
    println!("Token counts:");
    for (token, count) in &counts {
        println!("  {token} : {count}");
    }

    // Filter tokens that appear more than once
    let freq_filter = FrequencyFilter::from_counts(counts.clone(), 2);
    let filtered = freq_filter.apply(&tokens);
    println!("\nTokens that appear 2+ times: {filtered:?}");

    let filteredtext = freq_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 3. Filter by regex pattern
    println!("3. Regex Filtering");
    println!("-----------------");

    // Keep only tokens that contain a vowel followed by 'w' or 'r'
    let regex_filter = RegexFilter::new("[aeiou][wr]", true)?;
    let filtered = regex_filter.apply(&tokens);
    println!("Tokens containing a vowel followed by 'w' or 'r': {filtered:?}");

    let filteredtext = regex_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 4. Stopwords filtering
    println!("4. Stopwords Filtering");
    println!("---------------------");

    // Define common stopwords
    let stopwords = vec![
        "the".to_string(),
        "is".to_string(),
        "and".to_string(),
        "over".to_string(),
        "a".to_string(),
        "an".to_string(),
    ];

    let stopwords_filter = StopwordsFilter::new(stopwords, true);
    let filtered = stopwords_filter.apply(&tokens);
    println!("Tokens with stopwords removed: {filtered:?}");

    let filteredtext = stopwords_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 5. Composite filtering
    println!("5. Composite Filtering");
    println!("---------------------");

    // Create separate filters
    let length_filter = LengthFilter::new(3, 5);
    let regex_filter = RegexFilter::new("^[a-z]", true)?;

    // Apply filters sequentially
    let filtered_by_length = length_filter.apply(&tokens);
    let filtered = regex_filter.apply(&filtered_by_length);
    println!("Tokens with length 3-5 AND starting with lowercase letter: {filtered:?}");

    // First filter by length
    let text_with_length = length_filter.filtertext(text, &tokenizer)?;

    // Then apply regex filter to the already filtered text
    let filteredtext = regex_filter.filtertext(&text_with_length, &tokenizer)?;

    // We should see only words that are 3-5 chars AND start with lowercase
    println!("Filtered text: {filteredtext}\n");

    Ok(())
}

pub fn learn_from_corpus( texts: &[&str], tokenizer: &dyn Tokenizer, min_count: usize, ) -> Result<Self>

Learn token frequencies from a corpus

pub fn with_max_count(self, maxcount: usize) -> Self

Set the maximum count threshold

pub fn with_max_freq(self, maxfreq: f64) -> Result<Self>

Set the maximum frequency threshold (0.0 to 1.0)

Trait Implementations

impl Clone for FrequencyFilter

fn clone(&self) -> FrequencyFilter

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for FrequencyFilter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl TokenFilter for FrequencyFilter

fn apply(&self, tokens: &[String]) -> Vec<String>

Filter tokens based on the strategy

fn filtertext(&self, text: &str, tokenizer: &dyn Tokenizer) -> Result<String>

Apply the filter directly to text