Struct LengthFilter 

pub struct LengthFilter {
    pub min_length: usize,
    pub max_length: usize,
}

Filter tokens by length

Fields

min_length: usize

Minimum token length

max_length: usize

Maximum token length

Implementations

impl LengthFilter

pub fn new(_min_length: usize, maxlength: usize) -> Self

Create a new length filter

Examples found in repository

examples/token_filtering_demo.rs (line 24)

fn main() -> Result<()> {
    println!("Token Filtering Demo");
    println!("===================\n");

    // Create a sample text
    let text = "The quick brown fox jumps over the lazy dog. The fox is quick and brown.";
    println!("Original text: {text}\n");

    // Create a tokenizer
    let tokenizer = WordTokenizer::default();
    let tokens = tokenizer.tokenize(text)?;
    println!("Tokenized: {tokens:?}\n");

    // 1. Filter by length
    println!("1. Length Filtering");
    println!("------------------");
    let length_filter = LengthFilter::new(4, 6);
    let filtered = length_filter.apply(&tokens);
    println!("Tokens with length 4-6: {filtered:?}");

    let filteredtext = length_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 2. Filter by frequency
    println!("2. Frequency Filtering");
    println!("---------------------");

    // Create token counts
    let mut counts = HashMap::new();
    for token in &tokens {
        *counts.entry(token.clone()).or_insert(0) += 1;
    }

    // Print counts
    println!("Token counts:");
    for (token, count) in &counts {
        println!("  {token} : {count}");
    }

    // Filter tokens that appear more than once
    let freq_filter = FrequencyFilter::from_counts(counts.clone(), 2);
    let filtered = freq_filter.apply(&tokens);
    println!("\nTokens that appear 2+ times: {filtered:?}");

    let filteredtext = freq_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 3. Filter by regex pattern
    println!("3. Regex Filtering");
    println!("-----------------");

    // Keep only tokens that contain a vowel followed by 'w' or 'r'
    let regex_filter = RegexFilter::new("[aeiou][wr]", true)?;
    let filtered = regex_filter.apply(&tokens);
    println!("Tokens containing a vowel followed by 'w' or 'r': {filtered:?}");

    let filteredtext = regex_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 4. Stopwords filtering
    println!("4. Stopwords Filtering");
    println!("---------------------");

    // Define common stopwords
    let stopwords = vec![
        "the".to_string(),
        "is".to_string(),
        "and".to_string(),
        "over".to_string(),
        "a".to_string(),
        "an".to_string(),
    ];

    let stopwords_filter = StopwordsFilter::new(stopwords, true);
    let filtered = stopwords_filter.apply(&tokens);
    println!("Tokens with stopwords removed: {filtered:?}");

    let filteredtext = stopwords_filter.filtertext(text, &tokenizer)?;
    println!("Filtered text: {filteredtext}\n");

    // 5. Composite filtering
    println!("5. Composite Filtering");
    println!("---------------------");

    // Create separate filters
    let length_filter = LengthFilter::new(3, 5);
    let regex_filter = RegexFilter::new("^[a-z]", true)?;

    // Apply filters sequentially
    let filtered_by_length = length_filter.apply(&tokens);
    let filtered = regex_filter.apply(&filtered_by_length);
    println!("Tokens with length 3-5 AND starting with lowercase letter: {filtered:?}");

    // First filter by length
    let text_with_length = length_filter.filtertext(text, &tokenizer)?;

    // Then apply regex filter to the already filtered text
    let filteredtext = regex_filter.filtertext(&text_with_length, &tokenizer)?;

    // We should see only words that are 3-5 chars AND start with lowercase
    println!("Filtered text: {filteredtext}\n");

    Ok(())
}

pub fn with_min_length(self, minlength: usize) -> Self

Set minimum token length

pub fn with_max_length(self, maxlength: usize) -> Self

Set maximum token length

Trait Implementations

impl Clone for LengthFilter

fn clone(&self) -> LengthFilter

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LengthFilter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LengthFilter

fn default() -> Self

Returns the “default value” for a type.

impl TokenFilter for LengthFilter

fn apply(&self, tokens: &[String]) -> Vec<String>

Filter tokens based on the strategy

fn filtertext(&self, text: &str, tokenizer: &dyn Tokenizer) -> Result<String>

Apply the filter directly to text