Struct FrequencyFilter 

pub struct FrequencyFilter {
    pub min_count: usize,
    pub max_count: Option<usize>,
    pub max_freq: Option<f64>,
    /* private fields */
}

Filter tokens by frequency in a corpus

Fields

min_count: usize

Minimum token frequency

max_count: Option<usize>

Maximum token frequency (absolute count)

max_freq: Option<f64>

Maximum token frequency (as a fraction of total)

Implementations

impl FrequencyFilter

pub fn from_tokens_with_vocabulary( tokens: &[String], vocabulary: &Vocabulary, min_count: usize, ) -> Self

Create a new frequency filter from tokens with a vocabulary for reference

pub fn from_counts( _token_counts: HashMap<String, usize>, mincount: usize, ) -> Self

Create a new frequency filter from token counts

pub fn learn_from_corpus( texts: &[&str], tokenizer: &dyn Tokenizer, min_count: usize, ) -> Result<Self>

Learn token frequencies from a corpus

pub fn with_max_count(self, maxcount: usize) -> Self

Set the maximum count threshold

pub fn with_max_freq(self, maxfreq: f64) -> Result<Self>

Set the maximum frequency threshold (0.0 to 1.0)

Trait Implementations

impl Clone for FrequencyFilter

fn clone(&self) -> FrequencyFilter

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for FrequencyFilter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl TokenFilter for FrequencyFilter

fn apply(&self, tokens: &[String]) -> Vec<String>

Filter tokens based on the strategy

fn filtertext(&self, text: &str, tokenizer: &dyn Tokenizer) -> Result<String>

Apply the filter directly to text