Struct LengthFilter 

pub struct LengthFilter {
    pub min_length: usize,
    pub max_length: usize,
}

Filter tokens by length

Fields

min_length: usize

Minimum token length

max_length: usize

Maximum token length

Implementations

impl LengthFilter

pub fn new(_min_length: usize, maxlength: usize) -> Self

Create a new length filter

pub fn with_min_length(self, minlength: usize) -> Self

Set minimum token length

pub fn with_max_length(self, maxlength: usize) -> Self

Set maximum token length

Trait Implementations

impl Clone for LengthFilter

fn clone(&self) -> LengthFilter

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LengthFilter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LengthFilter

fn default() -> Self

Returns the “default value” for a type.

impl TokenFilter for LengthFilter

fn apply(&self, tokens: &[String]) -> Vec<String>

Filter tokens based on the strategy

fn filtertext(&self, text: &str, tokenizer: &dyn Tokenizer) -> Result<String>

Apply the filter directly to text