Trait TokenFilter 

pub trait TokenFilter:
    'static
    + Send
    + Sync {
    type Tokenizer<T: Tokenizer>: Tokenizer;

    // Required method
    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>;
}

Filter that wraps a tokenizer with additional token-processing behavior.

Required Associated Types

type Tokenizer<T: Tokenizer>: Tokenizer

Tokenizer produced by this filter.

Required Methods

fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>

Wrap the tokenizer.

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors

impl TokenFilter for AlphaNumOnlyFilter

type Tokenizer<T: Tokenizer> = AlphaNumOnlyFilterWrapper<T>

impl TokenFilter for AsciiFoldingFilter

type Tokenizer<T: Tokenizer> = AsciiFoldingFilterWrapper<T>

impl TokenFilter for LowerCaser

type Tokenizer<T: Tokenizer> = LowerCaserFilter<T>

impl TokenFilter for RemoveLongFilter

type Tokenizer<T: Tokenizer> = RemoveLongFilterWrapper<T>

impl TokenFilter for Stemmer

type Tokenizer<T: Tokenizer> = StemmerFilter<T>

impl TokenFilter for StopWordFilter

type Tokenizer<T: Tokenizer> = StopWordFilterWrapper<T>