Trait TokenFilter 

pub trait TokenFilter:
    'static
    + Send
    + Sync {
    type Tokenizer<T: Tokenizer>: Tokenizer;

    // Required method
    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>;
}

Filter that wraps a tokenizer with additional token-processing behavior.

Required Associated Types

type Tokenizer<T: Tokenizer>: Tokenizer

Tokenizer produced by this filter.

Required Methods

fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>

Wrap the tokenizer.

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.

Implementors

impl TokenFilter for AlphaNumOnlyFilter

type Tokenizer<T: Tokenizer> = AlphaNumOnlyFilterWrapper<T>

impl TokenFilter for AsciiFoldingFilter

type Tokenizer<T: Tokenizer> = AsciiFoldingFilterWrapper<T>

impl TokenFilter for LowerCaser

type Tokenizer<T: Tokenizer> = LowerCaserFilter<T>

impl TokenFilter for RemoveLongFilter

type Tokenizer<T: Tokenizer> = RemoveLongFilterWrapper<T>

impl TokenFilter for Stemmer

type Tokenizer<T: Tokenizer> = StemmerFilter<T>

impl TokenFilter for StopWordFilter

type Tokenizer<T: Tokenizer> = StopWordFilterWrapper<T>