Trait tantivy::tokenizer::TokenFilter

pub trait TokenFilter: 'static + Send + Sync {
    type Tokenizer<T: Tokenizer>: Tokenizer;

    // Required method
    fn transform<T>(self, tokenizer: T) -> Self::Tokenizer<T>
       where T: Tokenizer;
}

Trait for the pluggable components of Tokenizers.

Required Associated Types

type Tokenizer<T: Tokenizer>: Tokenizer

The Tokenizer type returned by this filter, typically parametrized by the underlying Tokenizer.

Required Methods

fn transform<T>(self, tokenizer: T) -> Self::Tokenizer<T>

where T: Tokenizer,

Wraps a Tokenizer and returns a new one.

Object Safety

This trait is not object safe.

Implementors

impl TokenFilter for AlphaNumOnlyFilter

type Tokenizer<T: Tokenizer> = AlphaNumOnlyFilterWrapper<T>

impl TokenFilter for AsciiFoldingFilter

type Tokenizer<T: Tokenizer> = AsciiFoldingFilterWrapper<T>

impl TokenFilter for LowerCaser

type Tokenizer<T: Tokenizer> = LowerCaserFilter<T>

impl TokenFilter for RemoveLongFilter

type Tokenizer<T: Tokenizer> = RemoveLongFilterWrapper<T>

impl TokenFilter for SplitCompoundWords

type Tokenizer<T: Tokenizer> = SplitCompoundWordsFilter<T>

impl TokenFilter for Stemmer

type Tokenizer<T: Tokenizer> = StemmerFilter<T>

impl TokenFilter for StopWordFilter

type Tokenizer<T: Tokenizer> = StopWordFilterWrapper<T>