Trait tantivy::tokenizer::TokenFilter

pub trait TokenFilter<TailTokenStream: TokenStream>: Clone {
    type ResultTokenStream: TokenStream;
    fn transform(
        &self, 
        token_stream: TailTokenStream
    ) -> Self::ResultTokenStream;
}

Trait for the pluggable components of Tokenizers.

Associated Types

type ResultTokenStream: TokenStream

The resulting TokenStream type.

Loading content...

Required methods

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

Wraps a token stream and returns the modified one.

Loading content...

Implementors

impl<TailTokenStream> TokenFilter<TailTokenStream> for AlphaNumOnlyFilter where
    TailTokenStream: TokenStream, 

type ResultTokenStream = AlphaNumOnlyFilterStream<TailTokenStream>

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

impl<TailTokenStream> TokenFilter<TailTokenStream> for AsciiFoldingFilter where
    TailTokenStream: TokenStream, 

type ResultTokenStream = AsciiFoldingFilterTokenStream<TailTokenStream>

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

impl<TailTokenStream> TokenFilter<TailTokenStream> for LowerCaser where
    TailTokenStream: TokenStream, 

type ResultTokenStream = LowerCaserTokenStream<TailTokenStream>

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

impl<TailTokenStream> TokenFilter<TailTokenStream> for RemoveLongFilter where
    TailTokenStream: TokenStream, 

type ResultTokenStream = RemoveLongFilterStream<TailTokenStream>

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

impl<TailTokenStream> TokenFilter<TailTokenStream> for Stemmer where
    TailTokenStream: TokenStream, 

type ResultTokenStream = StemmerTokenStream<TailTokenStream>

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

impl<TailTokenStream> TokenFilter<TailTokenStream> for StopWordFilter where
    TailTokenStream: TokenStream, 

type ResultTokenStream = StopWordFilterStream<TailTokenStream>

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

Loading content...