Trait tantivy::tokenizer::TokenFilter

pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
    fn transform<'a>(
        &self, 
        token_stream: BoxTokenStream<'a>
    ) -> BoxTokenStream<'a>;
}

Trait for the pluggable components of Tokenizers.

Required methods

fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>

Wraps a token stream and returns the modified one.

Loading content...

Implementors

impl TokenFilter for AlphaNumOnlyFilter

fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>

impl TokenFilter for AsciiFoldingFilter

fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>

impl TokenFilter for LowerCaser

fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>

impl TokenFilter for RemoveLongFilter

fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>

impl TokenFilter for Stemmer

fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>

impl TokenFilter for StopWordFilter

fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>

Loading content...