Struct tantivy_analysis_contrib::commons::LimitTokenCountFilter

pub struct LimitTokenCountFilter { /* private fields */ }

Available on crate feature commons only.

TokenFilter that limit the number of tokens

use tantivy_analysis_contrib::commons::LimitTokenCountFilter;

let filter:LimitTokenCountFilter = LimitTokenCountFilter::new(5);

Example

use tantivy::tokenizer::{WhitespaceTokenizer, TextAnalyzer, Token};
use tantivy_analysis_contrib::commons::LimitTokenCountFilter;

let mut tmp = TextAnalyzer::builder(WhitespaceTokenizer::default())
   .filter(LimitTokenCountFilter::from(3))
   .build();
let mut token_stream = tmp.token_stream("There will be 3 tokens in the end");

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "There".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "will".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "be".to_string());

assert_eq!(None, token_stream.next());

Implementations

impl LimitTokenCountFilter

pub fn new(max_tokens: usize) -> Self

Create a new LimitTokenCountFilter.

Parameters :

• max_tokens : maximum number of tokens that will be indexed

Trait Implementations

impl Clone for LimitTokenCountFilter

fn clone(&self) -> LimitTokenCountFilter

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LimitTokenCountFilter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl From<usize> for LimitTokenCountFilter

fn from(max_tokens: usize) -> Self

Converts to this type from the input type.

impl TokenFilter for LimitTokenCountFilter

type Tokenizer<T: Tokenizer> = LimitTokenCountFilterWrapper<T>

The Tokenizer type returned by this filter, typically parametrized by the underlying Tokenizer.

fn transform<T: Tokenizer>(self, token_stream: T) -> Self::Tokenizer<T>

Wraps a Tokenizer and returns a new one.

impl Copy for LimitTokenCountFilter