Struct tantivy_analysis_contrib::commons::ElisionTokenFilter

pub struct ElisionTokenFilter {
    pub elisions: FxHashSet<String>,
    pub ignore_case: bool,
}

Available on crate feature commons only.

A token filter that removes elision from a token. For example, the token l'avion will become avion.

use tantivy_analysis_contrib::commons::ElisionTokenFilter;

let filter = ElisionTokenFilter::from_iter_str(vec!["l", "m", "t", "qu", "n", "s", "j", "d", "c", "jusqu", "quoiqu", "lorsqu", "puisqu"], true);

Example

This example shows produced token by ElisionTokenFilter.

All starting l' and m' are removed from tokens whatever the case is.

use tantivy::tokenizer::{WhitespaceTokenizer, TextAnalyzer, Token};
use tantivy_analysis_contrib::commons::ElisionTokenFilter;

let mut tmp = TextAnalyzer::builder(WhitespaceTokenizer::default())
   .filter(ElisionTokenFilter::from_iter_str(vec!["L", "M"], true))
   .build();
let mut token_stream = tmp.token_stream("Plop, juste pour voir l'embrouille avec O'brian. m'enfin.");

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "Plop,".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "juste".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "pour".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "voir".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "embrouille".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "avec".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "O'brian.".to_string());

let token = token_stream.next().expect("A token should be present.");
assert_eq!(token.text, "enfin.".to_string());

assert_eq!(None, token_stream.next());

Fields

elisions: FxHashSet<String>

Set of elisions

ignore_case: bool

Indicates that elisions are case-insensitive

Implementations

impl ElisionTokenFilter

pub fn from_iter_string( elisions: impl IntoIterator<Item = String>, ignore_case: bool ) -> Self

Construct a new ElisionTokenFilter from an iterator over String and a bool.

Parameters :

• elisions: list of elision to remove from tokens
• ignore_case: indicate that elisions are case-insensitive

pub fn from_iter_str<'a>( elisions: impl IntoIterator<Item = &'a str>, ignore_case: bool ) -> Self

Construct a new ElisionTokenFilter from an iterator over str and a bool.

Parameters :

• elisions: list of elision to remove from tokens
• ignore_case: indicate that elisions are case-insensitive

Trait Implementations

impl Clone for ElisionTokenFilter

fn clone(&self) -> ElisionTokenFilter

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ElisionTokenFilter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl PartialEq for ElisionTokenFilter

fn eq(&self, other: &ElisionTokenFilter) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl TokenFilter for ElisionTokenFilter

type Tokenizer<T: Tokenizer> = ElisionFilterWrapper<T>

The Tokenizer type returned by this filter, typically parametrized by the underlying Tokenizer.

fn transform<T: Tokenizer>(self, token_stream: T) -> Self::Tokenizer<T>

Wraps a Tokenizer and returns a new one.

impl Eq for ElisionTokenFilter

impl StructuralPartialEq for ElisionTokenFilter