Struct WordDelimiterFilter 

pub struct WordDelimiterFilter { /* private fields */ }

Splits code identifiers into subword tokens.

The original identifier can be preserved at the first subword position. If a token is not a compound identifier then it is passed through unchanged.

Examples

use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream, WordDelimiterFilter};

let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false))
    .filter(WordDelimiterFilter::new(true, true))
    .build();
let mut stream = analyzer.token_stream("parseHTML2JSON");

let mut tokens = Vec::new();
stream.process(&mut |token| tokens.push(token.text.clone()));
assert_eq!(tokens, vec!["parseHTML2JSON", "parse", "HTML", "2", "JSON"]);

Implementations

impl WordDelimiterFilter

pub fn new(preserve_original: bool, split_on_numerics: bool) -> Self

Trait Implementations

impl Clone for WordDelimiterFilter

fn clone(&self) -> WordDelimiterFilter

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl TokenFilter for WordDelimiterFilter

type Tokenizer<T: Tokenizer> = WordDelimiterFilterWrapper<T>

Tokenizer produced by this filter.

fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>

Wrap the tokenizer.