lance-tokenizer 6.0.0

Tokenizer abstractions and implementations for Lance
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

mod alphanum_only;
mod analyzer;
mod ascii_folding_filter;
#[cfg(feature = "tokenizer-jieba")]
mod jieba;
mod lower_caser;
mod ngram_tokenizer;
mod raw_tokenizer;
mod remove_long;
mod simple_tokenizer;
mod stemmer;
mod stop_word_filter;
mod tokenizer_api;
mod whitespace_tokenizer;

#[cfg(feature = "tokenizer-lindera")]
mod lindera;

pub use alphanum_only::AlphaNumOnlyFilter;
pub use analyzer::{TextAnalyzer, TextAnalyzerBuilder};
pub use ascii_folding_filter::AsciiFoldingFilter;
#[cfg(feature = "tokenizer-jieba")]
pub use jieba::JiebaTokenizer;
#[cfg(feature = "tokenizer-lindera")]
pub use lindera::LinderaTokenizer;
pub use lower_caser::LowerCaser;
pub use ngram_tokenizer::NgramTokenizer;
pub use raw_tokenizer::RawTokenizer;
pub use remove_long::RemoveLongFilter;
pub use simple_tokenizer::{SimpleTokenStream, SimpleTokenizer};
pub use stemmer::{Language, Stemmer};
pub use stop_word_filter::StopWordFilter;
pub use tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream, Tokenizer};
pub use whitespace_tokenizer::WhitespaceTokenizer;