khive-text 0.10.0

Text analysis primitives for khive: tokenization, normalization, filtering
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13

//! Text analysis primitives: tokenization, normalization, filtering.

pub mod analyzer;
pub mod filter;
pub mod identifier;
pub mod lang;
pub mod preset;
pub mod tokenizer;
mod traits;

pub use analyzer::StandardAnalyzer;
pub use lang::{contains_cjk, contains_cjk_f32, is_cjk_char, is_meaningful_query, ScriptProfile};
pub use traits::{Analyzer, BoxedAnalyzer, BoxedTokenizer, TokenFilter, Tokenizer};