khive_text/lib.rs
1//! Text analysis primitives: tokenization, normalization, filtering.
2
3pub mod analyzer;
4pub mod filter;
5pub mod identifier;
6pub mod lang;
7pub mod preset;
8pub mod tokenizer;
9mod traits;
10
11pub use analyzer::StandardAnalyzer;
12pub use lang::{contains_cjk, contains_cjk_f32, is_cjk_char, is_meaningful_query, ScriptProfile};
13pub use traits::{Analyzer, BoxedAnalyzer, BoxedTokenizer, TokenFilter, Tokenizer};