khive_text/traits.rs
1//! Core text analysis traits and shared type aliases.
2
3use std::sync::Arc;
4
5/// Splits a string into raw tokens. Must be deterministic and stateless.
6pub trait Tokenizer: Send + Sync {
7 fn tokenize(&self, text: &str) -> Vec<String>;
8}
9
10/// Transforms or drops a single token. Returns None to drop.
11pub trait TokenFilter: Send + Sync {
12 fn apply(&self, token: String) -> Option<String>;
13}
14
15/// Full analysis pipeline: tokenize + filter chain.
16pub trait Analyzer: Send + Sync {
17 fn analyze(&self, text: &str) -> Vec<String>;
18}
19
20pub type BoxedAnalyzer = Arc<dyn Analyzer>;
21pub type BoxedTokenizer = Arc<dyn Tokenizer>;