khive-text 0.10.0

Text analysis primitives for khive: tokenization, normalization, filtering
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

//! Core text analysis traits and shared type aliases.

use std::sync::Arc;

/// Splits a string into raw tokens. Must be deterministic and stateless.
pub trait Tokenizer: Send + Sync {
    fn tokenize(&self, text: &str) -> Vec<String>;
}

/// Transforms or drops a single token. Returns None to drop.
pub trait TokenFilter: Send + Sync {
    fn apply(&self, token: String) -> Option<String>;
}

/// Full analysis pipeline: tokenize + filter chain.
pub trait Analyzer: Send + Sync {
    fn analyze(&self, text: &str) -> Vec<String>;
}

pub type BoxedAnalyzer = Arc<dyn Analyzer>;
pub type BoxedTokenizer = Arc<dyn Tokenizer>;