plugmem_core/tokenizer/
policy.rs

1//! Search policies for tokenizer folding.
2
3/// Controls search-specific folding without changing the scanner or its
4/// allocation behavior.
5///
6/// The default preserves the tokenizer's existing index contract: Latin
7/// diacritics are folded for recall and Russian `ё` is treated as `е`. Use
8/// [`TokenizerPolicy::unicode`] when callers need Unicode lowercase and
9/// normalization without either language/search-specific equivalence.
10#[derive(Debug, Clone, Copy, Eq, PartialEq)]
11pub struct TokenizerPolicy {
12    /// Fold Latin precomposed diacritics to their ASCII base.
13    pub fold_latin_diacritics: bool,
14    /// Treat Russian small letter `ё` as `е` after lowercase.
15    pub fold_russian_yo: bool,
16}
17
18impl TokenizerPolicy {
19    /// A language-neutral policy: Unicode lowercase and normalization only.
20    pub const fn unicode() -> Self {
21        Self {
22            fold_latin_diacritics: false,
23            fold_russian_yo: false,
24        }
25    }
26
27    /// The search policy used by [`crate::tokenizer::Tokenizer::new`] and
28    /// existing indexes.
29    pub const fn search() -> Self {
30        Self {
31            fold_latin_diacritics: true,
32            fold_russian_yo: true,
33        }
34    }
35}
36
37impl Default for TokenizerPolicy {
38    fn default() -> Self {
39        Self::search()
40    }
41}
plugmem_core/tokenizer/policy.rs

plugmem_core/tokenizer/
policy.rs
