zeppelin_embed/fts/tokenizer/
mod.rs

1//! The versioned analysis pipeline and its epoch identity.
2//!
3//! # The product feature
4//!
5//! Text goes in; tokens come out carrying a term, a position, and the byte
6//! range they came from. The pipeline that produces them — segmentation,
7//! folding, stopwords, stemming, number words, and a user vocabulary — is
8//! identified by one digest, the **tokenizer epoch**. Same epoch, same bytes
9//! in, same tokens out, forever.
10//!
11//! That identity is the point. Analyzer drift silently invalidating an index
12//! is failure class U11 (`research/03:594`), and no off-the-shelf tokenizer
13//! versions its own behaviour as index metadata (`research/05:202`). The
14//! digest goes into `EpochMeta.tokenizer`, which task 07 already reserved,
15//! so no format change is needed to carry it.
16//!
17//! # Query-time and index-time are one code path
18//!
19//! [`Analyzer::analyze`] is used for both. tantivy's missing search-analyzer
20//! is a known gap this design avoids by construction. A query-side extras
21//! hook is deliberately absent in v1 rather than stubbed.
22
23pub(crate) mod fold;
24mod fold_table;
25pub(crate) mod numbers;
26mod pipeline;
27pub mod profiles;
28#[cfg(test)]
29mod properties;
30pub(crate) mod segment;
31pub(crate) mod stemmer;
32pub mod stopwords;
33pub mod vocab;
34
35pub use profiles::Profile;
36pub use stopwords::StopwordList;
37pub use vocab::{MAX_SURFACE_TERMS, VocabError, Vocabulary};
38
39/// Bit flags carried by every token.
40///
41/// Task 15 consumes these; defining the byte now avoids a format-shaped
42/// change later (task 12 D5).
43#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
44pub struct TokenFlags(u8);
45
46impl TokenFlags {
47    /// The token must never be fuzzy-matched: an identifier, number, or SKU.
48    ///
49    /// A fuzzy match on `i-485` or a ticket id is almost always wrong.
50    pub const NO_FUZZY: Self = Self(1);
51    /// The token is a stacked variant, not a surface word.
52    ///
53    /// Variants come from the vocabulary, number normalization, or word
54    /// decomposition. Diagnostics use this so a match can always explain
55    /// which form produced it.
56    pub const VARIANT: Self = Self(2);
57
58    /// Returns the empty flag set.
59    #[must_use]
60    pub const fn empty() -> Self {
61        Self(0)
62    }
63
64    /// Returns the raw bits.
65    #[must_use]
66    pub const fn bits(self) -> u8 {
67        self.0
68    }
69
70    /// Returns true when every bit of `other` is set.
71    #[must_use]
72    pub const fn contains(self, other: Self) -> bool {
73        self.0 & other.0 == other.0
74    }
75
76    /// Returns the union of two flag sets.
77    #[must_use]
78    pub const fn union(self, other: Self) -> Self {
79        Self(self.0 | other.0)
80    }
81}
82
83/// The byte range a token came from, into the analyzed text.
84#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
85pub struct TokenOffset {
86    /// Inclusive start byte offset.
87    pub start: u32,
88    /// Exclusive end byte offset.
89    pub end: u32,
90}
91
92impl TokenOffset {
93    /// Slices the token's surface form back out of the analyzed text.
94    ///
95    /// Returns `None` when the range is not a character boundary, which the
96    /// pipeline never produces but a hand-built offset might.
97    #[must_use]
98    pub fn slice<'text>(&self, text: &'text str) -> Option<&'text str> {
99        let start = usize::try_from(self.start).ok()?;
100        let end = usize::try_from(self.end).ok()?;
101        text.get(start..end)
102    }
103}
104
105/// One analyzed token.
106#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
107pub struct Token {
108    /// The indexed term.
109    pub term: String,
110    /// The token position; stacked variants share a position.
111    pub position: u32,
112    /// The byte range the token came from.
113    pub offset: TokenOffset,
114    /// Token flags.
115    pub flags: TokenFlags,
116}
117
118/// Word segmentation strategy; an epoch-digest input.
119#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
120#[repr(u16)]
121pub enum Segmenter {
122    /// Identifier-preserving segmentation with CJK bigram fallback.
123    IdentifierPreserving = 1,
124}
125
126/// Character folding form; an epoch-digest input.
127#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
128#[repr(u16)]
129pub enum FoldingForm {
130    /// No folding at all; terms keep their original case and marks.
131    None = 0,
132    /// NFKC, case folding, and diacritic removal from the generated table.
133    NfkcSearchFold = 1,
134}
135
136/// Stemming algorithm; an epoch-digest input.
137#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
138#[repr(u16)]
139pub enum Stemmer {
140    /// No stemming.
141    None = 0,
142    /// English Snowball (Porter2).
143    EnglishPorter2 = 1,
144}
145
146/// Analysis rejection at the engine boundary.
147#[derive(Clone, Debug, Eq, PartialEq)]
148pub enum TokenizerError {
149    /// The supplied bytes were not valid UTF-8.
150    InvalidUtf8 {
151        /// Byte offset of the first invalid sequence.
152        offset: usize,
153    },
154    /// The analyzed text exceeded [`MAX_TEXT_BYTES`].
155    TextTooLong {
156        /// The rejected byte length.
157        bytes: usize,
158        /// The configured limit.
159        limit: usize,
160    },
161    /// The vocabulary was rejected.
162    Vocab(VocabError),
163}
164
165impl std::fmt::Display for TokenizerError {
166    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
167        match self {
168            Self::InvalidUtf8 { offset } => {
169                write!(formatter, "input is not valid UTF-8 at byte {offset}")
170            }
171            Self::TextTooLong { bytes, limit } => write!(
172                formatter,
173                "analyzed text is {bytes} bytes, over the {limit}-byte limit"
174            ),
175            Self::Vocab(error) => write!(formatter, "vocabulary rejected: {error}"),
176        }
177    }
178}
179
180impl std::error::Error for TokenizerError {}
181
182impl From<VocabError> for TokenizerError {
183    fn from(error: VocabError) -> Self {
184        Self::Vocab(error)
185    }
186}
187
188/// The largest text one call will analyze.
189///
190/// Offsets are `u32`, so this bound is what keeps every offset
191/// representable. It is not a truncation limit: task 13 forbids truncating
192/// documents before indexing, so an oversized field is a typed rejection the
193/// caller must split, never a silent cut.
194pub const MAX_TEXT_BYTES: usize = u32::MAX as usize;
195
196/// The complete analysis configuration.
197///
198/// Every field is an epoch-digest input. Changing any one of them produces a
199/// different epoch and therefore a different index meaning.
200#[derive(Clone, Debug, Eq, PartialEq)]
201pub struct TokenizerConfig {
202    /// The named profile this configuration started from.
203    pub profile: Profile,
204    /// Word segmentation strategy.
205    pub segmenter: Segmenter,
206    /// Character folding form.
207    pub folding: FoldingForm,
208    /// Stopword list.
209    pub stopwords: StopwordList,
210    /// Stemming algorithm.
211    pub stemmer: Stemmer,
212    /// Split words at joiners, case changes, and letter/digit transitions.
213    pub decompose_words: bool,
214    /// Emit the joiner-free catenation of a decomposed word.
215    pub emit_catenation: bool,
216    /// Emit digit and word variants for spelled numbers.
217    pub number_words: bool,
218    /// Drop single-LETTER decomposition parts.
219    ///
220    /// The `k` of `401k`, the `t` of `don't`, the `s` of `investor's`:
221    /// one-letter parts of a decomposed word are stop-level noise in
222    /// linguistic text — they carry no identity the whole word and the
223    /// catenation do not already carry, and they pollute document length
224    /// and document frequency. One-DIGIT parts are kept: `type-2` and
225    /// `SARS-CoV-2` are discriminated by exactly that digit. Dropping a
226    /// part leaves its position spent, exactly as a removed stopword
227    /// does, so phrase adjacency and the analyzed length are unchanged.
228    /// Identifier-heavy profiles keep every part: the `x` of `x_max` is
229    /// a real search target in code.
230    pub drop_single_char_parts: bool,
231    /// User vocabulary and synonyms.
232    pub vocabulary: Vocabulary,
233}
234
235impl TokenizerConfig {
236    /// The general-purpose text profile.
237    #[must_use]
238    pub fn text_default() -> Self {
239        Profile::TextDefault.config()
240    }
241
242    /// The identifier-heavy profile for source code and tickets.
243    #[must_use]
244    pub fn code() -> Self {
245        Profile::Code.config()
246    }
247
248    /// The spoken-text profile.
249    #[must_use]
250    pub fn voice() -> Self {
251        Profile::Voice.config()
252    }
253
254    /// Replaces the user vocabulary.
255    #[must_use]
256    pub fn with_vocabulary(mut self, vocabulary: Vocabulary) -> Self {
257        self.vocabulary = vocabulary;
258        self
259    }
260
261    /// Computes the epoch digest for this configuration.
262    #[must_use]
263    pub fn epoch(&self) -> TokenizerEpoch {
264        TokenizerEpoch::of(self)
265    }
266}
267
268/// A tokenizer epoch: the identity of one analysis behaviour.
269///
270/// Stored as a little-endian u64 in `EpochMeta.tokenizer`.
271#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
272pub struct TokenizerEpoch(u64);
273
274/// Digest-input framing magic. Changing it changes every epoch.
275const EPOCH_MAGIC: &[u8; 8] = b"ZETOKEP1";
276
277impl TokenizerEpoch {
278    pub(crate) const fn from_value(value: u64) -> Self {
279        Self(value)
280    }
281
282    /// Computes the epoch of a configuration.
283    ///
284    /// # The canonical digest input
285    ///
286    /// Little-endian throughout, length-prefixed for every variable field,
287    /// in exactly this order: magic, segmenter id, folding form id, folding
288    /// table version, Unicode version string, stopword list id, stopword
289    /// term count, stemmer id, the three boolean knobs, profile id, then the
290    /// vocabulary. The vocabulary contributes its entry count followed by
291    /// every (surface form, canonical term) pair in sorted order, so two
292    /// vocabularies built by different insertion orders digest identically.
293    ///
294    /// This layout is a persisted-meaning contract frozen by
295    /// `golden_streams_cannot_change_without_an_epoch_bump`.
296    #[must_use]
297    pub fn of(config: &TokenizerConfig) -> Self {
298        let mut input: Vec<u8> = Vec::with_capacity(256);
299        input.extend_from_slice(EPOCH_MAGIC);
300        push_u16(&mut input, config.segmenter as u16);
301        push_u16(&mut input, config.folding as u16);
302        push_u16(&mut input, fold::FOLDING_TABLE_VERSION);
303        push_str(&mut input, fold::FOLDING_UNICODE_VERSION);
304        push_u16(&mut input, config.stopwords.id());
305        push_u32(
306            &mut input,
307            u32::try_from(config.stopwords.terms().len()).unwrap_or(u32::MAX),
308        );
309        push_u16(&mut input, config.stemmer as u16);
310        input.push(u8::from(config.decompose_words));
311        input.push(u8::from(config.emit_catenation));
312        input.push(u8::from(config.number_words));
313        input.push(u8::from(config.drop_single_char_parts));
314        push_u16(&mut input, config.profile.id());
315        push_u32(
316            &mut input,
317            u32::try_from(config.vocabulary.len()).unwrap_or(u32::MAX),
318        );
319        for (surface, canonical) in config.vocabulary.entries() {
320            push_u32(&mut input, u32::try_from(surface.len()).unwrap_or(u32::MAX));
321            for term in surface {
322                push_str(&mut input, term);
323            }
324            push_str(&mut input, canonical);
325        }
326        Self(xxhash_rust::xxh3::xxh3_64(&input))
327    }
328
329    /// Returns the raw digest.
330    #[must_use]
331    pub const fn value(self) -> u64 {
332        self.0
333    }
334
335    /// Returns the lowercase hex form used in diagnostics.
336    #[must_use]
337    pub fn to_hex(self) -> String {
338        format!("{:016x}", self.0)
339    }
340}
341
342impl std::fmt::Display for TokenizerEpoch {
343    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
344        write!(formatter, "{:016x}", self.0)
345    }
346}
347
348fn push_u16(buffer: &mut Vec<u8>, value: u16) {
349    buffer.extend_from_slice(&value.to_le_bytes());
350}
351
352fn push_u32(buffer: &mut Vec<u8>, value: u32) {
353    buffer.extend_from_slice(&value.to_le_bytes());
354}
355
356fn push_str(buffer: &mut Vec<u8>, value: &str) {
357    push_u32(buffer, u32::try_from(value.len()).unwrap_or(u32::MAX));
358    buffer.extend_from_slice(value.as_bytes());
359}
360
361/// A compiled analyzer.
362///
363/// Construction validates the configuration once so [`Analyzer::analyze`]
364/// cannot fail on well-formed UTF-8.
365#[derive(Clone, Debug)]
366pub struct Analyzer {
367    config: TokenizerConfig,
368    epoch: TokenizerEpoch,
369}
370
371impl Analyzer {
372    /// Compiles a configuration.
373    ///
374    /// # Errors
375    ///
376    /// Returns [`TokenizerError::Vocab`] when the vocabulary is malformed.
377    pub fn new(config: TokenizerConfig) -> Result<Self, TokenizerError> {
378        let epoch = TokenizerEpoch::of(&config);
379        Ok(Self { config, epoch })
380    }
381
382    /// Returns the epoch identifying this analyzer's behaviour.
383    #[must_use]
384    pub const fn epoch(&self) -> TokenizerEpoch {
385        self.epoch
386    }
387
388    /// Returns the configuration.
389    #[must_use]
390    pub const fn config(&self) -> &TokenizerConfig {
391        &self.config
392    }
393
394    /// Analyzes text into tokens.
395    ///
396    /// Index-time and query-time analysis are this same call.
397    #[must_use]
398    pub fn analyze(&self, text: &str) -> Vec<Token> {
399        pipeline::analyze(&self.config, text)
400    }
401
402    /// Analyzes bytes, rejecting invalid UTF-8 with a typed error.
403    ///
404    /// # Errors
405    ///
406    /// Returns [`TokenizerError::InvalidUtf8`] rather than lossily
407    /// converting, and [`TokenizerError::TextTooLong`] rather than
408    /// truncating.
409    pub fn analyze_bytes(&self, bytes: &[u8]) -> Result<Vec<Token>, TokenizerError> {
410        if bytes.len() > MAX_TEXT_BYTES {
411            return Err(TokenizerError::TextTooLong {
412                bytes: bytes.len(),
413                limit: MAX_TEXT_BYTES,
414            });
415        }
416        let text = std::str::from_utf8(bytes).map_err(|error| TokenizerError::InvalidUtf8 {
417            offset: error.valid_up_to(),
418        })?;
419        Ok(self.analyze(text))
420    }
421}
422
423#[cfg(test)]
424#[allow(
425    clippy::expect_used,
426    clippy::indexing_slicing,
427    clippy::panic,
428    clippy::unwrap_used
429)]
430mod tests {
431    use super::*;
432
433    fn analyzer() -> Analyzer {
434        Analyzer::new(TokenizerConfig::text_default()).expect("valid config")
435    }
436
437    fn terms(tokens: &[Token]) -> Vec<&str> {
438        tokens.iter().map(|token| token.term.as_str()).collect()
439    }
440
441    #[test]
442    fn invalid_utf8_is_rejected_with_a_typed_error_at_the_boundary() {
443        let error = analyzer()
444            .analyze_bytes(&[b'o', b'k', 0xFF, 0xFE])
445            .expect_err("invalid UTF-8 must be refused");
446        assert_eq!(error, TokenizerError::InvalidUtf8 { offset: 2 });
447    }
448
449    #[test]
450    fn valid_utf8_bytes_analyze_identically_to_the_string_path() {
451        let text = "Café put_if_match";
452        let analyzer = analyzer();
453        assert_eq!(
454            analyzer
455                .analyze_bytes(text.as_bytes())
456                .expect("valid UTF-8"),
457            analyzer.analyze(text)
458        );
459    }
460
461    #[test]
462    fn changing_any_pipeline_knob_changes_the_epoch_digest() {
463        let base = TokenizerConfig::text_default();
464        let baseline = base.epoch();
465
466        let mut changed = base.clone();
467        changed.stemmer = Stemmer::None;
468        assert_ne!(changed.epoch(), baseline, "stemmer must affect the epoch");
469
470        let mut changed = base.clone();
471        changed.stopwords = StopwordList::None;
472        assert_ne!(changed.epoch(), baseline, "stopwords must affect the epoch");
473
474        let mut changed = base.clone();
475        changed.folding = FoldingForm::None;
476        assert_ne!(changed.epoch(), baseline, "folding must affect the epoch");
477
478        let mut changed = base.clone();
479        changed.decompose_words = !changed.decompose_words;
480        assert_ne!(changed.epoch(), baseline, "decomposition must affect it");
481
482        let mut changed = base.clone();
483        changed.emit_catenation = !changed.emit_catenation;
484        assert_ne!(changed.epoch(), baseline, "catenation must affect it");
485
486        let mut changed = base.clone();
487        changed.number_words = !changed.number_words;
488        assert_ne!(changed.epoch(), baseline, "number words must affect it");
489
490        let mut changed = base;
491        changed.profile = Profile::Code;
492        assert_ne!(changed.epoch(), baseline, "profile must affect the epoch");
493    }
494
495    #[test]
496    fn removing_a_vocab_entry_changes_the_epoch_digest() {
497        let mut vocabulary = Vocabulary::new();
498        vocabulary
499            .declare("put_if_match", &[&["put", "if", "match"][..]])
500            .expect("valid declaration");
501        let with_entry = TokenizerConfig::text_default().with_vocabulary(vocabulary.clone());
502        let populated = with_entry.epoch();
503
504        vocabulary.remove("put_if_match");
505        let without_entry = TokenizerConfig::text_default().with_vocabulary(vocabulary);
506        assert_ne!(populated, without_entry.epoch());
507        assert_eq!(
508            without_entry.epoch(),
509            TokenizerConfig::text_default().epoch()
510        );
511    }
512
513    #[test]
514    fn the_epoch_does_not_depend_on_vocabulary_insertion_order() {
515        let mut first = Vocabulary::new();
516        first
517            .declare("alpha", &[&["a", "one"][..]])
518            .expect("valid declaration");
519        first
520            .declare("beta", &[&["b", "two"][..]])
521            .expect("valid declaration");
522
523        let mut second = Vocabulary::new();
524        second
525            .declare("beta", &[&["b", "two"][..]])
526            .expect("valid declaration");
527        second
528            .declare("alpha", &[&["a", "one"][..]])
529            .expect("valid declaration");
530
531        assert_eq!(
532            TokenizerConfig::text_default()
533                .with_vocabulary(first)
534                .epoch(),
535            TokenizerConfig::text_default()
536                .with_vocabulary(second)
537                .epoch()
538        );
539    }
540
541    #[test]
542    fn the_epoch_hex_form_is_stable_and_sixteen_digits() {
543        let epoch = TokenizerConfig::text_default().epoch();
544        let hex = epoch.to_hex();
545        assert_eq!(hex.len(), 16);
546        assert_eq!(hex, epoch.to_string());
547        assert!(hex.chars().all(|value| value.is_ascii_hexdigit()));
548    }
549
550    #[test]
551    fn adding_a_vocab_entry_unifies_all_three_surface_forms_at_one_position() {
552        let mut vocabulary = Vocabulary::new();
553        vocabulary
554            .declare(
555                "put_if_match",
556                &[&["put", "if", "match"][..], &["putifmatch"][..]],
557            )
558            .expect("valid declaration");
559        let analyzer = Analyzer::new(TokenizerConfig::text_default().with_vocabulary(vocabulary))
560            .expect("valid config");
561
562        for surface in ["put_if_match", "put if match", "putifmatch"] {
563            let tokens = analyzer.analyze(surface);
564            let canonical = tokens
565                .iter()
566                .find(|token| token.term == "put_if_match")
567                .unwrap_or_else(|| panic!("{surface:?} did not emit the canonical term"));
568            assert_eq!(
569                canonical.position, 0,
570                "{surface:?} emitted the canonical term off position zero"
571            );
572        }
573    }
574
575    #[test]
576    fn identifier_and_number_tokens_carry_the_no_fuzzy_flag() {
577        let tokens = analyzer().analyze("ticket i-485 filed 2026 by anup");
578        for token in &tokens {
579            let expected =
580                token.term.contains('-') || token.term.chars().any(|value| value.is_ascii_digit());
581            if expected {
582                assert!(
583                    token.flags.contains(TokenFlags::NO_FUZZY),
584                    "{:?} must be flagged no_fuzzy",
585                    token.term
586                );
587            }
588        }
589        assert!(tokens.iter().any(|token| token.term == "i-485"));
590    }
591
592    #[test]
593    fn plain_words_are_not_flagged_no_fuzzy() {
594        let tokens = analyzer().analyze("engine");
595        assert_eq!(terms(&tokens), vec!["engin"]);
596        assert!(
597            !tokens
598                .first()
599                .expect("one token")
600                .flags
601                .contains(TokenFlags::NO_FUZZY)
602        );
603    }
604
605    #[test]
606    fn oversized_input_is_refused_rather_than_truncated() {
607        // The limit is u32::MAX; constructing it is wasteful, so this only
608        // pins that the typed error exists and names both numbers.
609        let error = TokenizerError::TextTooLong {
610            bytes: MAX_TEXT_BYTES + 1,
611            limit: MAX_TEXT_BYTES,
612        };
613        assert!(error.to_string().contains("over the"));
614    }
615
616    #[test]
617    fn token_offsets_slice_back_to_the_surface_form() {
618        let text = "Café put_if_match";
619        for token in analyzer().analyze(text) {
620            assert!(
621                token.offset.slice(text).is_some(),
622                "offset for {:?} is not a character boundary",
623                token.term
624            );
625        }
626    }
627
628    #[test]
629    fn flag_algebra_is_consistent() {
630        let both = TokenFlags::NO_FUZZY.union(TokenFlags::VARIANT);
631        assert!(both.contains(TokenFlags::NO_FUZZY));
632        assert!(both.contains(TokenFlags::VARIANT));
633        assert!(!TokenFlags::NO_FUZZY.contains(TokenFlags::VARIANT));
634        assert!(TokenFlags::empty().bits() == 0);
635        assert!(both.contains(TokenFlags::empty()));
636    }
637}
zeppelin_embed/fts/tokenizer/mod.rs

zeppelin_embed/fts/tokenizer/
mod.rs
