zeppelin_embed/fts/tokenizer/
mod.rs

1//! The versioned analysis pipeline and its epoch identity.
2//!
3//! # The product feature
4//!
5//! Text goes in; tokens come out carrying a term, a position, and the byte
6//! range they came from. The pipeline that produces them — segmentation,
7//! folding, stopwords, stemming, number words, and a user vocabulary — is
8//! identified by one digest, the **tokenizer epoch**. Same epoch, same bytes
9//! in, same tokens out, forever.
10//!
11//! That identity is the point. Analyzer drift silently invalidating an index
12//! is failure class U11 (`research/03:594`), and no off-the-shelf tokenizer
13//! versions its own behaviour as index metadata (`research/05:202`). The
14//! digest goes into `EpochMeta.tokenizer`, which task 07 already reserved,
15//! so no format change is needed to carry it.
16//!
17//! # Query-time and index-time are one code path
18//!
19//! [`Analyzer::analyze`] is used for both. tantivy's missing search-analyzer
20//! is a known gap this design avoids by construction. A query-side extras
21//! hook is deliberately absent in v1 rather than stubbed.
22
23pub(crate) mod fold;
24mod fold_table;
25pub(crate) mod numbers;
26mod pipeline;
27
28pub mod profiles;
29#[cfg(test)]
30mod properties;
31pub(crate) mod segment;
32pub(crate) mod stemmer;
33pub mod stopwords;
34pub mod vocab;
35
36pub use profiles::Profile;
37pub use stopwords::StopwordList;
38pub use vocab::{MAX_SURFACE_TERMS, VocabError, Vocabulary};
39
40/// Bit flags carried by every token.
41///
42/// Task 15 consumes these; defining the byte now avoids a format-shaped
43/// change later (task 12 D5).
44#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
45pub struct TokenFlags(u8);
46
47impl TokenFlags {
48    /// The token must never be fuzzy-matched: an identifier, number, or SKU.
49    ///
50    /// A fuzzy match on `i-485` or a ticket id is almost always wrong.
51    pub const NO_FUZZY: Self = Self(1);
52    /// The token is a stacked variant, not a surface word.
53    ///
54    /// Variants come from the vocabulary, number normalization, or word
55    /// decomposition. Diagnostics use this so a match can always explain
56    /// which form produced it.
57    pub const VARIANT: Self = Self(2);
58
59    /// Returns the empty flag set.
60    #[must_use]
61    pub const fn empty() -> Self {
62        Self(0)
63    }
64
65    /// Returns the raw bits.
66    #[must_use]
67    pub const fn bits(self) -> u8 {
68        self.0
69    }
70
71    /// Returns true when every bit of `other` is set.
72    #[must_use]
73    pub const fn contains(self, other: Self) -> bool {
74        self.0 & other.0 == other.0
75    }
76
77    /// Returns the union of two flag sets.
78    #[must_use]
79    pub const fn union(self, other: Self) -> Self {
80        Self(self.0 | other.0)
81    }
82}
83
84/// The byte range a token came from, into the analyzed text.
85#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
86pub struct TokenOffset {
87    /// Inclusive start byte offset.
88    pub start: u32,
89    /// Exclusive end byte offset.
90    pub end: u32,
91}
92
93impl TokenOffset {
94    /// Slices the token's surface form back out of the analyzed text.
95    ///
96    /// Returns `None` when the range is not a character boundary, which the
97    /// pipeline never produces but a hand-built offset might.
98    #[must_use]
99    pub fn slice<'text>(&self, text: &'text str) -> Option<&'text str> {
100        let start = usize::try_from(self.start).ok()?;
101        let end = usize::try_from(self.end).ok()?;
102        text.get(start..end)
103    }
104}
105
106/// One analyzed token.
107#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
108pub struct Token {
109    /// The indexed term.
110    pub term: String,
111    /// The token position; stacked variants share a position.
112    pub position: u32,
113    /// The byte range the token came from.
114    pub offset: TokenOffset,
115    /// Token flags.
116    pub flags: TokenFlags,
117}
118
119/// Word segmentation strategy; an epoch-digest input.
120#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
121#[repr(u16)]
122pub enum Segmenter {
123    /// Identifier-preserving segmentation with CJK bigram fallback.
124    IdentifierPreserving = 1,
125}
126
127/// Character folding form; an epoch-digest input.
128#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
129#[repr(u16)]
130pub enum FoldingForm {
131    /// No folding at all; terms keep their original case and marks.
132    None = 0,
133    /// NFKC, case folding, and diacritic removal from the generated table.
134    NfkcSearchFold = 1,
135}
136
137/// Stemming algorithm; an epoch-digest input.
138#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
139#[repr(u16)]
140pub enum Stemmer {
141    /// No stemming.
142    None = 0,
143    /// English Snowball (Porter2).
144    EnglishPorter2 = 1,
145}
146
147/// Analysis rejection at the engine boundary.
148#[derive(Clone, Debug, Eq, PartialEq)]
149pub enum TokenizerError {
150    /// The supplied bytes were not valid UTF-8.
151    InvalidUtf8 {
152        /// Byte offset of the first invalid sequence.
153        offset: usize,
154    },
155    /// The analyzed text exceeded [`MAX_TEXT_BYTES`].
156    TextTooLong {
157        /// The rejected byte length.
158        bytes: usize,
159        /// The configured limit.
160        limit: usize,
161    },
162    /// The vocabulary was rejected.
163    Vocab(VocabError),
164}
165
166impl std::fmt::Display for TokenizerError {
167    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
168        match self {
169            Self::InvalidUtf8 { offset } => {
170                write!(formatter, "input is not valid UTF-8 at byte {offset}")
171            }
172            Self::TextTooLong { bytes, limit } => write!(
173                formatter,
174                "analyzed text is {bytes} bytes, over the {limit}-byte limit"
175            ),
176            Self::Vocab(error) => write!(formatter, "vocabulary rejected: {error}"),
177        }
178    }
179}
180
181impl std::error::Error for TokenizerError {}
182
183impl From<VocabError> for TokenizerError {
184    fn from(error: VocabError) -> Self {
185        Self::Vocab(error)
186    }
187}
188
189/// The largest text one call will analyze.
190///
191/// Offsets are `u32`, so this bound is what keeps every offset
192/// representable. It is not a truncation limit: task 13 forbids truncating
193/// documents before indexing, so an oversized field is a typed rejection the
194/// caller must split, never a silent cut.
195pub const MAX_TEXT_BYTES: usize = u32::MAX as usize;
196
197/// The complete analysis configuration.
198///
199/// Every field is an epoch-digest input. Changing any one of them produces a
200/// different epoch and therefore a different index meaning.
201#[derive(Clone, Debug, Eq, PartialEq)]
202pub struct TokenizerConfig {
203    /// The named profile this configuration started from.
204    pub profile: Profile,
205    /// Word segmentation strategy.
206    pub segmenter: Segmenter,
207    /// Character folding form.
208    pub folding: FoldingForm,
209    /// Stopword list.
210    pub stopwords: StopwordList,
211    /// Stemming algorithm.
212    pub stemmer: Stemmer,
213    /// Split words at joiners, case changes, and letter/digit transitions.
214    pub decompose_words: bool,
215    /// Emit the joiner-free catenation of a decomposed word.
216    pub emit_catenation: bool,
217    /// Emit digit and word variants for spelled numbers.
218    pub number_words: bool,
219    /// Drop single-LETTER decomposition parts.
220    ///
221    /// The `k` of `401k`, the `t` of `don't`, the `s` of `investor's`:
222    /// one-letter parts of a decomposed word are stop-level noise in
223    /// linguistic text — they carry no identity the whole word and the
224    /// catenation do not already carry, and they pollute document length
225    /// and document frequency. One-DIGIT parts are kept: `type-2` and
226    /// `SARS-CoV-2` are discriminated by exactly that digit. Dropping a
227    /// part leaves its position spent, exactly as a removed stopword
228    /// does, so phrase adjacency and the analyzed length are unchanged.
229    /// Identifier-heavy profiles keep every part: the `x` of `x_max` is
230    /// a real search target in code.
231    pub drop_single_char_parts: bool,
232    /// User vocabulary and synonyms.
233    pub vocabulary: Vocabulary,
234}
235
236impl TokenizerConfig {
237    /// The general-purpose text profile.
238    #[must_use]
239    pub fn text_default() -> Self {
240        Profile::TextDefault.config()
241    }
242
243    /// The identifier-heavy profile for source code and tickets.
244    #[must_use]
245    pub fn code() -> Self {
246        Profile::Code.config()
247    }
248
249    /// The spoken-text profile.
250    #[must_use]
251    pub fn voice() -> Self {
252        Profile::Voice.config()
253    }
254
255    /// Replaces the user vocabulary.
256    #[must_use]
257    pub fn with_vocabulary(mut self, vocabulary: Vocabulary) -> Self {
258        self.vocabulary = vocabulary;
259        self
260    }
261
262    /// Computes the epoch digest for this configuration.
263    #[must_use]
264    pub fn epoch(&self) -> TokenizerEpoch {
265        TokenizerEpoch::of(self)
266    }
267}
268
269/// A tokenizer epoch: the identity of one analysis behaviour.
270///
271/// Stored as a little-endian u64 in `EpochMeta.tokenizer`.
272#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
273pub struct TokenizerEpoch(u64);
274
275/// Digest-input framing magic. Changing it changes every epoch.
276const EPOCH_MAGIC: &[u8; 8] = b"ZETOKEP1";
277
278impl TokenizerEpoch {
279    pub(crate) const fn from_value(value: u64) -> Self {
280        Self(value)
281    }
282
283    /// Computes the epoch of a configuration.
284    ///
285    /// # The canonical digest input
286    ///
287    /// Little-endian throughout, length-prefixed for every variable field,
288    /// in exactly this order: magic, segmenter id, folding form id, folding
289    /// table version, Unicode version string, stopword list id, stopword
290    /// term count, stemmer id, the three boolean knobs, profile id, then the
291    /// vocabulary. The vocabulary contributes its entry count followed by
292    /// every (surface form, canonical term) pair in sorted order, so two
293    /// vocabularies built by different insertion orders digest identically.
294    ///
295    /// This layout is a persisted-meaning contract frozen by
296    /// `golden_streams_cannot_change_without_an_epoch_bump`.
297    #[must_use]
298    pub fn of(config: &TokenizerConfig) -> Self {
299        let mut input: Vec<u8> = Vec::with_capacity(256);
300        input.extend_from_slice(EPOCH_MAGIC);
301        push_u16(&mut input, config.segmenter as u16);
302        push_u16(&mut input, config.folding as u16);
303        push_u16(&mut input, fold::FOLDING_TABLE_VERSION);
304        push_str(&mut input, fold::FOLDING_UNICODE_VERSION);
305        push_u16(&mut input, config.stopwords.id());
306        push_u32(
307            &mut input,
308            u32::try_from(config.stopwords.terms().len()).unwrap_or(u32::MAX),
309        );
310        push_u16(&mut input, config.stemmer as u16);
311        input.push(u8::from(config.decompose_words));
312        input.push(u8::from(config.emit_catenation));
313        input.push(u8::from(config.number_words));
314        input.push(u8::from(config.drop_single_char_parts));
315        push_u16(&mut input, config.profile.id());
316        push_u32(
317            &mut input,
318            u32::try_from(config.vocabulary.len()).unwrap_or(u32::MAX),
319        );
320        for (surface, canonical) in config.vocabulary.entries() {
321            push_u32(&mut input, u32::try_from(surface.len()).unwrap_or(u32::MAX));
322            for term in surface {
323                push_str(&mut input, term);
324            }
325            push_str(&mut input, canonical);
326        }
327        Self(xxhash_rust::xxh3::xxh3_64(&input))
328    }
329
330    /// Returns the raw digest.
331    #[must_use]
332    pub const fn value(self) -> u64 {
333        self.0
334    }
335
336    /// Returns the lowercase hex form used in diagnostics.
337    #[must_use]
338    pub fn to_hex(self) -> String {
339        format!("{:016x}", self.0)
340    }
341}
342
343impl std::fmt::Display for TokenizerEpoch {
344    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
345        write!(formatter, "{:016x}", self.0)
346    }
347}
348
349fn push_u16(buffer: &mut Vec<u8>, value: u16) {
350    buffer.extend_from_slice(&value.to_le_bytes());
351}
352
353fn push_u32(buffer: &mut Vec<u8>, value: u32) {
354    buffer.extend_from_slice(&value.to_le_bytes());
355}
356
357fn push_str(buffer: &mut Vec<u8>, value: &str) {
358    push_u32(buffer, u32::try_from(value.len()).unwrap_or(u32::MAX));
359    buffer.extend_from_slice(value.as_bytes());
360}
361
362/// A compiled analyzer.
363///
364/// Construction validates the configuration once so [`Analyzer::analyze`]
365/// cannot fail on well-formed UTF-8.
366#[derive(Clone, Debug)]
367pub struct Analyzer {
368    config: TokenizerConfig,
369    epoch: TokenizerEpoch,
370}
371
372impl Analyzer {
373    /// Compiles a configuration.
374    ///
375    /// # Errors
376    ///
377    /// Returns [`TokenizerError::Vocab`] when the vocabulary is malformed.
378    pub fn new(config: TokenizerConfig) -> Result<Self, TokenizerError> {
379        let epoch = TokenizerEpoch::of(&config);
380        Ok(Self { config, epoch })
381    }
382
383    /// Returns the epoch identifying this analyzer's behaviour.
384    #[must_use]
385    pub const fn epoch(&self) -> TokenizerEpoch {
386        self.epoch
387    }
388
389    /// Returns the configuration.
390    #[must_use]
391    pub const fn config(&self) -> &TokenizerConfig {
392        &self.config
393    }
394
395    /// Analyzes text into tokens.
396    ///
397    /// Index-time and query-time analysis are this same call.
398    #[must_use]
399    pub fn analyze(&self, text: &str) -> Vec<Token> {
400        pipeline::analyze(&self.config, text)
401    }
402
403    /// Analyzes bytes, rejecting invalid UTF-8 with a typed error.
404    ///
405    /// # Errors
406    ///
407    /// Returns [`TokenizerError::InvalidUtf8`] rather than lossily
408    /// converting, and [`TokenizerError::TextTooLong`] rather than
409    /// truncating.
410    pub fn analyze_bytes(&self, bytes: &[u8]) -> Result<Vec<Token>, TokenizerError> {
411        if bytes.len() > MAX_TEXT_BYTES {
412            return Err(TokenizerError::TextTooLong {
413                bytes: bytes.len(),
414                limit: MAX_TEXT_BYTES,
415            });
416        }
417        let text = std::str::from_utf8(bytes).map_err(|error| TokenizerError::InvalidUtf8 {
418            offset: error.valid_up_to(),
419        })?;
420        Ok(self.analyze(text))
421    }
422}
423
424#[cfg(test)]
425#[allow(
426    clippy::expect_used,
427    clippy::indexing_slicing,
428    clippy::panic,
429    clippy::unwrap_used
430)]
431mod tests {
432    use super::*;
433
434    fn analyzer() -> Analyzer {
435        Analyzer::new(TokenizerConfig::text_default()).expect("valid config")
436    }
437
438    fn terms(tokens: &[Token]) -> Vec<&str> {
439        tokens.iter().map(|token| token.term.as_str()).collect()
440    }
441
442    #[test]
443    fn invalid_utf8_is_rejected_with_a_typed_error_at_the_boundary() {
444        let error = analyzer()
445            .analyze_bytes(&[b'o', b'k', 0xFF, 0xFE])
446            .expect_err("invalid UTF-8 must be refused");
447        assert_eq!(error, TokenizerError::InvalidUtf8 { offset: 2 });
448    }
449
450    #[test]
451    fn valid_utf8_bytes_analyze_identically_to_the_string_path() {
452        let text = "Café put_if_match";
453        let analyzer = analyzer();
454        assert_eq!(
455            analyzer
456                .analyze_bytes(text.as_bytes())
457                .expect("valid UTF-8"),
458            analyzer.analyze(text)
459        );
460    }
461
462    #[test]
463    fn changing_any_pipeline_knob_changes_the_epoch_digest() {
464        let base = TokenizerConfig::text_default();
465        let baseline = base.epoch();
466
467        let mut changed = base.clone();
468        changed.stemmer = Stemmer::None;
469        assert_ne!(changed.epoch(), baseline, "stemmer must affect the epoch");
470
471        let mut changed = base.clone();
472        changed.stopwords = StopwordList::None;
473        assert_ne!(changed.epoch(), baseline, "stopwords must affect the epoch");
474
475        let mut changed = base.clone();
476        changed.folding = FoldingForm::None;
477        assert_ne!(changed.epoch(), baseline, "folding must affect the epoch");
478
479        let mut changed = base.clone();
480        changed.decompose_words = !changed.decompose_words;
481        assert_ne!(changed.epoch(), baseline, "decomposition must affect it");
482
483        let mut changed = base.clone();
484        changed.emit_catenation = !changed.emit_catenation;
485        assert_ne!(changed.epoch(), baseline, "catenation must affect it");
486
487        let mut changed = base.clone();
488        changed.number_words = !changed.number_words;
489        assert_ne!(changed.epoch(), baseline, "number words must affect it");
490
491        let mut changed = base;
492        changed.profile = Profile::Code;
493        assert_ne!(changed.epoch(), baseline, "profile must affect the epoch");
494    }
495
496    #[test]
497    fn removing_a_vocab_entry_changes_the_epoch_digest() {
498        let mut vocabulary = Vocabulary::new();
499        vocabulary
500            .declare("put_if_match", &[&["put", "if", "match"][..]])
501            .expect("valid declaration");
502        let with_entry = TokenizerConfig::text_default().with_vocabulary(vocabulary.clone());
503        let populated = with_entry.epoch();
504
505        vocabulary.remove("put_if_match");
506        let without_entry = TokenizerConfig::text_default().with_vocabulary(vocabulary);
507        assert_ne!(populated, without_entry.epoch());
508        assert_eq!(
509            without_entry.epoch(),
510            TokenizerConfig::text_default().epoch()
511        );
512    }
513
514    #[test]
515    fn the_epoch_does_not_depend_on_vocabulary_insertion_order() {
516        let mut first = Vocabulary::new();
517        first
518            .declare("alpha", &[&["a", "one"][..]])
519            .expect("valid declaration");
520        first
521            .declare("beta", &[&["b", "two"][..]])
522            .expect("valid declaration");
523
524        let mut second = Vocabulary::new();
525        second
526            .declare("beta", &[&["b", "two"][..]])
527            .expect("valid declaration");
528        second
529            .declare("alpha", &[&["a", "one"][..]])
530            .expect("valid declaration");
531
532        assert_eq!(
533            TokenizerConfig::text_default()
534                .with_vocabulary(first)
535                .epoch(),
536            TokenizerConfig::text_default()
537                .with_vocabulary(second)
538                .epoch()
539        );
540    }
541
542    #[test]
543    fn the_epoch_hex_form_is_stable_and_sixteen_digits() {
544        let epoch = TokenizerConfig::text_default().epoch();
545        let hex = epoch.to_hex();
546        assert_eq!(hex.len(), 16);
547        assert_eq!(hex, epoch.to_string());
548        assert!(hex.chars().all(|value| value.is_ascii_hexdigit()));
549    }
550
551    #[test]
552    fn adding_a_vocab_entry_unifies_all_three_surface_forms_at_one_position() {
553        let mut vocabulary = Vocabulary::new();
554        vocabulary
555            .declare(
556                "put_if_match",
557                &[&["put", "if", "match"][..], &["putifmatch"][..]],
558            )
559            .expect("valid declaration");
560        let analyzer = Analyzer::new(TokenizerConfig::text_default().with_vocabulary(vocabulary))
561            .expect("valid config");
562
563        for surface in ["put_if_match", "put if match", "putifmatch"] {
564            let tokens = analyzer.analyze(surface);
565            let canonical = tokens
566                .iter()
567                .find(|token| token.term == "put_if_match")
568                .unwrap_or_else(|| panic!("{surface:?} did not emit the canonical term"));
569            assert_eq!(
570                canonical.position, 0,
571                "{surface:?} emitted the canonical term off position zero"
572            );
573        }
574    }
575
576    #[test]
577    fn identifier_and_number_tokens_carry_the_no_fuzzy_flag() {
578        let tokens = analyzer().analyze("ticket i-485 filed 2026 by anup");
579        for token in &tokens {
580            let expected =
581                token.term.contains('-') || token.term.chars().any(|value| value.is_ascii_digit());
582            if expected {
583                assert!(
584                    token.flags.contains(TokenFlags::NO_FUZZY),
585                    "{:?} must be flagged no_fuzzy",
586                    token.term
587                );
588            }
589        }
590        assert!(tokens.iter().any(|token| token.term == "i-485"));
591        let numeral = analyzer().analyze("\u{2160}");
592        assert_eq!(numeral.len(), 1);
593        assert!(numeral[0].flags.contains(TokenFlags::NO_FUZZY));
594    }
595
596    #[test]
597    fn plain_words_are_not_flagged_no_fuzzy() {
598        let tokens = analyzer().analyze("engine");
599        assert_eq!(terms(&tokens), vec!["engin"]);
600        assert!(
601            !tokens
602                .first()
603                .expect("one token")
604                .flags
605                .contains(TokenFlags::NO_FUZZY)
606        );
607    }
608
609    #[test]
610    fn oversized_input_is_refused_rather_than_truncated() {
611        // The limit is u32::MAX; constructing it is wasteful, so this only
612        // pins that the typed error exists and names both numbers.
613        let error = TokenizerError::TextTooLong {
614            bytes: MAX_TEXT_BYTES + 1,
615            limit: MAX_TEXT_BYTES,
616        };
617        assert!(error.to_string().contains("over the"));
618    }
619
620    #[test]
621    fn token_offsets_slice_back_to_the_surface_form() {
622        let text = "Café put_if_match";
623        for token in analyzer().analyze(text) {
624            assert!(
625                token.offset.slice(text).is_some(),
626                "offset for {:?} is not a character boundary",
627                token.term
628            );
629        }
630    }
631
632    #[test]
633    fn flag_algebra_is_consistent() {
634        let both = TokenFlags::NO_FUZZY.union(TokenFlags::VARIANT);
635        assert!(both.contains(TokenFlags::NO_FUZZY));
636        assert!(both.contains(TokenFlags::VARIANT));
637        assert!(!TokenFlags::NO_FUZZY.contains(TokenFlags::VARIANT));
638        assert!(TokenFlags::empty().bits() == 0);
639        assert!(both.contains(TokenFlags::empty()));
640    }
641}
zeppelin_embed/fts/tokenizer/mod.rs

zeppelin_embed/fts/tokenizer/
mod.rs
