zeppelin_embed/fts/tokenizer/
mod.rs

1//! The versioned analysis pipeline and its epoch identity.
2//!
3//! # The product feature
4//!
5//! Text goes in; tokens come out carrying a term, a position, and the byte
6//! range they came from. The pipeline that produces them — segmentation,
7//! folding, stopwords, stemming, number words, and a user vocabulary — is
8//! identified by one digest, the **tokenizer epoch**. Same epoch, same bytes
9//! in, same tokens out, forever.
10//!
11//! That identity is the point. Analyzer drift silently invalidating an index
12//! is failure class U11 (`research/03:594`), and no off-the-shelf tokenizer
13//! versions its own behaviour as index metadata (`research/05:202`). The
14//! digest goes into `EpochMeta.tokenizer`, which task 07 already reserved,
15//! so no format change is needed to carry it.
16//!
17//! # Query-time and index-time are one code path
18//!
19//! [`Analyzer::analyze`] is used for both. tantivy's missing search-analyzer
20//! is a known gap this design avoids by construction. A query-side extras
21//! hook is deliberately absent in v1 rather than stubbed.
22
23pub(crate) mod fold;
24mod fold_table;
25pub(crate) mod numbers;
26mod pipeline;
27pub(crate) use pipeline::ControlledToken;
28pub mod profiles;
29#[cfg(test)]
30mod properties;
31pub(crate) mod segment;
32pub(crate) mod stemmer;
33pub mod stopwords;
34pub mod vocab;
35
36pub use profiles::Profile;
37pub use stopwords::StopwordList;
38pub use vocab::{MAX_SURFACE_TERMS, VocabError, Vocabulary};
39
40/// Bit flags carried by every token.
41///
42/// Task 15 consumes these; defining the byte now avoids a format-shaped
43/// change later (task 12 D5).
44#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
45pub struct TokenFlags(u8);
46
47impl TokenFlags {
48    /// The token must never be fuzzy-matched: an identifier, number, or SKU.
49    ///
50    /// A fuzzy match on `i-485` or a ticket id is almost always wrong.
51    pub const NO_FUZZY: Self = Self(1);
52    /// The token is a stacked variant, not a surface word.
53    ///
54    /// Variants come from the vocabulary, number normalization, or word
55    /// decomposition. Diagnostics use this so a match can always explain
56    /// which form produced it.
57    pub const VARIANT: Self = Self(2);
58
59    /// Returns the empty flag set.
60    #[must_use]
61    pub const fn empty() -> Self {
62        Self(0)
63    }
64
65    /// Returns the raw bits.
66    #[must_use]
67    pub const fn bits(self) -> u8 {
68        self.0
69    }
70
71    /// Returns true when every bit of `other` is set.
72    #[must_use]
73    pub const fn contains(self, other: Self) -> bool {
74        self.0 & other.0 == other.0
75    }
76
77    /// Returns the union of two flag sets.
78    #[must_use]
79    pub const fn union(self, other: Self) -> Self {
80        Self(self.0 | other.0)
81    }
82}
83
84/// The byte range a token came from, into the analyzed text.
85#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
86pub struct TokenOffset {
87    /// Inclusive start byte offset.
88    pub start: u32,
89    /// Exclusive end byte offset.
90    pub end: u32,
91}
92
93impl TokenOffset {
94    /// Slices the token's surface form back out of the analyzed text.
95    ///
96    /// Returns `None` when the range is not a character boundary, which the
97    /// pipeline never produces but a hand-built offset might.
98    #[must_use]
99    pub fn slice<'text>(&self, text: &'text str) -> Option<&'text str> {
100        let start = usize::try_from(self.start).ok()?;
101        let end = usize::try_from(self.end).ok()?;
102        text.get(start..end)
103    }
104}
105
106/// One analyzed token.
107#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
108pub struct Token {
109    /// The indexed term.
110    pub term: String,
111    /// The token position; stacked variants share a position.
112    pub position: u32,
113    /// The byte range the token came from.
114    pub offset: TokenOffset,
115    /// Token flags.
116    pub flags: TokenFlags,
117}
118
119/// Word segmentation strategy; an epoch-digest input.
120#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
121#[repr(u16)]
122pub enum Segmenter {
123    /// Identifier-preserving segmentation with CJK bigram fallback.
124    IdentifierPreserving = 1,
125}
126
127/// Character folding form; an epoch-digest input.
128#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
129#[repr(u16)]
130pub enum FoldingForm {
131    /// No folding at all; terms keep their original case and marks.
132    None = 0,
133    /// NFKC, case folding, and diacritic removal from the generated table.
134    NfkcSearchFold = 1,
135}
136
137/// Stemming algorithm; an epoch-digest input.
138#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
139#[repr(u16)]
140pub enum Stemmer {
141    /// No stemming.
142    None = 0,
143    /// English Snowball (Porter2).
144    EnglishPorter2 = 1,
145}
146
147/// Analysis rejection at the engine boundary.
148#[derive(Clone, Debug, Eq, PartialEq)]
149pub enum TokenizerError {
150    /// The supplied bytes were not valid UTF-8.
151    InvalidUtf8 {
152        /// Byte offset of the first invalid sequence.
153        offset: usize,
154    },
155    /// The analyzed text exceeded [`MAX_TEXT_BYTES`].
156    TextTooLong {
157        /// The rejected byte length.
158        bytes: usize,
159        /// The configured limit.
160        limit: usize,
161    },
162    /// The vocabulary was rejected.
163    Vocab(VocabError),
164}
165
166impl std::fmt::Display for TokenizerError {
167    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
168        match self {
169            Self::InvalidUtf8 { offset } => {
170                write!(formatter, "input is not valid UTF-8 at byte {offset}")
171            }
172            Self::TextTooLong { bytes, limit } => write!(
173                formatter,
174                "analyzed text is {bytes} bytes, over the {limit}-byte limit"
175            ),
176            Self::Vocab(error) => write!(formatter, "vocabulary rejected: {error}"),
177        }
178    }
179}
180
181impl std::error::Error for TokenizerError {}
182
183impl From<VocabError> for TokenizerError {
184    fn from(error: VocabError) -> Self {
185        Self::Vocab(error)
186    }
187}
188
189/// The largest text one call will analyze.
190///
191/// Offsets are `u32`, so this bound is what keeps every offset
192/// representable. It is not a truncation limit: task 13 forbids truncating
193/// documents before indexing, so an oversized field is a typed rejection the
194/// caller must split, never a silent cut.
195pub const MAX_TEXT_BYTES: usize = u32::MAX as usize;
196
197/// The complete analysis configuration.
198///
199/// Every field is an epoch-digest input. Changing any one of them produces a
200/// different epoch and therefore a different index meaning.
201#[derive(Clone, Debug, Eq, PartialEq)]
202pub struct TokenizerConfig {
203    /// The named profile this configuration started from.
204    pub profile: Profile,
205    /// Word segmentation strategy.
206    pub segmenter: Segmenter,
207    /// Character folding form.
208    pub folding: FoldingForm,
209    /// Stopword list.
210    pub stopwords: StopwordList,
211    /// Stemming algorithm.
212    pub stemmer: Stemmer,
213    /// Split words at joiners, case changes, and letter/digit transitions.
214    pub decompose_words: bool,
215    /// Emit the joiner-free catenation of a decomposed word.
216    pub emit_catenation: bool,
217    /// Emit digit and word variants for spelled numbers.
218    pub number_words: bool,
219    /// Drop single-LETTER decomposition parts.
220    ///
221    /// The `k` of `401k`, the `t` of `don't`, the `s` of `investor's`:
222    /// one-letter parts of a decomposed word are stop-level noise in
223    /// linguistic text — they carry no identity the whole word and the
224    /// catenation do not already carry, and they pollute document length
225    /// and document frequency. One-DIGIT parts are kept: `type-2` and
226    /// `SARS-CoV-2` are discriminated by exactly that digit. Dropping a
227    /// part leaves its position spent, exactly as a removed stopword
228    /// does, so phrase adjacency and the analyzed length are unchanged.
229    /// Identifier-heavy profiles keep every part: the `x` of `x_max` is
230    /// a real search target in code.
231    pub drop_single_char_parts: bool,
232    /// User vocabulary and synonyms.
233    pub vocabulary: Vocabulary,
234}
235
236impl TokenizerConfig {
237    /// The general-purpose text profile.
238    #[must_use]
239    pub fn text_default() -> Self {
240        Profile::TextDefault.config()
241    }
242
243    /// The identifier-heavy profile for source code and tickets.
244    #[must_use]
245    pub fn code() -> Self {
246        Profile::Code.config()
247    }
248
249    /// The spoken-text profile.
250    #[must_use]
251    pub fn voice() -> Self {
252        Profile::Voice.config()
253    }
254
255    /// Replaces the user vocabulary.
256    #[must_use]
257    pub fn with_vocabulary(mut self, vocabulary: Vocabulary) -> Self {
258        self.vocabulary = vocabulary;
259        self
260    }
261
262    /// Computes the epoch digest for this configuration.
263    #[must_use]
264    pub fn epoch(&self) -> TokenizerEpoch {
265        TokenizerEpoch::of(self)
266    }
267}
268
269/// A tokenizer epoch: the identity of one analysis behaviour.
270///
271/// Stored as a little-endian u64 in `EpochMeta.tokenizer`.
272#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
273pub struct TokenizerEpoch(u64);
274
275/// Digest-input framing magic. Changing it changes every epoch.
276const EPOCH_MAGIC: &[u8; 8] = b"ZETOKEP1";
277
278impl TokenizerEpoch {
279    pub(crate) const fn from_value(value: u64) -> Self {
280        Self(value)
281    }
282
283    /// Computes the epoch of a configuration.
284    ///
285    /// # The canonical digest input
286    ///
287    /// Little-endian throughout, length-prefixed for every variable field,
288    /// in exactly this order: magic, segmenter id, folding form id, folding
289    /// table version, Unicode version string, stopword list id, stopword
290    /// term count, stemmer id, the three boolean knobs, profile id, then the
291    /// vocabulary. The vocabulary contributes its entry count followed by
292    /// every (surface form, canonical term) pair in sorted order, so two
293    /// vocabularies built by different insertion orders digest identically.
294    ///
295    /// This layout is a persisted-meaning contract frozen by
296    /// `golden_streams_cannot_change_without_an_epoch_bump`.
297    #[must_use]
298    pub fn of(config: &TokenizerConfig) -> Self {
299        let mut input: Vec<u8> = Vec::with_capacity(256);
300        input.extend_from_slice(EPOCH_MAGIC);
301        push_u16(&mut input, config.segmenter as u16);
302        push_u16(&mut input, config.folding as u16);
303        push_u16(&mut input, fold::FOLDING_TABLE_VERSION);
304        push_str(&mut input, fold::FOLDING_UNICODE_VERSION);
305        push_u16(&mut input, config.stopwords.id());
306        push_u32(
307            &mut input,
308            u32::try_from(config.stopwords.terms().len()).unwrap_or(u32::MAX),
309        );
310        push_u16(&mut input, config.stemmer as u16);
311        input.push(u8::from(config.decompose_words));
312        input.push(u8::from(config.emit_catenation));
313        input.push(u8::from(config.number_words));
314        input.push(u8::from(config.drop_single_char_parts));
315        push_u16(&mut input, config.profile.id());
316        push_u32(
317            &mut input,
318            u32::try_from(config.vocabulary.len()).unwrap_or(u32::MAX),
319        );
320        for (surface, canonical) in config.vocabulary.entries() {
321            push_u32(&mut input, u32::try_from(surface.len()).unwrap_or(u32::MAX));
322            for term in surface {
323                push_str(&mut input, term);
324            }
325            push_str(&mut input, canonical);
326        }
327        Self(xxhash_rust::xxh3::xxh3_64(&input))
328    }
329
330    /// Returns the raw digest.
331    #[must_use]
332    pub const fn value(self) -> u64 {
333        self.0
334    }
335
336    /// Returns the lowercase hex form used in diagnostics.
337    #[must_use]
338    pub fn to_hex(self) -> String {
339        format!("{:016x}", self.0)
340    }
341}
342
343impl std::fmt::Display for TokenizerEpoch {
344    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
345        write!(formatter, "{:016x}", self.0)
346    }
347}
348
349fn push_u16(buffer: &mut Vec<u8>, value: u16) {
350    buffer.extend_from_slice(&value.to_le_bytes());
351}
352
353fn push_u32(buffer: &mut Vec<u8>, value: u32) {
354    buffer.extend_from_slice(&value.to_le_bytes());
355}
356
357fn push_str(buffer: &mut Vec<u8>, value: &str) {
358    push_u32(buffer, u32::try_from(value.len()).unwrap_or(u32::MAX));
359    buffer.extend_from_slice(value.as_bytes());
360}
361
362/// A compiled analyzer.
363///
364/// Construction validates the configuration once so [`Analyzer::analyze`]
365/// cannot fail on well-formed UTF-8.
366#[derive(Clone, Debug)]
367pub struct Analyzer {
368    config: TokenizerConfig,
369    epoch: TokenizerEpoch,
370}
371
372impl Analyzer {
373    /// Compiles a configuration.
374    ///
375    /// # Errors
376    ///
377    /// Returns [`TokenizerError::Vocab`] when the vocabulary is malformed.
378    pub fn new(config: TokenizerConfig) -> Result<Self, TokenizerError> {
379        let epoch = TokenizerEpoch::of(&config);
380        Ok(Self { config, epoch })
381    }
382
383    /// Returns the epoch identifying this analyzer's behaviour.
384    #[must_use]
385    pub const fn epoch(&self) -> TokenizerEpoch {
386        self.epoch
387    }
388
389    /// Returns the configuration.
390    #[must_use]
391    pub const fn config(&self) -> &TokenizerConfig {
392        &self.config
393    }
394
395    /// Analyzes text into tokens.
396    ///
397    /// Index-time and query-time analysis are this same call.
398    #[must_use]
399    pub fn analyze(&self, text: &str) -> Vec<Token> {
400        pipeline::analyze(&self.config, text)
401    }
402
403    pub(crate) fn analyze_with_policy<'m, P: super::control::BuildPolicy<'m>>(
404        &self,
405        text: &str,
406        policy: &mut P,
407    ) -> Result<pipeline::ControlledTokens<'m, P::Charge>, P::Error>
408    where
409        P::Error: From<TokenizerError>,
410    {
411        if text.len() > MAX_TEXT_BYTES {
412            return Err(P::Error::from(TokenizerError::TextTooLong {
413                bytes: text.len(),
414                limit: MAX_TEXT_BYTES,
415            }));
416        }
417        pipeline::analyze_with_policy(&self.config, text, policy)
418    }
419
420    /// Analyzes bytes, rejecting invalid UTF-8 with a typed error.
421    ///
422    /// # Errors
423    ///
424    /// Returns [`TokenizerError::InvalidUtf8`] rather than lossily
425    /// converting, and [`TokenizerError::TextTooLong`] rather than
426    /// truncating.
427    pub fn analyze_bytes(&self, bytes: &[u8]) -> Result<Vec<Token>, TokenizerError> {
428        if bytes.len() > MAX_TEXT_BYTES {
429            return Err(TokenizerError::TextTooLong {
430                bytes: bytes.len(),
431                limit: MAX_TEXT_BYTES,
432            });
433        }
434        let text = std::str::from_utf8(bytes).map_err(|error| TokenizerError::InvalidUtf8 {
435            offset: error.valid_up_to(),
436        })?;
437        Ok(self.analyze(text))
438    }
439}
440
441#[cfg(test)]
442#[allow(
443    clippy::expect_used,
444    clippy::indexing_slicing,
445    clippy::panic,
446    clippy::unwrap_used
447)]
448mod tests {
449    use super::*;
450
451    fn analyzer() -> Analyzer {
452        Analyzer::new(TokenizerConfig::text_default()).expect("valid config")
453    }
454
455    fn terms(tokens: &[Token]) -> Vec<&str> {
456        tokens.iter().map(|token| token.term.as_str()).collect()
457    }
458
459    #[test]
460    fn invalid_utf8_is_rejected_with_a_typed_error_at_the_boundary() {
461        let error = analyzer()
462            .analyze_bytes(&[b'o', b'k', 0xFF, 0xFE])
463            .expect_err("invalid UTF-8 must be refused");
464        assert_eq!(error, TokenizerError::InvalidUtf8 { offset: 2 });
465    }
466
467    #[test]
468    fn valid_utf8_bytes_analyze_identically_to_the_string_path() {
469        let text = "Café put_if_match";
470        let analyzer = analyzer();
471        assert_eq!(
472            analyzer
473                .analyze_bytes(text.as_bytes())
474                .expect("valid UTF-8"),
475            analyzer.analyze(text)
476        );
477    }
478
479    #[test]
480    fn changing_any_pipeline_knob_changes_the_epoch_digest() {
481        let base = TokenizerConfig::text_default();
482        let baseline = base.epoch();
483
484        let mut changed = base.clone();
485        changed.stemmer = Stemmer::None;
486        assert_ne!(changed.epoch(), baseline, "stemmer must affect the epoch");
487
488        let mut changed = base.clone();
489        changed.stopwords = StopwordList::None;
490        assert_ne!(changed.epoch(), baseline, "stopwords must affect the epoch");
491
492        let mut changed = base.clone();
493        changed.folding = FoldingForm::None;
494        assert_ne!(changed.epoch(), baseline, "folding must affect the epoch");
495
496        let mut changed = base.clone();
497        changed.decompose_words = !changed.decompose_words;
498        assert_ne!(changed.epoch(), baseline, "decomposition must affect it");
499
500        let mut changed = base.clone();
501        changed.emit_catenation = !changed.emit_catenation;
502        assert_ne!(changed.epoch(), baseline, "catenation must affect it");
503
504        let mut changed = base.clone();
505        changed.number_words = !changed.number_words;
506        assert_ne!(changed.epoch(), baseline, "number words must affect it");
507
508        let mut changed = base;
509        changed.profile = Profile::Code;
510        assert_ne!(changed.epoch(), baseline, "profile must affect the epoch");
511    }
512
513    #[test]
514    fn removing_a_vocab_entry_changes_the_epoch_digest() {
515        let mut vocabulary = Vocabulary::new();
516        vocabulary
517            .declare("put_if_match", &[&["put", "if", "match"][..]])
518            .expect("valid declaration");
519        let with_entry = TokenizerConfig::text_default().with_vocabulary(vocabulary.clone());
520        let populated = with_entry.epoch();
521
522        vocabulary.remove("put_if_match");
523        let without_entry = TokenizerConfig::text_default().with_vocabulary(vocabulary);
524        assert_ne!(populated, without_entry.epoch());
525        assert_eq!(
526            without_entry.epoch(),
527            TokenizerConfig::text_default().epoch()
528        );
529    }
530
531    #[test]
532    fn the_epoch_does_not_depend_on_vocabulary_insertion_order() {
533        let mut first = Vocabulary::new();
534        first
535            .declare("alpha", &[&["a", "one"][..]])
536            .expect("valid declaration");
537        first
538            .declare("beta", &[&["b", "two"][..]])
539            .expect("valid declaration");
540
541        let mut second = Vocabulary::new();
542        second
543            .declare("beta", &[&["b", "two"][..]])
544            .expect("valid declaration");
545        second
546            .declare("alpha", &[&["a", "one"][..]])
547            .expect("valid declaration");
548
549        assert_eq!(
550            TokenizerConfig::text_default()
551                .with_vocabulary(first)
552                .epoch(),
553            TokenizerConfig::text_default()
554                .with_vocabulary(second)
555                .epoch()
556        );
557    }
558
559    #[test]
560    fn the_epoch_hex_form_is_stable_and_sixteen_digits() {
561        let epoch = TokenizerConfig::text_default().epoch();
562        let hex = epoch.to_hex();
563        assert_eq!(hex.len(), 16);
564        assert_eq!(hex, epoch.to_string());
565        assert!(hex.chars().all(|value| value.is_ascii_hexdigit()));
566    }
567
568    #[test]
569    fn adding_a_vocab_entry_unifies_all_three_surface_forms_at_one_position() {
570        let mut vocabulary = Vocabulary::new();
571        vocabulary
572            .declare(
573                "put_if_match",
574                &[&["put", "if", "match"][..], &["putifmatch"][..]],
575            )
576            .expect("valid declaration");
577        let analyzer = Analyzer::new(TokenizerConfig::text_default().with_vocabulary(vocabulary))
578            .expect("valid config");
579
580        for surface in ["put_if_match", "put if match", "putifmatch"] {
581            let tokens = analyzer.analyze(surface);
582            let canonical = tokens
583                .iter()
584                .find(|token| token.term == "put_if_match")
585                .unwrap_or_else(|| panic!("{surface:?} did not emit the canonical term"));
586            assert_eq!(
587                canonical.position, 0,
588                "{surface:?} emitted the canonical term off position zero"
589            );
590        }
591    }
592
593    #[test]
594    fn identifier_and_number_tokens_carry_the_no_fuzzy_flag() {
595        let tokens = analyzer().analyze("ticket i-485 filed 2026 by anup");
596        for token in &tokens {
597            let expected =
598                token.term.contains('-') || token.term.chars().any(|value| value.is_ascii_digit());
599            if expected {
600                assert!(
601                    token.flags.contains(TokenFlags::NO_FUZZY),
602                    "{:?} must be flagged no_fuzzy",
603                    token.term
604                );
605            }
606        }
607        assert!(tokens.iter().any(|token| token.term == "i-485"));
608        let numeral = analyzer().analyze("\u{2160}");
609        assert_eq!(numeral.len(), 1);
610        assert!(numeral[0].flags.contains(TokenFlags::NO_FUZZY));
611    }
612
613    #[test]
614    fn plain_words_are_not_flagged_no_fuzzy() {
615        let tokens = analyzer().analyze("engine");
616        assert_eq!(terms(&tokens), vec!["engin"]);
617        assert!(
618            !tokens
619                .first()
620                .expect("one token")
621                .flags
622                .contains(TokenFlags::NO_FUZZY)
623        );
624    }
625
626    #[test]
627    fn oversized_input_is_refused_rather_than_truncated() {
628        // The limit is u32::MAX; constructing it is wasteful, so this only
629        // pins that the typed error exists and names both numbers.
630        let error = TokenizerError::TextTooLong {
631            bytes: MAX_TEXT_BYTES + 1,
632            limit: MAX_TEXT_BYTES,
633        };
634        assert!(error.to_string().contains("over the"));
635    }
636
637    #[test]
638    fn token_offsets_slice_back_to_the_surface_form() {
639        let text = "Café put_if_match";
640        for token in analyzer().analyze(text) {
641            assert!(
642                token.offset.slice(text).is_some(),
643                "offset for {:?} is not a character boundary",
644                token.term
645            );
646        }
647    }
648
649    #[test]
650    fn flag_algebra_is_consistent() {
651        let both = TokenFlags::NO_FUZZY.union(TokenFlags::VARIANT);
652        assert!(both.contains(TokenFlags::NO_FUZZY));
653        assert!(both.contains(TokenFlags::VARIANT));
654        assert!(!TokenFlags::NO_FUZZY.contains(TokenFlags::VARIANT));
655        assert!(TokenFlags::empty().bits() == 0);
656        assert!(both.contains(TokenFlags::empty()));
657    }
658}
zeppelin_embed/fts/tokenizer/mod.rs

zeppelin_embed/fts/tokenizer/
mod.rs
