zeppelin_embed/fts/tokenizer/
profiles.rs

1//! Named analysis profiles.
2//!
3//! A profile is a starting configuration, not a separate code path: every
4//! profile runs the same pipeline with different knobs, and the profile id
5//! is itself an epoch-digest input so two profiles never collide.
6
7use super::stopwords::StopwordList;
8use super::vocab::Vocabulary;
9use super::{FoldingForm, Segmenter, Stemmer, TokenizerConfig};
10
11/// A named starting configuration.
12#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
13#[repr(u16)]
14pub enum Profile {
15    /// General-purpose English text.
16    ///
17    /// Stemming and stopwords on, matching the Lucene `EnglishAnalyzer`
18    /// shape that the task-13 BEIR targets were produced with.
19    TextDefault = 1,
20    /// Source code, identifiers, tickets, and SKUs.
21    ///
22    /// Stemming and stopwords off: stemming an identifier produces a term
23    /// nobody queries, and `no`/`not`/`if` are meaningful in code.
24    Code = 2,
25    /// Spoken text and transcripts.
26    ///
27    /// Number-word normalization matters most here, because a transcript
28    /// spells what a document digitizes.
29    Voice = 3,
30}
31
32impl Profile {
33    /// Returns the permanent numeric identifier.
34    #[must_use]
35    pub const fn id(self) -> u16 {
36        self as u16
37    }
38
39    /// Returns the profile's starting configuration.
40    #[must_use]
41    pub fn config(self) -> TokenizerConfig {
42        let (stopwords, stemmer, number_words) = match self {
43            Self::TextDefault => (StopwordList::LuceneEnglish, Stemmer::EnglishPorter2, true),
44            Self::Code => (StopwordList::None, Stemmer::None, false),
45            Self::Voice => (StopwordList::LuceneEnglish, Stemmer::EnglishPorter2, true),
46        };
47        // Linguistic profiles drop one-character decomposition parts --
48        // stop-level noise in prose. The code profile keeps them: the `x`
49        // of `x_max` is a real search target in an identifier.
50        let drop_single_char_parts = !matches!(self, Self::Code);
51        TokenizerConfig {
52            profile: self,
53            segmenter: Segmenter::IdentifierPreserving,
54            folding: FoldingForm::NfkcSearchFold,
55            stopwords,
56            stemmer,
57            decompose_words: true,
58            emit_catenation: true,
59            number_words,
60            drop_single_char_parts,
61            vocabulary: Vocabulary::new(),
62        }
63    }
64}
65
66#[cfg(test)]
67#[allow(clippy::expect_used, clippy::indexing_slicing, clippy::unwrap_used)]
68mod tests {
69    use super::*;
70
71    #[test]
72    fn every_profile_has_a_distinct_epoch() {
73        let epochs = [Profile::TextDefault, Profile::Code, Profile::Voice]
74            .map(|profile| profile.config().epoch());
75        assert_ne!(epochs[0], epochs[1]);
76        assert_ne!(epochs[1], epochs[2]);
77        assert_ne!(epochs[0], epochs[2]);
78    }
79
80    #[test]
81    fn the_code_profile_neither_stems_nor_drops_stopwords() {
82        let config = Profile::Code.config();
83        assert_eq!(config.stemmer, Stemmer::None);
84        assert_eq!(config.stopwords, StopwordList::None);
85    }
86
87    #[test]
88    fn profile_ids_are_stable() {
89        assert_eq!(Profile::TextDefault.id(), 1);
90        assert_eq!(Profile::Code.id(), 2);
91        assert_eq!(Profile::Voice.id(), 3);
92    }
93}
zeppelin_embed/fts/tokenizer/profiles.rs

zeppelin_embed/fts/tokenizer/
profiles.rs
