riffdb_types/
text_analyzer.rs

1//! ADR-0173's closed, versioned tokenized-text analyzers.
2//!
3//! Analyzer output becomes durable index identity. Both profiles are therefore
4//! frozen functions: changing tokenization or normalization adds a new profile
5//! and rebuilds the index; it never changes either v1 implementation in place.
6
7use unicode_segmentation::{UnicodeSegmentation, UnicodeWords};
8
9use crate::unicode_fold_v1;
10
11/// Unicode revision frozen by both `standard_v1` segmentation and folding.
12pub const TEXT_ANALYZER_V1_UNICODE_VERSION: (u64, u64, u64) = (17, 0, 0);
13/// Immutable semantic schema digest of tokenized provider-state V1.
14pub const TOKENIZED_TEXT_PROVIDER_STATE_SCHEMA_HASH_V1: [u8; 32] = [
15    0x8e, 0x32, 0x7e, 0x0e, 0x53, 0xa3, 0x01, 0xe0, 0x3c, 0xe3, 0xbe, 0x04, 0xec, 0xe5, 0x7c, 0x3a,
16    0x94, 0x87, 0x83, 0x94, 0x9a, 0xc1, 0x01, 0x3d, 0x52, 0x16, 0x7f, 0x88, 0xd1, 0xe4, 0xd3, 0x0c,
17];
18
19/// Closed tokenized-text analyzer vocabulary for ADR-0173 v1.
20#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
21#[repr(u8)]
22pub enum TextAnalyzerV1 {
23    /// Preserve the complete source value as exactly one term.
24    KeywordV1 = 1,
25    /// Unicode 17.0.0 UAX #29 word segmentation, then `unicode_fold_v1`.
26    StandardV1 = 2,
27}
28
29/// Closed result model admitted by the declaration stage.
30///
31/// Ranking is a later separately shippable stage. Keeping the model explicit
32/// prevents that stage from silently changing an existing index identity.
33#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
34#[repr(u8)]
35pub enum TextSearchResultModelV1 {
36    /// Exact boolean membership with no relevance ordering.
37    BooleanV1 = 1,
38}
39
40impl TextSearchResultModelV1 {
41    /// Canonical contract spelling.
42    #[must_use]
43    pub const fn as_str(self) -> &'static str {
44        match self {
45            Self::BooleanV1 => "boolean_v1",
46        }
47    }
48}
49
50impl TextAnalyzerV1 {
51    /// Decodes the byte-frozen analyzer discriminant.
52    #[must_use]
53    pub const fn from_discriminant(value: u8) -> Option<Self> {
54        match value {
55            1 => Some(Self::KeywordV1),
56            2 => Some(Self::StandardV1),
57            _ => None,
58        }
59    }
60
61    /// Canonical contract spelling.
62    #[must_use]
63    pub const fn as_str(self) -> &'static str {
64        match self {
65            Self::KeywordV1 => "keyword_v1",
66            Self::StandardV1 => "standard_v1",
67        }
68    }
69
70    /// Analyzes one already-bounded source value without retaining the result.
71    ///
72    /// `keyword_v1` emits once even for the empty string. `standard_v1`
73    /// segments before folding, as ADR-0173 specifies, and assigns consecutive
74    /// zero-based positions to emitted words.
75    #[must_use]
76    pub fn analyze(self, value: &str) -> TextAnalyzerTermsV1<'_> {
77        match self {
78            Self::KeywordV1 => TextAnalyzerTermsV1 {
79                inner: TextAnalyzerTermsInner::Keyword(Some(value)),
80            },
81            Self::StandardV1 => TextAnalyzerTermsV1 {
82                inner: TextAnalyzerTermsInner::Standard(value.unicode_words().enumerate()),
83            },
84        }
85    }
86}
87
88/// One analyzed term and its zero-based token position.
89#[derive(Clone, Debug, Eq, PartialEq)]
90pub struct AnalyzedTermV1 {
91    position: usize,
92    term: String,
93}
94
95impl AnalyzedTermV1 {
96    /// Zero-based token position in analyzer output.
97    #[must_use]
98    pub const fn position(&self) -> usize {
99        self.position
100    }
101
102    /// Exact analyzed term stored in postings.
103    #[must_use]
104    pub fn term(&self) -> &str {
105        &self.term
106    }
107}
108
109/// Streaming analyzed terms for one source value.
110#[derive(Debug)]
111pub struct TextAnalyzerTermsV1<'a> {
112    inner: TextAnalyzerTermsInner<'a>,
113}
114
115#[derive(Debug)]
116enum TextAnalyzerTermsInner<'a> {
117    Keyword(Option<&'a str>),
118    Standard(std::iter::Enumerate<UnicodeWords<'a>>),
119}
120
121impl Iterator for TextAnalyzerTermsV1<'_> {
122    type Item = AnalyzedTermV1;
123
124    fn next(&mut self) -> Option<Self::Item> {
125        match &mut self.inner {
126            TextAnalyzerTermsInner::Keyword(value) => value.take().map(|value| AnalyzedTermV1 {
127                position: 0,
128                term: value.to_owned(),
129            }),
130            TextAnalyzerTermsInner::Standard(words) => {
131                words.next().map(|(position, word)| AnalyzedTermV1 {
132                    position,
133                    term: unicode_fold_v1(word),
134                })
135            }
136        }
137    }
138}
139
140// Exhaustive non-durable ownership inventories for exact-provider admission.
141crate::inspect_owned_capacity_copy_v1!(TextAnalyzerV1);
142
143#[cfg(test)]
144mod tests {
145    use super::TEXT_ANALYZER_V1_UNICODE_VERSION;
146
147    #[test]
148    fn segmentation_and_folding_use_the_same_unicode_revision() {
149        assert_eq!(
150            unicode_segmentation::UNICODE_VERSION,
151            TEXT_ANALYZER_V1_UNICODE_VERSION,
152            "tokenizer tables moved; add a new analyzer identity instead"
153        );
154        assert_eq!(
155            crate::UNICODE_FOLD_V1_UNICODE_VERSION,
156            (17, 0, 0),
157            "fold and tokenizer revisions must remain aligned"
158        );
159    }
160}
161
162crate::inspect_owned_capacity_copy_v1!(TextSearchResultModelV1);
riffdb_types/text_analyzer.rs

riffdb_types/
text_analyzer.rs
