nodedb_fts/index/
analyzer_config.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Per-collection analyzer configuration stored in backend metadata.
4//!
5//! Uses structural `(tid, collection, subkey)` meta blobs:
6//! - `subkey = "analyzer"` → analyzer name (e.g. "german", "cjk_bigram")
7//! - `subkey = "language"` → lang code (e.g. "de", "ja")
8//!
9//! Applied automatically at both index time and query time.
10
11use crate::analyzer::language::stemmer::{LanguageAnalyzer, NoStemAnalyzer};
12use crate::analyzer::pipeline::{TextAnalyzer, analyze};
13use crate::analyzer::standard::StandardAnalyzer;
14use crate::backend::FtsBackend;
15use crate::index::FtsIndex;
16
17impl<B: FtsBackend> FtsIndex<B> {
18    /// Set the analyzer for a collection. Persists to backend metadata.
19    pub fn set_collection_analyzer(
20        &self,
21        database_id: u64,
22        tid: u64,
23        collection: &str,
24        analyzer_name: &str,
25    ) -> Result<(), B::Error> {
26        self.backend.write_meta(
27            database_id,
28            tid,
29            collection,
30            "analyzer",
31            analyzer_name.as_bytes(),
32        )
33    }
34
35    /// Set the language for a collection. Persists to backend metadata.
36    pub fn set_collection_language(
37        &self,
38        database_id: u64,
39        tid: u64,
40        collection: &str,
41        lang_code: &str,
42    ) -> Result<(), B::Error> {
43        self.backend.write_meta(
44            database_id,
45            tid,
46            collection,
47            "language",
48            lang_code.as_bytes(),
49        )
50    }
51
52    /// Get the configured analyzer name for a collection.
53    pub fn get_collection_analyzer(
54        &self,
55        database_id: u64,
56        tid: u64,
57        collection: &str,
58    ) -> Result<Option<String>, B::Error> {
59        match self
60            .backend
61            .read_meta(database_id, tid, collection, "analyzer")?
62        {
63            Some(bytes) => Ok(std::str::from_utf8(&bytes).ok().map(String::from)),
64            None => Ok(None),
65        }
66    }
67
68    /// Get the configured language for a collection.
69    pub fn get_collection_language(
70        &self,
71        database_id: u64,
72        tid: u64,
73        collection: &str,
74    ) -> Result<Option<String>, B::Error> {
75        match self
76            .backend
77            .read_meta(database_id, tid, collection, "language")?
78        {
79            Some(bytes) => Ok(std::str::from_utf8(&bytes).ok().map(String::from)),
80            None => Ok(None),
81        }
82    }
83
84    /// Analyze text using the collection's configured analyzer.
85    ///
86    /// Falls back to the standard English analyzer if no analyzer is configured.
87    pub fn analyze_for_collection(
88        &self,
89        database_id: u64,
90        tid: u64,
91        collection: &str,
92        text: &str,
93    ) -> Result<Vec<String>, B::Error> {
94        let analyzer_name = self.get_collection_analyzer(database_id, tid, collection)?;
95        match analyzer_name.as_deref() {
96            Some(name) => Ok(resolve_analyzer(name).analyze(text)),
97            None => Ok(analyze(text)),
98        }
99    }
100
101    /// Tokenize text WITHOUT stemming for fuzzy matching.
102    pub fn tokenize_raw_for_collection(
103        &self,
104        database_id: u64,
105        tid: u64,
106        collection: &str,
107        text: &str,
108    ) -> Result<Vec<String>, B::Error> {
109        let lang = self.get_collection_language(database_id, tid, collection)?;
110        let lang_code = lang.as_deref().unwrap_or("en");
111        let stop_list = crate::analyzer::language::stop_words::stop_words(lang_code);
112        Ok(crate::analyzer::pipeline::tokenize_raw(
113            text, lang_code, stop_list,
114        ))
115    }
116}
117
118/// Resolve an analyzer name to a `Box<dyn TextAnalyzer>`.
119fn resolve_analyzer(name: &str) -> Box<dyn TextAnalyzer> {
120    match name {
121        "standard" => Box::new(StandardAnalyzer),
122        _ => {
123            if let Some(a) = LanguageAnalyzer::new(name) {
124                Box::new(a)
125            } else if let Some(a) = NoStemAnalyzer::new(name) {
126                Box::new(a)
127            } else {
128                Box::new(StandardAnalyzer)
129            }
130        }
131    }
132}
133
134#[cfg(test)]
135mod tests {
136    use crate::backend::memory::MemoryBackend;
137    use crate::index::FtsIndex;
138
139    const DB: u64 = 0;
140    const T: u64 = 1;
141
142    #[test]
143    fn default_analyzer() {
144        let idx = FtsIndex::new(MemoryBackend::new());
145        let tokens = idx
146            .analyze_for_collection(DB, T, "col", "The quick brown fox")
147            .unwrap();
148        assert!(tokens.contains(&"quick".to_string()));
149        assert!(!tokens.contains(&"the".to_string()));
150    }
151
152    #[test]
153    fn configured_german_analyzer() {
154        let idx = FtsIndex::new(MemoryBackend::new());
155        idx.set_collection_analyzer(DB, T, "col", "german").unwrap();
156
157        let tokens = idx
158            .analyze_for_collection(DB, T, "col", "Die Datenbanken sind schnell")
159            .unwrap();
160        assert!(!tokens.iter().any(|t| t == "die" || t == "sind"));
161        assert!(!tokens.is_empty());
162    }
163
164    #[test]
165    fn configured_hindi_no_stem() {
166        let idx = FtsIndex::new(MemoryBackend::new());
167        idx.set_collection_analyzer(DB, T, "col", "hindi").unwrap();
168
169        let tokens = idx
170            .analyze_for_collection(DB, T, "col", "यह एक परीक्षा है")
171            .unwrap();
172        assert!(!tokens.iter().any(|t| t == "यह" || t == "है"));
173    }
174
175    #[test]
176    fn analyzer_persists() {
177        let idx = FtsIndex::new(MemoryBackend::new());
178        idx.set_collection_analyzer(DB, T, "col", "french").unwrap();
179        idx.set_collection_language(DB, T, "col", "fr").unwrap();
180
181        assert_eq!(
182            idx.get_collection_analyzer(DB, T, "col")
183                .unwrap()
184                .as_deref(),
185            Some("french")
186        );
187        assert_eq!(
188            idx.get_collection_language(DB, T, "col")
189                .unwrap()
190                .as_deref(),
191            Some("fr")
192        );
193    }
194}
nodedb_fts/index/analyzer_config.rs

nodedb_fts/index/
analyzer_config.rs
