nodedb_fts/index/
synonym_groups.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Tenant-scoped synonym group persistence in the FTS backend.
4//!
5//! Synonym groups are stored in backend meta using a sentinel collection
6//! `"_synonym_groups"` with:
7//! - subkey `"_index"` → JSON array of all group names for the tenant (the index)
8//! - subkey `<group_name>` → JSON-serialized `SynonymGroupRecord`
9//!
10//! At query time, all groups for the tenant are loaded, merged into a
11//! `SynonymMap`, and applied to the query token stream.
12//!
13//! **OR-expansion semantics**: for a group `{a, b, c}`, querying any term
14//! expands to all other terms. A query for `db` with group `{db, database,
15//! datastore}` matches documents containing `database` or `datastore` as
16//! well. This is the only sensible default for synonym search.
17
18use crate::analyzer::synonym::SynonymMap;
19use crate::backend::FtsBackend;
20use crate::index::writer::FtsIndex;
21use crate::scope::IndexScope;
22
23/// Sentinel index whose metadata holds the synonym groups.
24const SYNONYM_GROUPS_COLLECTION: IndexScope<'static> = IndexScope::document("_synonym_groups");
25
26/// Special meta subkey that holds the JSON array of all group names.
27const INDEX_SUBKEY: &str = "_index";
28
29/// Serialized group record: name + terms list.
30#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
31pub struct SynonymGroupRecord {
32    pub name: String,
33    pub terms: Vec<String>,
34    pub created_at: u64,
35}
36
37impl<B: FtsBackend> FtsIndex<B> {
38    /// Persist a synonym group. Overwrites any existing group with the same name.
39    pub fn put_synonym_group(
40        &self,
41        database_id: u64,
42        tid: u64,
43        record: &SynonymGroupRecord,
44    ) -> Result<(), B::Error> {
45        // Write the record itself.
46        let bytes = sonic_rs::to_vec(record).unwrap_or_default();
47        self.backend.write_meta(
48            database_id,
49            tid,
50            SYNONYM_GROUPS_COLLECTION,
51            &record.name,
52            &bytes,
53        )?;
54
55        // Update the name index.
56        let mut names = self.read_name_index(database_id, tid)?;
57        if !names.contains(&record.name) {
58            names.push(record.name.clone());
59            self.write_name_index(database_id, tid, &names)?;
60        }
61        Ok(())
62    }
63
64    /// Delete a synonym group. Returns `true` if it existed.
65    pub fn delete_synonym_group(
66        &self,
67        database_id: u64,
68        tid: u64,
69        name: &str,
70    ) -> Result<bool, B::Error> {
71        // An empty byte slice is a tombstone written by a prior delete.
72        // Only treat a non-empty (non-tombstoned) record as "existing".
73        let existed = self
74            .backend
75            .read_meta(database_id, tid, SYNONYM_GROUPS_COLLECTION, name)?
76            .is_some_and(|b| !b.is_empty());
77
78        if existed {
79            // Tombstone the record.
80            self.backend
81                .write_meta(database_id, tid, SYNONYM_GROUPS_COLLECTION, name, &[])?;
82            // Remove from the name index.
83            let mut names = self.read_name_index(database_id, tid)?;
84            names.retain(|n| n != name);
85            self.write_name_index(database_id, tid, &names)?;
86        }
87        Ok(existed)
88    }
89
90    /// Read a single synonym group by name. Returns `None` if not found or tombstoned.
91    pub fn get_synonym_group(
92        &self,
93        database_id: u64,
94        tid: u64,
95        name: &str,
96    ) -> Result<Option<SynonymGroupRecord>, B::Error> {
97        match self
98            .backend
99            .read_meta(database_id, tid, SYNONYM_GROUPS_COLLECTION, name)?
100        {
101            None => Ok(None),
102            Some(bytes) if bytes.is_empty() => Ok(None),
103            Some(bytes) => Ok(sonic_rs::from_slice::<SynonymGroupRecord>(&bytes).ok()),
104        }
105    }
106
107    /// List all synonym group records for a tenant.
108    pub fn list_synonym_groups(
109        &self,
110        database_id: u64,
111        tid: u64,
112    ) -> Result<Vec<SynonymGroupRecord>, B::Error> {
113        let names = self.read_name_index(database_id, tid)?;
114        let mut groups = Vec::with_capacity(names.len());
115        for name in &names {
116            if let Some(rec) = self.get_synonym_group(database_id, tid, name)? {
117                groups.push(rec);
118            }
119        }
120        Ok(groups)
121    }
122
123    /// Build an in-memory `SynonymMap` from a slice of synonym group records.
124    ///
125    /// Each term in every group maps to all other terms in that group
126    /// (bidirectional OR-expansion). Terms are analyzed with the default
127    /// analyzer so synonym keys match the stemmed tokens produced at query
128    /// time by `search_with_params`.
129    pub fn build_synonym_map_for_tenant(
130        &self,
131        _database_id: u64,
132        _tid: u64,
133        all_groups: &[SynonymGroupRecord],
134    ) -> SynonymMap {
135        let mut map = SynonymMap::new();
136        for group in all_groups {
137            if group.terms.len() < 2 {
138                continue;
139            }
140            // Analyze each term through the same pipeline used at query time
141            // so that synonym keys align with stemmed query tokens.
142            let analyzed: Vec<Vec<String>> = group
143                .terms
144                .iter()
145                .map(|t| crate::analyzer::pipeline::analyze(t))
146                .collect();
147
148            for (i, my_tokens) in analyzed.iter().enumerate() {
149                let other_tokens: Vec<&str> = analyzed
150                    .iter()
151                    .enumerate()
152                    .filter(|(j, _)| *j != i)
153                    .flat_map(|(_, ts)| ts.iter().map(|s| s.as_str()))
154                    .collect();
155                for my_token in my_tokens {
156                    map.add(my_token, &other_tokens);
157                }
158            }
159        }
160        map
161    }
162
163    /// Load all synonym groups for a tenant and build the expansion map.
164    ///
165    /// Called at FTS query time inside `search_with_params` to expand query
166    /// tokens before BM25 scoring.
167    pub fn expand_query_with_synonyms(
168        &self,
169        database_id: u64,
170        tid: u64,
171        tokens: Vec<String>,
172    ) -> Result<Vec<String>, B::Error> {
173        let groups = self.list_synonym_groups(database_id, tid)?;
174        if groups.is_empty() {
175            return Ok(tokens);
176        }
177        let map = self.build_synonym_map_for_tenant(database_id, tid, &groups);
178        let expanded = map.expand(&tokens);
179        Ok(expanded)
180    }
181
182    /// Expand each analyzed query token into its word group: the token
183    /// first, then its distinct synonyms. One group per token, in token
184    /// order. A document matches a word when it holds any term of the group.
185    pub fn expand_query_groups(
186        &self,
187        database_id: u64,
188        tid: u64,
189        tokens: Vec<String>,
190    ) -> Result<Vec<Vec<String>>, B::Error> {
191        let groups = self.list_synonym_groups(database_id, tid)?;
192        if groups.is_empty() {
193            return Ok(tokens.into_iter().map(|token| vec![token]).collect());
194        }
195        let map = self.build_synonym_map_for_tenant(database_id, tid, &groups);
196        Ok(tokens
197            .into_iter()
198            .map(|token| {
199                let mut group = map.expand(std::slice::from_ref(&token));
200                let mut seen = std::collections::HashSet::new();
201                group.retain(|term| seen.insert(term.clone()));
202                group
203            })
204            .collect())
205    }
206
207    // ── internal helpers ──────────────────────────────────────────────────────
208
209    fn read_name_index(&self, database_id: u64, tid: u64) -> Result<Vec<String>, B::Error> {
210        match self
211            .backend
212            .read_meta(database_id, tid, SYNONYM_GROUPS_COLLECTION, INDEX_SUBKEY)?
213        {
214            None => Ok(Vec::new()),
215            Some(bytes) if bytes.is_empty() => Ok(Vec::new()),
216            Some(bytes) => Ok(sonic_rs::from_slice::<Vec<String>>(&bytes).unwrap_or_default()),
217        }
218    }
219
220    fn write_name_index(
221        &self,
222        database_id: u64,
223        tid: u64,
224        names: &[String],
225    ) -> Result<(), B::Error> {
226        let bytes = sonic_rs::to_vec(names).unwrap_or_default();
227        self.backend.write_meta(
228            database_id,
229            tid,
230            SYNONYM_GROUPS_COLLECTION,
231            INDEX_SUBKEY,
232            &bytes,
233        )
234    }
235}
236
237#[cfg(test)]
238mod tests {
239    use crate::backend::memory::MemoryBackend;
240    use crate::index::writer::FtsIndex;
241    use crate::test_support::test_governor;
242
243    use super::SynonymGroupRecord;
244
245    const DB: u64 = 0;
246    const T: u64 = 1;
247
248    fn idx() -> FtsIndex<MemoryBackend> {
249        FtsIndex::new(MemoryBackend::new(), test_governor())
250    }
251
252    fn rec(name: &str, terms: &[&str]) -> SynonymGroupRecord {
253        SynonymGroupRecord {
254            name: name.to_string(),
255            terms: terms.iter().map(|s| s.to_string()).collect(),
256            created_at: 0,
257        }
258    }
259
260    #[test]
261    fn put_and_get() {
262        let i = idx();
263        i.put_synonym_group(DB, T, &rec("db_terms", &["database", "db", "datastore"]))
264            .unwrap();
265        let got = i.get_synonym_group(DB, T, "db_terms").unwrap().unwrap();
266        assert_eq!(got.name, "db_terms");
267        assert_eq!(got.terms.len(), 3);
268    }
269
270    #[test]
271    fn delete_removes() {
272        let i = idx();
273        i.put_synonym_group(DB, T, &rec("g1", &["a", "b"])).unwrap();
274        assert!(i.delete_synonym_group(DB, T, "g1").unwrap());
275        assert!(!i.delete_synonym_group(DB, T, "g1").unwrap());
276        assert!(i.get_synonym_group(DB, T, "g1").unwrap().is_none());
277    }
278
279    #[test]
280    fn list_reflects_puts_and_deletes() {
281        let i = idx();
282        i.put_synonym_group(DB, T, &rec("g1", &["a", "b"])).unwrap();
283        i.put_synonym_group(DB, T, &rec("g2", &["x", "y"])).unwrap();
284        let names: Vec<String> = i
285            .list_synonym_groups(DB, T)
286            .unwrap()
287            .into_iter()
288            .map(|r| r.name)
289            .collect();
290        assert_eq!(names.len(), 2);
291
292        i.delete_synonym_group(DB, T, "g1").unwrap();
293        let names2: Vec<String> = i
294            .list_synonym_groups(DB, T)
295            .unwrap()
296            .into_iter()
297            .map(|r| r.name)
298            .collect();
299        assert_eq!(names2, vec!["g2"]);
300    }
301
302    #[test]
303    fn synonym_map_bidirectional() {
304        let i = idx();
305        let recs = vec![rec("db_terms", &["db", "database", "datastore"])];
306        let map = i.build_synonym_map_for_tenant(DB, T, &recs);
307
308        // Terms are analyzed/stemmed before building the map:
309        // "database" → "databas", "datastore" → "datastor", "db" → "db"
310        let expanded = map.expand(&["db".to_string()]);
311        assert!(expanded.contains(&"databas".to_string()));
312        assert!(expanded.contains(&"datastor".to_string()));
313
314        let expanded2 = map.expand(&["databas".to_string()]);
315        assert!(expanded2.contains(&"db".to_string()));
316        assert!(expanded2.contains(&"datastor".to_string()));
317    }
318
319    #[test]
320    fn expand_query_with_synonyms_no_groups() {
321        let i = idx();
322        let tokens = vec!["hello".to_string(), "world".to_string()];
323        let expanded = i.expand_query_with_synonyms(DB, T, tokens.clone()).unwrap();
324        assert_eq!(expanded, tokens);
325    }
326
327    #[test]
328    fn expand_query_expands_matching_token() {
329        let i = idx();
330        i.put_synonym_group(DB, T, &rec("db_terms", &["db", "database", "datastore"]))
331            .unwrap();
332        // expand_query_with_synonyms receives already-analyzed tokens ("databas" not "database")
333        // because search_with_params analyzes first, then expands.
334        // The synonym map stores analyzed stems: "database" → "databas".
335        let tokens = vec!["db".to_string(), "perform".to_string()];
336        let expanded = i.expand_query_with_synonyms(DB, T, tokens).unwrap();
337        assert!(expanded.contains(&"db".to_string()));
338        assert!(expanded.contains(&"databas".to_string()));
339        assert!(expanded.contains(&"datastor".to_string()));
340        assert!(expanded.contains(&"perform".to_string()));
341    }
342}
nodedb_fts/index/synonym_groups.rs

nodedb_fts/index/
synonym_groups.rs
