uqa_storage/inverted_index/
contract.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7use super::{
8    counter_error, usize_to_u64, Analyzer, Arc, BTreeMap, BlockMaxScorer, DocId, FieldName,
9    IndexStats, PostingEntry, PostingList, StorageBackendError, StorageBackendResult,
10};
11use crate::clustered_postings::{MaterializedPostingCursor, PostingCursor, PostingScore};
12
13/// Which side of the index/search pipeline a field analyzer applies to.
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum AnalyzerPhase {
16    /// Run only when *adding* documents.
17    Index,
18    /// Run only when *querying* documents (e.g. through `TermOperator`).
19    Search,
20    /// Run on both phases (the default).
21    Both,
22}
23
24impl AnalyzerPhase {
25    pub fn parse(s: &str) -> Result<Self, String> {
26        match s {
27            "index" => Ok(AnalyzerPhase::Index),
28            "search" | "query" => Ok(AnalyzerPhase::Search),
29            "both" => Ok(AnalyzerPhase::Both),
30            _ => Err(format!("phase must be 'index'|'search'|'both', got `{s}`")),
31        }
32    }
33}
34
35impl std::str::FromStr for AnalyzerPhase {
36    type Err = String;
37
38    fn from_str(s: &str) -> Result<Self, Self::Err> {
39        Self::parse(s)
40    }
41}
42
43pub trait InvertedIndex: Send + Sync {
44    fn analyzer(&self) -> &Analyzer;
45
46    fn add_document(
47        &mut self,
48        doc_id: DocId,
49        fields: BTreeMap<FieldName, String>,
50    ) -> StorageBackendResult<()>;
51
52    fn try_add_document(
53        &mut self,
54        doc_id: DocId,
55        fields: BTreeMap<FieldName, String>,
56    ) -> StorageBackendResult<()> {
57        self.add_document(doc_id, fields)
58    }
59
60    /// Add or replace several documents in input order. The default preserves the point-mutation contract for custom backends; transactional persistent backends can override this to make the call atomic and coalesce writes that share physical posting clusters.
61    fn try_add_documents(
62        &mut self,
63        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
64    ) -> StorageBackendResult<()> {
65        for (doc_id, fields) in documents {
66            self.try_add_document(doc_id, fields)?;
67        }
68        Ok(())
69    }
70
71    fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()>;
72
73    fn try_remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
74        self.remove_document(doc_id)
75    }
76
77    fn clear(&mut self) -> StorageBackendResult<()>;
78
79    fn try_clear(&mut self) -> StorageBackendResult<()> {
80        self.clear()
81    }
82
83    fn try_rebuild_documents(
84        &mut self,
85        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
86    ) -> StorageBackendResult<()> {
87        self.try_clear()?;
88        for (doc_id, fields) in documents {
89            if !fields.is_empty() {
90                self.try_add_document(doc_id, fields)?;
91            }
92        }
93        Ok(())
94    }
95
96    fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList>;
97
98    fn get_posting_lists_bulk(
99        &self,
100        field: &str,
101        terms: &[String],
102    ) -> StorageBackendResult<Vec<PostingList>> {
103        terms
104            .iter()
105            .map(|term| self.get_posting_list(field, term))
106            .collect()
107    }
108
109    /// Open a doc-id ordered score cursor for one term.
110    ///
111    /// The cursor carries term frequency and document length directly so
112    /// ranking does not need positional payloads or per-document length
113    /// lookups. Persistent backends override this with lazy clustered
114    /// cursors; the default preserves compatibility for custom backends.
115    fn posting_cursor(
116        &self,
117        field: &str,
118        term: &str,
119    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
120        let posting_list = self.get_posting_list(field, term)?;
121        let mut entries = Vec::with_capacity(posting_list.len());
122        for posting in posting_list {
123            let term_freq = usize_to_u64(posting.payload.positions.len().max(1), "term frequency")?;
124            entries.push(PostingScore {
125                doc_id: posting.doc_id,
126                term_freq,
127                doc_length: self.get_doc_length(posting.doc_id, field)?.max(term_freq),
128            });
129        }
130        Ok(Box::new(MaterializedPostingCursor::new(entries)?))
131    }
132
133    fn posting_cursors_bulk(
134        &self,
135        field: &str,
136        terms: &[String],
137    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
138        terms
139            .iter()
140            .map(|term| self.posting_cursor(field, term))
141            .collect()
142    }
143
144    /// Persist scorer-specific block maxima for every term in `field`.
145    ///
146    /// Backends that do not provide durable auxiliary indexes return `false`.
147    /// The fingerprint must include every scorer and corpus statistic that can
148    /// affect a term contribution; reads only expose rows with an exact match.
149    fn rebuild_persisted_block_max(
150        &mut self,
151        _field: &str,
152        _scorer: &dyn BlockMaxScorer,
153        _scorer_fingerprint: &str,
154    ) -> StorageBackendResult<bool> {
155        Ok(false)
156    }
157
158    /// Load scorer-versioned block maxima for one posting list. `None` means
159    /// the backend has no complete, valid materialization for this scorer.
160    fn persisted_block_max_scores(
161        &self,
162        _field: &str,
163        _term: &str,
164        _scorer_fingerprint: &str,
165    ) -> StorageBackendResult<Option<Vec<f64>>> {
166        Ok(None)
167    }
168
169    /// Load scorer-versioned block maxima for several terms while preserving input order; persistent backends override this to avoid one storage round trip per term.
170    fn persisted_block_max_scores_bulk(
171        &self,
172        field: &str,
173        terms: &[String],
174        scorer_fingerprint: &str,
175    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
176        terms
177            .iter()
178            .map(|term| self.persisted_block_max_scores(field, term, scorer_fingerprint))
179            .collect()
180    }
181
182    /// Visit every posting entry for `(field, term)` in ascending
183    /// doc-id order without handing out an owned list.
184    ///
185    /// [`InvertedIndex::get_posting_list`] deep-copies each entry's
186    /// payload (positions vector included), which costs one heap
187    /// allocation per matching document. Read-only scoring walks use
188    /// this instead; backends whose postings already live in memory
189    /// override it to iterate in place.
190    fn for_each_posting(
191        &self,
192        field: &str,
193        term: &str,
194        visit: &mut dyn FnMut(&PostingEntry),
195    ) -> StorageBackendResult<()> {
196        for entry in &self.get_posting_list(field, term)? {
197            visit(entry);
198        }
199        Ok(())
200    }
201
202    /// Visit `(doc_id, term_frequency)` pairs without requiring callers to
203    /// materialize or decode payload details they do not use. The default
204    /// keeps every backend compatible through the posting-list contract;
205    /// persistent backends can stream compact frequency projections.
206    fn for_each_term_freq(
207        &self,
208        field: &str,
209        term: &str,
210        visit: &mut dyn FnMut(DocId, u64),
211    ) -> StorageBackendResult<()> {
212        for entry in &self.get_posting_list(field, term)? {
213            visit(
214                entry.doc_id,
215                usize_to_u64(entry.payload.positions.len(), "term frequency")?,
216            );
217        }
218        Ok(())
219    }
220
221    fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64>;
222
223    fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64>;
224
225    fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64>;
226
227    fn doc_count(&self) -> StorageBackendResult<u64>;
228
229    fn total_field_length(&self, field: &str) -> StorageBackendResult<u64>;
230
231    /// Number of documents that have indexed content for `field`.
232    fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
233        self.doc_length_count(Some(field))
234    }
235
236    /// Field-specific statistics for BM25 scoring.
237    ///
238    /// BM25 length normalization and IDF collection size are defined for
239    /// one field. Reusing table-wide totals mixes unrelated field lengths
240    /// and produces scores that cannot match a field-scoped BM25 scorer.
241    fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
242        let mut stats = self.stats()?;
243        let field_docs = self.field_doc_count(field)?;
244        stats.total_docs = field_docs;
245        stats.avg_doc_length = if field_docs > 0 {
246            self.total_field_length(field)? as f64 / field_docs as f64
247        } else {
248            0.0
249        };
250        Ok(stats)
251    }
252
253    /// [`InvertedIndex::field_stats`] without the vocabulary-wide
254    /// document-frequency map.
255    ///
256    /// Query execution that already knows its terms' document
257    /// frequencies (it read them off the posting lists) only needs the
258    /// field's document count and average length; copying the whole
259    /// term dictionary per query is O(vocabulary) for nothing.
260    fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
261        let mut stats = IndexStats::default();
262        let field_docs = self.field_doc_count(field)?;
263        stats.total_docs = field_docs;
264        stats.avg_doc_length = if field_docs > 0 {
265            self.total_field_length(field)? as f64 / field_docs as f64
266        } else {
267            0.0
268        };
269        Ok(stats)
270    }
271
272    /// Sorted unique indexed terms for `field`.
273    ///
274    /// Backends implement this from their term dictionary rather than by
275    /// re-analyzing stored documents. This is the source used by Bayesian
276    /// calibration reservoir sampling.
277    fn vocabulary_terms(&self, _field: &str) -> StorageBackendResult<Vec<String>> {
278        Ok(Vec::new())
279    }
280
281    /// Fully-populated [`IndexStats`] snapshot for the cost model and
282    /// scoring layer. Implementations may cache this between mutations.
283    fn stats(&self) -> StorageBackendResult<IndexStats>;
284
285    /// Number of posting rows. With `field = Some(..)`, limits the count
286    /// to one indexed field.
287    fn posting_count(&self, _field: Option<&str>) -> StorageBackendResult<u64> {
288        Ok(0)
289    }
290
291    /// Number of `(doc_id, field)` length rows. With `field = Some(..)`,
292    /// this is the number of documents indexed for that field.
293    fn doc_length_count(&self, _field: Option<&str>) -> StorageBackendResult<u64> {
294        Ok(0)
295    }
296
297    /// Number of distinct indexed terms. With `field = Some(..)`, limits
298    /// the count to one indexed field.
299    fn term_count(&self, _field: Option<&str>) -> StorageBackendResult<u64> {
300        Ok(0)
301    }
302
303    /// Read-only handle suitable for an `ExecutionContext`.
304    fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>>;
305
306    /// Independent writable copy used to restore an in-memory engine
307    /// transaction without reconstructing analyzer state from documents.
308    fn writable_snapshot(&self) -> StorageBackendResult<Box<dyn InvertedIndex>> {
309        Err(StorageBackendError::Other(
310            "writable inverted-index snapshots are not supported by this backend".into(),
311        ))
312    }
313
314    // -- Extended inverted-index surface ---
315
316    /// Names of every field with at least one indexed document.
317    /// Default implementation walks the [`IndexStats`] snapshot's
318    /// total-length map. Backends with a richer schema can override.
319    fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
320        Ok(Vec::new())
321    }
322
323    /// Posting list for `term` across every indexed field, unioned
324    /// together. Default implementation sums per-field posting lists
325    /// via [`PostingList::merge_union`].
326    fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
327        let mut result = PostingList::new();
328        for field in self.field_names()? {
329            let pl = self.get_posting_list(&field, term)?;
330            result = result.merge_union(&pl);
331        }
332        Ok(result)
333    }
334
335    /// Document frequency of `term` across every indexed field.
336    fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
337        let mut total = 0_u64;
338        for field in self.field_names()? {
339            total = total
340                .checked_add(self.doc_freq(&field, term)?)
341                .ok_or_else(|| counter_error("document frequency"))?;
342        }
343        Ok(total)
344    }
345
346    /// Sum of all per-field token lengths for a single doc.
347    fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
348        let mut total = 0_u64;
349        for field in self.field_names()? {
350            total = total
351                .checked_add(self.get_doc_length(doc_id, &field)?)
352                .ok_or_else(|| counter_error("document length"))?;
353        }
354        Ok(total)
355    }
356
357    /// Bulk doc-length lookup. Default falls back to per-id calls.
358    fn get_doc_lengths_bulk(
359        &self,
360        doc_ids: &[DocId],
361        field: &str,
362    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
363        let mut out = BTreeMap::new();
364        for doc_id in doc_ids {
365            out.insert(*doc_id, self.get_doc_length(*doc_id, field)?);
366        }
367        Ok(out)
368    }
369
370    /// Bulk term-frequency lookup. Default falls back to per-id calls.
371    fn get_term_freqs_bulk(
372        &self,
373        doc_ids: &[DocId],
374        field: &str,
375        term: &str,
376    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
377        let mut out = BTreeMap::new();
378        for doc_id in doc_ids {
379            out.insert(*doc_id, self.get_term_freq(*doc_id, field, term)?);
380        }
381        Ok(out)
382    }
383
384    /// Fetch the document length and one term frequency per query term for
385    /// every requested document. Results stay aligned with `doc_ids`.
386    /// Persistent backends override this to collapse the scoring loop's
387    /// per-document point reads into a small number of set-oriented queries.
388    fn get_scoring_inputs_bulk(
389        &self,
390        doc_ids: &[DocId],
391        field: &str,
392        terms: &[String],
393    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
394        let mut out = Vec::with_capacity(doc_ids.len());
395        for doc_id in doc_ids {
396            let mut term_freqs = Vec::with_capacity(terms.len());
397            for term in terms {
398                term_freqs.push(self.get_term_freq(*doc_id, field, term)?);
399            }
400            out.push((self.get_doc_length(*doc_id, field)?, term_freqs));
401        }
402        Ok(out)
403    }
404
405    /// Total term frequency for a doc summed across every indexed
406    /// field.
407    fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
408        let mut total = 0_u64;
409        for field in self.field_names()? {
410            total = total
411                .checked_add(self.get_term_freq(doc_id, &field, term)?)
412                .ok_or_else(|| counter_error("term frequency"))?;
413        }
414        Ok(total)
415    }
416
417    /// Bind an analyzer to a single field for the given phase.
418    /// `Both` writes to both the index-side and search-side maps; the
419    /// default impl errors so backends that don't support per-field
420    /// analyzers fail loud rather than silently dropping the request.
421    fn set_field_analyzer(
422        &mut self,
423        _field: &str,
424        _analyzer: Analyzer,
425        _phase: AnalyzerPhase,
426    ) -> Result<(), String> {
427        Err("set_field_analyzer not supported by this InvertedIndex backend".into())
428    }
429
430    /// Remove every per-field analyzer override for `field`.  This is the
431    /// inverse of `set_field_analyzer(..., Both)` and is required when the
432    /// final logical FTS index for a field is dropped.  The default errors so
433    /// a backend cannot silently retain stale analysis behavior.
434    fn remove_field_analyzers(&mut self, _field: &str) -> Result<(), String> {
435        Err("remove_field_analyzers not supported by this InvertedIndex backend".into())
436    }
437
438    /// Index-time analyzer for `field`; falls back to
439    /// [`InvertedIndex::analyzer`] when no override is set.
440    fn get_field_analyzer(&self, _field: &str) -> Analyzer {
441        self.analyzer().clone()
442    }
443
444    /// Search-time analyzer for `field`; falls back to the index-time analyzer,
445    /// then to the default.
446    fn get_search_analyzer(&self, field: &str) -> Analyzer {
447        self.get_field_analyzer(field)
448    }
449}
uqa_storage/inverted_index/contract.rs

uqa_storage/inverted_index/
contract.rs
