Skip to main content

uqa_execution/serializable/text/
read.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Demand-driven observations preserve native posting cursors and bulk reads.
8
9use std::{collections::BTreeMap, ops::Deref, sync::Arc};
10use uqa_analysis::{Analyzer, CompiledAnalyzer};
11use uqa_core::{
12    memory::Budgeted, DocId, FieldName, IndexStats, PostingEntry, PostingList, TokenOccurrence,
13};
14use uqa_sql::ast::ColumnDef;
15use uqa_storage::{
16    clustered_postings::{
17        BudgetedPostingReadCursor, OccurrencePosting, PostingCursor, PostingReadCursor,
18        ScoreClusterVisitor,
19    },
20    inverted_index::IndexedFieldMetadata,
21    read_control::StorageReadControl,
22    BlockMaxScorer, InvertedIndex, StorageBackendError, StorageBackendResult, TokenTermKey,
23};
24
25use super::{TextObservation, DOCUMENT, POSTING, STATISTICS};
26use crate::serializable::SerializableRelationRead;
27
28/// A borrowed live index or owned retained snapshot with its original participant. Construction and analyzer metadata access do not observe a query.
29pub struct ObservedTextIndex<I> {
30    index: I,
31    observation: Option<TextObservation>,
32}
33
34impl<I> ObservedTextIndex<I> {
35    pub fn new(
36        index: I,
37        read: Option<SerializableRelationRead>,
38        columns: Arc<Vec<ColumnDef>>,
39    ) -> Self {
40        Self {
41            index,
42            observation: read.map(|read| TextObservation { read, columns }),
43        }
44    }
45
46    fn observe(
47        &self,
48        operation: impl FnOnce(&TextObservation) -> StorageBackendResult<()>,
49    ) -> StorageBackendResult<()> {
50        self.observation.as_ref().map_or(Ok(()), operation)
51    }
52}
53
54impl<I: Deref<Target = dyn InvertedIndex>> ObservedTextIndex<I> {
55    fn observe_fields(
56        &self,
57        operation: impl Fn(&TextObservation, &str) -> StorageBackendResult<()>,
58    ) -> StorageBackendResult<()> {
59        self.observe(|observation| {
60            observation.field_range(STATISTICS, None)?;
61            for field in self.index.field_names()? {
62                operation(observation, &field)?;
63            }
64            Ok(())
65        })
66    }
67}
68
69pub fn observe_snapshot(
70    read: Option<&SerializableRelationRead>,
71    columns: Arc<Vec<ColumnDef>>,
72    index: Arc<dyn InvertedIndex>,
73) -> Arc<dyn InvertedIndex> {
74    match read {
75        Some(read) => Arc::new(ObservedTextIndex::new(index, Some(read.clone()), columns)),
76        None => index,
77    }
78}
79
80fn read_only() -> StorageBackendError {
81    StorageBackendError::Other("cannot write a retained serializable text reader".into())
82}
83
84#[cfg(test)]
85mod tests;
86
87impl<I: Deref<Target = dyn InvertedIndex> + Send + Sync> InvertedIndex for ObservedTextIndex<I> {
88    fn analyzer(&self) -> &Analyzer {
89        self.index.analyzer()
90    }
91    fn default_analyzer_binding(
92        &self,
93    ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
94        self.index.default_analyzer_binding()
95    }
96    fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
97        self.index.source_rebuild_required()
98    }
99    fn get_field_analyzer(&self, field: &str) -> Analyzer {
100        self.index.get_field_analyzer(field)
101    }
102    fn get_search_analyzer(&self, field: &str) -> Analyzer {
103        self.index.get_search_analyzer(field)
104    }
105    fn index_analyzer_revision(&self, field: &str) -> StorageBackendResult<Arc<CompiledAnalyzer>> {
106        self.index.index_analyzer_revision(field)
107    }
108    fn search_analyzer_revision(&self, field: &str) -> StorageBackendResult<Arc<CompiledAnalyzer>> {
109        self.index.search_analyzer_revision(field)
110    }
111    fn add_document(
112        &mut self,
113        _: DocId,
114        _: BTreeMap<FieldName, String>,
115    ) -> StorageBackendResult<()> {
116        Err(read_only())
117    }
118    fn try_add_documents(
119        &mut self,
120        _: Vec<(DocId, BTreeMap<FieldName, String>)>,
121    ) -> StorageBackendResult<()> {
122        Err(read_only())
123    }
124    fn remove_document(&mut self, _: DocId) -> StorageBackendResult<()> {
125        Err(read_only())
126    }
127    fn clear(&mut self) -> StorageBackendResult<()> {
128        Err(read_only())
129    }
130    fn rebuild_persisted_block_max(
131        &mut self,
132        _: &str,
133        _: &dyn BlockMaxScorer,
134        _: &str,
135    ) -> StorageBackendResult<bool> {
136        Err(read_only())
137    }
138
139    fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
140        Ok(Arc::new(ObservedTextIndex {
141            index: self.index.snapshot()?,
142            observation: self.observation.clone(),
143        }))
144    }
145
146    fn snapshot_with_control(
147        &self,
148        control: &StorageReadControl,
149    ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
150        Ok(Arc::new(ObservedTextIndex {
151            index: self.index.snapshot_with_control(control)?,
152            observation: self.observation.clone(),
153        }))
154    }
155
156    fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
157        self.observe(|o| o.scalar_term(field, term, None))?;
158        self.index.get_posting_list(field, term)
159    }
160    fn get_posting_list_key(
161        &self,
162        field: &str,
163        term: &TokenTermKey,
164    ) -> StorageBackendResult<PostingList> {
165        self.observe(|o| o.term(field, term, None))?;
166        self.index.get_posting_list_key(field, term)
167    }
168    fn get_posting_lists_bulk(
169        &self,
170        field: &str,
171        terms: &[String],
172    ) -> StorageBackendResult<Vec<PostingList>> {
173        self.observe(|o| {
174            terms
175                .iter()
176                .try_for_each(|term| o.scalar_term(field, term, None))
177        })?;
178        self.index.get_posting_lists_bulk(field, terms)
179    }
180    fn get_posting_lists_keys_bulk(
181        &self,
182        field: &str,
183        terms: &[TokenTermKey],
184    ) -> StorageBackendResult<Vec<PostingList>> {
185        self.observe(|o| terms.iter().try_for_each(|term| o.term(field, term, None)))?;
186        self.index.get_posting_lists_keys_bulk(field, terms)
187    }
188    fn posting_cursor(
189        &self,
190        field: &str,
191        term: &str,
192    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
193        self.observe(|o| o.scalar_term(field, term, None))?;
194        self.index.posting_cursor(field, term)
195    }
196    fn posting_cursor_key(
197        &self,
198        field: &str,
199        term: &TokenTermKey,
200    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
201        self.observe(|o| o.term(field, term, None))?;
202        self.index.posting_cursor_key(field, term)
203    }
204    fn posting_cursors_bulk(
205        &self,
206        field: &str,
207        terms: &[String],
208    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
209        self.observe(|o| {
210            terms
211                .iter()
212                .try_for_each(|term| o.scalar_term(field, term, None))
213        })?;
214        self.index.posting_cursors_bulk(field, terms)
215    }
216    fn posting_cursors_keys_bulk(
217        &self,
218        field: &str,
219        terms: &[TokenTermKey],
220    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
221        self.observe(|o| terms.iter().try_for_each(|term| o.term(field, term, None)))?;
222        self.index.posting_cursors_keys_bulk(field, terms)
223    }
224    fn posting_read_cursor_key<'a>(
225        &'a self,
226        field: &'a str,
227        term: &TokenTermKey,
228    ) -> StorageBackendResult<Box<dyn PostingReadCursor + 'a>> {
229        self.observe(|o| o.term(field, term, None))?;
230        self.index.posting_read_cursor_key(field, term)
231    }
232    fn posting_read_cursor_key_budgeted<'a>(
233        &'a self,
234        field: &'a str,
235        term: &'a TokenTermKey,
236        control: &StorageReadControl,
237    ) -> StorageBackendResult<BudgetedPostingReadCursor<'a>> {
238        self.observe(|o| o.term(field, term, None))?;
239        self.index
240            .posting_read_cursor_key_budgeted(field, term, control)
241    }
242    fn visit_score_clusters(
243        &self,
244        field: &str,
245        term: &TokenTermKey,
246        after: Option<u64>,
247        limit: usize,
248        control: &StorageReadControl,
249        visit: &mut ScoreClusterVisitor<'_>,
250    ) -> StorageBackendResult<()> {
251        if limit != 0 {
252            self.observe(|o| o.term(field, term, None))?;
253        }
254        self.index
255            .visit_score_clusters(field, term, after, limit, control, visit)
256    }
257    fn get_occurrence_postings(
258        &self,
259        field: &str,
260        term: &TokenTermKey,
261    ) -> StorageBackendResult<Vec<OccurrencePosting>> {
262        self.observe(|o| o.term(field, term, None))?;
263        self.index.get_occurrence_postings(field, term)
264    }
265    fn get_occurrences(
266        &self,
267        doc: DocId,
268        field: &str,
269        term: &TokenTermKey,
270    ) -> StorageBackendResult<Vec<TokenOccurrence>> {
271        self.observe(|o| o.term(field, term, Some(doc)))?;
272        self.index.get_occurrences(doc, field, term)
273    }
274    fn get_occurrences_budgeted(
275        &self,
276        doc: DocId,
277        field: &str,
278        term: &TokenTermKey,
279        control: &StorageReadControl,
280    ) -> StorageBackendResult<Budgeted<Vec<TokenOccurrence>>> {
281        self.observe(|o| o.term(field, term, Some(doc)))?;
282        self.index
283            .get_occurrences_budgeted(doc, field, term, control)
284    }
285    fn indexed_field_metadata(
286        &self,
287        doc: DocId,
288        field: &str,
289    ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
290        self.observe(|o| o.document(field, doc))?;
291        self.index.indexed_field_metadata(doc, field)
292    }
293    fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
294        self.observe(|o| o.scalar_term(field, term, None))?;
295        self.index.doc_freq(field, term)
296    }
297    fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
298        self.observe(|o| o.term(field, term, None))?;
299        self.index.doc_freq_key(field, term)
300    }
301    fn get_term_freq(&self, doc: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
302        self.observe(|o| o.scalar_term(field, term, Some(doc)))?;
303        self.index.get_term_freq(doc, field, term)
304    }
305    fn get_term_freq_key(
306        &self,
307        doc: DocId,
308        field: &str,
309        term: &TokenTermKey,
310    ) -> StorageBackendResult<u64> {
311        self.observe(|o| o.term(field, term, Some(doc)))?;
312        self.index.get_term_freq_key(doc, field, term)
313    }
314    fn get_doc_length(&self, doc: DocId, field: &str) -> StorageBackendResult<u64> {
315        self.observe(|o| o.document(field, doc))?;
316        self.index.get_doc_length(doc, field)
317    }
318    fn doc_count(&self) -> StorageBackendResult<u64> {
319        self.observe(|o| o.field_range(DOCUMENT, None))?;
320        self.index.doc_count()
321    }
322    fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
323        self.observe(|o| o.statistics(field))?;
324        self.index.total_field_length(field)
325    }
326    fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
327        self.observe(|o| o.statistics(field))?;
328        self.index.field_doc_count(field)
329    }
330    fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
331        self.observe(|o| o.statistics(field))?;
332        self.index.field_stats_scalar(field)
333    }
334    fn field_stats_scalar_budgeted(
335        &self,
336        field: &str,
337        control: &StorageReadControl,
338    ) -> StorageBackendResult<IndexStats> {
339        self.observe(|o| o.statistics(field))?;
340        self.index.field_stats_scalar_budgeted(field, control)
341    }
342    fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
343        self.observe(TextObservation::all)?;
344        self.index.field_stats(field)
345    }
346    fn stats(&self) -> StorageBackendResult<IndexStats> {
347        self.observe(TextObservation::all)?;
348        self.index.stats()
349    }
350    fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
351        self.observe(|o| o.field_range(POSTING, Some(field)))?;
352        self.index.vocabulary_terms(field)
353    }
354    fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
355        self.observe(|o| o.field_range(POSTING, Some(field)))?;
356        self.index.vocabulary_keys(field)
357    }
358    fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
359        self.observe(|o| o.field_range(POSTING, field))?;
360        self.index.posting_count(field)
361    }
362    fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
363        self.observe(|o| o.field_range(DOCUMENT, field))?;
364        self.index.doc_length_count(field)
365    }
366    fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
367        self.observe(|o| o.field_range(POSTING, field))?;
368        self.index.term_count(field)
369    }
370    fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
371        self.observe(|o| o.field_range(STATISTICS, None))?;
372        self.index.field_names()
373    }
374    fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
375        self.observe_fields(|o, field| o.scalar_term(field, term, None))?;
376        self.index.get_posting_list_any_field(term)
377    }
378    fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
379        self.observe_fields(|o, field| o.scalar_term(field, term, None))?;
380        self.index.doc_freq_any_field(term)
381    }
382    fn get_total_doc_length(&self, doc: DocId) -> StorageBackendResult<u64> {
383        self.observe_fields(|o, field| o.document(field, doc))?;
384        self.index.get_total_doc_length(doc)
385    }
386    fn get_total_term_freq(&self, doc: DocId, term: &str) -> StorageBackendResult<u64> {
387        self.observe_fields(|o, field| o.scalar_term(field, term, Some(doc)))?;
388        self.index.get_total_term_freq(doc, term)
389    }
390    fn for_each_posting(
391        &self,
392        field: &str,
393        term: &str,
394        visit: &mut dyn FnMut(&PostingEntry),
395    ) -> StorageBackendResult<()> {
396        self.observe(|o| o.scalar_term(field, term, None))?;
397        self.index.for_each_posting(field, term, visit)
398    }
399    fn for_each_term_freq(
400        &self,
401        field: &str,
402        term: &str,
403        visit: &mut dyn FnMut(DocId, u64),
404    ) -> StorageBackendResult<()> {
405        self.observe(|o| o.scalar_term(field, term, None))?;
406        self.index.for_each_term_freq(field, term, visit)
407    }
408    fn persisted_block_max_scores(
409        &self,
410        field: &str,
411        term: &str,
412        fingerprint: &str,
413    ) -> StorageBackendResult<Option<Vec<f64>>> {
414        self.observe(|o| o.scalar_term(field, term, None))?;
415        self.index
416            .persisted_block_max_scores(field, term, fingerprint)
417    }
418    fn persisted_block_max_scores_bulk(
419        &self,
420        field: &str,
421        terms: &[String],
422        fingerprint: &str,
423    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
424        self.observe(|o| {
425            terms
426                .iter()
427                .try_for_each(|term| o.scalar_term(field, term, None))
428        })?;
429        self.index
430            .persisted_block_max_scores_bulk(field, terms, fingerprint)
431    }
432    fn persisted_block_max_scores_keys_bulk(
433        &self,
434        field: &str,
435        terms: &[TokenTermKey],
436        fingerprint: &str,
437    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
438        self.observe(|o| terms.iter().try_for_each(|term| o.term(field, term, None)))?;
439        self.index
440            .persisted_block_max_scores_keys_bulk(field, terms, fingerprint)
441    }
442    fn get_doc_lengths_bulk(
443        &self,
444        docs: &[DocId],
445        field: &str,
446    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
447        self.observe(|o| docs.iter().try_for_each(|doc| o.document(field, *doc)))?;
448        self.index.get_doc_lengths_bulk(docs, field)
449    }
450    fn get_term_freqs_bulk(
451        &self,
452        docs: &[DocId],
453        field: &str,
454        term: &str,
455    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
456        self.observe(|o| {
457            docs.iter()
458                .try_for_each(|doc| o.scalar_term(field, term, Some(*doc)))
459        })?;
460        self.index.get_term_freqs_bulk(docs, field, term)
461    }
462    fn get_scoring_inputs_bulk(
463        &self,
464        docs: &[DocId],
465        field: &str,
466        terms: &[String],
467    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
468        self.observe(|o| {
469            docs.iter().try_for_each(|doc| {
470                o.document(field, *doc)?;
471                terms
472                    .iter()
473                    .try_for_each(|term| o.scalar_term(field, term, Some(*doc)))
474            })
475        })?;
476        self.index.get_scoring_inputs_bulk(docs, field, terms)
477    }
478    fn get_scoring_inputs_keys_bulk(
479        &self,
480        docs: &[DocId],
481        field: &str,
482        terms: &[TokenTermKey],
483    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
484        self.observe(|o| {
485            docs.iter().try_for_each(|doc| {
486                o.document(field, *doc)?;
487                terms
488                    .iter()
489                    .try_for_each(|term| o.term(field, term, Some(*doc)))
490            })
491        })?;
492        self.index.get_scoring_inputs_keys_bulk(docs, field, terms)
493    }
494}