Skip to main content

uqa_storage_sqlite/inverted_index/
trait_impl.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! `InvertedIndex` trait implementation and read/statistics surface.
8
9use super::queries::project_postings;
10use super::{
11    clustered_result, decode_index_u64, encode_index_u64, invalidate_posting_accelerators, params,
12    params_from_iter, quote_ident, Analyzer, AnalyzerPhase, Arc, BTreeMap, BlockMaxScorer, DocId,
13    FieldName, IndexStats, InvertedIndex, OptionalExtension, PostingCursor, PostingList,
14    SQLiteError, SQLiteInvertedIndex, SqlValue, StorageBackendResult,
15};
16use super::{IndexedFieldMetadata, TokenTermKey};
17use uqa_storage::clustered_postings::{cluster_id, decode_all_scores, OccurrencePosting};
18
19impl InvertedIndex for SQLiteInvertedIndex {
20    fn posting_read_cursor_key_budgeted<'a>(
21        &'a self,
22        field: &'a str,
23        term: &'a TokenTermKey,
24        control: &uqa_storage::read_control::StorageReadControl,
25    ) -> StorageBackendResult<uqa_storage::clustered_postings::BudgetedPostingReadCursor<'a>> {
26        control.check()?;
27        if let Some(index) = self.native_index() {
28            return uqa_storage::clustered_postings::open_controlled_cursor(
29                index.snapshot()?,
30                field,
31                term,
32                control,
33            );
34        }
35        uqa_storage::clustered_postings::open_controlled_cursor(self, field, term, control)
36    }
37
38    fn visit_score_clusters(
39        &self,
40        field: &str,
41        term: &TokenTermKey,
42        after: Option<u64>,
43        limit: usize,
44        control: &uqa_storage::read_control::StorageReadControl,
45        visit: &mut uqa_storage::clustered_postings::ScoreClusterVisitor<'_>,
46    ) -> StorageBackendResult<()> {
47        if let Some(index) = self.native_index() {
48            return index.visit_score_clusters(field, term, after, limit, control, visit);
49        }
50        Ok(self.visit_clusters_budgeted(field, term, after, limit, control, visit)?)
51    }
52
53    fn get_occurrences_budgeted(
54        &self,
55        doc_id: DocId,
56        field: &str,
57        term: &TokenTermKey,
58        control: &uqa_storage::read_control::StorageReadControl,
59    ) -> StorageBackendResult<uqa_core::memory::Budgeted<Vec<uqa_core::TokenOccurrence>>> {
60        if let Some(index) = self.native_index() {
61            return index.get_occurrences_budgeted(doc_id, field, term, control);
62        }
63        Ok(self.occurrences_budgeted(doc_id, field, term, control)?)
64    }
65
66    fn field_stats_scalar_budgeted(
67        &self,
68        field: &str,
69        control: &uqa_storage::read_control::StorageReadControl,
70    ) -> StorageBackendResult<IndexStats> {
71        if let Some(index) = self.native_index() {
72            return index.field_stats_scalar_budgeted(field, control);
73        }
74        Ok(self.scalar_stats_budgeted(field, control)?)
75    }
76
77    fn analyzer(&self) -> &Analyzer {
78        self.bindings.default_configuration()
79    }
80
81    fn default_analyzer_binding(
82        &self,
83    ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
84        Ok(self.bindings.default_binding())
85    }
86
87    fn add_document(
88        &mut self,
89        doc_id: DocId,
90        fields: BTreeMap<FieldName, String>,
91    ) -> StorageBackendResult<()> {
92        if let Some(mut index) = self.native_index() {
93            return index.add_document(doc_id, fields);
94        }
95        Ok(self.add_document_inner(doc_id, fields)?)
96    }
97
98    fn try_add_documents(
99        &mut self,
100        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
101    ) -> StorageBackendResult<()> {
102        if let Some(mut index) = self.native_index() {
103            return index.try_add_documents(documents);
104        }
105        Ok(self.add_documents_inner(documents)?)
106    }
107
108    fn try_add_documents_observed(
109        &mut self,
110        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
111        visit: &mut uqa_storage::inverted_index::InvertedIndexChangeVisitor<'_>,
112    ) -> StorageBackendResult<()> {
113        if let Some(mut index) = self.native_index() {
114            return index.try_add_documents_observed(documents, visit);
115        }
116        Ok(self.add_documents_observed(documents, Some(visit))?)
117    }
118
119    fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
120        if let Some(mut index) = self.native_index() {
121            return index.remove_document(doc_id);
122        }
123        Ok(self.remove_document_inner(doc_id)?)
124    }
125
126    fn clear(&mut self) -> StorageBackendResult<()> {
127        if let Some(mut index) = self.native_index() {
128            return index.clear();
129        }
130        self.conn.with_mut(|conn| {
131            let tx = conn.savepoint()?;
132            invalidate_posting_accelerators(&tx, &self.table)?;
133            self.clear_index_on(&tx)?;
134            tx.commit()?;
135            Ok(())
136        })?;
137        Ok(())
138    }
139
140    fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
141        if let Some(index) = self.native_index() {
142            return index.source_rebuild_required();
143        }
144        Ok(self.conn.with(|conn| self.needs_source_rebuild_on(conn))?)
145    }
146
147    fn try_rebuild_documents(
148        &mut self,
149        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
150    ) -> StorageBackendResult<()> {
151        if let Some(mut index) = self.native_index() {
152            return index.try_rebuild_documents(documents);
153        }
154        Ok(self.rebuild_documents_inner(documents)?)
155    }
156
157    fn try_rebuild_documents_cancellable(
158        &mut self,
159        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
160        cancellation: &uqa_core::CancellationToken,
161    ) -> StorageBackendResult<()> {
162        if let Some(mut index) = self.native_index() {
163            return index.try_rebuild_documents_cancellable(documents, cancellation);
164        }
165        Ok(self.rebuild_documents_with_cancellation(documents, Some(cancellation))?)
166    }
167
168    fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
169        if let Some(index) = self.native_index() {
170            return index.get_posting_list(field, term);
171        }
172        self.get_posting_list_key(field, &TokenTermKey::from_text(term))
173    }
174
175    fn get_posting_list_key(
176        &self,
177        field: &str,
178        term: &TokenTermKey,
179    ) -> StorageBackendResult<PostingList> {
180        if let Some(index) = self.native_index() {
181            return index.get_posting_list_key(field, term);
182        }
183        Ok(project_postings(self.get_occurrence_postings(field, term)?))
184    }
185
186    fn get_occurrence_postings(
187        &self,
188        field: &str,
189        term: &TokenTermKey,
190    ) -> StorageBackendResult<Vec<OccurrencePosting>> {
191        if let Some(index) = self.native_index() {
192            return index.get_occurrence_postings(field, term);
193        }
194        Ok(self
195            .occurrence_postings_bulk(field, std::slice::from_ref(term))?
196            .pop()
197            .unwrap_or_default())
198    }
199
200    fn get_occurrences(
201        &self,
202        doc_id: DocId,
203        field: &str,
204        term: &TokenTermKey,
205    ) -> StorageBackendResult<Vec<uqa_core::TokenOccurrence>> {
206        if let Some(index) = self.native_index() {
207            return index.get_occurrences(doc_id, field, term);
208        }
209        Ok(self.conn.with(|conn| {
210            self.require_graph_format_on(conn)?;
211            let entries = super::load_cluster(conn, &self.table, field, term, cluster_id(doc_id))?;
212            let Some(posting) = entries.into_iter().find(|entry| entry.doc_id == doc_id) else {
213                return Ok(Vec::new());
214            };
215            self.validate_posting_metadata_on(conn, field, &posting)?;
216            Ok(posting.occurrences)
217        })?)
218    }
219
220    fn indexed_field_metadata(
221        &self,
222        doc_id: DocId,
223        field: &str,
224    ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
225        if let Some(index) = self.native_index() {
226            return index.indexed_field_metadata(doc_id, field);
227        }
228        let doc_id = encode_index_u64("document", doc_id)?;
229        Ok(self.conn.with(|conn| {
230            self.require_graph_format_on(conn)?;
231            self.read_field_metadata_on(conn, doc_id, field)
232        })?)
233    }
234
235    fn get_posting_lists_bulk(
236        &self,
237        field: &str,
238        terms: &[String],
239    ) -> StorageBackendResult<Vec<PostingList>> {
240        if let Some(index) = self.native_index() {
241            return index.get_posting_lists_bulk(field, terms);
242        }
243        let keys = terms
244            .iter()
245            .map(|term| TokenTermKey::from_text(term))
246            .collect::<Vec<_>>();
247        Ok(self
248            .occurrence_postings_bulk(field, &keys)?
249            .into_iter()
250            .map(project_postings)
251            .collect())
252    }
253
254    fn posting_cursor(
255        &self,
256        field: &str,
257        term: &str,
258    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
259        if let Some(index) = self.native_index() {
260            return index.posting_cursor(field, term);
261        }
262        self.posting_cursor_key(field, &TokenTermKey::from_text(term))
263    }
264
265    fn posting_cursor_key(
266        &self,
267        field: &str,
268        term: &TokenTermKey,
269    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
270        if let Some(index) = self.native_index() {
271            return index.posting_cursor_key(field, term);
272        }
273        self.cursor_for_term(field, term)
274    }
275
276    fn posting_cursors_bulk(
277        &self,
278        field: &str,
279        terms: &[String],
280    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
281        if let Some(index) = self.native_index() {
282            return index.posting_cursors_bulk(field, terms);
283        }
284        let keys = terms
285            .iter()
286            .map(|term| TokenTermKey::from_text(term))
287            .collect::<Vec<_>>();
288        self.cursors_for_terms(field, &keys)
289    }
290
291    fn posting_cursors_keys_bulk(
292        &self,
293        field: &str,
294        terms: &[TokenTermKey],
295    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
296        if let Some(index) = self.native_index() {
297            return index.posting_cursors_keys_bulk(field, terms);
298        }
299        self.cursors_for_terms(field, terms)
300    }
301
302    fn get_posting_lists_keys_bulk(
303        &self,
304        field: &str,
305        terms: &[TokenTermKey],
306    ) -> StorageBackendResult<Vec<PostingList>> {
307        if let Some(index) = self.native_index() {
308            return index.get_posting_lists_keys_bulk(field, terms);
309        }
310        Ok(self
311            .occurrence_postings_bulk(field, terms)?
312            .into_iter()
313            .map(project_postings)
314            .collect())
315    }
316
317    fn persisted_block_max_scores_keys_bulk(
318        &self,
319        field: &str,
320        terms: &[TokenTermKey],
321        scorer_fingerprint: &str,
322    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
323        if scorer_fingerprint.is_empty() {
324            return Ok(vec![None; terms.len()]);
325        }
326        self.get_versioned_block_max_scores_keys_bulk(field, terms, scorer_fingerprint)
327    }
328
329    fn rebuild_persisted_block_max(
330        &mut self,
331        field: &str,
332        scorer: &dyn BlockMaxScorer,
333        scorer_fingerprint: &str,
334    ) -> StorageBackendResult<bool> {
335        if let Some(mut index) = self.native_index() {
336            return index.rebuild_persisted_block_max(field, scorer, scorer_fingerprint);
337        }
338        if scorer_fingerprint.is_empty() {
339            return Err(SQLiteError::StorageBackend(
340                "persisted block-max scorer fingerprint must not be empty".into(),
341            )
342            .into());
343        }
344        let terms = self.vocabulary_keys(field)?;
345        self.ensure_aux_tables(field)?;
346        let table = self.blockmax_table_name(field);
347        self.conn.with_mut(|conn| {
348            conn.execute(&format!("DELETE FROM {}", quote_ident(&table)), [])?;
349            Ok(())
350        })?;
351        for term in terms {
352            self.build_block_max_scores_key(field, &term, scorer, scorer_fingerprint)?;
353        }
354        Ok(true)
355    }
356
357    fn persisted_block_max_scores(
358        &self,
359        field: &str,
360        term: &str,
361        scorer_fingerprint: &str,
362    ) -> StorageBackendResult<Option<Vec<f64>>> {
363        if scorer_fingerprint.is_empty() {
364            return Ok(None);
365        }
366        self.get_versioned_block_max_scores(field, term, scorer_fingerprint)
367    }
368
369    fn persisted_block_max_scores_bulk(
370        &self,
371        field: &str,
372        terms: &[String],
373        scorer_fingerprint: &str,
374    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
375        if scorer_fingerprint.is_empty() {
376            return Ok(vec![None; terms.len()]);
377        }
378        self.get_versioned_block_max_scores_bulk(field, terms, scorer_fingerprint)
379    }
380
381    fn for_each_term_freq(
382        &self,
383        field: &str,
384        term: &str,
385        visit: &mut dyn FnMut(DocId, u64),
386    ) -> StorageBackendResult<()> {
387        if let Some(index) = self.native_index() {
388            return index.for_each_term_freq(field, term, visit);
389        }
390        let mut cursor = self.posting_cursor(field, term)?;
391        while let Some(entry) = cursor.current() {
392            visit(entry.doc_id, entry.term_freq);
393            cursor.advance()?;
394        }
395        Ok(())
396    }
397
398    fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
399        if let Some(index) = self.native_index() {
400            return index.doc_freq(field, term);
401        }
402        self.doc_freq_key(field, &TokenTermKey::from_text(term))
403    }
404
405    fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
406        if let Some(index) = self.native_index() {
407            return index.doc_freq_key(field, term);
408        }
409        Ok(self.posting_cursor_key(field, term)?.doc_freq())
410    }
411
412    fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64> {
413        if let Some(index) = self.native_index() {
414            return index.get_doc_length(doc_id, field);
415        }
416        Ok(self
417            .get_doc_lengths_bulk(&[doc_id], field)?
418            .get(&doc_id)
419            .copied()
420            .unwrap_or(0))
421    }
422
423    fn get_doc_lengths_bulk(
424        &self,
425        doc_ids: &[DocId],
426        field: &str,
427    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
428        if let Some(index) = self.native_index() {
429            return index.get_doc_lengths_bulk(doc_ids, field);
430        }
431        Ok(self.conn.with(|conn| {
432            self.require_graph_format_on(conn)?;
433            let mut out = BTreeMap::new();
434            for chunk in doc_ids.chunks(900) {
435                let ids = (3..chunk.len()+3).map(|parameter| format!("?{parameter}")).collect::<Vec<_>>().join(", ");
436                let sql = format!("WITH lengths AS (SELECT doc_id, length FROM _occurrence_lengths WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})), documents AS (SELECT doc_id, metadata_blob FROM _occurrence_documents WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})) SELECT COALESCE(lengths.doc_id, documents.doc_id), length, metadata_blob FROM lengths FULL OUTER JOIN documents USING(doc_id)");
437                let mut values = vec![SqlValue::Text(self.table.clone()), SqlValue::Text(field.into())];
438                for doc_id in chunk {
439                    values.push(SqlValue::Integer(encode_index_u64("document", *doc_id)?));
440                }
441                let mut statement = conn.prepare(&sql)?;
442                let rows = statement.query_map(params_from_iter(values), |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option<i64>>(1)?, row.get::<_, Option<Vec<u8>>>(2)?)))?;
443                for row in rows {
444                    let (doc_id, length, metadata) = row?;
445                    let doc_id = decode_index_u64("document id", doc_id)?;
446                    let length = length.map(|length| decode_index_u64("document length", length)).transpose()?;
447                    let metadata = metadata.map(|bytes| clustered_result(IndexedFieldMetadata::from_bytes(&bytes))).transpose()?;
448                    let (Some(length), Some(metadata)) = (length, metadata) else {
449                        return Err(SQLiteError::StorageBackend("indexed field length and source metadata disagree".into()));
450                    };
451                    let stats = self.stored_field_stats_on(conn, field)?.ok_or_else(|| SQLiteError::StorageBackend("indexed field revision is missing".into()))?;
452                    if metadata.length != length || metadata.revision() != stats.revision {
453                        return Err(SQLiteError::StorageBackend("indexed field length or revision disagrees with source metadata".into()));
454                    }
455                    out.insert(doc_id, length);
456                }
457            }
458            Ok(out)
459        })?)
460    }
461
462    fn get_scoring_inputs_bulk(
463        &self,
464        doc_ids: &[DocId],
465        field: &str,
466        terms: &[String],
467    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
468        if let Some(index) = self.native_index() {
469            return index.get_scoring_inputs_bulk(doc_ids, field, terms);
470        }
471        let keys = terms
472            .iter()
473            .map(|term| TokenTermKey::from_text(term))
474            .collect::<Vec<_>>();
475        self.get_scoring_inputs_keys_bulk(doc_ids, field, &keys)
476    }
477
478    fn get_scoring_inputs_keys_bulk(
479        &self,
480        doc_ids: &[DocId],
481        field: &str,
482        terms: &[TokenTermKey],
483    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
484        if let Some(index) = self.native_index() {
485            return index.get_scoring_inputs_keys_bulk(doc_ids, field, terms);
486        }
487        if doc_ids.is_empty() {
488            return Ok(Vec::new());
489        }
490
491        let doc_lengths = self.get_doc_lengths_bulk(doc_ids, field)?;
492        let mut inputs: Vec<(u64, Vec<u64>)> = doc_ids
493            .iter()
494            .map(|doc_id| {
495                (
496                    doc_lengths.get(doc_id).copied().unwrap_or(0),
497                    vec![0; terms.len()],
498                )
499            })
500            .collect();
501        if terms.is_empty() {
502            return Ok(inputs);
503        }
504
505        let mut output_positions = BTreeMap::<DocId, Vec<usize>>::new();
506        for (position, doc_id) in doc_ids.iter().copied().enumerate() {
507            output_positions.entry(doc_id).or_default().push(position);
508        }
509        for (term_index, mut cursor) in self
510            .posting_cursors_keys_bulk(field, terms)?
511            .into_iter()
512            .enumerate()
513        {
514            while let Some(entry) = cursor.current() {
515                if let Some(positions) = output_positions.get(&entry.doc_id) {
516                    for position in positions {
517                        inputs[*position].1[term_index] = entry.term_freq;
518                        inputs[*position].0 = entry.doc_length;
519                    }
520                }
521                cursor.advance()?;
522            }
523        }
524        Ok(inputs)
525    }
526
527    fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
528        if let Some(index) = self.native_index() {
529            return index.get_term_freq(doc_id, field, term);
530        }
531        self.get_term_freq_key(doc_id, field, &TokenTermKey::from_text(term))
532    }
533
534    fn get_term_freq_key(
535        &self,
536        doc_id: DocId,
537        field: &str,
538        term: &TokenTermKey,
539    ) -> StorageBackendResult<u64> {
540        if let Some(index) = self.native_index() {
541            return index.get_term_freq_key(doc_id, field, term);
542        }
543        let cluster = encode_index_u64("posting cluster", cluster_id(doc_id))?;
544        Ok(self.conn.with(|conn| {
545            self.require_graph_format_on(conn)?;
546            let row: Option<(i64, Vec<u8>)> = conn.query_row("SELECT posting_count, score_blob FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 AND term = ?3 AND cluster_id = ?4", params![self.table, field, term.as_bytes(), cluster], |row| Ok((row.get(0)?, row.get(1)?))).optional()?;
547            match row {
548                Some((count, bytes)) => {
549                    // Validate the version and redundant row count before decoding only the requested cluster.
550                    super::posting_cursor_from_rows(vec![(cluster, count, bytes.clone())])?;
551                    let scores = clustered_result(decode_all_scores(cluster_id(doc_id), &bytes))?;
552                    Ok(scores.binary_search_by_key(&doc_id, |entry| entry.doc_id).ok().map_or(0, |position| scores[position].term_freq))
553                }
554                None => Ok(0),
555            }
556        })?)
557    }
558
559    fn doc_count(&self) -> StorageBackendResult<u64> {
560        if let Some(index) = self.native_index() {
561            return index.doc_count();
562        }
563        Ok(self.conn.with(|c| {
564            self.require_graph_format_on(c)?;
565            let n: i64 = c.query_row(
566                "SELECT COUNT(DISTINCT doc_id) FROM _occurrence_lengths
567                     WHERE table_name = ?1",
568                params![self.table],
569                |r| r.get(0),
570            )?;
571            decode_index_u64("document count", n)
572        })?)
573    }
574
575    fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
576        if let Some(index) = self.native_index() {
577            return index.total_field_length(field);
578        }
579        Ok(self.conn.with(|conn| {
580            self.require_graph_format_on(conn)?;
581            Ok(self
582                .stored_field_stats_on(conn, field)?
583                .map_or(0, |stats| stats.total_length))
584        })?)
585    }
586
587    fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
588        if let Some(index) = self.native_index() {
589            return index.vocabulary_terms(field);
590        }
591        self.terms_for_field(field)
592    }
593
594    fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
595        if let Some(index) = self.native_index() {
596            return index.vocabulary_keys(field);
597        }
598        Ok(self.conn.with(|conn| {
599            self.require_graph_format_on(conn)?;
600            let mut statement = conn.prepare("SELECT DISTINCT term FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 ORDER BY term")?;
601            let rows = statement.query_map(params![self.table, field], |row| row.get::<_, Vec<u8>>(0))?;
602            rows.map(|row| clustered_result(TokenTermKey::from_bytes(row?))).collect()
603        })?)
604    }
605
606    fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
607        if let Some(index) = self.native_index() {
608            return index.field_doc_count(field);
609        }
610        Ok(self.conn.with(|conn| {
611            self.require_graph_format_on(conn)?;
612            Ok(self
613                .stored_field_stats_on(conn, field)?
614                .map_or(0, |stats| stats.doc_count))
615        })?)
616    }
617
618    fn stats(&self) -> StorageBackendResult<IndexStats> {
619        if let Some(index) = self.native_index() {
620            return index.stats();
621        }
622        let doc_count = self.doc_count()?;
623        let mut s = IndexStats::default();
624        s.total_docs = doc_count;
625        if doc_count > 0 {
626            let total: u64 = self.conn.with(|c| {
627                self.require_graph_format_on(c)?;
628                let n: i64 = c.query_row(
629                    "SELECT COALESCE(SUM(total_length), 0) FROM _occurrence_fields
630                         WHERE table_name = ?1",
631                    params![self.table],
632                    |r| r.get(0),
633                )?;
634                decode_index_u64("total indexed length", n)
635            })?;
636            s.avg_doc_length = total as f64 / doc_count as f64;
637        }
638        let pairs = self
639            .conn
640            .with(|conn| self.term_frequencies_on(conn, None))?;
641        for ((field, term), df) in pairs {
642            let term = term.to_term();
643            if let Some(text) = term.as_str() {
644                s.set_doc_freq(field, text, df);
645            } else {
646                s.set_doc_freq_utf16(field, term.into_utf16(), df);
647            }
648        }
649        Ok(s)
650    }
651
652    fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
653        if let Some(index) = self.native_index() {
654            return index.posting_count(field);
655        }
656        Ok(self.conn.with(|conn| {
657            self.term_frequencies_on(conn, field)?
658                .into_values()
659                .try_fold(0_u64, |total, count| {
660                    total
661                        .checked_add(count)
662                        .ok_or_else(|| SQLiteError::StorageBackend("posting count overflow".into()))
663                })
664        })?)
665    }
666
667    fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
668        if let Some(index) = self.native_index() {
669            return index.doc_length_count(field);
670        }
671        Ok(self.conn.with(|c| {
672            self.require_graph_format_on(c)?;
673            let n: i64 = if let Some(field) = field {
674                c.query_row(
675                    "SELECT COUNT(*) FROM _occurrence_lengths
676                         WHERE table_name = ?1 AND field = ?2",
677                    params![self.table, field],
678                    |r| r.get(0),
679                )?
680            } else {
681                c.query_row(
682                    "SELECT COUNT(*) FROM _occurrence_lengths WHERE table_name = ?1",
683                    params![self.table],
684                    |r| r.get(0),
685                )?
686            };
687            decode_index_u64("document length count", n)
688        })?)
689    }
690
691    fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
692        if let Some(index) = self.native_index() {
693            return index.term_count(field);
694        }
695        Ok(self.conn.with(|conn| {
696            let terms = self
697                .term_frequencies_on(conn, field)?
698                .into_keys()
699                .map(|(_, term)| term)
700                .collect::<std::collections::BTreeSet<_>>();
701            Ok(terms.len() as u64)
702        })?)
703    }
704
705    fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
706        if let Some(index) = self.native_retained_index()? {
707            return Ok(index);
708        }
709        self.physical_snapshot(&self.retention_control)
710    }
711
712    fn snapshot_with_control(
713        &self,
714        control: &uqa_storage::read_control::StorageReadControl,
715    ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
716        control.check()?;
717        if let Some(index) = self.native_retained_index()? {
718            control.check()?;
719            return Ok(index);
720        }
721        self.physical_snapshot(control)
722    }
723
724    fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
725        if let Some(index) = self.native_index() {
726            return index.field_names();
727        }
728        Ok(self.conn.with(|c| {
729            self.require_graph_format_on(c)?;
730            let mut stmt =
731                c.prepare("SELECT DISTINCT field FROM _occurrence_lengths WHERE table_name = ?1")?;
732            let rows = stmt.query_map([&self.table], |row| row.get::<_, String>(0))?;
733            let mut fields = Vec::new();
734            for row in rows {
735                fields.push(row?);
736            }
737            Ok(fields)
738        })?)
739    }
740
741    fn set_field_analyzer(
742        &mut self,
743        field: &str,
744        analyzer: Analyzer,
745        phase: AnalyzerPhase,
746    ) -> Result<(), String> {
747        if let Some(mut index) = self.native_index() {
748            index.set_field_analyzer(field, analyzer, phase)?;
749            self.bindings = index.analyzer_bindings().clone();
750            return Ok(());
751        }
752        let mut candidate = self.bindings.clone();
753        candidate
754            .bind(field, &analyzer, phase)
755            .map_err(|error| error.to_string())?;
756        self.validate_index_revision_change(field, &candidate)
757            .map_err(|error| error.to_string())?;
758        self.bindings = candidate;
759        Ok(())
760    }
761
762    fn remove_field_analyzers(&mut self, field: &str) -> Result<(), String> {
763        if let Some(mut index) = self.native_index() {
764            index.remove_field_analyzers(field)?;
765            self.bindings = index.analyzer_bindings().clone();
766            return Ok(());
767        }
768        let mut candidate = self.bindings.clone();
769        candidate.remove(field);
770        self.validate_index_revision_change(field, &candidate)
771            .map_err(|error| error.to_string())?;
772        self.bindings = candidate;
773        Ok(())
774    }
775
776    fn get_field_analyzer(&self, field: &str) -> Analyzer {
777        self.bindings.index_configuration(field).clone()
778    }
779    fn get_search_analyzer(&self, field: &str) -> Analyzer {
780        self.bindings.search_configuration(field).clone()
781    }
782    fn index_analyzer_revision(
783        &self,
784        field: &str,
785    ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
786        Ok(self.bindings.index_revision(field)?)
787    }
788    fn search_analyzer_revision(
789        &self,
790        field: &str,
791    ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
792        Ok(self.bindings.search_revision(field)?)
793    }
794
795    fn set_field_analyzer_revision(
796        &mut self,
797        field: &str,
798        revision: Arc<uqa_analysis::CompiledAnalyzer>,
799        phase: AnalyzerPhase,
800    ) -> Result<(), String> {
801        if let Some(mut index) = self.native_index() {
802            index.set_field_analyzer_revision(field, revision, phase)?;
803            self.bindings = index.analyzer_bindings().clone();
804            return Ok(());
805        }
806        let mut candidate = self.bindings.clone();
807        candidate
808            .bind_revision(field, revision, phase)
809            .map_err(|error| error.to_string())?;
810        self.validate_index_revision_change(field, &candidate)
811            .map_err(|error| error.to_string())?;
812        self.bindings = candidate;
813        Ok(())
814    }
815
816    fn set_field_analyzer_revisions(
817        &mut self,
818        field: &str,
819        index: Arc<uqa_analysis::CompiledAnalyzer>,
820        search: Arc<uqa_analysis::CompiledAnalyzer>,
821    ) -> Result<(), String> {
822        if let Some(mut native) = self.native_index() {
823            native.set_field_analyzer_revisions(field, index, search)?;
824            self.bindings = native.analyzer_bindings().clone();
825            return Ok(());
826        }
827        let mut candidate = self.bindings.clone();
828        candidate
829            .bind_revisions(field, index, search)
830            .map_err(|error| error.to_string())?;
831        self.validate_index_revision_change(field, &candidate)
832            .map_err(|error| error.to_string())?;
833        self.bindings = candidate;
834        Ok(())
835    }
836
837    fn rebuild_with_analyzer_revision(
838        &mut self,
839        field: &str,
840        revision: Arc<uqa_analysis::CompiledAnalyzer>,
841        phase: AnalyzerPhase,
842        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
843    ) -> StorageBackendResult<()> {
844        if let Some(mut index) = self.native_index() {
845            index.rebuild_with_analyzer_revision(field, revision, phase, documents)?;
846            self.bindings = index.analyzer_bindings().clone();
847            return Ok(());
848        }
849        let mut replacement = self.clone();
850        replacement.bindings.bind_revision(field, revision, phase)?;
851        replacement.rebuild_documents_inner(documents)?;
852        *self = replacement;
853        Ok(())
854    }
855
856    fn rebuild_with_analyzer_revision_cancellable(
857        &mut self,
858        field: &str,
859        revision: Arc<uqa_analysis::CompiledAnalyzer>,
860        phase: AnalyzerPhase,
861        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
862        cancellation: &uqa_core::CancellationToken,
863    ) -> StorageBackendResult<()> {
864        if let Some(mut index) = self.native_index() {
865            index.rebuild_with_analyzer_revision_cancellable(
866                field,
867                revision,
868                phase,
869                documents,
870                cancellation,
871            )?;
872            self.bindings = index.analyzer_bindings().clone();
873            return Ok(());
874        }
875        cancellation.check()?;
876        let mut replacement = self.clone();
877        replacement.bindings.bind_revision(field, revision, phase)?;
878        replacement.rebuild_documents_with_cancellation(documents, Some(cancellation))?;
879        *self = replacement;
880        Ok(())
881    }
882    fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
883        if let Some(index) = self.native_index() {
884            return index.doc_freq_any_field(term);
885        }
886        uqa_storage::inverted_index::defaults::doc_freq_any_field(self, term)
887    }
888    fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
889        if let Some(index) = self.native_index() {
890            return index.field_stats(field);
891        }
892        uqa_storage::inverted_index::defaults::field_stats(self, field)
893    }
894    fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
895        if let Some(index) = self.native_index() {
896            return index.field_stats_scalar(field);
897        }
898        uqa_storage::inverted_index::defaults::field_stats_scalar(self, field)
899    }
900    fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
901        if let Some(index) = self.native_index() {
902            return index.get_posting_list_any_field(term);
903        }
904        uqa_storage::inverted_index::defaults::get_posting_list_any_field(self, term)
905    }
906    fn get_term_freqs_bulk(
907        &self,
908        doc_ids: &[DocId],
909        field: &str,
910        term: &str,
911    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
912        if let Some(index) = self.native_index() {
913            return index.get_term_freqs_bulk(doc_ids, field, term);
914        }
915        uqa_storage::inverted_index::defaults::get_term_freqs_bulk(self, doc_ids, field, term)
916    }
917    fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
918        if let Some(index) = self.native_index() {
919            return index.get_total_doc_length(doc_id);
920        }
921        uqa_storage::inverted_index::defaults::get_total_doc_length(self, doc_id)
922    }
923    fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
924        if let Some(index) = self.native_index() {
925            return index.get_total_term_freq(doc_id, term);
926        }
927        uqa_storage::inverted_index::defaults::get_total_term_freq(self, doc_id, term)
928    }
929}