Skip to main content

uqa_storage_sqlite/inverted_index/
trait_impl.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! `InvertedIndex` trait implementation and read/statistics surface.
8
9use super::queries::project_postings;
10use super::{
11    clustered_result, decode_index_u64, encode_index_u64, invalidate_posting_accelerators, params,
12    params_from_iter, quote_ident, Analyzer, AnalyzerPhase, Arc, BTreeMap, BlockMaxScorer, DocId,
13    FieldName, IndexStats, InvertedIndex, OptionalExtension, PostingCursor, PostingList,
14    SQLiteError, SQLiteInvertedIndex, SqlValue, StorageBackendResult,
15};
16use super::{IndexedFieldMetadata, TokenTermKey};
17use uqa_storage::clustered_postings::{cluster_id, decode_all_scores, OccurrencePosting};
18use uqa_storage::inverted_index::TextIndexSource;
19
20impl InvertedIndex for SQLiteInvertedIndex {
21    fn posting_read_cursor_key_budgeted<'a>(
22        &'a self,
23        field: &'a str,
24        term: &'a TokenTermKey,
25        control: &uqa_storage::read_control::StorageReadControl,
26    ) -> StorageBackendResult<uqa_storage::clustered_postings::BudgetedPostingReadCursor<'a>> {
27        control.check()?;
28        if let Some(index) = self.native_index() {
29            return uqa_storage::clustered_postings::open_controlled_cursor(
30                index.snapshot()?,
31                field,
32                term,
33                control,
34            );
35        }
36        uqa_storage::clustered_postings::open_controlled_cursor(self, field, term, control)
37    }
38
39    fn visit_score_clusters(
40        &self,
41        field: &str,
42        term: &TokenTermKey,
43        after: Option<u64>,
44        limit: usize,
45        control: &uqa_storage::read_control::StorageReadControl,
46        visit: &mut uqa_storage::clustered_postings::ScoreClusterVisitor<'_>,
47    ) -> StorageBackendResult<()> {
48        if let Some(index) = self.native_index() {
49            return index.visit_score_clusters(field, term, after, limit, control, visit);
50        }
51        Ok(self.visit_clusters_budgeted(field, term, after, limit, control, visit)?)
52    }
53
54    fn get_occurrences_budgeted(
55        &self,
56        doc_id: DocId,
57        field: &str,
58        term: &TokenTermKey,
59        control: &uqa_storage::read_control::StorageReadControl,
60    ) -> StorageBackendResult<uqa_core::memory::Budgeted<Vec<uqa_core::TokenOccurrence>>> {
61        if let Some(index) = self.native_index() {
62            return index.get_occurrences_budgeted(doc_id, field, term, control);
63        }
64        Ok(self.occurrences_budgeted(doc_id, field, term, control)?)
65    }
66
67    fn field_stats_scalar_budgeted(
68        &self,
69        field: &str,
70        control: &uqa_storage::read_control::StorageReadControl,
71    ) -> StorageBackendResult<IndexStats> {
72        if let Some(index) = self.native_index() {
73            return index.field_stats_scalar_budgeted(field, control);
74        }
75        Ok(self.scalar_stats_budgeted(field, control)?)
76    }
77
78    fn analyzer(&self) -> &Analyzer {
79        self.bindings.default_configuration()
80    }
81
82    fn default_analyzer_binding(
83        &self,
84    ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
85        Ok(self.bindings.default_binding())
86    }
87
88    fn add_document(
89        &mut self,
90        doc_id: DocId,
91        fields: BTreeMap<FieldName, String>,
92    ) -> StorageBackendResult<()> {
93        if let Some(mut index) = self.native_index() {
94            return index.add_document(doc_id, fields);
95        }
96        Ok(self.add_document_inner(doc_id, fields)?)
97    }
98
99    fn try_add_documents(
100        &mut self,
101        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
102    ) -> StorageBackendResult<()> {
103        if let Some(mut index) = self.native_index() {
104            return index.try_add_documents(documents);
105        }
106        Ok(self.add_documents_inner(documents)?)
107    }
108
109    fn try_add_documents_observed(
110        &mut self,
111        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
112        visit: &mut uqa_storage::inverted_index::InvertedIndexChangeVisitor<'_>,
113    ) -> StorageBackendResult<()> {
114        if let Some(mut index) = self.native_index() {
115            return index.try_add_documents_observed(documents, visit);
116        }
117        Ok(self.add_documents_observed(documents, Some(visit))?)
118    }
119
120    fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
121        if let Some(mut index) = self.native_index() {
122            return index.remove_document(doc_id);
123        }
124        Ok(self.remove_document_inner(doc_id)?)
125    }
126
127    fn clear(&mut self) -> StorageBackendResult<()> {
128        if let Some(mut index) = self.native_index() {
129            return index.clear();
130        }
131        self.conn.with_mut(|conn| {
132            let tx = conn.savepoint()?;
133            invalidate_posting_accelerators(&tx, &self.table)?;
134            self.clear_index_on(&tx)?;
135            tx.commit()?;
136            Ok(())
137        })?;
138        Ok(())
139    }
140
141    fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
142        if let Some(index) = self.native_index() {
143            return index.source_rebuild_required();
144        }
145        Ok(self.conn.with(|conn| self.needs_source_rebuild_on(conn))?)
146    }
147
148    fn try_rebuild_documents(
149        &mut self,
150        source: &mut dyn TextIndexSource,
151    ) -> StorageBackendResult<()> {
152        if let Some(mut index) = self.native_index() {
153            return index.try_rebuild_documents(source);
154        }
155        Ok(self.rebuild_documents_inner(source)?)
156    }
157
158    fn try_rebuild_documents_cancellable(
159        &mut self,
160        source: &mut dyn TextIndexSource,
161        cancellation: &uqa_core::CancellationToken,
162    ) -> StorageBackendResult<()> {
163        if let Some(mut index) = self.native_index() {
164            return index.try_rebuild_documents_cancellable(source, cancellation);
165        }
166        Ok(self.rebuild_documents_with_cancellation(source, Some(cancellation))?)
167    }
168
169    fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
170        if let Some(index) = self.native_index() {
171            return index.get_posting_list(field, term);
172        }
173        self.get_posting_list_key(field, &TokenTermKey::from_text(term))
174    }
175
176    fn get_posting_list_key(
177        &self,
178        field: &str,
179        term: &TokenTermKey,
180    ) -> StorageBackendResult<PostingList> {
181        if let Some(index) = self.native_index() {
182            return index.get_posting_list_key(field, term);
183        }
184        Ok(project_postings(self.get_occurrence_postings(field, term)?))
185    }
186
187    fn get_occurrence_postings(
188        &self,
189        field: &str,
190        term: &TokenTermKey,
191    ) -> StorageBackendResult<Vec<OccurrencePosting>> {
192        if let Some(index) = self.native_index() {
193            return index.get_occurrence_postings(field, term);
194        }
195        Ok(self
196            .occurrence_postings_bulk(field, std::slice::from_ref(term))?
197            .pop()
198            .unwrap_or_default())
199    }
200
201    fn get_occurrences(
202        &self,
203        doc_id: DocId,
204        field: &str,
205        term: &TokenTermKey,
206    ) -> StorageBackendResult<Vec<uqa_core::TokenOccurrence>> {
207        if let Some(index) = self.native_index() {
208            return index.get_occurrences(doc_id, field, term);
209        }
210        Ok(self.conn.with(|conn| {
211            self.require_graph_format_on(conn)?;
212            let entries = super::load_cluster(conn, &self.table, field, term, cluster_id(doc_id))?;
213            let Some(posting) = entries.into_iter().find(|entry| entry.doc_id == doc_id) else {
214                return Ok(Vec::new());
215            };
216            self.validate_posting_metadata_on(conn, field, &posting)?;
217            Ok(posting.occurrences)
218        })?)
219    }
220
221    fn indexed_field_metadata(
222        &self,
223        doc_id: DocId,
224        field: &str,
225    ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
226        if let Some(index) = self.native_index() {
227            return index.indexed_field_metadata(doc_id, field);
228        }
229        let doc_id = encode_index_u64("document", doc_id)?;
230        Ok(self.conn.with(|conn| {
231            self.require_graph_format_on(conn)?;
232            self.read_field_metadata_on(conn, doc_id, field)
233        })?)
234    }
235
236    fn get_posting_lists_bulk(
237        &self,
238        field: &str,
239        terms: &[String],
240    ) -> StorageBackendResult<Vec<PostingList>> {
241        if let Some(index) = self.native_index() {
242            return index.get_posting_lists_bulk(field, terms);
243        }
244        let keys = terms
245            .iter()
246            .map(|term| TokenTermKey::from_text(term))
247            .collect::<Vec<_>>();
248        Ok(self
249            .occurrence_postings_bulk(field, &keys)?
250            .into_iter()
251            .map(project_postings)
252            .collect())
253    }
254
255    fn posting_cursor(
256        &self,
257        field: &str,
258        term: &str,
259    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
260        if let Some(index) = self.native_index() {
261            return index.posting_cursor(field, term);
262        }
263        self.posting_cursor_key(field, &TokenTermKey::from_text(term))
264    }
265
266    fn posting_cursor_key(
267        &self,
268        field: &str,
269        term: &TokenTermKey,
270    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
271        if let Some(index) = self.native_index() {
272            return index.posting_cursor_key(field, term);
273        }
274        self.cursor_for_term(field, term)
275    }
276
277    fn posting_cursors_bulk(
278        &self,
279        field: &str,
280        terms: &[String],
281    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
282        if let Some(index) = self.native_index() {
283            return index.posting_cursors_bulk(field, terms);
284        }
285        let keys = terms
286            .iter()
287            .map(|term| TokenTermKey::from_text(term))
288            .collect::<Vec<_>>();
289        self.cursors_for_terms(field, &keys)
290    }
291
292    fn posting_cursors_keys_bulk(
293        &self,
294        field: &str,
295        terms: &[TokenTermKey],
296    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
297        if let Some(index) = self.native_index() {
298            return index.posting_cursors_keys_bulk(field, terms);
299        }
300        self.cursors_for_terms(field, terms)
301    }
302
303    fn get_posting_lists_keys_bulk(
304        &self,
305        field: &str,
306        terms: &[TokenTermKey],
307    ) -> StorageBackendResult<Vec<PostingList>> {
308        if let Some(index) = self.native_index() {
309            return index.get_posting_lists_keys_bulk(field, terms);
310        }
311        Ok(self
312            .occurrence_postings_bulk(field, terms)?
313            .into_iter()
314            .map(project_postings)
315            .collect())
316    }
317
318    fn persisted_block_max_scores_keys_bulk(
319        &self,
320        field: &str,
321        terms: &[TokenTermKey],
322        scorer_fingerprint: &str,
323    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
324        if scorer_fingerprint.is_empty() {
325            return Ok(vec![None; terms.len()]);
326        }
327        self.get_versioned_block_max_scores_keys_bulk(field, terms, scorer_fingerprint)
328    }
329
330    fn rebuild_persisted_block_max(
331        &mut self,
332        field: &str,
333        scorer: &dyn BlockMaxScorer,
334        scorer_fingerprint: &str,
335    ) -> StorageBackendResult<bool> {
336        if let Some(mut index) = self.native_index() {
337            return index.rebuild_persisted_block_max(field, scorer, scorer_fingerprint);
338        }
339        if scorer_fingerprint.is_empty() {
340            return Err(SQLiteError::StorageBackend(
341                "persisted block-max scorer fingerprint must not be empty".into(),
342            )
343            .into());
344        }
345        let terms = self.vocabulary_keys(field)?;
346        self.ensure_aux_tables(field)?;
347        let table = self.blockmax_table_name(field);
348        self.conn.with_mut(|conn| {
349            conn.execute(&format!("DELETE FROM {}", quote_ident(&table)), [])?;
350            Ok(())
351        })?;
352        for term in terms {
353            self.build_block_max_scores_key(field, &term, scorer, scorer_fingerprint)?;
354        }
355        Ok(true)
356    }
357
358    fn persisted_block_max_scores(
359        &self,
360        field: &str,
361        term: &str,
362        scorer_fingerprint: &str,
363    ) -> StorageBackendResult<Option<Vec<f64>>> {
364        if scorer_fingerprint.is_empty() {
365            return Ok(None);
366        }
367        self.get_versioned_block_max_scores(field, term, scorer_fingerprint)
368    }
369
370    fn persisted_block_max_scores_bulk(
371        &self,
372        field: &str,
373        terms: &[String],
374        scorer_fingerprint: &str,
375    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
376        if scorer_fingerprint.is_empty() {
377            return Ok(vec![None; terms.len()]);
378        }
379        self.get_versioned_block_max_scores_bulk(field, terms, scorer_fingerprint)
380    }
381
382    fn for_each_term_freq(
383        &self,
384        field: &str,
385        term: &str,
386        visit: &mut dyn FnMut(DocId, u64),
387    ) -> StorageBackendResult<()> {
388        if let Some(index) = self.native_index() {
389            return index.for_each_term_freq(field, term, visit);
390        }
391        let mut cursor = self.posting_cursor(field, term)?;
392        while let Some(entry) = cursor.current() {
393            visit(entry.doc_id, entry.term_freq);
394            cursor.advance()?;
395        }
396        Ok(())
397    }
398
399    fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
400        if let Some(index) = self.native_index() {
401            return index.doc_freq(field, term);
402        }
403        self.doc_freq_key(field, &TokenTermKey::from_text(term))
404    }
405
406    fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
407        if let Some(index) = self.native_index() {
408            return index.doc_freq_key(field, term);
409        }
410        Ok(self.posting_cursor_key(field, term)?.doc_freq())
411    }
412
413    fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64> {
414        if let Some(index) = self.native_index() {
415            return index.get_doc_length(doc_id, field);
416        }
417        Ok(self
418            .get_doc_lengths_bulk(&[doc_id], field)?
419            .get(&doc_id)
420            .copied()
421            .unwrap_or(0))
422    }
423
424    fn get_doc_lengths_bulk(
425        &self,
426        doc_ids: &[DocId],
427        field: &str,
428    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
429        if let Some(index) = self.native_index() {
430            return index.get_doc_lengths_bulk(doc_ids, field);
431        }
432        Ok(self.conn.with(|conn| {
433            self.require_graph_format_on(conn)?;
434            let mut out = BTreeMap::new();
435            for chunk in doc_ids.chunks(900) {
436                let ids = (3..chunk.len()+3).map(|parameter| format!("?{parameter}")).collect::<Vec<_>>().join(", ");
437                let sql = format!("WITH lengths AS (SELECT doc_id, length FROM _occurrence_lengths WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})), documents AS (SELECT doc_id, metadata_blob FROM _occurrence_documents WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})) SELECT COALESCE(lengths.doc_id, documents.doc_id), length, metadata_blob FROM lengths FULL OUTER JOIN documents USING(doc_id)");
438                let mut values = vec![SqlValue::Text(self.table.clone()), SqlValue::Text(field.into())];
439                for doc_id in chunk {
440                    values.push(SqlValue::Integer(encode_index_u64("document", *doc_id)?));
441                }
442                let mut statement = conn.prepare(&sql)?;
443                let rows = statement.query_map(params_from_iter(values), |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option<i64>>(1)?, row.get::<_, Option<Vec<u8>>>(2)?)))?;
444                for row in rows {
445                    let (doc_id, length, metadata) = row?;
446                    let doc_id = decode_index_u64("document id", doc_id)?;
447                    let length = length.map(|length| decode_index_u64("document length", length)).transpose()?;
448                    let metadata = metadata.map(|bytes| clustered_result(IndexedFieldMetadata::from_bytes(&bytes))).transpose()?;
449                    let (Some(length), Some(metadata)) = (length, metadata) else {
450                        return Err(SQLiteError::StorageBackend("indexed field length and source metadata disagree".into()));
451                    };
452                    let stats = self.stored_field_stats_on(conn, field)?.ok_or_else(|| SQLiteError::StorageBackend("indexed field revision is missing".into()))?;
453                    if metadata.length != length || metadata.revision() != stats.revision {
454                        return Err(SQLiteError::StorageBackend("indexed field length or revision disagrees with source metadata".into()));
455                    }
456                    out.insert(doc_id, length);
457                }
458            }
459            Ok(out)
460        })?)
461    }
462
463    fn get_scoring_inputs_bulk(
464        &self,
465        doc_ids: &[DocId],
466        field: &str,
467        terms: &[String],
468    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
469        if let Some(index) = self.native_index() {
470            return index.get_scoring_inputs_bulk(doc_ids, field, terms);
471        }
472        let keys = terms
473            .iter()
474            .map(|term| TokenTermKey::from_text(term))
475            .collect::<Vec<_>>();
476        self.get_scoring_inputs_keys_bulk(doc_ids, field, &keys)
477    }
478
479    fn get_scoring_inputs_keys_bulk(
480        &self,
481        doc_ids: &[DocId],
482        field: &str,
483        terms: &[TokenTermKey],
484    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
485        if let Some(index) = self.native_index() {
486            return index.get_scoring_inputs_keys_bulk(doc_ids, field, terms);
487        }
488        if doc_ids.is_empty() {
489            return Ok(Vec::new());
490        }
491
492        let doc_lengths = self.get_doc_lengths_bulk(doc_ids, field)?;
493        let mut inputs: Vec<(u64, Vec<u64>)> = doc_ids
494            .iter()
495            .map(|doc_id| {
496                (
497                    doc_lengths.get(doc_id).copied().unwrap_or(0),
498                    vec![0; terms.len()],
499                )
500            })
501            .collect();
502        if terms.is_empty() {
503            return Ok(inputs);
504        }
505
506        let mut output_positions = BTreeMap::<DocId, Vec<usize>>::new();
507        for (position, doc_id) in doc_ids.iter().copied().enumerate() {
508            output_positions.entry(doc_id).or_default().push(position);
509        }
510        for (term_index, mut cursor) in self
511            .posting_cursors_keys_bulk(field, terms)?
512            .into_iter()
513            .enumerate()
514        {
515            while let Some(entry) = cursor.current() {
516                if let Some(positions) = output_positions.get(&entry.doc_id) {
517                    for position in positions {
518                        inputs[*position].1[term_index] = entry.term_freq;
519                        inputs[*position].0 = entry.doc_length;
520                    }
521                }
522                cursor.advance()?;
523            }
524        }
525        Ok(inputs)
526    }
527
528    fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
529        if let Some(index) = self.native_index() {
530            return index.get_term_freq(doc_id, field, term);
531        }
532        self.get_term_freq_key(doc_id, field, &TokenTermKey::from_text(term))
533    }
534
535    fn get_term_freq_key(
536        &self,
537        doc_id: DocId,
538        field: &str,
539        term: &TokenTermKey,
540    ) -> StorageBackendResult<u64> {
541        if let Some(index) = self.native_index() {
542            return index.get_term_freq_key(doc_id, field, term);
543        }
544        let cluster = encode_index_u64("posting cluster", cluster_id(doc_id))?;
545        Ok(self.conn.with(|conn| {
546            self.require_graph_format_on(conn)?;
547            let row: Option<(i64, Vec<u8>)> = conn.query_row("SELECT posting_count, score_blob FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 AND term = ?3 AND cluster_id = ?4", params![self.table, field, term.as_bytes(), cluster], |row| Ok((row.get(0)?, row.get(1)?))).optional()?;
548            match row {
549                Some((count, bytes)) => {
550                    // Validate the version and redundant row count before decoding only the requested cluster.
551                    super::posting_cursor_from_rows(vec![(cluster, count, bytes.clone())])?;
552                    let scores = clustered_result(decode_all_scores(cluster_id(doc_id), &bytes))?;
553                    Ok(scores.binary_search_by_key(&doc_id, |entry| entry.doc_id).ok().map_or(0, |position| scores[position].term_freq))
554                }
555                None => Ok(0),
556            }
557        })?)
558    }
559
560    fn doc_count(&self) -> StorageBackendResult<u64> {
561        if let Some(index) = self.native_index() {
562            return index.doc_count();
563        }
564        Ok(self.conn.with(|c| {
565            self.require_graph_format_on(c)?;
566            let n: i64 = c.query_row(
567                "SELECT COUNT(DISTINCT doc_id) FROM _occurrence_lengths
568                     WHERE table_name = ?1",
569                params![self.table],
570                |r| r.get(0),
571            )?;
572            decode_index_u64("document count", n)
573        })?)
574    }
575
576    fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
577        if let Some(index) = self.native_index() {
578            return index.total_field_length(field);
579        }
580        Ok(self.conn.with(|conn| {
581            self.require_graph_format_on(conn)?;
582            Ok(self
583                .stored_field_stats_on(conn, field)?
584                .map_or(0, |stats| stats.total_length))
585        })?)
586    }
587
588    fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
589        if let Some(index) = self.native_index() {
590            return index.vocabulary_terms(field);
591        }
592        self.terms_for_field(field)
593    }
594
595    fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
596        if let Some(index) = self.native_index() {
597            return index.vocabulary_keys(field);
598        }
599        Ok(self.conn.with(|conn| {
600            self.require_graph_format_on(conn)?;
601            let mut statement = conn.prepare("SELECT DISTINCT term FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 ORDER BY term")?;
602            let rows = statement.query_map(params![self.table, field], |row| row.get::<_, Vec<u8>>(0))?;
603            rows.map(|row| clustered_result(TokenTermKey::from_bytes(row?))).collect()
604        })?)
605    }
606
607    fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
608        if let Some(index) = self.native_index() {
609            return index.field_doc_count(field);
610        }
611        Ok(self.conn.with(|conn| {
612            self.require_graph_format_on(conn)?;
613            Ok(self
614                .stored_field_stats_on(conn, field)?
615                .map_or(0, |stats| stats.doc_count))
616        })?)
617    }
618
619    fn stats(&self) -> StorageBackendResult<IndexStats> {
620        if let Some(index) = self.native_index() {
621            return index.stats();
622        }
623        let doc_count = self.doc_count()?;
624        let mut s = IndexStats::default();
625        s.total_docs = doc_count;
626        if doc_count > 0 {
627            let total: u64 = self.conn.with(|c| {
628                self.require_graph_format_on(c)?;
629                let n: i64 = c.query_row(
630                    "SELECT COALESCE(SUM(total_length), 0) FROM _occurrence_fields
631                         WHERE table_name = ?1",
632                    params![self.table],
633                    |r| r.get(0),
634                )?;
635                decode_index_u64("total indexed length", n)
636            })?;
637            s.avg_doc_length = total as f64 / doc_count as f64;
638        }
639        let pairs = self
640            .conn
641            .with(|conn| self.term_frequencies_on(conn, None))?;
642        for ((field, term), df) in pairs {
643            let term = term.to_term();
644            if let Some(text) = term.as_str() {
645                s.set_doc_freq(field, text, df);
646            } else {
647                s.set_doc_freq_utf16(field, term.into_utf16(), df);
648            }
649        }
650        Ok(s)
651    }
652
653    fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
654        if let Some(index) = self.native_index() {
655            return index.posting_count(field);
656        }
657        Ok(self.conn.with(|conn| {
658            self.term_frequencies_on(conn, field)?
659                .into_values()
660                .try_fold(0_u64, |total, count| {
661                    total
662                        .checked_add(count)
663                        .ok_or_else(|| SQLiteError::StorageBackend("posting count overflow".into()))
664                })
665        })?)
666    }
667
668    fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
669        if let Some(index) = self.native_index() {
670            return index.doc_length_count(field);
671        }
672        Ok(self.conn.with(|c| {
673            self.require_graph_format_on(c)?;
674            let n: i64 = if let Some(field) = field {
675                c.query_row(
676                    "SELECT COUNT(*) FROM _occurrence_lengths
677                         WHERE table_name = ?1 AND field = ?2",
678                    params![self.table, field],
679                    |r| r.get(0),
680                )?
681            } else {
682                c.query_row(
683                    "SELECT COUNT(*) FROM _occurrence_lengths WHERE table_name = ?1",
684                    params![self.table],
685                    |r| r.get(0),
686                )?
687            };
688            decode_index_u64("document length count", n)
689        })?)
690    }
691
692    fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
693        if let Some(index) = self.native_index() {
694            return index.term_count(field);
695        }
696        Ok(self.conn.with(|conn| {
697            let terms = self
698                .term_frequencies_on(conn, field)?
699                .into_keys()
700                .map(|(_, term)| term)
701                .collect::<std::collections::BTreeSet<_>>();
702            Ok(terms.len() as u64)
703        })?)
704    }
705
706    fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
707        if let Some(index) = self.native_retained_index()? {
708            return Ok(index);
709        }
710        self.physical_snapshot(&self.retention_control)
711    }
712
713    fn snapshot_with_control(
714        &self,
715        control: &uqa_storage::read_control::StorageReadControl,
716    ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
717        control.check()?;
718        if let Some(index) = self.native_retained_index()? {
719            control.check()?;
720            return Ok(index);
721        }
722        self.physical_snapshot(control)
723    }
724
725    fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
726        if let Some(index) = self.native_index() {
727            return index.field_names();
728        }
729        Ok(self.conn.with(|c| {
730            self.require_graph_format_on(c)?;
731            let mut stmt =
732                c.prepare("SELECT DISTINCT field FROM _occurrence_lengths WHERE table_name = ?1")?;
733            let rows = stmt.query_map([&self.table], |row| row.get::<_, String>(0))?;
734            let mut fields = Vec::new();
735            for row in rows {
736                fields.push(row?);
737            }
738            Ok(fields)
739        })?)
740    }
741
742    fn set_field_analyzer(
743        &mut self,
744        field: &str,
745        analyzer: Analyzer,
746        phase: AnalyzerPhase,
747    ) -> Result<(), String> {
748        if let Some(mut index) = self.native_index() {
749            index.set_field_analyzer(field, analyzer, phase)?;
750            self.bindings = index.analyzer_bindings().clone();
751            return Ok(());
752        }
753        let mut candidate = self.bindings.clone();
754        candidate
755            .bind(field, &analyzer, phase)
756            .map_err(|error| error.to_string())?;
757        self.validate_index_revision_change(field, &candidate)
758            .map_err(|error| error.to_string())?;
759        self.bindings = candidate;
760        Ok(())
761    }
762
763    fn remove_field_analyzers(&mut self, field: &str) -> Result<(), String> {
764        if let Some(mut index) = self.native_index() {
765            index.remove_field_analyzers(field)?;
766            self.bindings = index.analyzer_bindings().clone();
767            return Ok(());
768        }
769        let mut candidate = self.bindings.clone();
770        candidate.remove(field);
771        self.validate_index_revision_change(field, &candidate)
772            .map_err(|error| error.to_string())?;
773        self.bindings = candidate;
774        Ok(())
775    }
776
777    fn get_field_analyzer(&self, field: &str) -> Analyzer {
778        self.bindings.index_configuration(field).clone()
779    }
780    fn get_search_analyzer(&self, field: &str) -> Analyzer {
781        self.bindings.search_configuration(field).clone()
782    }
783    fn index_analyzer_revision(
784        &self,
785        field: &str,
786    ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
787        Ok(self.bindings.index_revision(field)?)
788    }
789    fn search_analyzer_revision(
790        &self,
791        field: &str,
792    ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
793        Ok(self.bindings.search_revision(field)?)
794    }
795
796    fn set_field_analyzer_revision(
797        &mut self,
798        field: &str,
799        revision: Arc<uqa_analysis::CompiledAnalyzer>,
800        phase: AnalyzerPhase,
801    ) -> Result<(), String> {
802        if let Some(mut index) = self.native_index() {
803            index.set_field_analyzer_revision(field, revision, phase)?;
804            self.bindings = index.analyzer_bindings().clone();
805            return Ok(());
806        }
807        let mut candidate = self.bindings.clone();
808        candidate
809            .bind_revision(field, revision, phase)
810            .map_err(|error| error.to_string())?;
811        self.validate_index_revision_change(field, &candidate)
812            .map_err(|error| error.to_string())?;
813        self.bindings = candidate;
814        Ok(())
815    }
816
817    fn set_field_analyzer_revisions(
818        &mut self,
819        field: &str,
820        index: Arc<uqa_analysis::CompiledAnalyzer>,
821        search: Arc<uqa_analysis::CompiledAnalyzer>,
822    ) -> Result<(), String> {
823        if let Some(mut native) = self.native_index() {
824            native.set_field_analyzer_revisions(field, index, search)?;
825            self.bindings = native.analyzer_bindings().clone();
826            return Ok(());
827        }
828        let mut candidate = self.bindings.clone();
829        candidate
830            .bind_revisions(field, index, search)
831            .map_err(|error| error.to_string())?;
832        self.validate_index_revision_change(field, &candidate)
833            .map_err(|error| error.to_string())?;
834        self.bindings = candidate;
835        Ok(())
836    }
837
838    fn rebuild_with_analyzer_revision(
839        &mut self,
840        field: &str,
841        revision: Arc<uqa_analysis::CompiledAnalyzer>,
842        phase: AnalyzerPhase,
843        source: &mut dyn TextIndexSource,
844    ) -> StorageBackendResult<()> {
845        if let Some(mut index) = self.native_index() {
846            index.rebuild_with_analyzer_revision(field, revision, phase, source)?;
847            self.bindings = index.analyzer_bindings().clone();
848            return Ok(());
849        }
850        let mut replacement = self.clone();
851        replacement.bindings.bind_revision(field, revision, phase)?;
852        replacement.rebuild_documents_inner(source)?;
853        *self = replacement;
854        Ok(())
855    }
856
857    fn rebuild_with_analyzer_revision_cancellable(
858        &mut self,
859        field: &str,
860        revision: Arc<uqa_analysis::CompiledAnalyzer>,
861        phase: AnalyzerPhase,
862        source: &mut dyn TextIndexSource,
863        cancellation: &uqa_core::CancellationToken,
864    ) -> StorageBackendResult<()> {
865        if let Some(mut index) = self.native_index() {
866            index.rebuild_with_analyzer_revision_cancellable(
867                field,
868                revision,
869                phase,
870                source,
871                cancellation,
872            )?;
873            self.bindings = index.analyzer_bindings().clone();
874            return Ok(());
875        }
876        cancellation.check()?;
877        let mut replacement = self.clone();
878        replacement.bindings.bind_revision(field, revision, phase)?;
879        replacement.rebuild_documents_with_cancellation(source, Some(cancellation))?;
880        *self = replacement;
881        Ok(())
882    }
883    fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
884        if let Some(index) = self.native_index() {
885            return index.doc_freq_any_field(term);
886        }
887        uqa_storage::inverted_index::defaults::doc_freq_any_field(self, term)
888    }
889    fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
890        if let Some(index) = self.native_index() {
891            return index.field_stats(field);
892        }
893        uqa_storage::inverted_index::defaults::field_stats(self, field)
894    }
895    fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
896        if let Some(index) = self.native_index() {
897            return index.field_stats_scalar(field);
898        }
899        uqa_storage::inverted_index::defaults::field_stats_scalar(self, field)
900    }
901    fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
902        if let Some(index) = self.native_index() {
903            return index.get_posting_list_any_field(term);
904        }
905        uqa_storage::inverted_index::defaults::get_posting_list_any_field(self, term)
906    }
907    fn get_term_freqs_bulk(
908        &self,
909        doc_ids: &[DocId],
910        field: &str,
911        term: &str,
912    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
913        if let Some(index) = self.native_index() {
914            return index.get_term_freqs_bulk(doc_ids, field, term);
915        }
916        uqa_storage::inverted_index::defaults::get_term_freqs_bulk(self, doc_ids, field, term)
917    }
918    fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
919        if let Some(index) = self.native_index() {
920            return index.get_total_doc_length(doc_id);
921        }
922        uqa_storage::inverted_index::defaults::get_total_doc_length(self, doc_id)
923    }
924    fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
925        if let Some(index) = self.native_index() {
926            return index.get_total_term_freq(doc_id, term);
927        }
928        uqa_storage::inverted_index::defaults::get_total_term_freq(self, doc_id, term)
929    }
930}