Skip to main content

uqa_storage_sqlite/inverted_index/
trait_impl.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! `InvertedIndex` trait implementation and read/statistics surface.
8
9use super::queries::project_postings;
10use super::{
11    clustered_result, decode_index_u64, encode_index_u64, invalidate_posting_accelerators, params,
12    params_from_iter, quote_ident, Analyzer, AnalyzerPhase, Arc, BTreeMap, BlockMaxScorer, DocId,
13    FieldName, IndexStats, InvertedIndex, OptionalExtension, PostingCursor, PostingList,
14    SQLiteError, SQLiteInvertedIndex, SqlValue, StorageBackendResult,
15};
16use super::{IndexedFieldMetadata, TokenTermKey};
17use uqa_storage::clustered_postings::{
18    cluster_id, decode_all_scores, OccurrencePosting, PostingReadCursor,
19};
20use uqa_storage::inverted_index::TextIndexSource;
21
22impl InvertedIndex for SQLiteInvertedIndex {
23    fn posting_read_cursor_key_budgeted<'a>(
24        &'a self,
25        field: &'a str,
26        term: &'a TokenTermKey,
27        control: &uqa_storage::read_control::StorageReadControl,
28    ) -> StorageBackendResult<uqa_storage::clustered_postings::BudgetedPostingReadCursor<'a>> {
29        control.check()?;
30        if let Some(index) = self.native_index() {
31            return uqa_storage::clustered_postings::open_controlled_cursor(
32                index.snapshot()?,
33                field,
34                term,
35                control,
36            );
37        }
38        uqa_storage::clustered_postings::open_controlled_cursor(self, field, term, control)
39    }
40
41    fn visit_score_clusters(
42        &self,
43        field: &str,
44        term: &TokenTermKey,
45        after: Option<u64>,
46        limit: usize,
47        control: &uqa_storage::read_control::StorageReadControl,
48        visit: &mut uqa_storage::clustered_postings::ScoreClusterVisitor<'_>,
49    ) -> StorageBackendResult<()> {
50        if let Some(index) = self.native_index() {
51            return index.visit_score_clusters(field, term, after, limit, control, visit);
52        }
53        Ok(self.visit_clusters_budgeted(field, term, after, limit, control, visit)?)
54    }
55
56    fn get_occurrences_budgeted(
57        &self,
58        doc_id: DocId,
59        field: &str,
60        term: &TokenTermKey,
61        control: &uqa_storage::read_control::StorageReadControl,
62    ) -> StorageBackendResult<uqa_core::memory::Budgeted<Vec<uqa_core::TokenOccurrence>>> {
63        if let Some(index) = self.native_index() {
64            return index.get_occurrences_budgeted(doc_id, field, term, control);
65        }
66        Ok(self.occurrences_budgeted(doc_id, field, term, control)?)
67    }
68
69    fn field_stats_scalar_budgeted(
70        &self,
71        field: &str,
72        control: &uqa_storage::read_control::StorageReadControl,
73    ) -> StorageBackendResult<IndexStats> {
74        if let Some(index) = self.native_index() {
75            return index.field_stats_scalar_budgeted(field, control);
76        }
77        Ok(self.scalar_stats_budgeted(field, control)?)
78    }
79
80    fn analyzer(&self) -> &Analyzer {
81        self.bindings.default_configuration()
82    }
83
84    fn default_analyzer_binding(
85        &self,
86    ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
87        Ok(self.bindings.default_binding())
88    }
89
90    fn add_document(
91        &mut self,
92        doc_id: DocId,
93        fields: BTreeMap<FieldName, String>,
94    ) -> StorageBackendResult<()> {
95        if let Some(mut index) = self.native_index() {
96            return index.add_document(doc_id, fields);
97        }
98        Ok(self.add_document_inner(doc_id, fields)?)
99    }
100
101    fn try_add_documents(
102        &mut self,
103        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
104    ) -> StorageBackendResult<()> {
105        if let Some(mut index) = self.native_index() {
106            return index.try_add_documents(documents);
107        }
108        Ok(self.add_documents_inner(documents)?)
109    }
110
111    fn try_add_documents_observed(
112        &mut self,
113        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
114        visit: &mut uqa_storage::inverted_index::InvertedIndexChangeVisitor<'_>,
115    ) -> StorageBackendResult<()> {
116        if let Some(mut index) = self.native_index() {
117            return index.try_add_documents_observed(documents, visit);
118        }
119        Ok(self.add_documents_observed(documents, Some(visit))?)
120    }
121
122    fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
123        if let Some(mut index) = self.native_index() {
124            return index.remove_document(doc_id);
125        }
126        Ok(self.remove_document_inner(doc_id)?)
127    }
128
129    fn clear(&mut self) -> StorageBackendResult<()> {
130        if let Some(mut index) = self.native_index() {
131            return index.clear();
132        }
133        self.conn.with_mut(|conn| {
134            let tx = conn.savepoint()?;
135            invalidate_posting_accelerators(&tx, &self.table)?;
136            self.clear_index_on(&tx)?;
137            tx.commit()?;
138            Ok(())
139        })?;
140        Ok(())
141    }
142
143    fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
144        if let Some(index) = self.native_index() {
145            return index.source_rebuild_required();
146        }
147        Ok(self.conn.with(|conn| self.needs_source_rebuild_on(conn))?)
148    }
149
150    fn try_rebuild_documents(
151        &mut self,
152        source: &mut dyn TextIndexSource,
153    ) -> StorageBackendResult<()> {
154        if let Some(mut index) = self.native_index() {
155            return index.try_rebuild_documents(source);
156        }
157        Ok(self.rebuild_documents_inner(source)?)
158    }
159
160    fn try_rebuild_documents_cancellable(
161        &mut self,
162        source: &mut dyn TextIndexSource,
163        cancellation: &uqa_core::CancellationToken,
164    ) -> StorageBackendResult<()> {
165        if let Some(mut index) = self.native_index() {
166            return index.try_rebuild_documents_cancellable(source, cancellation);
167        }
168        Ok(self.rebuild_documents_with_cancellation(source, Some(cancellation))?)
169    }
170
171    fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
172        if let Some(index) = self.native_index() {
173            return index.get_posting_list(field, term);
174        }
175        self.get_posting_list_key(field, &TokenTermKey::from_text(term))
176    }
177
178    fn get_posting_list_key(
179        &self,
180        field: &str,
181        term: &TokenTermKey,
182    ) -> StorageBackendResult<PostingList> {
183        if let Some(index) = self.native_index() {
184            return index.get_posting_list_key(field, term);
185        }
186        Ok(project_postings(self.get_occurrence_postings(field, term)?))
187    }
188
189    fn get_occurrence_postings(
190        &self,
191        field: &str,
192        term: &TokenTermKey,
193    ) -> StorageBackendResult<Vec<OccurrencePosting>> {
194        if let Some(index) = self.native_index() {
195            return index.get_occurrence_postings(field, term);
196        }
197        Ok(self
198            .occurrence_postings_bulk(field, std::slice::from_ref(term))?
199            .pop()
200            .unwrap_or_default())
201    }
202
203    fn get_occurrences(
204        &self,
205        doc_id: DocId,
206        field: &str,
207        term: &TokenTermKey,
208    ) -> StorageBackendResult<Vec<uqa_core::TokenOccurrence>> {
209        if let Some(index) = self.native_index() {
210            return index.get_occurrences(doc_id, field, term);
211        }
212        Ok(self.conn.with(|conn| {
213            self.require_graph_format_on(conn)?;
214            let entries = super::load_cluster(conn, &self.table, field, term, cluster_id(doc_id))?;
215            let Some(posting) = entries.into_iter().find(|entry| entry.doc_id == doc_id) else {
216                return Ok(Vec::new());
217            };
218            self.validate_posting_metadata_on(conn, field, &posting)?;
219            Ok(posting.occurrences)
220        })?)
221    }
222
223    fn indexed_field_metadata(
224        &self,
225        doc_id: DocId,
226        field: &str,
227    ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
228        if let Some(index) = self.native_index() {
229            return index.indexed_field_metadata(doc_id, field);
230        }
231        let doc_id = encode_index_u64("document", doc_id)?;
232        Ok(self.conn.with(|conn| {
233            self.require_graph_format_on(conn)?;
234            self.read_field_metadata_on(conn, doc_id, field)
235        })?)
236    }
237
238    fn get_posting_lists_bulk(
239        &self,
240        field: &str,
241        terms: &[String],
242    ) -> StorageBackendResult<Vec<PostingList>> {
243        if let Some(index) = self.native_index() {
244            return index.get_posting_lists_bulk(field, terms);
245        }
246        let keys = terms
247            .iter()
248            .map(|term| TokenTermKey::from_text(term))
249            .collect::<Vec<_>>();
250        Ok(self
251            .occurrence_postings_bulk(field, &keys)?
252            .into_iter()
253            .map(project_postings)
254            .collect())
255    }
256
257    fn posting_cursor(
258        &self,
259        field: &str,
260        term: &str,
261    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
262        if let Some(index) = self.native_index() {
263            return index.posting_cursor(field, term);
264        }
265        self.posting_cursor_key(field, &TokenTermKey::from_text(term))
266    }
267
268    fn posting_cursor_key(
269        &self,
270        field: &str,
271        term: &TokenTermKey,
272    ) -> StorageBackendResult<Box<dyn PostingCursor>> {
273        if let Some(index) = self.native_index() {
274            return index.posting_cursor_key(field, term);
275        }
276        self.cursor_for_term(field, term)
277    }
278
279    fn posting_cursors_bulk(
280        &self,
281        field: &str,
282        terms: &[String],
283    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
284        if let Some(index) = self.native_index() {
285            return index.posting_cursors_bulk(field, terms);
286        }
287        let keys = terms
288            .iter()
289            .map(|term| TokenTermKey::from_text(term))
290            .collect::<Vec<_>>();
291        self.cursors_for_terms(field, &keys)
292    }
293
294    fn posting_cursors_keys_bulk(
295        &self,
296        field: &str,
297        terms: &[TokenTermKey],
298    ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
299        if let Some(index) = self.native_index() {
300            return index.posting_cursors_keys_bulk(field, terms);
301        }
302        self.cursors_for_terms(field, terms)
303    }
304
305    fn get_posting_lists_keys_bulk(
306        &self,
307        field: &str,
308        terms: &[TokenTermKey],
309    ) -> StorageBackendResult<Vec<PostingList>> {
310        if let Some(index) = self.native_index() {
311            return index.get_posting_lists_keys_bulk(field, terms);
312        }
313        Ok(self
314            .occurrence_postings_bulk(field, terms)?
315            .into_iter()
316            .map(project_postings)
317            .collect())
318    }
319
320    fn persisted_block_max_scores_keys_bulk(
321        &self,
322        field: &str,
323        terms: &[TokenTermKey],
324        scorer_fingerprint: &str,
325    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
326        if scorer_fingerprint.is_empty() {
327            return Ok(vec![None; terms.len()]);
328        }
329        self.get_versioned_block_max_scores_keys_bulk(field, terms, scorer_fingerprint)
330    }
331
332    fn rebuild_persisted_block_max(
333        &mut self,
334        field: &str,
335        scorer: &dyn BlockMaxScorer,
336        scorer_fingerprint: &str,
337    ) -> StorageBackendResult<bool> {
338        if let Some(mut index) = self.native_index() {
339            return index.rebuild_persisted_block_max(field, scorer, scorer_fingerprint);
340        }
341        if scorer_fingerprint.is_empty() {
342            return Err(SQLiteError::StorageBackend(
343                "persisted block-max scorer fingerprint must not be empty".into(),
344            )
345            .into());
346        }
347        let terms = self.vocabulary_keys(field)?;
348        self.ensure_aux_tables(field)?;
349        let table = self.blockmax_table_name(field);
350        self.conn.with_mut(|conn| {
351            conn.execute(&format!("DELETE FROM {}", quote_ident(&table)), [])?;
352            Ok(())
353        })?;
354        for term in terms {
355            self.build_block_max_scores_key(field, &term, scorer, scorer_fingerprint)?;
356        }
357        Ok(true)
358    }
359
360    fn persisted_block_max_scores(
361        &self,
362        field: &str,
363        term: &str,
364        scorer_fingerprint: &str,
365    ) -> StorageBackendResult<Option<Vec<f64>>> {
366        if scorer_fingerprint.is_empty() {
367            return Ok(None);
368        }
369        self.get_versioned_block_max_scores(field, term, scorer_fingerprint)
370    }
371
372    fn persisted_block_max_scores_bulk(
373        &self,
374        field: &str,
375        terms: &[String],
376        scorer_fingerprint: &str,
377    ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
378        if scorer_fingerprint.is_empty() {
379            return Ok(vec![None; terms.len()]);
380        }
381        self.get_versioned_block_max_scores_bulk(field, terms, scorer_fingerprint)
382    }
383
384    fn for_each_term_freq(
385        &self,
386        field: &str,
387        term: &str,
388        visit: &mut dyn FnMut(DocId, u64),
389    ) -> StorageBackendResult<()> {
390        if let Some(index) = self.native_index() {
391            return index.for_each_term_freq(field, term, visit);
392        }
393        let mut cursor = self.posting_cursor(field, term)?;
394        while let Some(entry) = cursor.current() {
395            visit(entry.doc_id, entry.term_freq);
396            cursor.advance()?;
397        }
398        Ok(())
399    }
400
401    fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
402        if let Some(index) = self.native_index() {
403            return index.doc_freq(field, term);
404        }
405        self.doc_freq_key(field, &TokenTermKey::from_text(term))
406    }
407
408    fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
409        if let Some(index) = self.native_index() {
410            return index.doc_freq_key(field, term);
411        }
412        Ok(self.posting_cursor_key(field, term)?.doc_freq())
413    }
414
415    fn doc_freq_key_budgeted(
416        &self,
417        field: &str,
418        term: &TokenTermKey,
419        control: &uqa_storage::read_control::StorageReadControl,
420    ) -> StorageBackendResult<u64> {
421        if let Some(index) = self.native_index() {
422            return index.doc_freq_key_budgeted(field, term, control);
423        }
424        let cursor = self.posting_read_cursor_key_budgeted(field, term, control)?;
425        Ok(cursor.doc_freq())
426    }
427
428    fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64> {
429        if let Some(index) = self.native_index() {
430            return index.get_doc_length(doc_id, field);
431        }
432        Ok(self
433            .get_doc_lengths_bulk(&[doc_id], field)?
434            .get(&doc_id)
435            .copied()
436            .unwrap_or(0))
437    }
438
439    fn get_doc_lengths_bulk(
440        &self,
441        doc_ids: &[DocId],
442        field: &str,
443    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
444        if let Some(index) = self.native_index() {
445            return index.get_doc_lengths_bulk(doc_ids, field);
446        }
447        Ok(self.conn.with(|conn| {
448            self.require_graph_format_on(conn)?;
449            let mut out = BTreeMap::new();
450            for chunk in doc_ids.chunks(900) {
451                let ids = (3..chunk.len()+3).map(|parameter| format!("?{parameter}")).collect::<Vec<_>>().join(", ");
452                let sql = format!("WITH lengths AS (SELECT doc_id, length FROM _occurrence_lengths WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})), documents AS (SELECT doc_id, metadata_blob FROM _occurrence_documents WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})) SELECT COALESCE(lengths.doc_id, documents.doc_id), length, metadata_blob FROM lengths FULL OUTER JOIN documents USING(doc_id)");
453                let mut values = vec![SqlValue::Text(self.table.clone()), SqlValue::Text(field.into())];
454                for doc_id in chunk {
455                    values.push(SqlValue::Integer(encode_index_u64("document", *doc_id)?));
456                }
457                let mut statement = conn.prepare(&sql)?;
458                let rows = statement.query_map(params_from_iter(values), |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option<i64>>(1)?, row.get::<_, Option<Vec<u8>>>(2)?)))?;
459                for row in rows {
460                    let (doc_id, length, metadata) = row?;
461                    let doc_id = decode_index_u64("document id", doc_id)?;
462                    let length = length.map(|length| decode_index_u64("document length", length)).transpose()?;
463                    let metadata = metadata.map(|bytes| clustered_result(IndexedFieldMetadata::from_bytes(&bytes))).transpose()?;
464                    let (Some(length), Some(metadata)) = (length, metadata) else {
465                        return Err(SQLiteError::StorageBackend("indexed field length and source metadata disagree".into()));
466                    };
467                    let stats = self.stored_field_stats_on(conn, field)?.ok_or_else(|| SQLiteError::StorageBackend("indexed field revision is missing".into()))?;
468                    if metadata.length != length || metadata.revision() != stats.revision {
469                        return Err(SQLiteError::StorageBackend("indexed field length or revision disagrees with source metadata".into()));
470                    }
471                    out.insert(doc_id, length);
472                }
473            }
474            Ok(out)
475        })?)
476    }
477
478    fn get_scoring_inputs_bulk(
479        &self,
480        doc_ids: &[DocId],
481        field: &str,
482        terms: &[String],
483    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
484        if let Some(index) = self.native_index() {
485            return index.get_scoring_inputs_bulk(doc_ids, field, terms);
486        }
487        let keys = terms
488            .iter()
489            .map(|term| TokenTermKey::from_text(term))
490            .collect::<Vec<_>>();
491        self.get_scoring_inputs_keys_bulk(doc_ids, field, &keys)
492    }
493
494    fn get_scoring_inputs_keys_bulk(
495        &self,
496        doc_ids: &[DocId],
497        field: &str,
498        terms: &[TokenTermKey],
499    ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
500        if let Some(index) = self.native_index() {
501            return index.get_scoring_inputs_keys_bulk(doc_ids, field, terms);
502        }
503        if doc_ids.is_empty() {
504            return Ok(Vec::new());
505        }
506
507        let doc_lengths = self.get_doc_lengths_bulk(doc_ids, field)?;
508        let mut inputs: Vec<(u64, Vec<u64>)> = doc_ids
509            .iter()
510            .map(|doc_id| {
511                (
512                    doc_lengths.get(doc_id).copied().unwrap_or(0),
513                    vec![0; terms.len()],
514                )
515            })
516            .collect();
517        if terms.is_empty() {
518            return Ok(inputs);
519        }
520
521        let mut output_positions = BTreeMap::<DocId, Vec<usize>>::new();
522        for (position, doc_id) in doc_ids.iter().copied().enumerate() {
523            output_positions.entry(doc_id).or_default().push(position);
524        }
525        for (term_index, mut cursor) in self
526            .posting_cursors_keys_bulk(field, terms)?
527            .into_iter()
528            .enumerate()
529        {
530            while let Some(entry) = cursor.current() {
531                if let Some(positions) = output_positions.get(&entry.doc_id) {
532                    for position in positions {
533                        inputs[*position].1[term_index] = entry.term_freq;
534                        inputs[*position].0 = entry.doc_length;
535                    }
536                }
537                cursor.advance()?;
538            }
539        }
540        Ok(inputs)
541    }
542
543    fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
544        if let Some(index) = self.native_index() {
545            return index.get_term_freq(doc_id, field, term);
546        }
547        self.get_term_freq_key(doc_id, field, &TokenTermKey::from_text(term))
548    }
549
550    fn get_term_freq_key(
551        &self,
552        doc_id: DocId,
553        field: &str,
554        term: &TokenTermKey,
555    ) -> StorageBackendResult<u64> {
556        if let Some(index) = self.native_index() {
557            return index.get_term_freq_key(doc_id, field, term);
558        }
559        let cluster = encode_index_u64("posting cluster", cluster_id(doc_id))?;
560        Ok(self.conn.with(|conn| {
561            self.require_graph_format_on(conn)?;
562            let row: Option<(i64, Vec<u8>)> = conn.query_row("SELECT posting_count, score_blob FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 AND term = ?3 AND cluster_id = ?4", params![self.table, field, term.as_bytes(), cluster], |row| Ok((row.get(0)?, row.get(1)?))).optional()?;
563            match row {
564                Some((count, bytes)) => {
565                    // Validate the version and redundant row count before decoding only the requested cluster.
566                    super::posting_cursor_from_rows(vec![(cluster, count, bytes.clone())])?;
567                    let scores = clustered_result(decode_all_scores(cluster_id(doc_id), &bytes))?;
568                    Ok(scores.binary_search_by_key(&doc_id, |entry| entry.doc_id).ok().map_or(0, |position| scores[position].term_freq))
569                }
570                None => Ok(0),
571            }
572        })?)
573    }
574
575    fn doc_count(&self) -> StorageBackendResult<u64> {
576        if let Some(index) = self.native_index() {
577            return index.doc_count();
578        }
579        Ok(self.conn.with(|c| {
580            self.require_graph_format_on(c)?;
581            let n: i64 = c.query_row(
582                "SELECT COUNT(DISTINCT doc_id) FROM _occurrence_lengths
583                     WHERE table_name = ?1",
584                params![self.table],
585                |r| r.get(0),
586            )?;
587            decode_index_u64("document count", n)
588        })?)
589    }
590
591    fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
592        if let Some(index) = self.native_index() {
593            return index.total_field_length(field);
594        }
595        Ok(self.conn.with(|conn| {
596            self.require_graph_format_on(conn)?;
597            Ok(self
598                .stored_field_stats_on(conn, field)?
599                .map_or(0, |stats| stats.total_length))
600        })?)
601    }
602
603    fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
604        if let Some(index) = self.native_index() {
605            return index.vocabulary_terms(field);
606        }
607        self.terms_for_field(field)
608    }
609
610    fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
611        if let Some(index) = self.native_index() {
612            return index.vocabulary_keys(field);
613        }
614        Ok(self.conn.with(|conn| {
615            self.require_graph_format_on(conn)?;
616            let mut statement = conn.prepare("SELECT DISTINCT term FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 ORDER BY term")?;
617            let rows = statement.query_map(params![self.table, field], |row| row.get::<_, Vec<u8>>(0))?;
618            rows.map(|row| clustered_result(TokenTermKey::from_bytes(row?))).collect()
619        })?)
620    }
621
622    fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
623        if let Some(index) = self.native_index() {
624            return index.field_doc_count(field);
625        }
626        Ok(self.conn.with(|conn| {
627            self.require_graph_format_on(conn)?;
628            Ok(self
629                .stored_field_stats_on(conn, field)?
630                .map_or(0, |stats| stats.doc_count))
631        })?)
632    }
633
634    fn stats(&self) -> StorageBackendResult<IndexStats> {
635        if let Some(index) = self.native_index() {
636            return index.stats();
637        }
638        let doc_count = self.doc_count()?;
639        let mut s = IndexStats::default();
640        s.total_docs = doc_count;
641        if doc_count > 0 {
642            let total: u64 = self.conn.with(|c| {
643                self.require_graph_format_on(c)?;
644                let n: i64 = c.query_row(
645                    "SELECT COALESCE(SUM(total_length), 0) FROM _occurrence_fields
646                         WHERE table_name = ?1",
647                    params![self.table],
648                    |r| r.get(0),
649                )?;
650                decode_index_u64("total indexed length", n)
651            })?;
652            s.avg_doc_length = total as f64 / doc_count as f64;
653        }
654        let pairs = self
655            .conn
656            .with(|conn| self.term_frequencies_on(conn, None))?;
657        for ((field, term), df) in pairs {
658            let term = term.to_term();
659            if let Some(text) = term.as_str() {
660                s.set_doc_freq(field, text, df);
661            } else {
662                s.set_doc_freq_utf16(field, term.into_utf16(), df);
663            }
664        }
665        Ok(s)
666    }
667
668    fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
669        if let Some(index) = self.native_index() {
670            return index.posting_count(field);
671        }
672        Ok(self.conn.with(|conn| {
673            self.term_frequencies_on(conn, field)?
674                .into_values()
675                .try_fold(0_u64, |total, count| {
676                    total
677                        .checked_add(count)
678                        .ok_or_else(|| SQLiteError::StorageBackend("posting count overflow".into()))
679                })
680        })?)
681    }
682
683    fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
684        if let Some(index) = self.native_index() {
685            return index.doc_length_count(field);
686        }
687        Ok(self.conn.with(|c| {
688            self.require_graph_format_on(c)?;
689            let n: i64 = if let Some(field) = field {
690                c.query_row(
691                    "SELECT COUNT(*) FROM _occurrence_lengths
692                         WHERE table_name = ?1 AND field = ?2",
693                    params![self.table, field],
694                    |r| r.get(0),
695                )?
696            } else {
697                c.query_row(
698                    "SELECT COUNT(*) FROM _occurrence_lengths WHERE table_name = ?1",
699                    params![self.table],
700                    |r| r.get(0),
701                )?
702            };
703            decode_index_u64("document length count", n)
704        })?)
705    }
706
707    fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
708        if let Some(index) = self.native_index() {
709            return index.term_count(field);
710        }
711        Ok(self.conn.with(|conn| {
712            let terms = self
713                .term_frequencies_on(conn, field)?
714                .into_keys()
715                .map(|(_, term)| term)
716                .collect::<std::collections::BTreeSet<_>>();
717            Ok(terms.len() as u64)
718        })?)
719    }
720
721    fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
722        if let Some(index) = self.native_retained_index()? {
723            return Ok(index);
724        }
725        self.physical_snapshot(&self.retention_control)
726    }
727
728    fn snapshot_with_control(
729        &self,
730        control: &uqa_storage::read_control::StorageReadControl,
731    ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
732        control.check()?;
733        if let Some(index) = self.native_retained_index()? {
734            control.check()?;
735            return Ok(index);
736        }
737        self.physical_snapshot(control)
738    }
739
740    fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
741        if let Some(index) = self.native_index() {
742            return index.field_names();
743        }
744        Ok(self.conn.with(|c| {
745            self.require_graph_format_on(c)?;
746            let mut stmt =
747                c.prepare("SELECT DISTINCT field FROM _occurrence_lengths WHERE table_name = ?1")?;
748            let rows = stmt.query_map([&self.table], |row| row.get::<_, String>(0))?;
749            let mut fields = Vec::new();
750            for row in rows {
751                fields.push(row?);
752            }
753            Ok(fields)
754        })?)
755    }
756
757    fn field_names_budgeted(
758        &self,
759        control: &uqa_storage::read_control::StorageReadControl,
760    ) -> StorageBackendResult<uqa_core::memory::BudgetedVec<uqa_core::memory::BudgetedString>> {
761        if let Some(index) = self.native_index() {
762            return index.field_names_budgeted(control);
763        }
764        Ok(SQLiteInvertedIndex::field_names_budgeted(self, control)?)
765    }
766
767    fn set_field_analyzer(
768        &mut self,
769        field: &str,
770        analyzer: Analyzer,
771        phase: AnalyzerPhase,
772    ) -> Result<(), String> {
773        if let Some(mut index) = self.native_index() {
774            index.set_field_analyzer(field, analyzer, phase)?;
775            self.bindings = index.analyzer_bindings().clone();
776            return Ok(());
777        }
778        let mut candidate = self.bindings.clone();
779        candidate
780            .bind(field, &analyzer, phase)
781            .map_err(|error| error.to_string())?;
782        self.validate_index_revision_change(field, &candidate)
783            .map_err(|error| error.to_string())?;
784        self.bindings = candidate;
785        Ok(())
786    }
787
788    fn remove_field_analyzers(&mut self, field: &str) -> Result<(), String> {
789        if let Some(mut index) = self.native_index() {
790            index.remove_field_analyzers(field)?;
791            self.bindings = index.analyzer_bindings().clone();
792            return Ok(());
793        }
794        let mut candidate = self.bindings.clone();
795        candidate.remove(field);
796        self.validate_index_revision_change(field, &candidate)
797            .map_err(|error| error.to_string())?;
798        self.bindings = candidate;
799        Ok(())
800    }
801
802    fn get_field_analyzer(&self, field: &str) -> Analyzer {
803        self.bindings.index_configuration(field).clone()
804    }
805    fn get_search_analyzer(&self, field: &str) -> Analyzer {
806        self.bindings.search_configuration(field).clone()
807    }
808    fn index_analyzer_revision(
809        &self,
810        field: &str,
811    ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
812        Ok(self.bindings.index_revision(field)?)
813    }
814    fn search_analyzer_revision(
815        &self,
816        field: &str,
817    ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
818        Ok(self.bindings.search_revision(field)?)
819    }
820
821    fn set_field_analyzer_revision(
822        &mut self,
823        field: &str,
824        revision: Arc<uqa_analysis::CompiledAnalyzer>,
825        phase: AnalyzerPhase,
826    ) -> Result<(), String> {
827        if let Some(mut index) = self.native_index() {
828            index.set_field_analyzer_revision(field, revision, phase)?;
829            self.bindings = index.analyzer_bindings().clone();
830            return Ok(());
831        }
832        let mut candidate = self.bindings.clone();
833        candidate
834            .bind_revision(field, revision, phase)
835            .map_err(|error| error.to_string())?;
836        self.validate_index_revision_change(field, &candidate)
837            .map_err(|error| error.to_string())?;
838        self.bindings = candidate;
839        Ok(())
840    }
841
842    fn set_field_analyzer_revisions(
843        &mut self,
844        field: &str,
845        index: Arc<uqa_analysis::CompiledAnalyzer>,
846        search: Arc<uqa_analysis::CompiledAnalyzer>,
847    ) -> Result<(), String> {
848        if let Some(mut native) = self.native_index() {
849            native.set_field_analyzer_revisions(field, index, search)?;
850            self.bindings = native.analyzer_bindings().clone();
851            return Ok(());
852        }
853        let mut candidate = self.bindings.clone();
854        candidate
855            .bind_revisions(field, index, search)
856            .map_err(|error| error.to_string())?;
857        self.validate_index_revision_change(field, &candidate)
858            .map_err(|error| error.to_string())?;
859        self.bindings = candidate;
860        Ok(())
861    }
862
863    fn rebuild_with_analyzer_revision(
864        &mut self,
865        field: &str,
866        revision: Arc<uqa_analysis::CompiledAnalyzer>,
867        phase: AnalyzerPhase,
868        source: &mut dyn TextIndexSource,
869    ) -> StorageBackendResult<()> {
870        if let Some(mut index) = self.native_index() {
871            index.rebuild_with_analyzer_revision(field, revision, phase, source)?;
872            self.bindings = index.analyzer_bindings().clone();
873            return Ok(());
874        }
875        let mut replacement = self.clone();
876        replacement.bindings.bind_revision(field, revision, phase)?;
877        replacement.rebuild_documents_inner(source)?;
878        *self = replacement;
879        Ok(())
880    }
881
882    fn rebuild_with_analyzer_revision_cancellable(
883        &mut self,
884        field: &str,
885        revision: Arc<uqa_analysis::CompiledAnalyzer>,
886        phase: AnalyzerPhase,
887        source: &mut dyn TextIndexSource,
888        cancellation: &uqa_core::CancellationToken,
889    ) -> StorageBackendResult<()> {
890        if let Some(mut index) = self.native_index() {
891            index.rebuild_with_analyzer_revision_cancellable(
892                field,
893                revision,
894                phase,
895                source,
896                cancellation,
897            )?;
898            self.bindings = index.analyzer_bindings().clone();
899            return Ok(());
900        }
901        cancellation.check()?;
902        let mut replacement = self.clone();
903        replacement.bindings.bind_revision(field, revision, phase)?;
904        replacement.rebuild_documents_with_cancellation(source, Some(cancellation))?;
905        *self = replacement;
906        Ok(())
907    }
908    fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
909        if let Some(index) = self.native_index() {
910            return index.doc_freq_any_field(term);
911        }
912        uqa_storage::inverted_index::defaults::doc_freq_any_field(self, term)
913    }
914    fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
915        if let Some(index) = self.native_index() {
916            return index.field_stats(field);
917        }
918        uqa_storage::inverted_index::defaults::field_stats(self, field)
919    }
920    fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
921        if let Some(index) = self.native_index() {
922            return index.field_stats_scalar(field);
923        }
924        uqa_storage::inverted_index::defaults::field_stats_scalar(self, field)
925    }
926    fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
927        if let Some(index) = self.native_index() {
928            return index.get_posting_list_any_field(term);
929        }
930        uqa_storage::inverted_index::defaults::get_posting_list_any_field(self, term)
931    }
932    fn get_term_freqs_bulk(
933        &self,
934        doc_ids: &[DocId],
935        field: &str,
936        term: &str,
937    ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
938        if let Some(index) = self.native_index() {
939            return index.get_term_freqs_bulk(doc_ids, field, term);
940        }
941        uqa_storage::inverted_index::defaults::get_term_freqs_bulk(self, doc_ids, field, term)
942    }
943    fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
944        if let Some(index) = self.native_index() {
945            return index.get_total_doc_length(doc_id);
946        }
947        uqa_storage::inverted_index::defaults::get_total_doc_length(self, doc_id)
948    }
949    fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
950        if let Some(index) = self.native_index() {
951            return index.get_total_term_freq(doc_id, term);
952        }
953        uqa_storage::inverted_index::defaults::get_total_term_freq(self, doc_id, term)
954    }
955}