1use super::queries::project_postings;
10use super::{
11 clustered_result, decode_index_u64, encode_index_u64, invalidate_posting_accelerators, params,
12 params_from_iter, quote_ident, Analyzer, AnalyzerPhase, Arc, BTreeMap, BlockMaxScorer, DocId,
13 FieldName, IndexStats, InvertedIndex, OptionalExtension, PostingCursor, PostingList,
14 SQLiteError, SQLiteInvertedIndex, SqlValue, StorageBackendResult,
15};
16use super::{IndexedFieldMetadata, TokenTermKey};
17use uqa_storage::clustered_postings::{
18 cluster_id, decode_all_scores, OccurrencePosting, PostingReadCursor,
19};
20use uqa_storage::inverted_index::TextIndexSource;
21
22impl InvertedIndex for SQLiteInvertedIndex {
23 fn posting_read_cursor_key_budgeted<'a>(
24 &'a self,
25 field: &'a str,
26 term: &'a TokenTermKey,
27 control: &uqa_storage::read_control::StorageReadControl,
28 ) -> StorageBackendResult<uqa_storage::clustered_postings::BudgetedPostingReadCursor<'a>> {
29 control.check()?;
30 if let Some(index) = self.native_index() {
31 return uqa_storage::clustered_postings::open_controlled_cursor(
32 index.snapshot()?,
33 field,
34 term,
35 control,
36 );
37 }
38 uqa_storage::clustered_postings::open_controlled_cursor(self, field, term, control)
39 }
40
41 fn visit_score_clusters(
42 &self,
43 field: &str,
44 term: &TokenTermKey,
45 after: Option<u64>,
46 limit: usize,
47 control: &uqa_storage::read_control::StorageReadControl,
48 visit: &mut uqa_storage::clustered_postings::ScoreClusterVisitor<'_>,
49 ) -> StorageBackendResult<()> {
50 if let Some(index) = self.native_index() {
51 return index.visit_score_clusters(field, term, after, limit, control, visit);
52 }
53 Ok(self.visit_clusters_budgeted(field, term, after, limit, control, visit)?)
54 }
55
56 fn get_occurrences_budgeted(
57 &self,
58 doc_id: DocId,
59 field: &str,
60 term: &TokenTermKey,
61 control: &uqa_storage::read_control::StorageReadControl,
62 ) -> StorageBackendResult<uqa_core::memory::Budgeted<Vec<uqa_core::TokenOccurrence>>> {
63 if let Some(index) = self.native_index() {
64 return index.get_occurrences_budgeted(doc_id, field, term, control);
65 }
66 Ok(self.occurrences_budgeted(doc_id, field, term, control)?)
67 }
68
69 fn field_stats_scalar_budgeted(
70 &self,
71 field: &str,
72 control: &uqa_storage::read_control::StorageReadControl,
73 ) -> StorageBackendResult<IndexStats> {
74 if let Some(index) = self.native_index() {
75 return index.field_stats_scalar_budgeted(field, control);
76 }
77 Ok(self.scalar_stats_budgeted(field, control)?)
78 }
79
80 fn analyzer(&self) -> &Analyzer {
81 self.bindings.default_configuration()
82 }
83
84 fn default_analyzer_binding(
85 &self,
86 ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
87 Ok(self.bindings.default_binding())
88 }
89
90 fn add_document(
91 &mut self,
92 doc_id: DocId,
93 fields: BTreeMap<FieldName, String>,
94 ) -> StorageBackendResult<()> {
95 if let Some(mut index) = self.native_index() {
96 return index.add_document(doc_id, fields);
97 }
98 Ok(self.add_document_inner(doc_id, fields)?)
99 }
100
101 fn try_add_documents(
102 &mut self,
103 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
104 ) -> StorageBackendResult<()> {
105 if let Some(mut index) = self.native_index() {
106 return index.try_add_documents(documents);
107 }
108 Ok(self.add_documents_inner(documents)?)
109 }
110
111 fn try_add_documents_observed(
112 &mut self,
113 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
114 visit: &mut uqa_storage::inverted_index::InvertedIndexChangeVisitor<'_>,
115 ) -> StorageBackendResult<()> {
116 if let Some(mut index) = self.native_index() {
117 return index.try_add_documents_observed(documents, visit);
118 }
119 Ok(self.add_documents_observed(documents, Some(visit))?)
120 }
121
122 fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
123 if let Some(mut index) = self.native_index() {
124 return index.remove_document(doc_id);
125 }
126 Ok(self.remove_document_inner(doc_id)?)
127 }
128
129 fn clear(&mut self) -> StorageBackendResult<()> {
130 if let Some(mut index) = self.native_index() {
131 return index.clear();
132 }
133 self.conn.with_mut(|conn| {
134 let tx = conn.savepoint()?;
135 invalidate_posting_accelerators(&tx, &self.table)?;
136 self.clear_index_on(&tx)?;
137 tx.commit()?;
138 Ok(())
139 })?;
140 Ok(())
141 }
142
143 fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
144 if let Some(index) = self.native_index() {
145 return index.source_rebuild_required();
146 }
147 Ok(self.conn.with(|conn| self.needs_source_rebuild_on(conn))?)
148 }
149
150 fn try_rebuild_documents(
151 &mut self,
152 source: &mut dyn TextIndexSource,
153 ) -> StorageBackendResult<()> {
154 if let Some(mut index) = self.native_index() {
155 return index.try_rebuild_documents(source);
156 }
157 Ok(self.rebuild_documents_inner(source)?)
158 }
159
160 fn try_rebuild_documents_cancellable(
161 &mut self,
162 source: &mut dyn TextIndexSource,
163 cancellation: &uqa_core::CancellationToken,
164 ) -> StorageBackendResult<()> {
165 if let Some(mut index) = self.native_index() {
166 return index.try_rebuild_documents_cancellable(source, cancellation);
167 }
168 Ok(self.rebuild_documents_with_cancellation(source, Some(cancellation))?)
169 }
170
171 fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
172 if let Some(index) = self.native_index() {
173 return index.get_posting_list(field, term);
174 }
175 self.get_posting_list_key(field, &TokenTermKey::from_text(term))
176 }
177
178 fn get_posting_list_key(
179 &self,
180 field: &str,
181 term: &TokenTermKey,
182 ) -> StorageBackendResult<PostingList> {
183 if let Some(index) = self.native_index() {
184 return index.get_posting_list_key(field, term);
185 }
186 Ok(project_postings(self.get_occurrence_postings(field, term)?))
187 }
188
189 fn get_occurrence_postings(
190 &self,
191 field: &str,
192 term: &TokenTermKey,
193 ) -> StorageBackendResult<Vec<OccurrencePosting>> {
194 if let Some(index) = self.native_index() {
195 return index.get_occurrence_postings(field, term);
196 }
197 Ok(self
198 .occurrence_postings_bulk(field, std::slice::from_ref(term))?
199 .pop()
200 .unwrap_or_default())
201 }
202
203 fn get_occurrences(
204 &self,
205 doc_id: DocId,
206 field: &str,
207 term: &TokenTermKey,
208 ) -> StorageBackendResult<Vec<uqa_core::TokenOccurrence>> {
209 if let Some(index) = self.native_index() {
210 return index.get_occurrences(doc_id, field, term);
211 }
212 Ok(self.conn.with(|conn| {
213 self.require_graph_format_on(conn)?;
214 let entries = super::load_cluster(conn, &self.table, field, term, cluster_id(doc_id))?;
215 let Some(posting) = entries.into_iter().find(|entry| entry.doc_id == doc_id) else {
216 return Ok(Vec::new());
217 };
218 self.validate_posting_metadata_on(conn, field, &posting)?;
219 Ok(posting.occurrences)
220 })?)
221 }
222
223 fn indexed_field_metadata(
224 &self,
225 doc_id: DocId,
226 field: &str,
227 ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
228 if let Some(index) = self.native_index() {
229 return index.indexed_field_metadata(doc_id, field);
230 }
231 let doc_id = encode_index_u64("document", doc_id)?;
232 Ok(self.conn.with(|conn| {
233 self.require_graph_format_on(conn)?;
234 self.read_field_metadata_on(conn, doc_id, field)
235 })?)
236 }
237
238 fn get_posting_lists_bulk(
239 &self,
240 field: &str,
241 terms: &[String],
242 ) -> StorageBackendResult<Vec<PostingList>> {
243 if let Some(index) = self.native_index() {
244 return index.get_posting_lists_bulk(field, terms);
245 }
246 let keys = terms
247 .iter()
248 .map(|term| TokenTermKey::from_text(term))
249 .collect::<Vec<_>>();
250 Ok(self
251 .occurrence_postings_bulk(field, &keys)?
252 .into_iter()
253 .map(project_postings)
254 .collect())
255 }
256
257 fn posting_cursor(
258 &self,
259 field: &str,
260 term: &str,
261 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
262 if let Some(index) = self.native_index() {
263 return index.posting_cursor(field, term);
264 }
265 self.posting_cursor_key(field, &TokenTermKey::from_text(term))
266 }
267
268 fn posting_cursor_key(
269 &self,
270 field: &str,
271 term: &TokenTermKey,
272 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
273 if let Some(index) = self.native_index() {
274 return index.posting_cursor_key(field, term);
275 }
276 self.cursor_for_term(field, term)
277 }
278
279 fn posting_cursors_bulk(
280 &self,
281 field: &str,
282 terms: &[String],
283 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
284 if let Some(index) = self.native_index() {
285 return index.posting_cursors_bulk(field, terms);
286 }
287 let keys = terms
288 .iter()
289 .map(|term| TokenTermKey::from_text(term))
290 .collect::<Vec<_>>();
291 self.cursors_for_terms(field, &keys)
292 }
293
294 fn posting_cursors_keys_bulk(
295 &self,
296 field: &str,
297 terms: &[TokenTermKey],
298 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
299 if let Some(index) = self.native_index() {
300 return index.posting_cursors_keys_bulk(field, terms);
301 }
302 self.cursors_for_terms(field, terms)
303 }
304
305 fn get_posting_lists_keys_bulk(
306 &self,
307 field: &str,
308 terms: &[TokenTermKey],
309 ) -> StorageBackendResult<Vec<PostingList>> {
310 if let Some(index) = self.native_index() {
311 return index.get_posting_lists_keys_bulk(field, terms);
312 }
313 Ok(self
314 .occurrence_postings_bulk(field, terms)?
315 .into_iter()
316 .map(project_postings)
317 .collect())
318 }
319
320 fn persisted_block_max_scores_keys_bulk(
321 &self,
322 field: &str,
323 terms: &[TokenTermKey],
324 scorer_fingerprint: &str,
325 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
326 if scorer_fingerprint.is_empty() {
327 return Ok(vec![None; terms.len()]);
328 }
329 self.get_versioned_block_max_scores_keys_bulk(field, terms, scorer_fingerprint)
330 }
331
332 fn rebuild_persisted_block_max(
333 &mut self,
334 field: &str,
335 scorer: &dyn BlockMaxScorer,
336 scorer_fingerprint: &str,
337 ) -> StorageBackendResult<bool> {
338 if let Some(mut index) = self.native_index() {
339 return index.rebuild_persisted_block_max(field, scorer, scorer_fingerprint);
340 }
341 if scorer_fingerprint.is_empty() {
342 return Err(SQLiteError::StorageBackend(
343 "persisted block-max scorer fingerprint must not be empty".into(),
344 )
345 .into());
346 }
347 let terms = self.vocabulary_keys(field)?;
348 self.ensure_aux_tables(field)?;
349 let table = self.blockmax_table_name(field);
350 self.conn.with_mut(|conn| {
351 conn.execute(&format!("DELETE FROM {}", quote_ident(&table)), [])?;
352 Ok(())
353 })?;
354 for term in terms {
355 self.build_block_max_scores_key(field, &term, scorer, scorer_fingerprint)?;
356 }
357 Ok(true)
358 }
359
360 fn persisted_block_max_scores(
361 &self,
362 field: &str,
363 term: &str,
364 scorer_fingerprint: &str,
365 ) -> StorageBackendResult<Option<Vec<f64>>> {
366 if scorer_fingerprint.is_empty() {
367 return Ok(None);
368 }
369 self.get_versioned_block_max_scores(field, term, scorer_fingerprint)
370 }
371
372 fn persisted_block_max_scores_bulk(
373 &self,
374 field: &str,
375 terms: &[String],
376 scorer_fingerprint: &str,
377 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
378 if scorer_fingerprint.is_empty() {
379 return Ok(vec![None; terms.len()]);
380 }
381 self.get_versioned_block_max_scores_bulk(field, terms, scorer_fingerprint)
382 }
383
384 fn for_each_term_freq(
385 &self,
386 field: &str,
387 term: &str,
388 visit: &mut dyn FnMut(DocId, u64),
389 ) -> StorageBackendResult<()> {
390 if let Some(index) = self.native_index() {
391 return index.for_each_term_freq(field, term, visit);
392 }
393 let mut cursor = self.posting_cursor(field, term)?;
394 while let Some(entry) = cursor.current() {
395 visit(entry.doc_id, entry.term_freq);
396 cursor.advance()?;
397 }
398 Ok(())
399 }
400
401 fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
402 if let Some(index) = self.native_index() {
403 return index.doc_freq(field, term);
404 }
405 self.doc_freq_key(field, &TokenTermKey::from_text(term))
406 }
407
408 fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
409 if let Some(index) = self.native_index() {
410 return index.doc_freq_key(field, term);
411 }
412 Ok(self.posting_cursor_key(field, term)?.doc_freq())
413 }
414
415 fn doc_freq_key_budgeted(
416 &self,
417 field: &str,
418 term: &TokenTermKey,
419 control: &uqa_storage::read_control::StorageReadControl,
420 ) -> StorageBackendResult<u64> {
421 if let Some(index) = self.native_index() {
422 return index.doc_freq_key_budgeted(field, term, control);
423 }
424 let cursor = self.posting_read_cursor_key_budgeted(field, term, control)?;
425 Ok(cursor.doc_freq())
426 }
427
428 fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64> {
429 if let Some(index) = self.native_index() {
430 return index.get_doc_length(doc_id, field);
431 }
432 Ok(self
433 .get_doc_lengths_bulk(&[doc_id], field)?
434 .get(&doc_id)
435 .copied()
436 .unwrap_or(0))
437 }
438
439 fn get_doc_lengths_bulk(
440 &self,
441 doc_ids: &[DocId],
442 field: &str,
443 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
444 if let Some(index) = self.native_index() {
445 return index.get_doc_lengths_bulk(doc_ids, field);
446 }
447 Ok(self.conn.with(|conn| {
448 self.require_graph_format_on(conn)?;
449 let mut out = BTreeMap::new();
450 for chunk in doc_ids.chunks(900) {
451 let ids = (3..chunk.len()+3).map(|parameter| format!("?{parameter}")).collect::<Vec<_>>().join(", ");
452 let sql = format!("WITH lengths AS (SELECT doc_id, length FROM _occurrence_lengths WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})), documents AS (SELECT doc_id, metadata_blob FROM _occurrence_documents WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})) SELECT COALESCE(lengths.doc_id, documents.doc_id), length, metadata_blob FROM lengths FULL OUTER JOIN documents USING(doc_id)");
453 let mut values = vec![SqlValue::Text(self.table.clone()), SqlValue::Text(field.into())];
454 for doc_id in chunk {
455 values.push(SqlValue::Integer(encode_index_u64("document", *doc_id)?));
456 }
457 let mut statement = conn.prepare(&sql)?;
458 let rows = statement.query_map(params_from_iter(values), |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option<i64>>(1)?, row.get::<_, Option<Vec<u8>>>(2)?)))?;
459 for row in rows {
460 let (doc_id, length, metadata) = row?;
461 let doc_id = decode_index_u64("document id", doc_id)?;
462 let length = length.map(|length| decode_index_u64("document length", length)).transpose()?;
463 let metadata = metadata.map(|bytes| clustered_result(IndexedFieldMetadata::from_bytes(&bytes))).transpose()?;
464 let (Some(length), Some(metadata)) = (length, metadata) else {
465 return Err(SQLiteError::StorageBackend("indexed field length and source metadata disagree".into()));
466 };
467 let stats = self.stored_field_stats_on(conn, field)?.ok_or_else(|| SQLiteError::StorageBackend("indexed field revision is missing".into()))?;
468 if metadata.length != length || metadata.revision() != stats.revision {
469 return Err(SQLiteError::StorageBackend("indexed field length or revision disagrees with source metadata".into()));
470 }
471 out.insert(doc_id, length);
472 }
473 }
474 Ok(out)
475 })?)
476 }
477
478 fn get_scoring_inputs_bulk(
479 &self,
480 doc_ids: &[DocId],
481 field: &str,
482 terms: &[String],
483 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
484 if let Some(index) = self.native_index() {
485 return index.get_scoring_inputs_bulk(doc_ids, field, terms);
486 }
487 let keys = terms
488 .iter()
489 .map(|term| TokenTermKey::from_text(term))
490 .collect::<Vec<_>>();
491 self.get_scoring_inputs_keys_bulk(doc_ids, field, &keys)
492 }
493
494 fn get_scoring_inputs_keys_bulk(
495 &self,
496 doc_ids: &[DocId],
497 field: &str,
498 terms: &[TokenTermKey],
499 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
500 if let Some(index) = self.native_index() {
501 return index.get_scoring_inputs_keys_bulk(doc_ids, field, terms);
502 }
503 if doc_ids.is_empty() {
504 return Ok(Vec::new());
505 }
506
507 let doc_lengths = self.get_doc_lengths_bulk(doc_ids, field)?;
508 let mut inputs: Vec<(u64, Vec<u64>)> = doc_ids
509 .iter()
510 .map(|doc_id| {
511 (
512 doc_lengths.get(doc_id).copied().unwrap_or(0),
513 vec![0; terms.len()],
514 )
515 })
516 .collect();
517 if terms.is_empty() {
518 return Ok(inputs);
519 }
520
521 let mut output_positions = BTreeMap::<DocId, Vec<usize>>::new();
522 for (position, doc_id) in doc_ids.iter().copied().enumerate() {
523 output_positions.entry(doc_id).or_default().push(position);
524 }
525 for (term_index, mut cursor) in self
526 .posting_cursors_keys_bulk(field, terms)?
527 .into_iter()
528 .enumerate()
529 {
530 while let Some(entry) = cursor.current() {
531 if let Some(positions) = output_positions.get(&entry.doc_id) {
532 for position in positions {
533 inputs[*position].1[term_index] = entry.term_freq;
534 inputs[*position].0 = entry.doc_length;
535 }
536 }
537 cursor.advance()?;
538 }
539 }
540 Ok(inputs)
541 }
542
543 fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
544 if let Some(index) = self.native_index() {
545 return index.get_term_freq(doc_id, field, term);
546 }
547 self.get_term_freq_key(doc_id, field, &TokenTermKey::from_text(term))
548 }
549
550 fn get_term_freq_key(
551 &self,
552 doc_id: DocId,
553 field: &str,
554 term: &TokenTermKey,
555 ) -> StorageBackendResult<u64> {
556 if let Some(index) = self.native_index() {
557 return index.get_term_freq_key(doc_id, field, term);
558 }
559 let cluster = encode_index_u64("posting cluster", cluster_id(doc_id))?;
560 Ok(self.conn.with(|conn| {
561 self.require_graph_format_on(conn)?;
562 let row: Option<(i64, Vec<u8>)> = conn.query_row("SELECT posting_count, score_blob FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 AND term = ?3 AND cluster_id = ?4", params![self.table, field, term.as_bytes(), cluster], |row| Ok((row.get(0)?, row.get(1)?))).optional()?;
563 match row {
564 Some((count, bytes)) => {
565 super::posting_cursor_from_rows(vec![(cluster, count, bytes.clone())])?;
567 let scores = clustered_result(decode_all_scores(cluster_id(doc_id), &bytes))?;
568 Ok(scores.binary_search_by_key(&doc_id, |entry| entry.doc_id).ok().map_or(0, |position| scores[position].term_freq))
569 }
570 None => Ok(0),
571 }
572 })?)
573 }
574
575 fn doc_count(&self) -> StorageBackendResult<u64> {
576 if let Some(index) = self.native_index() {
577 return index.doc_count();
578 }
579 Ok(self.conn.with(|c| {
580 self.require_graph_format_on(c)?;
581 let n: i64 = c.query_row(
582 "SELECT COUNT(DISTINCT doc_id) FROM _occurrence_lengths
583 WHERE table_name = ?1",
584 params![self.table],
585 |r| r.get(0),
586 )?;
587 decode_index_u64("document count", n)
588 })?)
589 }
590
591 fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
592 if let Some(index) = self.native_index() {
593 return index.total_field_length(field);
594 }
595 Ok(self.conn.with(|conn| {
596 self.require_graph_format_on(conn)?;
597 Ok(self
598 .stored_field_stats_on(conn, field)?
599 .map_or(0, |stats| stats.total_length))
600 })?)
601 }
602
603 fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
604 if let Some(index) = self.native_index() {
605 return index.vocabulary_terms(field);
606 }
607 self.terms_for_field(field)
608 }
609
610 fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
611 if let Some(index) = self.native_index() {
612 return index.vocabulary_keys(field);
613 }
614 Ok(self.conn.with(|conn| {
615 self.require_graph_format_on(conn)?;
616 let mut statement = conn.prepare("SELECT DISTINCT term FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 ORDER BY term")?;
617 let rows = statement.query_map(params![self.table, field], |row| row.get::<_, Vec<u8>>(0))?;
618 rows.map(|row| clustered_result(TokenTermKey::from_bytes(row?))).collect()
619 })?)
620 }
621
622 fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
623 if let Some(index) = self.native_index() {
624 return index.field_doc_count(field);
625 }
626 Ok(self.conn.with(|conn| {
627 self.require_graph_format_on(conn)?;
628 Ok(self
629 .stored_field_stats_on(conn, field)?
630 .map_or(0, |stats| stats.doc_count))
631 })?)
632 }
633
634 fn stats(&self) -> StorageBackendResult<IndexStats> {
635 if let Some(index) = self.native_index() {
636 return index.stats();
637 }
638 let doc_count = self.doc_count()?;
639 let mut s = IndexStats::default();
640 s.total_docs = doc_count;
641 if doc_count > 0 {
642 let total: u64 = self.conn.with(|c| {
643 self.require_graph_format_on(c)?;
644 let n: i64 = c.query_row(
645 "SELECT COALESCE(SUM(total_length), 0) FROM _occurrence_fields
646 WHERE table_name = ?1",
647 params![self.table],
648 |r| r.get(0),
649 )?;
650 decode_index_u64("total indexed length", n)
651 })?;
652 s.avg_doc_length = total as f64 / doc_count as f64;
653 }
654 let pairs = self
655 .conn
656 .with(|conn| self.term_frequencies_on(conn, None))?;
657 for ((field, term), df) in pairs {
658 let term = term.to_term();
659 if let Some(text) = term.as_str() {
660 s.set_doc_freq(field, text, df);
661 } else {
662 s.set_doc_freq_utf16(field, term.into_utf16(), df);
663 }
664 }
665 Ok(s)
666 }
667
668 fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
669 if let Some(index) = self.native_index() {
670 return index.posting_count(field);
671 }
672 Ok(self.conn.with(|conn| {
673 self.term_frequencies_on(conn, field)?
674 .into_values()
675 .try_fold(0_u64, |total, count| {
676 total
677 .checked_add(count)
678 .ok_or_else(|| SQLiteError::StorageBackend("posting count overflow".into()))
679 })
680 })?)
681 }
682
683 fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
684 if let Some(index) = self.native_index() {
685 return index.doc_length_count(field);
686 }
687 Ok(self.conn.with(|c| {
688 self.require_graph_format_on(c)?;
689 let n: i64 = if let Some(field) = field {
690 c.query_row(
691 "SELECT COUNT(*) FROM _occurrence_lengths
692 WHERE table_name = ?1 AND field = ?2",
693 params![self.table, field],
694 |r| r.get(0),
695 )?
696 } else {
697 c.query_row(
698 "SELECT COUNT(*) FROM _occurrence_lengths WHERE table_name = ?1",
699 params![self.table],
700 |r| r.get(0),
701 )?
702 };
703 decode_index_u64("document length count", n)
704 })?)
705 }
706
707 fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
708 if let Some(index) = self.native_index() {
709 return index.term_count(field);
710 }
711 Ok(self.conn.with(|conn| {
712 let terms = self
713 .term_frequencies_on(conn, field)?
714 .into_keys()
715 .map(|(_, term)| term)
716 .collect::<std::collections::BTreeSet<_>>();
717 Ok(terms.len() as u64)
718 })?)
719 }
720
721 fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
722 if let Some(index) = self.native_retained_index()? {
723 return Ok(index);
724 }
725 self.physical_snapshot(&self.retention_control)
726 }
727
728 fn snapshot_with_control(
729 &self,
730 control: &uqa_storage::read_control::StorageReadControl,
731 ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
732 control.check()?;
733 if let Some(index) = self.native_retained_index()? {
734 control.check()?;
735 return Ok(index);
736 }
737 self.physical_snapshot(control)
738 }
739
740 fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
741 if let Some(index) = self.native_index() {
742 return index.field_names();
743 }
744 Ok(self.conn.with(|c| {
745 self.require_graph_format_on(c)?;
746 let mut stmt =
747 c.prepare("SELECT DISTINCT field FROM _occurrence_lengths WHERE table_name = ?1")?;
748 let rows = stmt.query_map([&self.table], |row| row.get::<_, String>(0))?;
749 let mut fields = Vec::new();
750 for row in rows {
751 fields.push(row?);
752 }
753 Ok(fields)
754 })?)
755 }
756
757 fn field_names_budgeted(
758 &self,
759 control: &uqa_storage::read_control::StorageReadControl,
760 ) -> StorageBackendResult<uqa_core::memory::BudgetedVec<uqa_core::memory::BudgetedString>> {
761 if let Some(index) = self.native_index() {
762 return index.field_names_budgeted(control);
763 }
764 Ok(SQLiteInvertedIndex::field_names_budgeted(self, control)?)
765 }
766
767 fn set_field_analyzer(
768 &mut self,
769 field: &str,
770 analyzer: Analyzer,
771 phase: AnalyzerPhase,
772 ) -> Result<(), String> {
773 if let Some(mut index) = self.native_index() {
774 index.set_field_analyzer(field, analyzer, phase)?;
775 self.bindings = index.analyzer_bindings().clone();
776 return Ok(());
777 }
778 let mut candidate = self.bindings.clone();
779 candidate
780 .bind(field, &analyzer, phase)
781 .map_err(|error| error.to_string())?;
782 self.validate_index_revision_change(field, &candidate)
783 .map_err(|error| error.to_string())?;
784 self.bindings = candidate;
785 Ok(())
786 }
787
788 fn remove_field_analyzers(&mut self, field: &str) -> Result<(), String> {
789 if let Some(mut index) = self.native_index() {
790 index.remove_field_analyzers(field)?;
791 self.bindings = index.analyzer_bindings().clone();
792 return Ok(());
793 }
794 let mut candidate = self.bindings.clone();
795 candidate.remove(field);
796 self.validate_index_revision_change(field, &candidate)
797 .map_err(|error| error.to_string())?;
798 self.bindings = candidate;
799 Ok(())
800 }
801
802 fn get_field_analyzer(&self, field: &str) -> Analyzer {
803 self.bindings.index_configuration(field).clone()
804 }
805 fn get_search_analyzer(&self, field: &str) -> Analyzer {
806 self.bindings.search_configuration(field).clone()
807 }
808 fn index_analyzer_revision(
809 &self,
810 field: &str,
811 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
812 Ok(self.bindings.index_revision(field)?)
813 }
814 fn search_analyzer_revision(
815 &self,
816 field: &str,
817 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
818 Ok(self.bindings.search_revision(field)?)
819 }
820
821 fn set_field_analyzer_revision(
822 &mut self,
823 field: &str,
824 revision: Arc<uqa_analysis::CompiledAnalyzer>,
825 phase: AnalyzerPhase,
826 ) -> Result<(), String> {
827 if let Some(mut index) = self.native_index() {
828 index.set_field_analyzer_revision(field, revision, phase)?;
829 self.bindings = index.analyzer_bindings().clone();
830 return Ok(());
831 }
832 let mut candidate = self.bindings.clone();
833 candidate
834 .bind_revision(field, revision, phase)
835 .map_err(|error| error.to_string())?;
836 self.validate_index_revision_change(field, &candidate)
837 .map_err(|error| error.to_string())?;
838 self.bindings = candidate;
839 Ok(())
840 }
841
842 fn set_field_analyzer_revisions(
843 &mut self,
844 field: &str,
845 index: Arc<uqa_analysis::CompiledAnalyzer>,
846 search: Arc<uqa_analysis::CompiledAnalyzer>,
847 ) -> Result<(), String> {
848 if let Some(mut native) = self.native_index() {
849 native.set_field_analyzer_revisions(field, index, search)?;
850 self.bindings = native.analyzer_bindings().clone();
851 return Ok(());
852 }
853 let mut candidate = self.bindings.clone();
854 candidate
855 .bind_revisions(field, index, search)
856 .map_err(|error| error.to_string())?;
857 self.validate_index_revision_change(field, &candidate)
858 .map_err(|error| error.to_string())?;
859 self.bindings = candidate;
860 Ok(())
861 }
862
863 fn rebuild_with_analyzer_revision(
864 &mut self,
865 field: &str,
866 revision: Arc<uqa_analysis::CompiledAnalyzer>,
867 phase: AnalyzerPhase,
868 source: &mut dyn TextIndexSource,
869 ) -> StorageBackendResult<()> {
870 if let Some(mut index) = self.native_index() {
871 index.rebuild_with_analyzer_revision(field, revision, phase, source)?;
872 self.bindings = index.analyzer_bindings().clone();
873 return Ok(());
874 }
875 let mut replacement = self.clone();
876 replacement.bindings.bind_revision(field, revision, phase)?;
877 replacement.rebuild_documents_inner(source)?;
878 *self = replacement;
879 Ok(())
880 }
881
882 fn rebuild_with_analyzer_revision_cancellable(
883 &mut self,
884 field: &str,
885 revision: Arc<uqa_analysis::CompiledAnalyzer>,
886 phase: AnalyzerPhase,
887 source: &mut dyn TextIndexSource,
888 cancellation: &uqa_core::CancellationToken,
889 ) -> StorageBackendResult<()> {
890 if let Some(mut index) = self.native_index() {
891 index.rebuild_with_analyzer_revision_cancellable(
892 field,
893 revision,
894 phase,
895 source,
896 cancellation,
897 )?;
898 self.bindings = index.analyzer_bindings().clone();
899 return Ok(());
900 }
901 cancellation.check()?;
902 let mut replacement = self.clone();
903 replacement.bindings.bind_revision(field, revision, phase)?;
904 replacement.rebuild_documents_with_cancellation(source, Some(cancellation))?;
905 *self = replacement;
906 Ok(())
907 }
908 fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
909 if let Some(index) = self.native_index() {
910 return index.doc_freq_any_field(term);
911 }
912 uqa_storage::inverted_index::defaults::doc_freq_any_field(self, term)
913 }
914 fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
915 if let Some(index) = self.native_index() {
916 return index.field_stats(field);
917 }
918 uqa_storage::inverted_index::defaults::field_stats(self, field)
919 }
920 fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
921 if let Some(index) = self.native_index() {
922 return index.field_stats_scalar(field);
923 }
924 uqa_storage::inverted_index::defaults::field_stats_scalar(self, field)
925 }
926 fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
927 if let Some(index) = self.native_index() {
928 return index.get_posting_list_any_field(term);
929 }
930 uqa_storage::inverted_index::defaults::get_posting_list_any_field(self, term)
931 }
932 fn get_term_freqs_bulk(
933 &self,
934 doc_ids: &[DocId],
935 field: &str,
936 term: &str,
937 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
938 if let Some(index) = self.native_index() {
939 return index.get_term_freqs_bulk(doc_ids, field, term);
940 }
941 uqa_storage::inverted_index::defaults::get_term_freqs_bulk(self, doc_ids, field, term)
942 }
943 fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
944 if let Some(index) = self.native_index() {
945 return index.get_total_doc_length(doc_id);
946 }
947 uqa_storage::inverted_index::defaults::get_total_doc_length(self, doc_id)
948 }
949 fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
950 if let Some(index) = self.native_index() {
951 return index.get_total_term_freq(doc_id, term);
952 }
953 uqa_storage::inverted_index::defaults::get_total_term_freq(self, doc_id, term)
954 }
955}