1use super::queries::project_postings;
10use super::{
11 clustered_result, decode_index_u64, encode_index_u64, invalidate_posting_accelerators, params,
12 params_from_iter, quote_ident, Analyzer, AnalyzerPhase, Arc, BTreeMap, BlockMaxScorer, DocId,
13 FieldName, IndexStats, InvertedIndex, OptionalExtension, PostingCursor, PostingList,
14 SQLiteError, SQLiteInvertedIndex, SqlValue, StorageBackendResult,
15};
16use super::{IndexedFieldMetadata, TokenTermKey};
17use uqa_storage::clustered_postings::{cluster_id, decode_all_scores, OccurrencePosting};
18
19impl InvertedIndex for SQLiteInvertedIndex {
20 fn posting_read_cursor_key_budgeted<'a>(
21 &'a self,
22 field: &'a str,
23 term: &'a TokenTermKey,
24 control: &uqa_storage::read_control::StorageReadControl,
25 ) -> StorageBackendResult<uqa_storage::clustered_postings::BudgetedPostingReadCursor<'a>> {
26 control.check()?;
27 if let Some(index) = self.native_index() {
28 return uqa_storage::clustered_postings::open_controlled_cursor(
29 index.snapshot()?,
30 field,
31 term,
32 control,
33 );
34 }
35 uqa_storage::clustered_postings::open_controlled_cursor(self, field, term, control)
36 }
37
38 fn visit_score_clusters(
39 &self,
40 field: &str,
41 term: &TokenTermKey,
42 after: Option<u64>,
43 limit: usize,
44 control: &uqa_storage::read_control::StorageReadControl,
45 visit: &mut uqa_storage::clustered_postings::ScoreClusterVisitor<'_>,
46 ) -> StorageBackendResult<()> {
47 if let Some(index) = self.native_index() {
48 return index.visit_score_clusters(field, term, after, limit, control, visit);
49 }
50 Ok(self.visit_clusters_budgeted(field, term, after, limit, control, visit)?)
51 }
52
53 fn get_occurrences_budgeted(
54 &self,
55 doc_id: DocId,
56 field: &str,
57 term: &TokenTermKey,
58 control: &uqa_storage::read_control::StorageReadControl,
59 ) -> StorageBackendResult<uqa_core::memory::Budgeted<Vec<uqa_core::TokenOccurrence>>> {
60 if let Some(index) = self.native_index() {
61 return index.get_occurrences_budgeted(doc_id, field, term, control);
62 }
63 Ok(self.occurrences_budgeted(doc_id, field, term, control)?)
64 }
65
66 fn field_stats_scalar_budgeted(
67 &self,
68 field: &str,
69 control: &uqa_storage::read_control::StorageReadControl,
70 ) -> StorageBackendResult<IndexStats> {
71 if let Some(index) = self.native_index() {
72 return index.field_stats_scalar_budgeted(field, control);
73 }
74 Ok(self.scalar_stats_budgeted(field, control)?)
75 }
76
77 fn analyzer(&self) -> &Analyzer {
78 self.bindings.default_configuration()
79 }
80
81 fn default_analyzer_binding(
82 &self,
83 ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
84 Ok(self.bindings.default_binding())
85 }
86
87 fn add_document(
88 &mut self,
89 doc_id: DocId,
90 fields: BTreeMap<FieldName, String>,
91 ) -> StorageBackendResult<()> {
92 if let Some(mut index) = self.native_index() {
93 return index.add_document(doc_id, fields);
94 }
95 Ok(self.add_document_inner(doc_id, fields)?)
96 }
97
98 fn try_add_documents(
99 &mut self,
100 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
101 ) -> StorageBackendResult<()> {
102 if let Some(mut index) = self.native_index() {
103 return index.try_add_documents(documents);
104 }
105 Ok(self.add_documents_inner(documents)?)
106 }
107
108 fn try_add_documents_observed(
109 &mut self,
110 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
111 visit: &mut uqa_storage::inverted_index::InvertedIndexChangeVisitor<'_>,
112 ) -> StorageBackendResult<()> {
113 if let Some(mut index) = self.native_index() {
114 return index.try_add_documents_observed(documents, visit);
115 }
116 Ok(self.add_documents_observed(documents, Some(visit))?)
117 }
118
119 fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
120 if let Some(mut index) = self.native_index() {
121 return index.remove_document(doc_id);
122 }
123 Ok(self.remove_document_inner(doc_id)?)
124 }
125
126 fn clear(&mut self) -> StorageBackendResult<()> {
127 if let Some(mut index) = self.native_index() {
128 return index.clear();
129 }
130 self.conn.with_mut(|conn| {
131 let tx = conn.savepoint()?;
132 invalidate_posting_accelerators(&tx, &self.table)?;
133 self.clear_index_on(&tx)?;
134 tx.commit()?;
135 Ok(())
136 })?;
137 Ok(())
138 }
139
140 fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
141 if let Some(index) = self.native_index() {
142 return index.source_rebuild_required();
143 }
144 Ok(self.conn.with(|conn| self.needs_source_rebuild_on(conn))?)
145 }
146
147 fn try_rebuild_documents(
148 &mut self,
149 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
150 ) -> StorageBackendResult<()> {
151 if let Some(mut index) = self.native_index() {
152 return index.try_rebuild_documents(documents);
153 }
154 Ok(self.rebuild_documents_inner(documents)?)
155 }
156
157 fn try_rebuild_documents_cancellable(
158 &mut self,
159 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
160 cancellation: &uqa_core::CancellationToken,
161 ) -> StorageBackendResult<()> {
162 if let Some(mut index) = self.native_index() {
163 return index.try_rebuild_documents_cancellable(documents, cancellation);
164 }
165 Ok(self.rebuild_documents_with_cancellation(documents, Some(cancellation))?)
166 }
167
168 fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
169 if let Some(index) = self.native_index() {
170 return index.get_posting_list(field, term);
171 }
172 self.get_posting_list_key(field, &TokenTermKey::from_text(term))
173 }
174
175 fn get_posting_list_key(
176 &self,
177 field: &str,
178 term: &TokenTermKey,
179 ) -> StorageBackendResult<PostingList> {
180 if let Some(index) = self.native_index() {
181 return index.get_posting_list_key(field, term);
182 }
183 Ok(project_postings(self.get_occurrence_postings(field, term)?))
184 }
185
186 fn get_occurrence_postings(
187 &self,
188 field: &str,
189 term: &TokenTermKey,
190 ) -> StorageBackendResult<Vec<OccurrencePosting>> {
191 if let Some(index) = self.native_index() {
192 return index.get_occurrence_postings(field, term);
193 }
194 Ok(self
195 .occurrence_postings_bulk(field, std::slice::from_ref(term))?
196 .pop()
197 .unwrap_or_default())
198 }
199
200 fn get_occurrences(
201 &self,
202 doc_id: DocId,
203 field: &str,
204 term: &TokenTermKey,
205 ) -> StorageBackendResult<Vec<uqa_core::TokenOccurrence>> {
206 if let Some(index) = self.native_index() {
207 return index.get_occurrences(doc_id, field, term);
208 }
209 Ok(self.conn.with(|conn| {
210 self.require_graph_format_on(conn)?;
211 let entries = super::load_cluster(conn, &self.table, field, term, cluster_id(doc_id))?;
212 let Some(posting) = entries.into_iter().find(|entry| entry.doc_id == doc_id) else {
213 return Ok(Vec::new());
214 };
215 self.validate_posting_metadata_on(conn, field, &posting)?;
216 Ok(posting.occurrences)
217 })?)
218 }
219
220 fn indexed_field_metadata(
221 &self,
222 doc_id: DocId,
223 field: &str,
224 ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
225 if let Some(index) = self.native_index() {
226 return index.indexed_field_metadata(doc_id, field);
227 }
228 let doc_id = encode_index_u64("document", doc_id)?;
229 Ok(self.conn.with(|conn| {
230 self.require_graph_format_on(conn)?;
231 self.read_field_metadata_on(conn, doc_id, field)
232 })?)
233 }
234
235 fn get_posting_lists_bulk(
236 &self,
237 field: &str,
238 terms: &[String],
239 ) -> StorageBackendResult<Vec<PostingList>> {
240 if let Some(index) = self.native_index() {
241 return index.get_posting_lists_bulk(field, terms);
242 }
243 let keys = terms
244 .iter()
245 .map(|term| TokenTermKey::from_text(term))
246 .collect::<Vec<_>>();
247 Ok(self
248 .occurrence_postings_bulk(field, &keys)?
249 .into_iter()
250 .map(project_postings)
251 .collect())
252 }
253
254 fn posting_cursor(
255 &self,
256 field: &str,
257 term: &str,
258 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
259 if let Some(index) = self.native_index() {
260 return index.posting_cursor(field, term);
261 }
262 self.posting_cursor_key(field, &TokenTermKey::from_text(term))
263 }
264
265 fn posting_cursor_key(
266 &self,
267 field: &str,
268 term: &TokenTermKey,
269 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
270 if let Some(index) = self.native_index() {
271 return index.posting_cursor_key(field, term);
272 }
273 self.cursor_for_term(field, term)
274 }
275
276 fn posting_cursors_bulk(
277 &self,
278 field: &str,
279 terms: &[String],
280 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
281 if let Some(index) = self.native_index() {
282 return index.posting_cursors_bulk(field, terms);
283 }
284 let keys = terms
285 .iter()
286 .map(|term| TokenTermKey::from_text(term))
287 .collect::<Vec<_>>();
288 self.cursors_for_terms(field, &keys)
289 }
290
291 fn posting_cursors_keys_bulk(
292 &self,
293 field: &str,
294 terms: &[TokenTermKey],
295 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
296 if let Some(index) = self.native_index() {
297 return index.posting_cursors_keys_bulk(field, terms);
298 }
299 self.cursors_for_terms(field, terms)
300 }
301
302 fn get_posting_lists_keys_bulk(
303 &self,
304 field: &str,
305 terms: &[TokenTermKey],
306 ) -> StorageBackendResult<Vec<PostingList>> {
307 if let Some(index) = self.native_index() {
308 return index.get_posting_lists_keys_bulk(field, terms);
309 }
310 Ok(self
311 .occurrence_postings_bulk(field, terms)?
312 .into_iter()
313 .map(project_postings)
314 .collect())
315 }
316
317 fn persisted_block_max_scores_keys_bulk(
318 &self,
319 field: &str,
320 terms: &[TokenTermKey],
321 scorer_fingerprint: &str,
322 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
323 if scorer_fingerprint.is_empty() {
324 return Ok(vec![None; terms.len()]);
325 }
326 self.get_versioned_block_max_scores_keys_bulk(field, terms, scorer_fingerprint)
327 }
328
329 fn rebuild_persisted_block_max(
330 &mut self,
331 field: &str,
332 scorer: &dyn BlockMaxScorer,
333 scorer_fingerprint: &str,
334 ) -> StorageBackendResult<bool> {
335 if let Some(mut index) = self.native_index() {
336 return index.rebuild_persisted_block_max(field, scorer, scorer_fingerprint);
337 }
338 if scorer_fingerprint.is_empty() {
339 return Err(SQLiteError::StorageBackend(
340 "persisted block-max scorer fingerprint must not be empty".into(),
341 )
342 .into());
343 }
344 let terms = self.vocabulary_keys(field)?;
345 self.ensure_aux_tables(field)?;
346 let table = self.blockmax_table_name(field);
347 self.conn.with_mut(|conn| {
348 conn.execute(&format!("DELETE FROM {}", quote_ident(&table)), [])?;
349 Ok(())
350 })?;
351 for term in terms {
352 self.build_block_max_scores_key(field, &term, scorer, scorer_fingerprint)?;
353 }
354 Ok(true)
355 }
356
357 fn persisted_block_max_scores(
358 &self,
359 field: &str,
360 term: &str,
361 scorer_fingerprint: &str,
362 ) -> StorageBackendResult<Option<Vec<f64>>> {
363 if scorer_fingerprint.is_empty() {
364 return Ok(None);
365 }
366 self.get_versioned_block_max_scores(field, term, scorer_fingerprint)
367 }
368
369 fn persisted_block_max_scores_bulk(
370 &self,
371 field: &str,
372 terms: &[String],
373 scorer_fingerprint: &str,
374 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
375 if scorer_fingerprint.is_empty() {
376 return Ok(vec![None; terms.len()]);
377 }
378 self.get_versioned_block_max_scores_bulk(field, terms, scorer_fingerprint)
379 }
380
381 fn for_each_term_freq(
382 &self,
383 field: &str,
384 term: &str,
385 visit: &mut dyn FnMut(DocId, u64),
386 ) -> StorageBackendResult<()> {
387 if let Some(index) = self.native_index() {
388 return index.for_each_term_freq(field, term, visit);
389 }
390 let mut cursor = self.posting_cursor(field, term)?;
391 while let Some(entry) = cursor.current() {
392 visit(entry.doc_id, entry.term_freq);
393 cursor.advance()?;
394 }
395 Ok(())
396 }
397
398 fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
399 if let Some(index) = self.native_index() {
400 return index.doc_freq(field, term);
401 }
402 self.doc_freq_key(field, &TokenTermKey::from_text(term))
403 }
404
405 fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
406 if let Some(index) = self.native_index() {
407 return index.doc_freq_key(field, term);
408 }
409 Ok(self.posting_cursor_key(field, term)?.doc_freq())
410 }
411
412 fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64> {
413 if let Some(index) = self.native_index() {
414 return index.get_doc_length(doc_id, field);
415 }
416 Ok(self
417 .get_doc_lengths_bulk(&[doc_id], field)?
418 .get(&doc_id)
419 .copied()
420 .unwrap_or(0))
421 }
422
423 fn get_doc_lengths_bulk(
424 &self,
425 doc_ids: &[DocId],
426 field: &str,
427 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
428 if let Some(index) = self.native_index() {
429 return index.get_doc_lengths_bulk(doc_ids, field);
430 }
431 Ok(self.conn.with(|conn| {
432 self.require_graph_format_on(conn)?;
433 let mut out = BTreeMap::new();
434 for chunk in doc_ids.chunks(900) {
435 let ids = (3..chunk.len()+3).map(|parameter| format!("?{parameter}")).collect::<Vec<_>>().join(", ");
436 let sql = format!("WITH lengths AS (SELECT doc_id, length FROM _occurrence_lengths WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})), documents AS (SELECT doc_id, metadata_blob FROM _occurrence_documents WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})) SELECT COALESCE(lengths.doc_id, documents.doc_id), length, metadata_blob FROM lengths FULL OUTER JOIN documents USING(doc_id)");
437 let mut values = vec![SqlValue::Text(self.table.clone()), SqlValue::Text(field.into())];
438 for doc_id in chunk {
439 values.push(SqlValue::Integer(encode_index_u64("document", *doc_id)?));
440 }
441 let mut statement = conn.prepare(&sql)?;
442 let rows = statement.query_map(params_from_iter(values), |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option<i64>>(1)?, row.get::<_, Option<Vec<u8>>>(2)?)))?;
443 for row in rows {
444 let (doc_id, length, metadata) = row?;
445 let doc_id = decode_index_u64("document id", doc_id)?;
446 let length = length.map(|length| decode_index_u64("document length", length)).transpose()?;
447 let metadata = metadata.map(|bytes| clustered_result(IndexedFieldMetadata::from_bytes(&bytes))).transpose()?;
448 let (Some(length), Some(metadata)) = (length, metadata) else {
449 return Err(SQLiteError::StorageBackend("indexed field length and source metadata disagree".into()));
450 };
451 let stats = self.stored_field_stats_on(conn, field)?.ok_or_else(|| SQLiteError::StorageBackend("indexed field revision is missing".into()))?;
452 if metadata.length != length || metadata.revision() != stats.revision {
453 return Err(SQLiteError::StorageBackend("indexed field length or revision disagrees with source metadata".into()));
454 }
455 out.insert(doc_id, length);
456 }
457 }
458 Ok(out)
459 })?)
460 }
461
462 fn get_scoring_inputs_bulk(
463 &self,
464 doc_ids: &[DocId],
465 field: &str,
466 terms: &[String],
467 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
468 if let Some(index) = self.native_index() {
469 return index.get_scoring_inputs_bulk(doc_ids, field, terms);
470 }
471 let keys = terms
472 .iter()
473 .map(|term| TokenTermKey::from_text(term))
474 .collect::<Vec<_>>();
475 self.get_scoring_inputs_keys_bulk(doc_ids, field, &keys)
476 }
477
478 fn get_scoring_inputs_keys_bulk(
479 &self,
480 doc_ids: &[DocId],
481 field: &str,
482 terms: &[TokenTermKey],
483 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
484 if let Some(index) = self.native_index() {
485 return index.get_scoring_inputs_keys_bulk(doc_ids, field, terms);
486 }
487 if doc_ids.is_empty() {
488 return Ok(Vec::new());
489 }
490
491 let doc_lengths = self.get_doc_lengths_bulk(doc_ids, field)?;
492 let mut inputs: Vec<(u64, Vec<u64>)> = doc_ids
493 .iter()
494 .map(|doc_id| {
495 (
496 doc_lengths.get(doc_id).copied().unwrap_or(0),
497 vec![0; terms.len()],
498 )
499 })
500 .collect();
501 if terms.is_empty() {
502 return Ok(inputs);
503 }
504
505 let mut output_positions = BTreeMap::<DocId, Vec<usize>>::new();
506 for (position, doc_id) in doc_ids.iter().copied().enumerate() {
507 output_positions.entry(doc_id).or_default().push(position);
508 }
509 for (term_index, mut cursor) in self
510 .posting_cursors_keys_bulk(field, terms)?
511 .into_iter()
512 .enumerate()
513 {
514 while let Some(entry) = cursor.current() {
515 if let Some(positions) = output_positions.get(&entry.doc_id) {
516 for position in positions {
517 inputs[*position].1[term_index] = entry.term_freq;
518 inputs[*position].0 = entry.doc_length;
519 }
520 }
521 cursor.advance()?;
522 }
523 }
524 Ok(inputs)
525 }
526
527 fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
528 if let Some(index) = self.native_index() {
529 return index.get_term_freq(doc_id, field, term);
530 }
531 self.get_term_freq_key(doc_id, field, &TokenTermKey::from_text(term))
532 }
533
534 fn get_term_freq_key(
535 &self,
536 doc_id: DocId,
537 field: &str,
538 term: &TokenTermKey,
539 ) -> StorageBackendResult<u64> {
540 if let Some(index) = self.native_index() {
541 return index.get_term_freq_key(doc_id, field, term);
542 }
543 let cluster = encode_index_u64("posting cluster", cluster_id(doc_id))?;
544 Ok(self.conn.with(|conn| {
545 self.require_graph_format_on(conn)?;
546 let row: Option<(i64, Vec<u8>)> = conn.query_row("SELECT posting_count, score_blob FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 AND term = ?3 AND cluster_id = ?4", params![self.table, field, term.as_bytes(), cluster], |row| Ok((row.get(0)?, row.get(1)?))).optional()?;
547 match row {
548 Some((count, bytes)) => {
549 super::posting_cursor_from_rows(vec![(cluster, count, bytes.clone())])?;
551 let scores = clustered_result(decode_all_scores(cluster_id(doc_id), &bytes))?;
552 Ok(scores.binary_search_by_key(&doc_id, |entry| entry.doc_id).ok().map_or(0, |position| scores[position].term_freq))
553 }
554 None => Ok(0),
555 }
556 })?)
557 }
558
559 fn doc_count(&self) -> StorageBackendResult<u64> {
560 if let Some(index) = self.native_index() {
561 return index.doc_count();
562 }
563 Ok(self.conn.with(|c| {
564 self.require_graph_format_on(c)?;
565 let n: i64 = c.query_row(
566 "SELECT COUNT(DISTINCT doc_id) FROM _occurrence_lengths
567 WHERE table_name = ?1",
568 params![self.table],
569 |r| r.get(0),
570 )?;
571 decode_index_u64("document count", n)
572 })?)
573 }
574
575 fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
576 if let Some(index) = self.native_index() {
577 return index.total_field_length(field);
578 }
579 Ok(self.conn.with(|conn| {
580 self.require_graph_format_on(conn)?;
581 Ok(self
582 .stored_field_stats_on(conn, field)?
583 .map_or(0, |stats| stats.total_length))
584 })?)
585 }
586
587 fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
588 if let Some(index) = self.native_index() {
589 return index.vocabulary_terms(field);
590 }
591 self.terms_for_field(field)
592 }
593
594 fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
595 if let Some(index) = self.native_index() {
596 return index.vocabulary_keys(field);
597 }
598 Ok(self.conn.with(|conn| {
599 self.require_graph_format_on(conn)?;
600 let mut statement = conn.prepare("SELECT DISTINCT term FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 ORDER BY term")?;
601 let rows = statement.query_map(params![self.table, field], |row| row.get::<_, Vec<u8>>(0))?;
602 rows.map(|row| clustered_result(TokenTermKey::from_bytes(row?))).collect()
603 })?)
604 }
605
606 fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
607 if let Some(index) = self.native_index() {
608 return index.field_doc_count(field);
609 }
610 Ok(self.conn.with(|conn| {
611 self.require_graph_format_on(conn)?;
612 Ok(self
613 .stored_field_stats_on(conn, field)?
614 .map_or(0, |stats| stats.doc_count))
615 })?)
616 }
617
618 fn stats(&self) -> StorageBackendResult<IndexStats> {
619 if let Some(index) = self.native_index() {
620 return index.stats();
621 }
622 let doc_count = self.doc_count()?;
623 let mut s = IndexStats::default();
624 s.total_docs = doc_count;
625 if doc_count > 0 {
626 let total: u64 = self.conn.with(|c| {
627 self.require_graph_format_on(c)?;
628 let n: i64 = c.query_row(
629 "SELECT COALESCE(SUM(total_length), 0) FROM _occurrence_fields
630 WHERE table_name = ?1",
631 params![self.table],
632 |r| r.get(0),
633 )?;
634 decode_index_u64("total indexed length", n)
635 })?;
636 s.avg_doc_length = total as f64 / doc_count as f64;
637 }
638 let pairs = self
639 .conn
640 .with(|conn| self.term_frequencies_on(conn, None))?;
641 for ((field, term), df) in pairs {
642 let term = term.to_term();
643 if let Some(text) = term.as_str() {
644 s.set_doc_freq(field, text, df);
645 } else {
646 s.set_doc_freq_utf16(field, term.into_utf16(), df);
647 }
648 }
649 Ok(s)
650 }
651
652 fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
653 if let Some(index) = self.native_index() {
654 return index.posting_count(field);
655 }
656 Ok(self.conn.with(|conn| {
657 self.term_frequencies_on(conn, field)?
658 .into_values()
659 .try_fold(0_u64, |total, count| {
660 total
661 .checked_add(count)
662 .ok_or_else(|| SQLiteError::StorageBackend("posting count overflow".into()))
663 })
664 })?)
665 }
666
667 fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
668 if let Some(index) = self.native_index() {
669 return index.doc_length_count(field);
670 }
671 Ok(self.conn.with(|c| {
672 self.require_graph_format_on(c)?;
673 let n: i64 = if let Some(field) = field {
674 c.query_row(
675 "SELECT COUNT(*) FROM _occurrence_lengths
676 WHERE table_name = ?1 AND field = ?2",
677 params![self.table, field],
678 |r| r.get(0),
679 )?
680 } else {
681 c.query_row(
682 "SELECT COUNT(*) FROM _occurrence_lengths WHERE table_name = ?1",
683 params![self.table],
684 |r| r.get(0),
685 )?
686 };
687 decode_index_u64("document length count", n)
688 })?)
689 }
690
691 fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
692 if let Some(index) = self.native_index() {
693 return index.term_count(field);
694 }
695 Ok(self.conn.with(|conn| {
696 let terms = self
697 .term_frequencies_on(conn, field)?
698 .into_keys()
699 .map(|(_, term)| term)
700 .collect::<std::collections::BTreeSet<_>>();
701 Ok(terms.len() as u64)
702 })?)
703 }
704
705 fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
706 if let Some(index) = self.native_retained_index()? {
707 return Ok(index);
708 }
709 self.physical_snapshot(&self.retention_control)
710 }
711
712 fn snapshot_with_control(
713 &self,
714 control: &uqa_storage::read_control::StorageReadControl,
715 ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
716 control.check()?;
717 if let Some(index) = self.native_retained_index()? {
718 control.check()?;
719 return Ok(index);
720 }
721 self.physical_snapshot(control)
722 }
723
724 fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
725 if let Some(index) = self.native_index() {
726 return index.field_names();
727 }
728 Ok(self.conn.with(|c| {
729 self.require_graph_format_on(c)?;
730 let mut stmt =
731 c.prepare("SELECT DISTINCT field FROM _occurrence_lengths WHERE table_name = ?1")?;
732 let rows = stmt.query_map([&self.table], |row| row.get::<_, String>(0))?;
733 let mut fields = Vec::new();
734 for row in rows {
735 fields.push(row?);
736 }
737 Ok(fields)
738 })?)
739 }
740
741 fn set_field_analyzer(
742 &mut self,
743 field: &str,
744 analyzer: Analyzer,
745 phase: AnalyzerPhase,
746 ) -> Result<(), String> {
747 if let Some(mut index) = self.native_index() {
748 index.set_field_analyzer(field, analyzer, phase)?;
749 self.bindings = index.analyzer_bindings().clone();
750 return Ok(());
751 }
752 let mut candidate = self.bindings.clone();
753 candidate
754 .bind(field, &analyzer, phase)
755 .map_err(|error| error.to_string())?;
756 self.validate_index_revision_change(field, &candidate)
757 .map_err(|error| error.to_string())?;
758 self.bindings = candidate;
759 Ok(())
760 }
761
762 fn remove_field_analyzers(&mut self, field: &str) -> Result<(), String> {
763 if let Some(mut index) = self.native_index() {
764 index.remove_field_analyzers(field)?;
765 self.bindings = index.analyzer_bindings().clone();
766 return Ok(());
767 }
768 let mut candidate = self.bindings.clone();
769 candidate.remove(field);
770 self.validate_index_revision_change(field, &candidate)
771 .map_err(|error| error.to_string())?;
772 self.bindings = candidate;
773 Ok(())
774 }
775
776 fn get_field_analyzer(&self, field: &str) -> Analyzer {
777 self.bindings.index_configuration(field).clone()
778 }
779 fn get_search_analyzer(&self, field: &str) -> Analyzer {
780 self.bindings.search_configuration(field).clone()
781 }
782 fn index_analyzer_revision(
783 &self,
784 field: &str,
785 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
786 Ok(self.bindings.index_revision(field)?)
787 }
788 fn search_analyzer_revision(
789 &self,
790 field: &str,
791 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
792 Ok(self.bindings.search_revision(field)?)
793 }
794
795 fn set_field_analyzer_revision(
796 &mut self,
797 field: &str,
798 revision: Arc<uqa_analysis::CompiledAnalyzer>,
799 phase: AnalyzerPhase,
800 ) -> Result<(), String> {
801 if let Some(mut index) = self.native_index() {
802 index.set_field_analyzer_revision(field, revision, phase)?;
803 self.bindings = index.analyzer_bindings().clone();
804 return Ok(());
805 }
806 let mut candidate = self.bindings.clone();
807 candidate
808 .bind_revision(field, revision, phase)
809 .map_err(|error| error.to_string())?;
810 self.validate_index_revision_change(field, &candidate)
811 .map_err(|error| error.to_string())?;
812 self.bindings = candidate;
813 Ok(())
814 }
815
816 fn set_field_analyzer_revisions(
817 &mut self,
818 field: &str,
819 index: Arc<uqa_analysis::CompiledAnalyzer>,
820 search: Arc<uqa_analysis::CompiledAnalyzer>,
821 ) -> Result<(), String> {
822 if let Some(mut native) = self.native_index() {
823 native.set_field_analyzer_revisions(field, index, search)?;
824 self.bindings = native.analyzer_bindings().clone();
825 return Ok(());
826 }
827 let mut candidate = self.bindings.clone();
828 candidate
829 .bind_revisions(field, index, search)
830 .map_err(|error| error.to_string())?;
831 self.validate_index_revision_change(field, &candidate)
832 .map_err(|error| error.to_string())?;
833 self.bindings = candidate;
834 Ok(())
835 }
836
837 fn rebuild_with_analyzer_revision(
838 &mut self,
839 field: &str,
840 revision: Arc<uqa_analysis::CompiledAnalyzer>,
841 phase: AnalyzerPhase,
842 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
843 ) -> StorageBackendResult<()> {
844 if let Some(mut index) = self.native_index() {
845 index.rebuild_with_analyzer_revision(field, revision, phase, documents)?;
846 self.bindings = index.analyzer_bindings().clone();
847 return Ok(());
848 }
849 let mut replacement = self.clone();
850 replacement.bindings.bind_revision(field, revision, phase)?;
851 replacement.rebuild_documents_inner(documents)?;
852 *self = replacement;
853 Ok(())
854 }
855
856 fn rebuild_with_analyzer_revision_cancellable(
857 &mut self,
858 field: &str,
859 revision: Arc<uqa_analysis::CompiledAnalyzer>,
860 phase: AnalyzerPhase,
861 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
862 cancellation: &uqa_core::CancellationToken,
863 ) -> StorageBackendResult<()> {
864 if let Some(mut index) = self.native_index() {
865 index.rebuild_with_analyzer_revision_cancellable(
866 field,
867 revision,
868 phase,
869 documents,
870 cancellation,
871 )?;
872 self.bindings = index.analyzer_bindings().clone();
873 return Ok(());
874 }
875 cancellation.check()?;
876 let mut replacement = self.clone();
877 replacement.bindings.bind_revision(field, revision, phase)?;
878 replacement.rebuild_documents_with_cancellation(documents, Some(cancellation))?;
879 *self = replacement;
880 Ok(())
881 }
882 fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
883 if let Some(index) = self.native_index() {
884 return index.doc_freq_any_field(term);
885 }
886 uqa_storage::inverted_index::defaults::doc_freq_any_field(self, term)
887 }
888 fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
889 if let Some(index) = self.native_index() {
890 return index.field_stats(field);
891 }
892 uqa_storage::inverted_index::defaults::field_stats(self, field)
893 }
894 fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
895 if let Some(index) = self.native_index() {
896 return index.field_stats_scalar(field);
897 }
898 uqa_storage::inverted_index::defaults::field_stats_scalar(self, field)
899 }
900 fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
901 if let Some(index) = self.native_index() {
902 return index.get_posting_list_any_field(term);
903 }
904 uqa_storage::inverted_index::defaults::get_posting_list_any_field(self, term)
905 }
906 fn get_term_freqs_bulk(
907 &self,
908 doc_ids: &[DocId],
909 field: &str,
910 term: &str,
911 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
912 if let Some(index) = self.native_index() {
913 return index.get_term_freqs_bulk(doc_ids, field, term);
914 }
915 uqa_storage::inverted_index::defaults::get_term_freqs_bulk(self, doc_ids, field, term)
916 }
917 fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
918 if let Some(index) = self.native_index() {
919 return index.get_total_doc_length(doc_id);
920 }
921 uqa_storage::inverted_index::defaults::get_total_doc_length(self, doc_id)
922 }
923 fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
924 if let Some(index) = self.native_index() {
925 return index.get_total_term_freq(doc_id, term);
926 }
927 uqa_storage::inverted_index::defaults::get_total_term_freq(self, doc_id, term)
928 }
929}