1use super::queries::project_postings;
10use super::{
11 clustered_result, decode_index_u64, encode_index_u64, invalidate_posting_accelerators, params,
12 params_from_iter, quote_ident, Analyzer, AnalyzerPhase, Arc, BTreeMap, BlockMaxScorer, DocId,
13 FieldName, IndexStats, InvertedIndex, OptionalExtension, PostingCursor, PostingList,
14 SQLiteError, SQLiteInvertedIndex, SqlValue, StorageBackendResult,
15};
16use super::{IndexedFieldMetadata, TokenTermKey};
17use uqa_storage::clustered_postings::{cluster_id, decode_all_scores, OccurrencePosting};
18use uqa_storage::inverted_index::TextIndexSource;
19
20impl InvertedIndex for SQLiteInvertedIndex {
21 fn posting_read_cursor_key_budgeted<'a>(
22 &'a self,
23 field: &'a str,
24 term: &'a TokenTermKey,
25 control: &uqa_storage::read_control::StorageReadControl,
26 ) -> StorageBackendResult<uqa_storage::clustered_postings::BudgetedPostingReadCursor<'a>> {
27 control.check()?;
28 if let Some(index) = self.native_index() {
29 return uqa_storage::clustered_postings::open_controlled_cursor(
30 index.snapshot()?,
31 field,
32 term,
33 control,
34 );
35 }
36 uqa_storage::clustered_postings::open_controlled_cursor(self, field, term, control)
37 }
38
39 fn visit_score_clusters(
40 &self,
41 field: &str,
42 term: &TokenTermKey,
43 after: Option<u64>,
44 limit: usize,
45 control: &uqa_storage::read_control::StorageReadControl,
46 visit: &mut uqa_storage::clustered_postings::ScoreClusterVisitor<'_>,
47 ) -> StorageBackendResult<()> {
48 if let Some(index) = self.native_index() {
49 return index.visit_score_clusters(field, term, after, limit, control, visit);
50 }
51 Ok(self.visit_clusters_budgeted(field, term, after, limit, control, visit)?)
52 }
53
54 fn get_occurrences_budgeted(
55 &self,
56 doc_id: DocId,
57 field: &str,
58 term: &TokenTermKey,
59 control: &uqa_storage::read_control::StorageReadControl,
60 ) -> StorageBackendResult<uqa_core::memory::Budgeted<Vec<uqa_core::TokenOccurrence>>> {
61 if let Some(index) = self.native_index() {
62 return index.get_occurrences_budgeted(doc_id, field, term, control);
63 }
64 Ok(self.occurrences_budgeted(doc_id, field, term, control)?)
65 }
66
67 fn field_stats_scalar_budgeted(
68 &self,
69 field: &str,
70 control: &uqa_storage::read_control::StorageReadControl,
71 ) -> StorageBackendResult<IndexStats> {
72 if let Some(index) = self.native_index() {
73 return index.field_stats_scalar_budgeted(field, control);
74 }
75 Ok(self.scalar_stats_budgeted(field, control)?)
76 }
77
78 fn analyzer(&self) -> &Analyzer {
79 self.bindings.default_configuration()
80 }
81
82 fn default_analyzer_binding(
83 &self,
84 ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
85 Ok(self.bindings.default_binding())
86 }
87
88 fn add_document(
89 &mut self,
90 doc_id: DocId,
91 fields: BTreeMap<FieldName, String>,
92 ) -> StorageBackendResult<()> {
93 if let Some(mut index) = self.native_index() {
94 return index.add_document(doc_id, fields);
95 }
96 Ok(self.add_document_inner(doc_id, fields)?)
97 }
98
99 fn try_add_documents(
100 &mut self,
101 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
102 ) -> StorageBackendResult<()> {
103 if let Some(mut index) = self.native_index() {
104 return index.try_add_documents(documents);
105 }
106 Ok(self.add_documents_inner(documents)?)
107 }
108
109 fn try_add_documents_observed(
110 &mut self,
111 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
112 visit: &mut uqa_storage::inverted_index::InvertedIndexChangeVisitor<'_>,
113 ) -> StorageBackendResult<()> {
114 if let Some(mut index) = self.native_index() {
115 return index.try_add_documents_observed(documents, visit);
116 }
117 Ok(self.add_documents_observed(documents, Some(visit))?)
118 }
119
120 fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
121 if let Some(mut index) = self.native_index() {
122 return index.remove_document(doc_id);
123 }
124 Ok(self.remove_document_inner(doc_id)?)
125 }
126
127 fn clear(&mut self) -> StorageBackendResult<()> {
128 if let Some(mut index) = self.native_index() {
129 return index.clear();
130 }
131 self.conn.with_mut(|conn| {
132 let tx = conn.savepoint()?;
133 invalidate_posting_accelerators(&tx, &self.table)?;
134 self.clear_index_on(&tx)?;
135 tx.commit()?;
136 Ok(())
137 })?;
138 Ok(())
139 }
140
141 fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
142 if let Some(index) = self.native_index() {
143 return index.source_rebuild_required();
144 }
145 Ok(self.conn.with(|conn| self.needs_source_rebuild_on(conn))?)
146 }
147
148 fn try_rebuild_documents(
149 &mut self,
150 source: &mut dyn TextIndexSource,
151 ) -> StorageBackendResult<()> {
152 if let Some(mut index) = self.native_index() {
153 return index.try_rebuild_documents(source);
154 }
155 Ok(self.rebuild_documents_inner(source)?)
156 }
157
158 fn try_rebuild_documents_cancellable(
159 &mut self,
160 source: &mut dyn TextIndexSource,
161 cancellation: &uqa_core::CancellationToken,
162 ) -> StorageBackendResult<()> {
163 if let Some(mut index) = self.native_index() {
164 return index.try_rebuild_documents_cancellable(source, cancellation);
165 }
166 Ok(self.rebuild_documents_with_cancellation(source, Some(cancellation))?)
167 }
168
169 fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
170 if let Some(index) = self.native_index() {
171 return index.get_posting_list(field, term);
172 }
173 self.get_posting_list_key(field, &TokenTermKey::from_text(term))
174 }
175
176 fn get_posting_list_key(
177 &self,
178 field: &str,
179 term: &TokenTermKey,
180 ) -> StorageBackendResult<PostingList> {
181 if let Some(index) = self.native_index() {
182 return index.get_posting_list_key(field, term);
183 }
184 Ok(project_postings(self.get_occurrence_postings(field, term)?))
185 }
186
187 fn get_occurrence_postings(
188 &self,
189 field: &str,
190 term: &TokenTermKey,
191 ) -> StorageBackendResult<Vec<OccurrencePosting>> {
192 if let Some(index) = self.native_index() {
193 return index.get_occurrence_postings(field, term);
194 }
195 Ok(self
196 .occurrence_postings_bulk(field, std::slice::from_ref(term))?
197 .pop()
198 .unwrap_or_default())
199 }
200
201 fn get_occurrences(
202 &self,
203 doc_id: DocId,
204 field: &str,
205 term: &TokenTermKey,
206 ) -> StorageBackendResult<Vec<uqa_core::TokenOccurrence>> {
207 if let Some(index) = self.native_index() {
208 return index.get_occurrences(doc_id, field, term);
209 }
210 Ok(self.conn.with(|conn| {
211 self.require_graph_format_on(conn)?;
212 let entries = super::load_cluster(conn, &self.table, field, term, cluster_id(doc_id))?;
213 let Some(posting) = entries.into_iter().find(|entry| entry.doc_id == doc_id) else {
214 return Ok(Vec::new());
215 };
216 self.validate_posting_metadata_on(conn, field, &posting)?;
217 Ok(posting.occurrences)
218 })?)
219 }
220
221 fn indexed_field_metadata(
222 &self,
223 doc_id: DocId,
224 field: &str,
225 ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
226 if let Some(index) = self.native_index() {
227 return index.indexed_field_metadata(doc_id, field);
228 }
229 let doc_id = encode_index_u64("document", doc_id)?;
230 Ok(self.conn.with(|conn| {
231 self.require_graph_format_on(conn)?;
232 self.read_field_metadata_on(conn, doc_id, field)
233 })?)
234 }
235
236 fn get_posting_lists_bulk(
237 &self,
238 field: &str,
239 terms: &[String],
240 ) -> StorageBackendResult<Vec<PostingList>> {
241 if let Some(index) = self.native_index() {
242 return index.get_posting_lists_bulk(field, terms);
243 }
244 let keys = terms
245 .iter()
246 .map(|term| TokenTermKey::from_text(term))
247 .collect::<Vec<_>>();
248 Ok(self
249 .occurrence_postings_bulk(field, &keys)?
250 .into_iter()
251 .map(project_postings)
252 .collect())
253 }
254
255 fn posting_cursor(
256 &self,
257 field: &str,
258 term: &str,
259 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
260 if let Some(index) = self.native_index() {
261 return index.posting_cursor(field, term);
262 }
263 self.posting_cursor_key(field, &TokenTermKey::from_text(term))
264 }
265
266 fn posting_cursor_key(
267 &self,
268 field: &str,
269 term: &TokenTermKey,
270 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
271 if let Some(index) = self.native_index() {
272 return index.posting_cursor_key(field, term);
273 }
274 self.cursor_for_term(field, term)
275 }
276
277 fn posting_cursors_bulk(
278 &self,
279 field: &str,
280 terms: &[String],
281 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
282 if let Some(index) = self.native_index() {
283 return index.posting_cursors_bulk(field, terms);
284 }
285 let keys = terms
286 .iter()
287 .map(|term| TokenTermKey::from_text(term))
288 .collect::<Vec<_>>();
289 self.cursors_for_terms(field, &keys)
290 }
291
292 fn posting_cursors_keys_bulk(
293 &self,
294 field: &str,
295 terms: &[TokenTermKey],
296 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
297 if let Some(index) = self.native_index() {
298 return index.posting_cursors_keys_bulk(field, terms);
299 }
300 self.cursors_for_terms(field, terms)
301 }
302
303 fn get_posting_lists_keys_bulk(
304 &self,
305 field: &str,
306 terms: &[TokenTermKey],
307 ) -> StorageBackendResult<Vec<PostingList>> {
308 if let Some(index) = self.native_index() {
309 return index.get_posting_lists_keys_bulk(field, terms);
310 }
311 Ok(self
312 .occurrence_postings_bulk(field, terms)?
313 .into_iter()
314 .map(project_postings)
315 .collect())
316 }
317
318 fn persisted_block_max_scores_keys_bulk(
319 &self,
320 field: &str,
321 terms: &[TokenTermKey],
322 scorer_fingerprint: &str,
323 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
324 if scorer_fingerprint.is_empty() {
325 return Ok(vec![None; terms.len()]);
326 }
327 self.get_versioned_block_max_scores_keys_bulk(field, terms, scorer_fingerprint)
328 }
329
330 fn rebuild_persisted_block_max(
331 &mut self,
332 field: &str,
333 scorer: &dyn BlockMaxScorer,
334 scorer_fingerprint: &str,
335 ) -> StorageBackendResult<bool> {
336 if let Some(mut index) = self.native_index() {
337 return index.rebuild_persisted_block_max(field, scorer, scorer_fingerprint);
338 }
339 if scorer_fingerprint.is_empty() {
340 return Err(SQLiteError::StorageBackend(
341 "persisted block-max scorer fingerprint must not be empty".into(),
342 )
343 .into());
344 }
345 let terms = self.vocabulary_keys(field)?;
346 self.ensure_aux_tables(field)?;
347 let table = self.blockmax_table_name(field);
348 self.conn.with_mut(|conn| {
349 conn.execute(&format!("DELETE FROM {}", quote_ident(&table)), [])?;
350 Ok(())
351 })?;
352 for term in terms {
353 self.build_block_max_scores_key(field, &term, scorer, scorer_fingerprint)?;
354 }
355 Ok(true)
356 }
357
358 fn persisted_block_max_scores(
359 &self,
360 field: &str,
361 term: &str,
362 scorer_fingerprint: &str,
363 ) -> StorageBackendResult<Option<Vec<f64>>> {
364 if scorer_fingerprint.is_empty() {
365 return Ok(None);
366 }
367 self.get_versioned_block_max_scores(field, term, scorer_fingerprint)
368 }
369
370 fn persisted_block_max_scores_bulk(
371 &self,
372 field: &str,
373 terms: &[String],
374 scorer_fingerprint: &str,
375 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
376 if scorer_fingerprint.is_empty() {
377 return Ok(vec![None; terms.len()]);
378 }
379 self.get_versioned_block_max_scores_bulk(field, terms, scorer_fingerprint)
380 }
381
382 fn for_each_term_freq(
383 &self,
384 field: &str,
385 term: &str,
386 visit: &mut dyn FnMut(DocId, u64),
387 ) -> StorageBackendResult<()> {
388 if let Some(index) = self.native_index() {
389 return index.for_each_term_freq(field, term, visit);
390 }
391 let mut cursor = self.posting_cursor(field, term)?;
392 while let Some(entry) = cursor.current() {
393 visit(entry.doc_id, entry.term_freq);
394 cursor.advance()?;
395 }
396 Ok(())
397 }
398
399 fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
400 if let Some(index) = self.native_index() {
401 return index.doc_freq(field, term);
402 }
403 self.doc_freq_key(field, &TokenTermKey::from_text(term))
404 }
405
406 fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
407 if let Some(index) = self.native_index() {
408 return index.doc_freq_key(field, term);
409 }
410 Ok(self.posting_cursor_key(field, term)?.doc_freq())
411 }
412
413 fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64> {
414 if let Some(index) = self.native_index() {
415 return index.get_doc_length(doc_id, field);
416 }
417 Ok(self
418 .get_doc_lengths_bulk(&[doc_id], field)?
419 .get(&doc_id)
420 .copied()
421 .unwrap_or(0))
422 }
423
424 fn get_doc_lengths_bulk(
425 &self,
426 doc_ids: &[DocId],
427 field: &str,
428 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
429 if let Some(index) = self.native_index() {
430 return index.get_doc_lengths_bulk(doc_ids, field);
431 }
432 Ok(self.conn.with(|conn| {
433 self.require_graph_format_on(conn)?;
434 let mut out = BTreeMap::new();
435 for chunk in doc_ids.chunks(900) {
436 let ids = (3..chunk.len()+3).map(|parameter| format!("?{parameter}")).collect::<Vec<_>>().join(", ");
437 let sql = format!("WITH lengths AS (SELECT doc_id, length FROM _occurrence_lengths WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})), documents AS (SELECT doc_id, metadata_blob FROM _occurrence_documents WHERE table_name = ?1 AND field = ?2 AND doc_id IN ({ids})) SELECT COALESCE(lengths.doc_id, documents.doc_id), length, metadata_blob FROM lengths FULL OUTER JOIN documents USING(doc_id)");
438 let mut values = vec![SqlValue::Text(self.table.clone()), SqlValue::Text(field.into())];
439 for doc_id in chunk {
440 values.push(SqlValue::Integer(encode_index_u64("document", *doc_id)?));
441 }
442 let mut statement = conn.prepare(&sql)?;
443 let rows = statement.query_map(params_from_iter(values), |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option<i64>>(1)?, row.get::<_, Option<Vec<u8>>>(2)?)))?;
444 for row in rows {
445 let (doc_id, length, metadata) = row?;
446 let doc_id = decode_index_u64("document id", doc_id)?;
447 let length = length.map(|length| decode_index_u64("document length", length)).transpose()?;
448 let metadata = metadata.map(|bytes| clustered_result(IndexedFieldMetadata::from_bytes(&bytes))).transpose()?;
449 let (Some(length), Some(metadata)) = (length, metadata) else {
450 return Err(SQLiteError::StorageBackend("indexed field length and source metadata disagree".into()));
451 };
452 let stats = self.stored_field_stats_on(conn, field)?.ok_or_else(|| SQLiteError::StorageBackend("indexed field revision is missing".into()))?;
453 if metadata.length != length || metadata.revision() != stats.revision {
454 return Err(SQLiteError::StorageBackend("indexed field length or revision disagrees with source metadata".into()));
455 }
456 out.insert(doc_id, length);
457 }
458 }
459 Ok(out)
460 })?)
461 }
462
463 fn get_scoring_inputs_bulk(
464 &self,
465 doc_ids: &[DocId],
466 field: &str,
467 terms: &[String],
468 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
469 if let Some(index) = self.native_index() {
470 return index.get_scoring_inputs_bulk(doc_ids, field, terms);
471 }
472 let keys = terms
473 .iter()
474 .map(|term| TokenTermKey::from_text(term))
475 .collect::<Vec<_>>();
476 self.get_scoring_inputs_keys_bulk(doc_ids, field, &keys)
477 }
478
479 fn get_scoring_inputs_keys_bulk(
480 &self,
481 doc_ids: &[DocId],
482 field: &str,
483 terms: &[TokenTermKey],
484 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
485 if let Some(index) = self.native_index() {
486 return index.get_scoring_inputs_keys_bulk(doc_ids, field, terms);
487 }
488 if doc_ids.is_empty() {
489 return Ok(Vec::new());
490 }
491
492 let doc_lengths = self.get_doc_lengths_bulk(doc_ids, field)?;
493 let mut inputs: Vec<(u64, Vec<u64>)> = doc_ids
494 .iter()
495 .map(|doc_id| {
496 (
497 doc_lengths.get(doc_id).copied().unwrap_or(0),
498 vec![0; terms.len()],
499 )
500 })
501 .collect();
502 if terms.is_empty() {
503 return Ok(inputs);
504 }
505
506 let mut output_positions = BTreeMap::<DocId, Vec<usize>>::new();
507 for (position, doc_id) in doc_ids.iter().copied().enumerate() {
508 output_positions.entry(doc_id).or_default().push(position);
509 }
510 for (term_index, mut cursor) in self
511 .posting_cursors_keys_bulk(field, terms)?
512 .into_iter()
513 .enumerate()
514 {
515 while let Some(entry) = cursor.current() {
516 if let Some(positions) = output_positions.get(&entry.doc_id) {
517 for position in positions {
518 inputs[*position].1[term_index] = entry.term_freq;
519 inputs[*position].0 = entry.doc_length;
520 }
521 }
522 cursor.advance()?;
523 }
524 }
525 Ok(inputs)
526 }
527
528 fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
529 if let Some(index) = self.native_index() {
530 return index.get_term_freq(doc_id, field, term);
531 }
532 self.get_term_freq_key(doc_id, field, &TokenTermKey::from_text(term))
533 }
534
535 fn get_term_freq_key(
536 &self,
537 doc_id: DocId,
538 field: &str,
539 term: &TokenTermKey,
540 ) -> StorageBackendResult<u64> {
541 if let Some(index) = self.native_index() {
542 return index.get_term_freq_key(doc_id, field, term);
543 }
544 let cluster = encode_index_u64("posting cluster", cluster_id(doc_id))?;
545 Ok(self.conn.with(|conn| {
546 self.require_graph_format_on(conn)?;
547 let row: Option<(i64, Vec<u8>)> = conn.query_row("SELECT posting_count, score_blob FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 AND term = ?3 AND cluster_id = ?4", params![self.table, field, term.as_bytes(), cluster], |row| Ok((row.get(0)?, row.get(1)?))).optional()?;
548 match row {
549 Some((count, bytes)) => {
550 super::posting_cursor_from_rows(vec![(cluster, count, bytes.clone())])?;
552 let scores = clustered_result(decode_all_scores(cluster_id(doc_id), &bytes))?;
553 Ok(scores.binary_search_by_key(&doc_id, |entry| entry.doc_id).ok().map_or(0, |position| scores[position].term_freq))
554 }
555 None => Ok(0),
556 }
557 })?)
558 }
559
560 fn doc_count(&self) -> StorageBackendResult<u64> {
561 if let Some(index) = self.native_index() {
562 return index.doc_count();
563 }
564 Ok(self.conn.with(|c| {
565 self.require_graph_format_on(c)?;
566 let n: i64 = c.query_row(
567 "SELECT COUNT(DISTINCT doc_id) FROM _occurrence_lengths
568 WHERE table_name = ?1",
569 params![self.table],
570 |r| r.get(0),
571 )?;
572 decode_index_u64("document count", n)
573 })?)
574 }
575
576 fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
577 if let Some(index) = self.native_index() {
578 return index.total_field_length(field);
579 }
580 Ok(self.conn.with(|conn| {
581 self.require_graph_format_on(conn)?;
582 Ok(self
583 .stored_field_stats_on(conn, field)?
584 .map_or(0, |stats| stats.total_length))
585 })?)
586 }
587
588 fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
589 if let Some(index) = self.native_index() {
590 return index.vocabulary_terms(field);
591 }
592 self.terms_for_field(field)
593 }
594
595 fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
596 if let Some(index) = self.native_index() {
597 return index.vocabulary_keys(field);
598 }
599 Ok(self.conn.with(|conn| {
600 self.require_graph_format_on(conn)?;
601 let mut statement = conn.prepare("SELECT DISTINCT term FROM _occurrence_clusters WHERE table_name = ?1 AND field = ?2 ORDER BY term")?;
602 let rows = statement.query_map(params![self.table, field], |row| row.get::<_, Vec<u8>>(0))?;
603 rows.map(|row| clustered_result(TokenTermKey::from_bytes(row?))).collect()
604 })?)
605 }
606
607 fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
608 if let Some(index) = self.native_index() {
609 return index.field_doc_count(field);
610 }
611 Ok(self.conn.with(|conn| {
612 self.require_graph_format_on(conn)?;
613 Ok(self
614 .stored_field_stats_on(conn, field)?
615 .map_or(0, |stats| stats.doc_count))
616 })?)
617 }
618
619 fn stats(&self) -> StorageBackendResult<IndexStats> {
620 if let Some(index) = self.native_index() {
621 return index.stats();
622 }
623 let doc_count = self.doc_count()?;
624 let mut s = IndexStats::default();
625 s.total_docs = doc_count;
626 if doc_count > 0 {
627 let total: u64 = self.conn.with(|c| {
628 self.require_graph_format_on(c)?;
629 let n: i64 = c.query_row(
630 "SELECT COALESCE(SUM(total_length), 0) FROM _occurrence_fields
631 WHERE table_name = ?1",
632 params![self.table],
633 |r| r.get(0),
634 )?;
635 decode_index_u64("total indexed length", n)
636 })?;
637 s.avg_doc_length = total as f64 / doc_count as f64;
638 }
639 let pairs = self
640 .conn
641 .with(|conn| self.term_frequencies_on(conn, None))?;
642 for ((field, term), df) in pairs {
643 let term = term.to_term();
644 if let Some(text) = term.as_str() {
645 s.set_doc_freq(field, text, df);
646 } else {
647 s.set_doc_freq_utf16(field, term.into_utf16(), df);
648 }
649 }
650 Ok(s)
651 }
652
653 fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
654 if let Some(index) = self.native_index() {
655 return index.posting_count(field);
656 }
657 Ok(self.conn.with(|conn| {
658 self.term_frequencies_on(conn, field)?
659 .into_values()
660 .try_fold(0_u64, |total, count| {
661 total
662 .checked_add(count)
663 .ok_or_else(|| SQLiteError::StorageBackend("posting count overflow".into()))
664 })
665 })?)
666 }
667
668 fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
669 if let Some(index) = self.native_index() {
670 return index.doc_length_count(field);
671 }
672 Ok(self.conn.with(|c| {
673 self.require_graph_format_on(c)?;
674 let n: i64 = if let Some(field) = field {
675 c.query_row(
676 "SELECT COUNT(*) FROM _occurrence_lengths
677 WHERE table_name = ?1 AND field = ?2",
678 params![self.table, field],
679 |r| r.get(0),
680 )?
681 } else {
682 c.query_row(
683 "SELECT COUNT(*) FROM _occurrence_lengths WHERE table_name = ?1",
684 params![self.table],
685 |r| r.get(0),
686 )?
687 };
688 decode_index_u64("document length count", n)
689 })?)
690 }
691
692 fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
693 if let Some(index) = self.native_index() {
694 return index.term_count(field);
695 }
696 Ok(self.conn.with(|conn| {
697 let terms = self
698 .term_frequencies_on(conn, field)?
699 .into_keys()
700 .map(|(_, term)| term)
701 .collect::<std::collections::BTreeSet<_>>();
702 Ok(terms.len() as u64)
703 })?)
704 }
705
706 fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
707 if let Some(index) = self.native_retained_index()? {
708 return Ok(index);
709 }
710 self.physical_snapshot(&self.retention_control)
711 }
712
713 fn snapshot_with_control(
714 &self,
715 control: &uqa_storage::read_control::StorageReadControl,
716 ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
717 control.check()?;
718 if let Some(index) = self.native_retained_index()? {
719 control.check()?;
720 return Ok(index);
721 }
722 self.physical_snapshot(control)
723 }
724
725 fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
726 if let Some(index) = self.native_index() {
727 return index.field_names();
728 }
729 Ok(self.conn.with(|c| {
730 self.require_graph_format_on(c)?;
731 let mut stmt =
732 c.prepare("SELECT DISTINCT field FROM _occurrence_lengths WHERE table_name = ?1")?;
733 let rows = stmt.query_map([&self.table], |row| row.get::<_, String>(0))?;
734 let mut fields = Vec::new();
735 for row in rows {
736 fields.push(row?);
737 }
738 Ok(fields)
739 })?)
740 }
741
742 fn set_field_analyzer(
743 &mut self,
744 field: &str,
745 analyzer: Analyzer,
746 phase: AnalyzerPhase,
747 ) -> Result<(), String> {
748 if let Some(mut index) = self.native_index() {
749 index.set_field_analyzer(field, analyzer, phase)?;
750 self.bindings = index.analyzer_bindings().clone();
751 return Ok(());
752 }
753 let mut candidate = self.bindings.clone();
754 candidate
755 .bind(field, &analyzer, phase)
756 .map_err(|error| error.to_string())?;
757 self.validate_index_revision_change(field, &candidate)
758 .map_err(|error| error.to_string())?;
759 self.bindings = candidate;
760 Ok(())
761 }
762
763 fn remove_field_analyzers(&mut self, field: &str) -> Result<(), String> {
764 if let Some(mut index) = self.native_index() {
765 index.remove_field_analyzers(field)?;
766 self.bindings = index.analyzer_bindings().clone();
767 return Ok(());
768 }
769 let mut candidate = self.bindings.clone();
770 candidate.remove(field);
771 self.validate_index_revision_change(field, &candidate)
772 .map_err(|error| error.to_string())?;
773 self.bindings = candidate;
774 Ok(())
775 }
776
777 fn get_field_analyzer(&self, field: &str) -> Analyzer {
778 self.bindings.index_configuration(field).clone()
779 }
780 fn get_search_analyzer(&self, field: &str) -> Analyzer {
781 self.bindings.search_configuration(field).clone()
782 }
783 fn index_analyzer_revision(
784 &self,
785 field: &str,
786 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
787 Ok(self.bindings.index_revision(field)?)
788 }
789 fn search_analyzer_revision(
790 &self,
791 field: &str,
792 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
793 Ok(self.bindings.search_revision(field)?)
794 }
795
796 fn set_field_analyzer_revision(
797 &mut self,
798 field: &str,
799 revision: Arc<uqa_analysis::CompiledAnalyzer>,
800 phase: AnalyzerPhase,
801 ) -> Result<(), String> {
802 if let Some(mut index) = self.native_index() {
803 index.set_field_analyzer_revision(field, revision, phase)?;
804 self.bindings = index.analyzer_bindings().clone();
805 return Ok(());
806 }
807 let mut candidate = self.bindings.clone();
808 candidate
809 .bind_revision(field, revision, phase)
810 .map_err(|error| error.to_string())?;
811 self.validate_index_revision_change(field, &candidate)
812 .map_err(|error| error.to_string())?;
813 self.bindings = candidate;
814 Ok(())
815 }
816
817 fn set_field_analyzer_revisions(
818 &mut self,
819 field: &str,
820 index: Arc<uqa_analysis::CompiledAnalyzer>,
821 search: Arc<uqa_analysis::CompiledAnalyzer>,
822 ) -> Result<(), String> {
823 if let Some(mut native) = self.native_index() {
824 native.set_field_analyzer_revisions(field, index, search)?;
825 self.bindings = native.analyzer_bindings().clone();
826 return Ok(());
827 }
828 let mut candidate = self.bindings.clone();
829 candidate
830 .bind_revisions(field, index, search)
831 .map_err(|error| error.to_string())?;
832 self.validate_index_revision_change(field, &candidate)
833 .map_err(|error| error.to_string())?;
834 self.bindings = candidate;
835 Ok(())
836 }
837
838 fn rebuild_with_analyzer_revision(
839 &mut self,
840 field: &str,
841 revision: Arc<uqa_analysis::CompiledAnalyzer>,
842 phase: AnalyzerPhase,
843 source: &mut dyn TextIndexSource,
844 ) -> StorageBackendResult<()> {
845 if let Some(mut index) = self.native_index() {
846 index.rebuild_with_analyzer_revision(field, revision, phase, source)?;
847 self.bindings = index.analyzer_bindings().clone();
848 return Ok(());
849 }
850 let mut replacement = self.clone();
851 replacement.bindings.bind_revision(field, revision, phase)?;
852 replacement.rebuild_documents_inner(source)?;
853 *self = replacement;
854 Ok(())
855 }
856
857 fn rebuild_with_analyzer_revision_cancellable(
858 &mut self,
859 field: &str,
860 revision: Arc<uqa_analysis::CompiledAnalyzer>,
861 phase: AnalyzerPhase,
862 source: &mut dyn TextIndexSource,
863 cancellation: &uqa_core::CancellationToken,
864 ) -> StorageBackendResult<()> {
865 if let Some(mut index) = self.native_index() {
866 index.rebuild_with_analyzer_revision_cancellable(
867 field,
868 revision,
869 phase,
870 source,
871 cancellation,
872 )?;
873 self.bindings = index.analyzer_bindings().clone();
874 return Ok(());
875 }
876 cancellation.check()?;
877 let mut replacement = self.clone();
878 replacement.bindings.bind_revision(field, revision, phase)?;
879 replacement.rebuild_documents_with_cancellation(source, Some(cancellation))?;
880 *self = replacement;
881 Ok(())
882 }
883 fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
884 if let Some(index) = self.native_index() {
885 return index.doc_freq_any_field(term);
886 }
887 uqa_storage::inverted_index::defaults::doc_freq_any_field(self, term)
888 }
889 fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
890 if let Some(index) = self.native_index() {
891 return index.field_stats(field);
892 }
893 uqa_storage::inverted_index::defaults::field_stats(self, field)
894 }
895 fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
896 if let Some(index) = self.native_index() {
897 return index.field_stats_scalar(field);
898 }
899 uqa_storage::inverted_index::defaults::field_stats_scalar(self, field)
900 }
901 fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
902 if let Some(index) = self.native_index() {
903 return index.get_posting_list_any_field(term);
904 }
905 uqa_storage::inverted_index::defaults::get_posting_list_any_field(self, term)
906 }
907 fn get_term_freqs_bulk(
908 &self,
909 doc_ids: &[DocId],
910 field: &str,
911 term: &str,
912 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
913 if let Some(index) = self.native_index() {
914 return index.get_term_freqs_bulk(doc_ids, field, term);
915 }
916 uqa_storage::inverted_index::defaults::get_term_freqs_bulk(self, doc_ids, field, term)
917 }
918 fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
919 if let Some(index) = self.native_index() {
920 return index.get_total_doc_length(doc_id);
921 }
922 uqa_storage::inverted_index::defaults::get_total_doc_length(self, doc_id)
923 }
924 fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
925 if let Some(index) = self.native_index() {
926 return index.get_total_term_freq(doc_id, term);
927 }
928 uqa_storage::inverted_index::defaults::get_total_term_freq(self, doc_id, term)
929 }
930}