1use super::{
8 Analyzer, Arc, BTreeMap, BlockMaxScorer, DocId, FieldName, IndexStats, PostingEntry,
9 PostingList, StorageBackendError, StorageBackendResult,
10};
11use super::{IndexedFieldMetadata, TextIndexSource};
12use crate::clustered_postings::{BudgetedPostingReadCursor, PostingReadCursor};
13use crate::clustered_postings::{
14 MaterializedPostingCursor, OccurrencePosting, PostingCursor, PostingScore,
15};
16use crate::read_control::StorageReadControl;
17use crate::TokenTermKey;
18use uqa_core::memory::Budgeted;
19use uqa_core::TokenOccurrence;
20
21#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
23#[serde(rename_all = "snake_case")]
24pub enum AnalyzerPhase {
25 Index,
27 Search,
29 Both,
31}
32
33impl AnalyzerPhase {
34 pub fn parse(s: &str) -> Result<Self, String> {
35 match s {
36 "index" => Ok(AnalyzerPhase::Index),
37 "search" | "query" => Ok(AnalyzerPhase::Search),
38 "both" => Ok(AnalyzerPhase::Both),
39 _ => Err(format!("phase must be 'index'|'search'|'both', got `{s}`")),
40 }
41 }
42}
43
44impl std::str::FromStr for AnalyzerPhase {
45 type Err = String;
46
47 fn from_str(s: &str) -> Result<Self, Self::Err> {
48 Self::parse(s)
49 }
50}
51
52pub trait InvertedIndex: Send + Sync {
53 fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
55 Ok(false)
56 }
57
58 fn analyzer(&self) -> &Analyzer;
59
60 fn default_analyzer_binding(&self) -> StorageBackendResult<super::AnalyzerDefault> {
62 Err(StorageBackendError::Other(
63 "retained default analyzer bindings are not supported by this backend".into(),
64 ))
65 }
66
67 fn add_document(
68 &mut self,
69 doc_id: DocId,
70 fields: BTreeMap<FieldName, String>,
71 ) -> StorageBackendResult<()>;
72
73 fn try_add_document(
74 &mut self,
75 doc_id: DocId,
76 fields: BTreeMap<FieldName, String>,
77 ) -> StorageBackendResult<()> {
78 self.add_document(doc_id, fields)
79 }
80
81 fn try_add_documents(
83 &mut self,
84 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
85 ) -> StorageBackendResult<()> {
86 for (doc_id, fields) in documents {
87 self.try_add_document(doc_id, fields)?;
88 }
89 Ok(())
90 }
91
92 fn try_add_documents_observed(
94 &mut self,
95 _documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
96 _visit: &mut super::InvertedIndexChangeVisitor<'_>,
97 ) -> StorageBackendResult<()> {
98 Err(StorageBackendError::Other(
99 "evaluated text mutation observations are not supported by this backend".into(),
100 ))
101 }
102
103 fn try_remove_document_observed(
105 &mut self,
106 doc_id: DocId,
107 visit: &mut super::InvertedIndexChangeVisitor<'_>,
108 ) -> StorageBackendResult<()> {
109 self.try_add_documents_observed(vec![(doc_id, BTreeMap::new())], visit)
110 }
111
112 fn remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()>;
113
114 fn try_remove_document(&mut self, doc_id: DocId) -> StorageBackendResult<()> {
115 self.remove_document(doc_id)
116 }
117
118 fn clear(&mut self) -> StorageBackendResult<()>;
119
120 fn try_clear(&mut self) -> StorageBackendResult<()> {
121 self.clear()
122 }
123
124 fn try_rebuild_documents(
126 &mut self,
127 source: &mut dyn TextIndexSource,
128 ) -> StorageBackendResult<()> {
129 self.try_clear()?;
130 while let Some((doc_id, fields)) = source.next_document()? {
131 if !fields.is_empty() {
132 self.try_add_document(doc_id, fields)?;
133 }
134 }
135 Ok(())
136 }
137
138 fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList>;
139
140 fn get_posting_list_key(
142 &self,
143 field: &str,
144 term: &TokenTermKey,
145 ) -> StorageBackendResult<PostingList> {
146 self.get_posting_list(field, &term.to_term().into_string()?)
147 }
148
149 fn posting_cursor_key(
151 &self,
152 field: &str,
153 term: &TokenTermKey,
154 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
155 self.posting_cursor(field, &term.to_term().into_string()?)
156 }
157
158 fn posting_read_cursor_key<'a>(
160 &'a self,
161 field: &'a str,
162 term: &TokenTermKey,
163 ) -> StorageBackendResult<Box<dyn crate::clustered_postings::PostingReadCursor + 'a>> {
164 Ok(Box::new(crate::clustered_postings::OwnedPostingReadCursor(
165 self.posting_cursor_key(field, term)?,
166 )))
167 }
168
169 fn posting_read_cursor_key_budgeted<'a>(
171 &'a self,
172 field: &'a str,
173 term: &'a TokenTermKey,
174 control: &StorageReadControl,
175 ) -> StorageBackendResult<BudgetedPostingReadCursor<'a>> {
176 crate::clustered_postings::open_controlled_cursor(self, field, term, control)
177 }
178
179 fn visit_score_clusters(
181 &self,
182 _field: &str,
183 _term: &TokenTermKey,
184 _after: Option<u64>,
185 _limit: usize,
186 control: &StorageReadControl,
187 _visit: &mut crate::clustered_postings::ScoreClusterVisitor<'_>,
188 ) -> StorageBackendResult<()> {
189 control.check()?;
190 Err(StorageBackendError::Other(
191 "controlled score cluster reads are not supported by this backend".into(),
192 ))
193 }
194
195 fn get_occurrences_budgeted(
197 &self,
198 _doc_id: DocId,
199 _field: &str,
200 _term: &TokenTermKey,
201 control: &StorageReadControl,
202 ) -> StorageBackendResult<Budgeted<Vec<TokenOccurrence>>> {
203 control.check()?;
204 Err(StorageBackendError::Other(
205 "controlled occurrence reads are not supported by this backend".into(),
206 ))
207 }
208
209 fn get_occurrence_postings(
211 &self,
212 _field: &str,
213 _term: &TokenTermKey,
214 ) -> StorageBackendResult<Vec<OccurrencePosting>> {
215 Err(StorageBackendError::Other(
216 "lossless occurrence storage is not supported by this backend".into(),
217 ))
218 }
219
220 fn get_occurrences(
222 &self,
223 doc_id: DocId,
224 field: &str,
225 term: &TokenTermKey,
226 ) -> StorageBackendResult<Vec<TokenOccurrence>> {
227 Ok(self
228 .get_occurrence_postings(field, term)?
229 .into_iter()
230 .find(|posting| posting.doc_id == doc_id)
231 .map_or_else(Vec::new, |posting| posting.occurrences))
232 }
233
234 fn indexed_field_metadata(
236 &self,
237 _doc_id: DocId,
238 _field: &str,
239 ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
240 Err(StorageBackendError::Other(
241 "indexed field analysis metadata is not supported by this backend".into(),
242 ))
243 }
244
245 fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
246 self.doc_freq(field, &term.to_term().into_string()?)
247 }
248
249 fn doc_freq_key_budgeted(
251 &self,
252 field: &str,
253 term: &TokenTermKey,
254 control: &StorageReadControl,
255 ) -> StorageBackendResult<u64> {
256 control.check()?;
257 let cursor = self.posting_read_cursor_key_budgeted(field, term, control)?;
258 let frequency = cursor.doc_freq();
259 control.check()?;
260 Ok(frequency)
261 }
262
263 fn get_term_freq_key(
264 &self,
265 doc_id: DocId,
266 field: &str,
267 term: &TokenTermKey,
268 ) -> StorageBackendResult<u64> {
269 self.get_term_freq(doc_id, field, &term.to_term().into_string()?)
270 }
271
272 fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
274 Ok(self
275 .vocabulary_terms(field)?
276 .iter()
277 .map(|term| TokenTermKey::from_text(term))
278 .collect())
279 }
280
281 fn get_posting_lists_bulk(
282 &self,
283 field: &str,
284 terms: &[String],
285 ) -> StorageBackendResult<Vec<PostingList>> {
286 terms
287 .iter()
288 .map(|term| self.get_posting_list(field, term))
289 .collect()
290 }
291
292 fn posting_cursor(
299 &self,
300 field: &str,
301 term: &str,
302 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
303 let posting_list = self.get_posting_list(field, term)?;
304 let mut entries = Vec::with_capacity(posting_list.len());
305 for posting in posting_list {
306 let term_freq = self.get_term_freq(posting.doc_id, field, term)?;
307 entries.push(PostingScore {
308 doc_id: posting.doc_id,
309 term_freq,
310 doc_length: self.get_doc_length(posting.doc_id, field)?,
311 });
312 }
313 Ok(Box::new(MaterializedPostingCursor::new(entries)?))
314 }
315
316 fn posting_cursors_bulk(
317 &self,
318 field: &str,
319 terms: &[String],
320 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
321 terms
322 .iter()
323 .map(|term| self.posting_cursor(field, term))
324 .collect()
325 }
326
327 fn posting_cursors_keys_bulk(
329 &self,
330 field: &str,
331 terms: &[TokenTermKey],
332 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
333 if let Some(scalar) = terms
334 .iter()
335 .map(|key| key.as_str().map(str::to_owned))
336 .collect::<Option<Vec<_>>>()
337 {
338 return self.posting_cursors_bulk(field, &scalar);
339 }
340 terms
341 .iter()
342 .map(|term| self.posting_cursor_key(field, term))
343 .collect()
344 }
345
346 fn get_posting_lists_keys_bulk(
348 &self,
349 field: &str,
350 terms: &[TokenTermKey],
351 ) -> StorageBackendResult<Vec<PostingList>> {
352 if let Some(scalar) = terms
353 .iter()
354 .map(|key| key.as_str().map(str::to_owned))
355 .collect::<Option<Vec<_>>>()
356 {
357 return self.get_posting_lists_bulk(field, &scalar);
358 }
359 terms
360 .iter()
361 .map(|term| self.get_posting_list_key(field, term))
362 .collect()
363 }
364
365 fn persisted_block_max_scores_keys_bulk(
367 &self,
368 field: &str,
369 terms: &[TokenTermKey],
370 scorer_fingerprint: &str,
371 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
372 if let Some(scalar) = terms
373 .iter()
374 .map(|key| key.as_str().map(str::to_owned))
375 .collect::<Option<Vec<_>>>()
376 {
377 return self.persisted_block_max_scores_bulk(field, &scalar, scorer_fingerprint);
378 }
379 terms
380 .iter()
381 .map(|key| match key.as_str() {
382 Some(term) => self.persisted_block_max_scores(field, term, scorer_fingerprint),
383 None => Ok(None),
384 })
385 .collect()
386 }
387
388 fn get_scoring_inputs_keys_bulk(
390 &self,
391 doc_ids: &[DocId],
392 field: &str,
393 terms: &[TokenTermKey],
394 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
395 if let Some(scalar) = terms
396 .iter()
397 .map(|key| key.as_str().map(str::to_owned))
398 .collect::<Option<Vec<_>>>()
399 {
400 return self.get_scoring_inputs_bulk(doc_ids, field, &scalar);
401 }
402 doc_ids
403 .iter()
404 .map(|id| {
405 Ok((
406 self.get_doc_length(*id, field)?,
407 terms
408 .iter()
409 .map(|key| self.get_term_freq_key(*id, field, key))
410 .collect::<StorageBackendResult<_>>()?,
411 ))
412 })
413 .collect()
414 }
415
416 fn rebuild_persisted_block_max(
422 &mut self,
423 _field: &str,
424 _scorer: &dyn BlockMaxScorer,
425 _scorer_fingerprint: &str,
426 ) -> StorageBackendResult<bool> {
427 Ok(false)
428 }
429
430 fn persisted_block_max_scores(
433 &self,
434 _field: &str,
435 _term: &str,
436 _scorer_fingerprint: &str,
437 ) -> StorageBackendResult<Option<Vec<f64>>> {
438 Ok(None)
439 }
440
441 fn persisted_block_max_scores_bulk(
443 &self,
444 field: &str,
445 terms: &[String],
446 scorer_fingerprint: &str,
447 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
448 terms
449 .iter()
450 .map(|term| self.persisted_block_max_scores(field, term, scorer_fingerprint))
451 .collect()
452 }
453
454 fn for_each_posting(
458 &self,
459 field: &str,
460 term: &str,
461 visit: &mut dyn FnMut(&PostingEntry),
462 ) -> StorageBackendResult<()> {
463 for entry in &self.get_posting_list(field, term)? {
464 visit(entry);
465 }
466 Ok(())
467 }
468
469 fn for_each_term_freq(
474 &self,
475 field: &str,
476 term: &str,
477 visit: &mut dyn FnMut(DocId, u64),
478 ) -> StorageBackendResult<()> {
479 for entry in &self.get_posting_list(field, term)? {
480 visit(entry.doc_id, self.get_term_freq(entry.doc_id, field, term)?);
481 }
482 Ok(())
483 }
484
485 fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64>;
486
487 fn get_doc_length(&self, doc_id: DocId, field: &str) -> StorageBackendResult<u64>;
488
489 fn get_term_freq(&self, doc_id: DocId, field: &str, term: &str) -> StorageBackendResult<u64>;
490
491 fn doc_count(&self) -> StorageBackendResult<u64>;
492
493 fn total_field_length(&self, field: &str) -> StorageBackendResult<u64>;
494
495 fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
497 self.doc_length_count(Some(field))
498 }
499
500 fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
506 super::defaults::field_stats(self, field)
507 }
508
509 fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
517 super::defaults::field_stats_scalar(self, field)
518 }
519
520 fn field_stats_scalar_budgeted(
522 &self,
523 _field: &str,
524 control: &StorageReadControl,
525 ) -> StorageBackendResult<IndexStats> {
526 control.check()?;
527 Err(StorageBackendError::Other(
528 "controlled field statistics are not supported by this backend".into(),
529 ))
530 }
531
532 fn vocabulary_terms(&self, _field: &str) -> StorageBackendResult<Vec<String>> {
538 Ok(Vec::new())
539 }
540
541 fn stats(&self) -> StorageBackendResult<IndexStats>;
544
545 fn posting_count(&self, _field: Option<&str>) -> StorageBackendResult<u64> {
548 Ok(0)
549 }
550
551 fn doc_length_count(&self, _field: Option<&str>) -> StorageBackendResult<u64> {
554 Ok(0)
555 }
556
557 fn term_count(&self, _field: Option<&str>) -> StorageBackendResult<u64> {
560 Ok(0)
561 }
562
563 fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>>;
565
566 fn snapshot_with_control(
568 &self,
569 control: &crate::read_control::StorageReadControl,
570 ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
571 control.check()?;
572 let snapshot = self.snapshot()?;
573 control.check()?;
574 Ok(snapshot)
575 }
576
577 fn writable_snapshot(&self) -> StorageBackendResult<Box<dyn InvertedIndex>> {
580 Err(StorageBackendError::Other(
581 "writable inverted-index snapshots are not supported by this backend".into(),
582 ))
583 }
584
585 fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
591 Ok(Vec::new())
592 }
593
594 fn field_names_budgeted(
596 &self,
597 control: &StorageReadControl,
598 ) -> StorageBackendResult<uqa_core::memory::BudgetedVec<uqa_core::memory::BudgetedString>> {
599 control.check()?;
600 Err(StorageBackendError::Other(
601 "controlled field inventory is not supported by this backend".into(),
602 ))
603 }
604
605 fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
609 super::defaults::get_posting_list_any_field(self, term)
610 }
611
612 fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
614 super::defaults::doc_freq_any_field(self, term)
615 }
616
617 fn get_total_doc_length(&self, doc_id: DocId) -> StorageBackendResult<u64> {
619 super::defaults::get_total_doc_length(self, doc_id)
620 }
621
622 fn get_doc_lengths_bulk(
624 &self,
625 doc_ids: &[DocId],
626 field: &str,
627 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
628 let mut out = BTreeMap::new();
629 for doc_id in doc_ids {
630 out.insert(*doc_id, self.get_doc_length(*doc_id, field)?);
631 }
632 Ok(out)
633 }
634
635 fn get_term_freqs_bulk(
637 &self,
638 doc_ids: &[DocId],
639 field: &str,
640 term: &str,
641 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
642 super::defaults::get_term_freqs_bulk(self, doc_ids, field, term)
643 }
644
645 fn get_scoring_inputs_bulk(
650 &self,
651 doc_ids: &[DocId],
652 field: &str,
653 terms: &[String],
654 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
655 let mut out = Vec::with_capacity(doc_ids.len());
656 for doc_id in doc_ids {
657 let mut term_freqs = Vec::with_capacity(terms.len());
658 for term in terms {
659 term_freqs.push(self.get_term_freq(*doc_id, field, term)?);
660 }
661 out.push((self.get_doc_length(*doc_id, field)?, term_freqs));
662 }
663 Ok(out)
664 }
665
666 fn get_total_term_freq(&self, doc_id: DocId, term: &str) -> StorageBackendResult<u64> {
669 super::defaults::get_total_term_freq(self, doc_id, term)
670 }
671
672 fn set_field_analyzer(
677 &mut self,
678 _field: &str,
679 _analyzer: Analyzer,
680 _phase: AnalyzerPhase,
681 ) -> Result<(), String> {
682 Err("set_field_analyzer not supported by this InvertedIndex backend".into())
683 }
684
685 fn remove_field_analyzers(&mut self, _field: &str) -> Result<(), String> {
690 Err("remove_field_analyzers not supported by this InvertedIndex backend".into())
691 }
692
693 fn get_field_analyzer(&self, _field: &str) -> Analyzer {
696 self.analyzer().clone()
697 }
698
699 fn get_search_analyzer(&self, field: &str) -> Analyzer {
701 self.get_field_analyzer(field)
702 }
703
704 fn index_analyzer_revision(
706 &self,
707 field: &str,
708 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
709 Ok(self.get_field_analyzer(field).compile()?)
710 }
711
712 fn search_analyzer_revision(
714 &self,
715 field: &str,
716 ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
717 Ok(self.get_search_analyzer(field).compile()?)
718 }
719
720 fn set_field_analyzer_revision(
722 &mut self,
723 _field: &str,
724 _revision: Arc<uqa_analysis::CompiledAnalyzer>,
725 _phase: AnalyzerPhase,
726 ) -> Result<(), String> {
727 Err("immutable analyzer revisions are not supported by this backend".into())
728 }
729
730 fn set_field_analyzer_revisions(
732 &mut self,
733 _field: &str,
734 _index: Arc<uqa_analysis::CompiledAnalyzer>,
735 _search: Arc<uqa_analysis::CompiledAnalyzer>,
736 ) -> Result<(), String> {
737 Err("atomic analyzer revision pairs are not supported by this backend".into())
738 }
739
740 fn rebuild_with_analyzer_revision(
742 &mut self,
743 _field: &str,
744 _revision: Arc<uqa_analysis::CompiledAnalyzer>,
745 _phase: AnalyzerPhase,
746 _source: &mut dyn TextIndexSource,
747 ) -> StorageBackendResult<()> {
748 Err(StorageBackendError::Other(
749 "atomic analyzer revision rebuild is not supported by this backend".into(),
750 ))
751 }
752
753 fn try_rebuild_documents_cancellable(
755 &mut self,
756 _source: &mut dyn TextIndexSource,
757 cancellation: &uqa_core::CancellationToken,
758 ) -> StorageBackendResult<()> {
759 cancellation.check()?;
760 Err(StorageBackendError::Other(
761 "cancellable atomic index rebuild is not supported by this backend".into(),
762 ))
763 }
764
765 fn rebuild_with_analyzer_revision_cancellable(
767 &mut self,
768 _field: &str,
769 _revision: Arc<uqa_analysis::CompiledAnalyzer>,
770 _phase: AnalyzerPhase,
771 _source: &mut dyn TextIndexSource,
772 cancellation: &uqa_core::CancellationToken,
773 ) -> StorageBackendResult<()> {
774 cancellation.check()?;
775 Err(StorageBackendError::Other(
776 "cancellable atomic analyzer revision rebuild is not supported by this backend".into(),
777 ))
778 }
779}