1use std::{collections::BTreeMap, ops::Deref, sync::Arc};
10use uqa_analysis::{Analyzer, CompiledAnalyzer};
11use uqa_core::{
12 memory::Budgeted, DocId, FieldName, IndexStats, PostingEntry, PostingList, TokenOccurrence,
13};
14use uqa_sql::ast::ColumnDef;
15use uqa_storage::{
16 clustered_postings::{
17 BudgetedPostingReadCursor, OccurrencePosting, PostingCursor, PostingReadCursor,
18 ScoreClusterVisitor,
19 },
20 inverted_index::IndexedFieldMetadata,
21 read_control::StorageReadControl,
22 BlockMaxScorer, InvertedIndex, StorageBackendError, StorageBackendResult, TokenTermKey,
23};
24
25use super::{TextObservation, DOCUMENT, POSTING, STATISTICS};
26use crate::serializable::SerializableRelationRead;
27
28pub struct ObservedTextIndex<I> {
30 index: I,
31 observation: Option<TextObservation>,
32}
33
34impl<I> ObservedTextIndex<I> {
35 pub fn new(
36 index: I,
37 read: Option<SerializableRelationRead>,
38 columns: Arc<Vec<ColumnDef>>,
39 ) -> Self {
40 Self {
41 index,
42 observation: read.map(|read| TextObservation { read, columns }),
43 }
44 }
45
46 fn observe(
47 &self,
48 operation: impl FnOnce(&TextObservation) -> StorageBackendResult<()>,
49 ) -> StorageBackendResult<()> {
50 self.observation.as_ref().map_or(Ok(()), operation)
51 }
52}
53
54impl<I: Deref<Target = dyn InvertedIndex>> ObservedTextIndex<I> {
55 fn observe_fields(
56 &self,
57 operation: impl Fn(&TextObservation, &str) -> StorageBackendResult<()>,
58 ) -> StorageBackendResult<()> {
59 self.observe(|observation| {
60 observation.field_range(STATISTICS, None)?;
61 for field in self.index.field_names()? {
62 operation(observation, &field)?;
63 }
64 Ok(())
65 })
66 }
67}
68
69pub fn observe_snapshot(
70 read: Option<&SerializableRelationRead>,
71 columns: Arc<Vec<ColumnDef>>,
72 index: Arc<dyn InvertedIndex>,
73) -> Arc<dyn InvertedIndex> {
74 match read {
75 Some(read) => Arc::new(ObservedTextIndex::new(index, Some(read.clone()), columns)),
76 None => index,
77 }
78}
79
80fn read_only() -> StorageBackendError {
81 StorageBackendError::Other("cannot write a retained serializable text reader".into())
82}
83
84#[cfg(test)]
85mod tests;
86
87impl<I: Deref<Target = dyn InvertedIndex> + Send + Sync> InvertedIndex for ObservedTextIndex<I> {
88 fn analyzer(&self) -> &Analyzer {
89 self.index.analyzer()
90 }
91 fn default_analyzer_binding(
92 &self,
93 ) -> StorageBackendResult<uqa_storage::inverted_index::AnalyzerDefault> {
94 self.index.default_analyzer_binding()
95 }
96 fn source_rebuild_required(&self) -> StorageBackendResult<bool> {
97 self.index.source_rebuild_required()
98 }
99 fn get_field_analyzer(&self, field: &str) -> Analyzer {
100 self.index.get_field_analyzer(field)
101 }
102 fn get_search_analyzer(&self, field: &str) -> Analyzer {
103 self.index.get_search_analyzer(field)
104 }
105 fn index_analyzer_revision(&self, field: &str) -> StorageBackendResult<Arc<CompiledAnalyzer>> {
106 self.index.index_analyzer_revision(field)
107 }
108 fn search_analyzer_revision(&self, field: &str) -> StorageBackendResult<Arc<CompiledAnalyzer>> {
109 self.index.search_analyzer_revision(field)
110 }
111 fn add_document(
112 &mut self,
113 _: DocId,
114 _: BTreeMap<FieldName, String>,
115 ) -> StorageBackendResult<()> {
116 Err(read_only())
117 }
118 fn try_add_documents(
119 &mut self,
120 _: Vec<(DocId, BTreeMap<FieldName, String>)>,
121 ) -> StorageBackendResult<()> {
122 Err(read_only())
123 }
124 fn remove_document(&mut self, _: DocId) -> StorageBackendResult<()> {
125 Err(read_only())
126 }
127 fn clear(&mut self) -> StorageBackendResult<()> {
128 Err(read_only())
129 }
130 fn rebuild_persisted_block_max(
131 &mut self,
132 _: &str,
133 _: &dyn BlockMaxScorer,
134 _: &str,
135 ) -> StorageBackendResult<bool> {
136 Err(read_only())
137 }
138
139 fn snapshot(&self) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
140 Ok(Arc::new(ObservedTextIndex {
141 index: self.index.snapshot()?,
142 observation: self.observation.clone(),
143 }))
144 }
145
146 fn snapshot_with_control(
147 &self,
148 control: &StorageReadControl,
149 ) -> StorageBackendResult<Arc<dyn InvertedIndex>> {
150 Ok(Arc::new(ObservedTextIndex {
151 index: self.index.snapshot_with_control(control)?,
152 observation: self.observation.clone(),
153 }))
154 }
155
156 fn get_posting_list(&self, field: &str, term: &str) -> StorageBackendResult<PostingList> {
157 self.observe(|o| o.scalar_term(field, term, None))?;
158 self.index.get_posting_list(field, term)
159 }
160 fn get_posting_list_key(
161 &self,
162 field: &str,
163 term: &TokenTermKey,
164 ) -> StorageBackendResult<PostingList> {
165 self.observe(|o| o.term(field, term, None))?;
166 self.index.get_posting_list_key(field, term)
167 }
168 fn get_posting_lists_bulk(
169 &self,
170 field: &str,
171 terms: &[String],
172 ) -> StorageBackendResult<Vec<PostingList>> {
173 self.observe(|o| {
174 terms
175 .iter()
176 .try_for_each(|term| o.scalar_term(field, term, None))
177 })?;
178 self.index.get_posting_lists_bulk(field, terms)
179 }
180 fn get_posting_lists_keys_bulk(
181 &self,
182 field: &str,
183 terms: &[TokenTermKey],
184 ) -> StorageBackendResult<Vec<PostingList>> {
185 self.observe(|o| terms.iter().try_for_each(|term| o.term(field, term, None)))?;
186 self.index.get_posting_lists_keys_bulk(field, terms)
187 }
188 fn posting_cursor(
189 &self,
190 field: &str,
191 term: &str,
192 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
193 self.observe(|o| o.scalar_term(field, term, None))?;
194 self.index.posting_cursor(field, term)
195 }
196 fn posting_cursor_key(
197 &self,
198 field: &str,
199 term: &TokenTermKey,
200 ) -> StorageBackendResult<Box<dyn PostingCursor>> {
201 self.observe(|o| o.term(field, term, None))?;
202 self.index.posting_cursor_key(field, term)
203 }
204 fn posting_cursors_bulk(
205 &self,
206 field: &str,
207 terms: &[String],
208 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
209 self.observe(|o| {
210 terms
211 .iter()
212 .try_for_each(|term| o.scalar_term(field, term, None))
213 })?;
214 self.index.posting_cursors_bulk(field, terms)
215 }
216 fn posting_cursors_keys_bulk(
217 &self,
218 field: &str,
219 terms: &[TokenTermKey],
220 ) -> StorageBackendResult<Vec<Box<dyn PostingCursor>>> {
221 self.observe(|o| terms.iter().try_for_each(|term| o.term(field, term, None)))?;
222 self.index.posting_cursors_keys_bulk(field, terms)
223 }
224 fn posting_read_cursor_key<'a>(
225 &'a self,
226 field: &'a str,
227 term: &TokenTermKey,
228 ) -> StorageBackendResult<Box<dyn PostingReadCursor + 'a>> {
229 self.observe(|o| o.term(field, term, None))?;
230 self.index.posting_read_cursor_key(field, term)
231 }
232 fn posting_read_cursor_key_budgeted<'a>(
233 &'a self,
234 field: &'a str,
235 term: &'a TokenTermKey,
236 control: &StorageReadControl,
237 ) -> StorageBackendResult<BudgetedPostingReadCursor<'a>> {
238 self.observe(|o| o.term(field, term, None))?;
239 self.index
240 .posting_read_cursor_key_budgeted(field, term, control)
241 }
242 fn visit_score_clusters(
243 &self,
244 field: &str,
245 term: &TokenTermKey,
246 after: Option<u64>,
247 limit: usize,
248 control: &StorageReadControl,
249 visit: &mut ScoreClusterVisitor<'_>,
250 ) -> StorageBackendResult<()> {
251 if limit != 0 {
252 self.observe(|o| o.term(field, term, None))?;
253 }
254 self.index
255 .visit_score_clusters(field, term, after, limit, control, visit)
256 }
257 fn get_occurrence_postings(
258 &self,
259 field: &str,
260 term: &TokenTermKey,
261 ) -> StorageBackendResult<Vec<OccurrencePosting>> {
262 self.observe(|o| o.term(field, term, None))?;
263 self.index.get_occurrence_postings(field, term)
264 }
265 fn get_occurrences(
266 &self,
267 doc: DocId,
268 field: &str,
269 term: &TokenTermKey,
270 ) -> StorageBackendResult<Vec<TokenOccurrence>> {
271 self.observe(|o| o.term(field, term, Some(doc)))?;
272 self.index.get_occurrences(doc, field, term)
273 }
274 fn get_occurrences_budgeted(
275 &self,
276 doc: DocId,
277 field: &str,
278 term: &TokenTermKey,
279 control: &StorageReadControl,
280 ) -> StorageBackendResult<Budgeted<Vec<TokenOccurrence>>> {
281 self.observe(|o| o.term(field, term, Some(doc)))?;
282 self.index
283 .get_occurrences_budgeted(doc, field, term, control)
284 }
285 fn indexed_field_metadata(
286 &self,
287 doc: DocId,
288 field: &str,
289 ) -> StorageBackendResult<Option<IndexedFieldMetadata>> {
290 self.observe(|o| o.document(field, doc))?;
291 self.index.indexed_field_metadata(doc, field)
292 }
293 fn doc_freq(&self, field: &str, term: &str) -> StorageBackendResult<u64> {
294 self.observe(|o| o.scalar_term(field, term, None))?;
295 self.index.doc_freq(field, term)
296 }
297 fn doc_freq_key(&self, field: &str, term: &TokenTermKey) -> StorageBackendResult<u64> {
298 self.observe(|o| o.term(field, term, None))?;
299 self.index.doc_freq_key(field, term)
300 }
301 fn get_term_freq(&self, doc: DocId, field: &str, term: &str) -> StorageBackendResult<u64> {
302 self.observe(|o| o.scalar_term(field, term, Some(doc)))?;
303 self.index.get_term_freq(doc, field, term)
304 }
305 fn get_term_freq_key(
306 &self,
307 doc: DocId,
308 field: &str,
309 term: &TokenTermKey,
310 ) -> StorageBackendResult<u64> {
311 self.observe(|o| o.term(field, term, Some(doc)))?;
312 self.index.get_term_freq_key(doc, field, term)
313 }
314 fn get_doc_length(&self, doc: DocId, field: &str) -> StorageBackendResult<u64> {
315 self.observe(|o| o.document(field, doc))?;
316 self.index.get_doc_length(doc, field)
317 }
318 fn doc_count(&self) -> StorageBackendResult<u64> {
319 self.observe(|o| o.field_range(DOCUMENT, None))?;
320 self.index.doc_count()
321 }
322 fn total_field_length(&self, field: &str) -> StorageBackendResult<u64> {
323 self.observe(|o| o.statistics(field))?;
324 self.index.total_field_length(field)
325 }
326 fn field_doc_count(&self, field: &str) -> StorageBackendResult<u64> {
327 self.observe(|o| o.statistics(field))?;
328 self.index.field_doc_count(field)
329 }
330 fn field_stats_scalar(&self, field: &str) -> StorageBackendResult<IndexStats> {
331 self.observe(|o| o.statistics(field))?;
332 self.index.field_stats_scalar(field)
333 }
334 fn field_stats_scalar_budgeted(
335 &self,
336 field: &str,
337 control: &StorageReadControl,
338 ) -> StorageBackendResult<IndexStats> {
339 self.observe(|o| o.statistics(field))?;
340 self.index.field_stats_scalar_budgeted(field, control)
341 }
342 fn field_stats(&self, field: &str) -> StorageBackendResult<IndexStats> {
343 self.observe(TextObservation::all)?;
344 self.index.field_stats(field)
345 }
346 fn stats(&self) -> StorageBackendResult<IndexStats> {
347 self.observe(TextObservation::all)?;
348 self.index.stats()
349 }
350 fn vocabulary_terms(&self, field: &str) -> StorageBackendResult<Vec<String>> {
351 self.observe(|o| o.field_range(POSTING, Some(field)))?;
352 self.index.vocabulary_terms(field)
353 }
354 fn vocabulary_keys(&self, field: &str) -> StorageBackendResult<Vec<TokenTermKey>> {
355 self.observe(|o| o.field_range(POSTING, Some(field)))?;
356 self.index.vocabulary_keys(field)
357 }
358 fn posting_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
359 self.observe(|o| o.field_range(POSTING, field))?;
360 self.index.posting_count(field)
361 }
362 fn doc_length_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
363 self.observe(|o| o.field_range(DOCUMENT, field))?;
364 self.index.doc_length_count(field)
365 }
366 fn term_count(&self, field: Option<&str>) -> StorageBackendResult<u64> {
367 self.observe(|o| o.field_range(POSTING, field))?;
368 self.index.term_count(field)
369 }
370 fn field_names(&self) -> StorageBackendResult<Vec<FieldName>> {
371 self.observe(|o| o.field_range(STATISTICS, None))?;
372 self.index.field_names()
373 }
374 fn get_posting_list_any_field(&self, term: &str) -> StorageBackendResult<PostingList> {
375 self.observe_fields(|o, field| o.scalar_term(field, term, None))?;
376 self.index.get_posting_list_any_field(term)
377 }
378 fn doc_freq_any_field(&self, term: &str) -> StorageBackendResult<u64> {
379 self.observe_fields(|o, field| o.scalar_term(field, term, None))?;
380 self.index.doc_freq_any_field(term)
381 }
382 fn get_total_doc_length(&self, doc: DocId) -> StorageBackendResult<u64> {
383 self.observe_fields(|o, field| o.document(field, doc))?;
384 self.index.get_total_doc_length(doc)
385 }
386 fn get_total_term_freq(&self, doc: DocId, term: &str) -> StorageBackendResult<u64> {
387 self.observe_fields(|o, field| o.scalar_term(field, term, Some(doc)))?;
388 self.index.get_total_term_freq(doc, term)
389 }
390 fn for_each_posting(
391 &self,
392 field: &str,
393 term: &str,
394 visit: &mut dyn FnMut(&PostingEntry),
395 ) -> StorageBackendResult<()> {
396 self.observe(|o| o.scalar_term(field, term, None))?;
397 self.index.for_each_posting(field, term, visit)
398 }
399 fn for_each_term_freq(
400 &self,
401 field: &str,
402 term: &str,
403 visit: &mut dyn FnMut(DocId, u64),
404 ) -> StorageBackendResult<()> {
405 self.observe(|o| o.scalar_term(field, term, None))?;
406 self.index.for_each_term_freq(field, term, visit)
407 }
408 fn persisted_block_max_scores(
409 &self,
410 field: &str,
411 term: &str,
412 fingerprint: &str,
413 ) -> StorageBackendResult<Option<Vec<f64>>> {
414 self.observe(|o| o.scalar_term(field, term, None))?;
415 self.index
416 .persisted_block_max_scores(field, term, fingerprint)
417 }
418 fn persisted_block_max_scores_bulk(
419 &self,
420 field: &str,
421 terms: &[String],
422 fingerprint: &str,
423 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
424 self.observe(|o| {
425 terms
426 .iter()
427 .try_for_each(|term| o.scalar_term(field, term, None))
428 })?;
429 self.index
430 .persisted_block_max_scores_bulk(field, terms, fingerprint)
431 }
432 fn persisted_block_max_scores_keys_bulk(
433 &self,
434 field: &str,
435 terms: &[TokenTermKey],
436 fingerprint: &str,
437 ) -> StorageBackendResult<Vec<Option<Vec<f64>>>> {
438 self.observe(|o| terms.iter().try_for_each(|term| o.term(field, term, None)))?;
439 self.index
440 .persisted_block_max_scores_keys_bulk(field, terms, fingerprint)
441 }
442 fn get_doc_lengths_bulk(
443 &self,
444 docs: &[DocId],
445 field: &str,
446 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
447 self.observe(|o| docs.iter().try_for_each(|doc| o.document(field, *doc)))?;
448 self.index.get_doc_lengths_bulk(docs, field)
449 }
450 fn get_term_freqs_bulk(
451 &self,
452 docs: &[DocId],
453 field: &str,
454 term: &str,
455 ) -> StorageBackendResult<BTreeMap<DocId, u64>> {
456 self.observe(|o| {
457 docs.iter()
458 .try_for_each(|doc| o.scalar_term(field, term, Some(*doc)))
459 })?;
460 self.index.get_term_freqs_bulk(docs, field, term)
461 }
462 fn get_scoring_inputs_bulk(
463 &self,
464 docs: &[DocId],
465 field: &str,
466 terms: &[String],
467 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
468 self.observe(|o| {
469 docs.iter().try_for_each(|doc| {
470 o.document(field, *doc)?;
471 terms
472 .iter()
473 .try_for_each(|term| o.scalar_term(field, term, Some(*doc)))
474 })
475 })?;
476 self.index.get_scoring_inputs_bulk(docs, field, terms)
477 }
478 fn get_scoring_inputs_keys_bulk(
479 &self,
480 docs: &[DocId],
481 field: &str,
482 terms: &[TokenTermKey],
483 ) -> StorageBackendResult<Vec<(u64, Vec<u64>)>> {
484 self.observe(|o| {
485 docs.iter().try_for_each(|doc| {
486 o.document(field, *doc)?;
487 terms
488 .iter()
489 .try_for_each(|term| o.term(field, term, Some(*doc)))
490 })
491 })?;
492 self.index.get_scoring_inputs_keys_bulk(docs, field, terms)
493 }
494}