1use uqa_storage::InvertedIndex;
10
11use super::{
12 storage_sql_error, Arc, BayesianBM25Scorer, CalibrationMetrics, CalibrationReport, DocId,
13 Engine, Instant, OperatorTree, RawBm25Score, SQLError, ScoredEntry, ScoringMode,
14 TextScoringMode, TextSearchProfile,
15};
16
17impl Engine {
18 pub fn search(
20 &self,
21 table: &str,
22 field: &str,
23 query: &str,
24 mode: &ScoringMode,
25 top_k: usize,
26 ) -> Result<Vec<ScoredEntry>, SQLError> {
27 self.with_direct_table_read(table, |engine, name, _| {
28 let scoring = match mode {
29 ScoringMode::BM25(params) => TextScoringMode::CustomBM25(*params),
30 ScoringMode::BayesianBM25(params) => TextScoringMode::CustomBayesianBM25(*params),
31 };
32 let tree = engine.plan_text_top_k_tree(name, field, query, scoring, top_k)?;
33 let entries =
34 crate::operator_tree_bridge::execute_scored_tree(engine, name, table, &[], &tree)?;
35 Ok(uqa_scoring::rank_scored_entries_top_k(entries, top_k))
36 })
37 }
38
39 pub fn search_profiled(
43 &self,
44 table: &str,
45 field: &str,
46 query: &str,
47 mode: &ScoringMode,
48 top_k: usize,
49 ) -> Result<TextSearchProfile, SQLError> {
50 let started = Instant::now();
51 let mut profile = self.with_direct_table_read(table, |engine, name, _| {
52 let scoring = match mode {
53 ScoringMode::BM25(params) => TextScoringMode::CustomBM25(*params),
54 ScoringMode::BayesianBM25(params) => TextScoringMode::CustomBayesianBM25(*params),
55 };
56 let tree = engine.plan_text_top_k_tree(name, field, query, scoring, top_k)?;
57 let physical_top_k = match tree {
58 OperatorTree::Term { top_k, .. } => top_k,
59 _ => None,
60 };
61 engine.search_leaf_profiled(name, field, query, mode, top_k, physical_top_k)
62 })?;
63 profile.elapsed_ms = started.elapsed().as_secs_f64() * 1000.0;
64 Ok(profile)
65 }
66
67 pub fn calibration_report(
71 &self,
72 table: &str,
73 field: &str,
74 query: &str,
75 labels: &[u8],
76 ) -> Result<CalibrationReport, SQLError> {
77 self.with_direct_table_query(table, false, |engine, name, _| {
78 engine.calibration_report_in_transaction(name, table, field, query, labels)
79 })
80 }
81
82 fn calibration_report_in_transaction(
83 &self,
84 table: &str,
85 signal_table: &str,
86 field: &str,
87 query: &str,
88 labels: &[u8],
89 ) -> Result<CalibrationReport, SQLError> {
90 let table_state = self.require_query_table(table)?;
91 let doc_ids = self.query_table_doc_ids(table)?;
92 if labels.len() != doc_ids.len() {
93 return Err(SQLError::TypeMismatch(format!(
94 "labels length ({}) must match document count ({})",
95 labels.len(),
96 doc_ids.len()
97 )));
98 }
99 if labels.iter().any(|label| *label > 1) {
100 return Err(SQLError::TypeMismatch(
101 "labels must contain only 0 or 1".into(),
102 ));
103 }
104
105 let params = self.bayesian_params_for_signal(table, signal_table, field)?;
106 let (query_term_count, stats) = {
107 let index = table_state.inverted_index.read();
108 let index = uqa_execution::serializable::text::ObservedTextIndex::new(
109 index.as_ref(),
110 self.serializable_table_read(table)?,
111 table_state.columns.snapshot(),
112 );
113 let query_term_count = index
114 .search_analyzer_revision(field)
115 .map_err(|error| storage_sql_error("resolve calibration analyzer revision", error))?
116 .analyze_tokens(query)
117 .map_err(|error| storage_sql_error("analyze calibration query", error))?
118 .tokens()
119 .len();
120 let stats = Arc::new(
121 index
122 .field_stats(field)
123 .map_err(|error| storage_sql_error("read calibration field stats", error))?,
124 );
125 (query_term_count, stats)
126 };
127 let scaled_params = params.scaled_for_query_terms(query_term_count);
128 let non_match_probability = BayesianBM25Scorer::new(scaled_params, stats)
129 .map_err(|error| {
130 SQLError::Internal(format!("build calibration-report scorer: {error}"))
131 })?
132 .calibrate_raw_score(
133 RawBm25Score::new(0.0).expect("the raw BM25 score for a non-match is finite"),
134 )
135 .value();
136
137 let mode = ScoringMode::BayesianBM25(params);
138 let score_map: std::collections::BTreeMap<DocId, f64> = self
139 .search(table, field, query, &mode, usize::MAX)?
140 .into_iter()
141 .map(|entry| (entry.doc_id, entry.score))
142 .collect();
143 let probabilities: Vec<f64> = doc_ids
144 .iter()
145 .map(|doc_id| {
146 score_map
147 .get(doc_id)
148 .copied()
149 .unwrap_or(non_match_probability)
150 })
151 .collect();
152 CalibrationMetrics::report(&probabilities, labels, 10)
153 .map_err(|error| SQLError::Internal(format!("compute calibration report: {error}")))
154 }
155}