1use uqa_storage::InvertedIndex;
10
11use super::{
12 analyze_query_terms, storage_sql_error, BM25Params, BTreeMap, BayesianBM25Params, Engine,
13 SQLError, TokenTermKey, UnsupervisedBm25ScoreEstimator,
14};
15
16const ESTIMATE_STALENESS_FACTOR: f64 = 2.0;
19
20fn resolve_saved_params(params: &BTreeMap<String, f64>) -> BayesianBM25Params {
21 let mut resolved = BayesianBM25Params::default();
22 if let Some(alpha) = params
23 .get("alpha")
24 .copied()
25 .filter(|value| value.is_finite() && *value > 0.0)
26 {
27 resolved.alpha = alpha;
28 }
29 if let Some(beta) = params
30 .get("beta")
31 .copied()
32 .filter(|value| value.is_finite())
33 {
34 resolved.beta = beta;
35 }
36 if let Some(base_rate) = params
37 .get("base_rate")
38 .copied()
39 .filter(|value| value.is_finite() && (0.0..1.0).contains(value))
40 {
41 resolved.base_rate = base_rate;
42 }
43 if let Some(calibration_tokens) = params
44 .get("calibration_tokens")
45 .copied()
46 .filter(|value| value.is_finite() && *value > 0.0)
47 {
48 resolved.calibration_tokens = calibration_tokens;
49 if let Some(beta_slope) = params
50 .get("beta_slope")
51 .copied()
52 .filter(|value| value.is_finite())
53 {
54 resolved.beta_slope = beta_slope;
55 }
56 if let Some(sigma_slope) = params
57 .get("sigma_slope")
58 .copied()
59 .filter(|value| value.is_finite())
60 {
61 resolved.sigma_slope = sigma_slope;
62 }
63 }
64 resolved
65}
66
67impl Engine {
68 pub(crate) fn clear_bayesian_params_cache(&self) {
69 self.runtime.bayesian_params_cache.write().clear();
70 }
71
72 pub fn bayesian_params_for(
76 &self,
77 table: &str,
78 field: &str,
79 ) -> Result<BayesianBM25Params, SQLError> {
80 self.with_direct_table_query(table, false, |engine, name, _| {
81 engine.bayesian_params_for_signal(name, table, field)
82 })
83 }
84
85 pub(super) fn bayesian_params_for_signal(
86 &self,
87 table: &str,
88 signal_table: &str,
89 field: &str,
90 ) -> Result<BayesianBM25Params, SQLError> {
91 self.validate_text_search_field(table, field)?;
92 if let Some(params) = self.load_fresh_bayesian_params(table, signal_table, field)? {
93 return Ok(params);
94 }
95 if self.current_transaction_is_read_only() {
96 return self.resolve_missing_bayesian_params_in_transaction(table, signal_table, field);
97 }
98
99 self.with_implicit_transaction(|engine| {
105 engine.resolve_missing_bayesian_params_in_transaction(table, signal_table, field)
106 })
107 }
108
109 pub(crate) fn bayesian_params_for_in_execution(
117 &self,
118 table: &str,
119 field: &str,
120 ) -> Result<BayesianBM25Params, SQLError> {
121 self.bayesian_params_for_relation_in_execution(table, table, field)
122 }
123
124 pub(crate) fn bayesian_params_for_relation_in_execution(
126 &self,
127 table: &str,
128 signal_table: &str,
129 field: &str,
130 ) -> Result<BayesianBM25Params, SQLError> {
131 self.validate_text_search_field(table, field)?;
132 let key = format!("{signal_table}.{field}");
133 if let Some(params) = self.runtime.bayesian_params_cache.read().get(&key).copied() {
134 return Ok(params);
135 }
136 if let Some(params) = self.load_fresh_bayesian_params(table, signal_table, field)? {
137 self.runtime
138 .bayesian_params_cache
139 .write()
140 .insert(key, params);
141 return Ok(params);
142 }
143 if self.transaction_depth() == 0 {
144 return Err(SQLError::Internal(
145 "Bayesian calibration execution requires an active statement transaction".into(),
146 ));
147 }
148 let params =
149 self.resolve_missing_bayesian_params_in_transaction(table, signal_table, field)?;
150 if !self.current_transaction_is_read_only() {
151 self.runtime
152 .bayesian_params_cache
153 .write()
154 .insert(key, params);
155 }
156 Ok(params)
157 }
158
159 pub(super) fn resolve_missing_bayesian_params_in_transaction(
160 &self,
161 table: &str,
162 signal_table: &str,
163 field: &str,
164 ) -> Result<BayesianBM25Params, SQLError> {
165 self.validate_text_search_field(table, field)?;
166 if !self.current_transaction_is_read_only() {
167 self.lock_scoring_parameter_write(&format!("{signal_table}.{field}"))?;
168 }
169 if let Some(params) = self.load_fresh_bayesian_params(table, signal_table, field)? {
170 return Ok(params);
171 }
172 Ok(self
173 .auto_estimate_params(table, signal_table, field)?
174 .unwrap_or_default())
175 }
176
177 pub(super) fn load_fresh_bayesian_params(
178 &self,
179 table: &str,
180 signal_table: &str,
181 field: &str,
182 ) -> Result<Option<BayesianBM25Params>, SQLError> {
183 let key = format!("{signal_table}.{field}");
184 let saved = match self.try_load_scoring_params(&key)? {
185 Some(json) => Some(
186 serde_json::from_str::<BTreeMap<String, f64>>(&json).map_err(|error| {
187 SQLError::Internal(format!(
188 "decode persisted scoring parameters `{key}`: {error}"
189 ))
190 })?,
191 ),
192 None => None,
193 };
194 if let Some(params) = saved {
195 let stamp = params.get("estimated_doc_count").copied();
196 if !self.estimated_params_are_stale(table, stamp)? {
197 return Ok(Some(resolve_saved_params(¶ms)));
198 }
199 }
200 Ok(None)
201 }
202
203 pub(super) fn estimated_params_are_stale(
207 &self,
208 table: &str,
209 stamp: Option<f64>,
210 ) -> Result<bool, SQLError> {
211 let Some(stamped_doc_count) = stamp.filter(|value| value.is_finite() && *value > 0.0)
212 else {
213 return Ok(false);
214 };
215 let Some(table_state) = self
216 .try_query_table(table)
217 .map_err(|error| storage_sql_error("resolve search table", error))?
218 else {
219 return Err(SQLError::UnknownTable(table.to_string()));
220 };
221 let index = table_state.inverted_index.read();
222 let index = uqa_execution::serializable::text::ObservedTextIndex::new(
223 index.as_ref(),
224 self.serializable_table_read(table)?,
225 table_state.columns.snapshot(),
226 );
227 let current = index
228 .doc_count()
229 .map_err(|error| storage_sql_error("read indexed document count", error))?
230 as f64;
231 Ok(current >= stamped_doc_count * ESTIMATE_STALENESS_FACTOR
232 || current <= stamped_doc_count / ESTIMATE_STALENESS_FACTOR)
233 }
234
235 pub(super) fn sample_calibration_queries(
243 &self,
244 table: &str,
245 field: &str,
246 estimator: &UnsupervisedBm25ScoreEstimator,
247 ) -> Result<Vec<Vec<TokenTermKey>>, SQLError> {
248 let Some(table_state) = self
249 .try_query_table(table)
250 .map_err(|error| storage_sql_error("resolve calibration table", error))?
251 else {
252 return Err(SQLError::UnknownTable(table.to_string()));
253 };
254 let analyzer = table_state
255 .inverted_index
256 .read()
257 .search_analyzer_revision(field)
258 .map_err(|error| storage_sql_error("resolve calibration analyzer revision", error))?;
259 let store = table_state.document_store.read();
260 if let Some(read) = self.serializable_table_read(table)? {
261 read.observe_scan()?;
262 }
263 let doc_ids = store
264 .doc_ids()
265 .map_err(|error| storage_sql_error("read calibration document ids", error))?;
266 estimator.sample_document_queries(doc_ids, |doc_id| {
267 let Some(uqa_core::Value::Str(text)) = store
268 .get_field(doc_id, field)
269 .map_err(|error| storage_sql_error("read calibration document field", error))?
270 else {
271 return Ok(None);
272 };
273 analyze_query_terms(&analyzer, &text)
274 .map(Some)
275 .map_err(|error| storage_sql_error("analyze calibration document field", error))
276 })
277 }
278
279 pub(super) fn auto_estimate_params(
281 &self,
282 table: &str,
283 signal_table: &str,
284 field: &str,
285 ) -> Result<Option<BayesianBM25Params>, SQLError> {
286 let table_state = self.require_query_table(table)?;
287 let estimator = UnsupervisedBm25ScoreEstimator::default();
288 let queries = self.sample_calibration_queries(table, field, &estimator)?;
289 let (params, doc_count) = {
290 let index = table_state.inverted_index.read();
291 let index = uqa_execution::serializable::text::ObservedTextIndex::new(
292 index.as_ref(),
293 self.serializable_table_read(table)?,
294 table_state.columns.snapshot(),
295 );
296 if index
297 .doc_count()
298 .map_err(|error| storage_sql_error("read indexed document count", error))?
299 == 0
300 || index
301 .vocabulary_keys(field)
302 .map_err(|error| storage_sql_error("read indexed vocabulary", error))?
303 .is_empty()
304 {
305 return Ok(None);
306 }
307 let params = if queries.is_empty() {
308 estimator.estimate(&index, field, BM25Params::default())
309 } else {
310 estimator.estimate_with_query_keys(&index, field, BM25Params::default(), &queries)
311 }
312 .map_err(|error| {
313 storage_sql_error("estimate BM25 score-transform parameters", error)
314 })?;
315 let doc_count = index
316 .doc_count()
317 .map_err(|error| storage_sql_error("read indexed document count", error))?;
318 (params, doc_count)
319 };
320 if self.current_transaction_is_read_only() {
321 return Ok(Some(params));
322 }
323 let values = BTreeMap::from([
324 ("alpha".to_string(), params.alpha),
325 ("beta".to_string(), params.beta),
326 ("base_rate".to_string(), params.base_rate),
327 ("calibration_tokens".to_string(), params.calibration_tokens),
328 ("beta_slope".to_string(), params.beta_slope),
329 ("sigma_slope".to_string(), params.sigma_slope),
330 ("estimated_doc_count".to_string(), doc_count as f64),
331 ]);
332 let json = serde_json::to_string(&values)
333 .map_err(|error| SQLError::Internal(format!("serialize scoring params: {error}")))?;
334 self.save_scoring_params_inner(&format!("{signal_table}.{field}"), &json)?;
335 Ok(Some(params))
336 }
337}