Skip to main content

uqa_engine/search/
calibration.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Bayesian BM25 parameter loading, staleness, sampling, and estimation.
8
9use uqa_storage::InvertedIndex;
10
11use super::{
12    analyze_query_terms, storage_sql_error, BM25Params, BTreeMap, BayesianBM25Params, Engine,
13    SQLError, TokenTermKey, UnsupervisedBm25ScoreEstimator,
14};
15
16/// Auto-estimated parameters are refreshed once the corpus doubles or
17/// halves relative to the document count stamped at estimation time.
18const ESTIMATE_STALENESS_FACTOR: f64 = 2.0;
19
20fn resolve_saved_params(params: &BTreeMap<String, f64>) -> BayesianBM25Params {
21    let mut resolved = BayesianBM25Params::default();
22    if let Some(alpha) = params
23        .get("alpha")
24        .copied()
25        .filter(|value| value.is_finite() && *value > 0.0)
26    {
27        resolved.alpha = alpha;
28    }
29    if let Some(beta) = params
30        .get("beta")
31        .copied()
32        .filter(|value| value.is_finite())
33    {
34        resolved.beta = beta;
35    }
36    if let Some(base_rate) = params
37        .get("base_rate")
38        .copied()
39        .filter(|value| value.is_finite() && (0.0..1.0).contains(value))
40    {
41        resolved.base_rate = base_rate;
42    }
43    if let Some(calibration_tokens) = params
44        .get("calibration_tokens")
45        .copied()
46        .filter(|value| value.is_finite() && *value > 0.0)
47    {
48        resolved.calibration_tokens = calibration_tokens;
49        if let Some(beta_slope) = params
50            .get("beta_slope")
51            .copied()
52            .filter(|value| value.is_finite())
53        {
54            resolved.beta_slope = beta_slope;
55        }
56        if let Some(sigma_slope) = params
57            .get("sigma_slope")
58            .copied()
59            .filter(|value| value.is_finite())
60        {
61            resolved.sigma_slope = sigma_slope;
62        }
63    }
64    resolved
65}
66
67impl Engine {
68    pub(crate) fn clear_bayesian_params_cache(&self) {
69        self.runtime.bayesian_params_cache.write().clear();
70    }
71
72    /// Resolve the Bayesian BM25 calibration for `table.field`.
73    ///
74    /// Saved parameters win. Absent or stale auto-estimated parameters trigger a corpus-driven estimation. Writable transactions persist it for subsequent queries; read-only transactions use it without publication. The raw-score identity calibration (`alpha = 1, beta = 0`) never silently ships a score for a populated field. Parameters written by the online learner carry no `estimated_doc_count` stamp and are never overwritten automatically.
75    pub fn bayesian_params_for(
76        &self,
77        table: &str,
78        field: &str,
79    ) -> Result<BayesianBM25Params, SQLError> {
80        self.with_direct_table_query(table, false, |engine, name, _| {
81            engine.bayesian_params_for_signal(name, table, field)
82        })
83    }
84
85    pub(super) fn bayesian_params_for_signal(
86        &self,
87        table: &str,
88        signal_table: &str,
89        field: &str,
90    ) -> Result<BayesianBM25Params, SQLError> {
91        self.validate_text_search_field(table, field)?;
92        if let Some(params) = self.load_fresh_bayesian_params(table, signal_table, field)? {
93            return Ok(params);
94        }
95        if self.current_transaction_is_read_only() {
96            return self.resolve_missing_bayesian_params_in_transaction(table, signal_table, field);
97        }
98
99        // Estimation is a read/modify/write operation: reserve the durable
100        // writer before taking the corpus and parameter snapshots, then
101        // recheck in case another session populated a fresh estimate while
102        // this caller waited for the writer. The common cached read above
103        // deliberately avoids the writer lock.
104        self.with_implicit_transaction(|engine| {
105            engine.resolve_missing_bayesian_params_in_transaction(table, signal_table, field)
106        })
107    }
108
109    /// Resolve calibration while executing an operator tree inside a SQL or
110    /// direct-retrieval transaction that already owns the statement gate.
111    ///
112    /// Rayon branches run on different threads from that gate owner, so they
113    /// must not re-enter the public implicit-transaction wrapper. Requiring an
114    /// existing transaction keeps this path internal: an unguarded direct API
115    /// caller cannot silently join or create a writer transaction.
116    pub(crate) fn bayesian_params_for_in_execution(
117        &self,
118        table: &str,
119        field: &str,
120    ) -> Result<BayesianBM25Params, SQLError> {
121        self.bayesian_params_for_relation_in_execution(table, table, field)
122    }
123
124    /// Resolve calibration for a canonical storage relation while preserving the signal namespace selected by the SQL relation reference.
125    pub(crate) fn bayesian_params_for_relation_in_execution(
126        &self,
127        table: &str,
128        signal_table: &str,
129        field: &str,
130    ) -> Result<BayesianBM25Params, SQLError> {
131        self.validate_text_search_field(table, field)?;
132        let key = format!("{signal_table}.{field}");
133        if let Some(params) = self.runtime.bayesian_params_cache.read().get(&key).copied() {
134            return Ok(params);
135        }
136        if let Some(params) = self.load_fresh_bayesian_params(table, signal_table, field)? {
137            self.runtime
138                .bayesian_params_cache
139                .write()
140                .insert(key, params);
141            return Ok(params);
142        }
143        if self.transaction_depth() == 0 {
144            return Err(SQLError::Internal(
145                "Bayesian calibration execution requires an active statement transaction".into(),
146            ));
147        }
148        let params =
149            self.resolve_missing_bayesian_params_in_transaction(table, signal_table, field)?;
150        if !self.current_transaction_is_read_only() {
151            self.runtime
152                .bayesian_params_cache
153                .write()
154                .insert(key, params);
155        }
156        Ok(params)
157    }
158
159    pub(super) fn resolve_missing_bayesian_params_in_transaction(
160        &self,
161        table: &str,
162        signal_table: &str,
163        field: &str,
164    ) -> Result<BayesianBM25Params, SQLError> {
165        self.validate_text_search_field(table, field)?;
166        if !self.current_transaction_is_read_only() {
167            self.lock_scoring_parameter_write(&format!("{signal_table}.{field}"))?;
168        }
169        if let Some(params) = self.load_fresh_bayesian_params(table, signal_table, field)? {
170            return Ok(params);
171        }
172        Ok(self
173            .auto_estimate_params(table, signal_table, field)?
174            .unwrap_or_default())
175    }
176
177    pub(super) fn load_fresh_bayesian_params(
178        &self,
179        table: &str,
180        signal_table: &str,
181        field: &str,
182    ) -> Result<Option<BayesianBM25Params>, SQLError> {
183        let key = format!("{signal_table}.{field}");
184        let saved = match self.try_load_scoring_params(&key)? {
185            Some(json) => Some(
186                serde_json::from_str::<BTreeMap<String, f64>>(&json).map_err(|error| {
187                    SQLError::Internal(format!(
188                        "decode persisted scoring parameters `{key}`: {error}"
189                    ))
190                })?,
191            ),
192            None => None,
193        };
194        if let Some(params) = saved {
195            let stamp = params.get("estimated_doc_count").copied();
196            if !self.estimated_params_are_stale(table, stamp)? {
197                return Ok(Some(resolve_saved_params(&params)));
198            }
199        }
200        Ok(None)
201    }
202
203    /// A stamped estimate goes stale when the corpus grows or shrinks
204    /// past [`ESTIMATE_STALENESS_FACTOR`]. Unstamped parameters (online
205    /// learner output, hand-written values) never go stale.
206    pub(super) fn estimated_params_are_stale(
207        &self,
208        table: &str,
209        stamp: Option<f64>,
210    ) -> Result<bool, SQLError> {
211        let Some(stamped_doc_count) = stamp.filter(|value| value.is_finite() && *value > 0.0)
212        else {
213            return Ok(false);
214        };
215        let Some(table_state) = self
216            .try_query_table(table)
217            .map_err(|error| storage_sql_error("resolve search table", error))?
218        else {
219            return Err(SQLError::UnknownTable(table.to_string()));
220        };
221        let index = table_state.inverted_index.read();
222        let index = uqa_execution::serializable::text::ObservedTextIndex::new(
223            index.as_ref(),
224            self.serializable_table_read(table)?,
225            table_state.columns.snapshot(),
226        );
227        let current = index
228            .doc_count()
229            .map_err(|error| storage_sql_error("read indexed document count", error))?
230            as f64;
231        Ok(current >= stamped_doc_count * ESTIMATE_STALENESS_FACTOR
232            || current <= stamped_doc_count / ESTIMATE_STALENESS_FACTOR)
233    }
234
235    /// Build coherent calibration pseudo-queries by sampling stored
236    /// documents: each query takes the leading distinct analyzed terms
237    /// of one document, so its terms co-occur the way real query terms
238    /// do in relevant documents. Vocabulary-random sampling cannot
239    /// model that co-occurrence, which flattens the fitted length
240    /// slopes on sparse-vocabulary corpora. Documents are
241    /// stride-sampled, so the result is deterministic per corpus.
242    pub(super) fn sample_calibration_queries(
243        &self,
244        table: &str,
245        field: &str,
246        estimator: &UnsupervisedBm25ScoreEstimator,
247    ) -> Result<Vec<Vec<TokenTermKey>>, SQLError> {
248        let Some(table_state) = self
249            .try_query_table(table)
250            .map_err(|error| storage_sql_error("resolve calibration table", error))?
251        else {
252            return Err(SQLError::UnknownTable(table.to_string()));
253        };
254        let analyzer = table_state
255            .inverted_index
256            .read()
257            .search_analyzer_revision(field)
258            .map_err(|error| storage_sql_error("resolve calibration analyzer revision", error))?;
259        let store = table_state.document_store.read();
260        if let Some(read) = self.serializable_table_read(table)? {
261            read.observe_scan()?;
262        }
263        let doc_ids = store
264            .doc_ids()
265            .map_err(|error| storage_sql_error("read calibration document ids", error))?;
266        estimator.sample_document_queries(doc_ids, |doc_id| {
267            let Some(uqa_core::Value::Str(text)) = store
268                .get_field(doc_id, field)
269                .map_err(|error| storage_sql_error("read calibration document field", error))?
270            else {
271                return Ok(None);
272            };
273            analyze_query_terms(&analyzer, &text)
274                .map(Some)
275                .map_err(|error| storage_sql_error("analyze calibration document field", error))
276        })
277    }
278
279    /// Estimate unsupervised score-transform parameters from the field's indexed vocabulary and, in writable transactions, persist them with a document-count stamp. Returns `None` (without persisting) when the field has nothing to sample, so an empty table estimates on first real use.
280    pub(super) fn auto_estimate_params(
281        &self,
282        table: &str,
283        signal_table: &str,
284        field: &str,
285    ) -> Result<Option<BayesianBM25Params>, SQLError> {
286        let table_state = self.require_query_table(table)?;
287        let estimator = UnsupervisedBm25ScoreEstimator::default();
288        let queries = self.sample_calibration_queries(table, field, &estimator)?;
289        let (params, doc_count) = {
290            let index = table_state.inverted_index.read();
291            let index = uqa_execution::serializable::text::ObservedTextIndex::new(
292                index.as_ref(),
293                self.serializable_table_read(table)?,
294                table_state.columns.snapshot(),
295            );
296            if index
297                .doc_count()
298                .map_err(|error| storage_sql_error("read indexed document count", error))?
299                == 0
300                || index
301                    .vocabulary_keys(field)
302                    .map_err(|error| storage_sql_error("read indexed vocabulary", error))?
303                    .is_empty()
304            {
305                return Ok(None);
306            }
307            let params = if queries.is_empty() {
308                estimator.estimate(&index, field, BM25Params::default())
309            } else {
310                estimator.estimate_with_query_keys(&index, field, BM25Params::default(), &queries)
311            }
312            .map_err(|error| {
313                storage_sql_error("estimate BM25 score-transform parameters", error)
314            })?;
315            let doc_count = index
316                .doc_count()
317                .map_err(|error| storage_sql_error("read indexed document count", error))?;
318            (params, doc_count)
319        };
320        if self.current_transaction_is_read_only() {
321            return Ok(Some(params));
322        }
323        let values = BTreeMap::from([
324            ("alpha".to_string(), params.alpha),
325            ("beta".to_string(), params.beta),
326            ("base_rate".to_string(), params.base_rate),
327            ("calibration_tokens".to_string(), params.calibration_tokens),
328            ("beta_slope".to_string(), params.beta_slope),
329            ("sigma_slope".to_string(), params.sigma_slope),
330            ("estimated_doc_count".to_string(), doc_count as f64),
331        ]);
332        let json = serde_json::to_string(&values)
333            .map_err(|error| SQLError::Internal(format!("serialize scoring params: {error}")))?;
334        self.save_scoring_params_inner(&format!("{signal_table}.{field}"), &json)?;
335        Ok(Some(params))
336    }
337}