Skip to main content

uqa_engine/
models.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7use super::{DeepLearnOutput, DeepModel, DocId, Engine, LearnOptions, SQLError, TrainingSet};
8
9const VECTOR_CALIBRATION_MODEL_PREFIX: &str = "vector_calibration_model::";
10
11fn vector_calibration_model_key(name: &str) -> Result<String, SQLError> {
12    if name.trim().is_empty() {
13        return Err(SQLError::TypeMismatch(
14            "vector calibration model name must not be empty".into(),
15        ));
16    }
17    Ok(format!("{VECTOR_CALIBRATION_MODEL_PREFIX}{name}"))
18}
19
20impl Engine {
21    pub fn save_model(&self, name: &str, model: &DeepModel) -> Result<(), SQLError> {
22        self.with_implicit_transaction(|engine| engine.save_model_inner(name, model))
23    }
24
25    fn save_model_inner(&self, name: &str, model: &DeepModel) -> Result<(), SQLError> {
26        let json = serde_json::to_string(model)
27            .map_err(|e| SQLError::Internal(format!("model serialise: {e}")))?;
28        let mut models = self.durable.models.write();
29        if let Some(catalog) = self.storage.catalog.as_ref() {
30            catalog
31                .save_model(name, &json)
32                .map_err(|e| SQLError::Internal(format!("catalog save_model: {e}")))?;
33        }
34        models.insert(name.to_string(), model.clone());
35        drop(models);
36        self.note_catalog_registry_changed();
37        Ok(())
38    }
39
40    pub fn load_model(&self, name: &str) -> Result<Option<DeepModel>, SQLError> {
41        self.with_direct_read_snapshot(|engine| engine.load_model_in_execution(name))
42    }
43
44    pub(crate) fn load_model_in_execution(
45        &self,
46        name: &str,
47    ) -> Result<Option<DeepModel>, SQLError> {
48        let Some(catalog) = self.storage.catalog.as_ref() else {
49            return Ok(self.durable.models.read().get(name).cloned());
50        };
51        let json = catalog
52            .load_model(name)
53            .map_err(|err| SQLError::Internal(format!("catalog load_model: {err}")))?;
54        let model = json
55            .as_deref()
56            .map(serde_json::from_str::<DeepModel>)
57            .transpose()
58            .map_err(|err| SQLError::Internal(format!("catalog model decode: {err}")))?;
59        let mut cache = self.durable.models.write();
60        match model.as_ref() {
61            Some(model) => {
62                cache.insert(name.to_string(), model.clone());
63            }
64            None => {
65                cache.remove(name);
66            }
67        }
68        Ok(model)
69    }
70
71    pub fn drop_model(&self, name: &str) -> Result<bool, SQLError> {
72        self.with_implicit_transaction(|engine| engine.drop_model_inner(name))
73    }
74
75    fn drop_model_inner(&self, name: &str) -> Result<bool, SQLError> {
76        if self.load_model_in_execution(name)?.is_none() {
77            return Ok(false);
78        }
79        let mut models = self.durable.models.write();
80        if let Some(catalog) = self.storage.catalog.as_ref() {
81            catalog
82                .drop_model(name)
83                .map_err(|err| SQLError::Internal(format!("catalog drop_model: {err}")))?;
84        }
85        models.remove(name);
86        drop(models);
87        self.note_catalog_registry_changed();
88        Ok(true)
89    }
90
91    /// compatibility alias for [`Engine::drop_model`].
92    pub fn delete_model(&self, name: &str) -> Result<bool, SQLError> {
93        self.drop_model(name)
94    }
95
96    /// Train an analytical deep model and persist it under `name`.
97    pub fn deep_learn(
98        &self,
99        name: &str,
100        training_set: &TrainingSet,
101        options: &LearnOptions,
102    ) -> Result<DeepLearnOutput, SQLError> {
103        self.model_training_context()
104            .train(name, training_set, options)
105    }
106
107    /// Parse a JSON [`TrainingSet`], train it, and persist the model.
108    pub fn deep_learn_json(
109        &self,
110        name: &str,
111        training_json: &str,
112        options: &LearnOptions,
113    ) -> Result<DeepLearnOutput, SQLError> {
114        self.model_training_context()
115            .train_json(name, training_json, options)
116    }
117
118    /// Train from a table containing `features` and `label` columns.
119    pub fn deep_learn_table(
120        &self,
121        name: &str,
122        table: &str,
123        options: &LearnOptions,
124    ) -> Result<DeepLearnOutput, SQLError> {
125        self.model_training_context()
126            .train_table(name, table, options)
127    }
128
129    /// Persist Bayesian calibration parameters for a named signal. The
130    /// Parameters arrive serialized as JSON so callers can persist arbitrary
131    /// `(alpha, beta, base_rate, ...)` shapes without a fixed struct.
132    pub fn save_scoring_params(&self, name: &str, params_json: &str) -> Result<(), SQLError> {
133        self.with_implicit_transaction(|engine| engine.save_scoring_params_inner(name, params_json))
134    }
135
136    pub(crate) fn save_scoring_params_inner(
137        &self,
138        name: &str,
139        params_json: &str,
140    ) -> Result<(), SQLError> {
141        // Physical retrieval workers publish through this adapter without reentering the public mutation scope.
142        if self.current_transaction_is_read_only() {
143            return Err(SQLError::Routine {
144                sqlstate: "25006".into(),
145                message: "cannot save scoring parameters in a read-only transaction".into(),
146            });
147        }
148        self.lock_scoring_parameter_write(name)?;
149        let mut scoring_params = self.durable.scoring_params.write();
150        if let Some(catalog) = self.storage.catalog.as_ref() {
151            catalog
152                .save_scoring_params(name, params_json)
153                .map_err(|e| SQLError::Internal(format!("catalog save_scoring_params: {e}")))?;
154        }
155        scoring_params.insert(name.to_string(), params_json.to_string());
156        drop(scoring_params);
157        self.note_catalog_registry_changed();
158        Ok(())
159    }
160
161    /// Load persisted scoring parameters for a single signal. Falls back to
162    /// the in-memory cache when the engine is not catalog-backed.
163    pub fn load_scoring_params(&self, name: &str) -> Result<Option<String>, SQLError> {
164        let Some(catalog) = self.storage.catalog.as_ref() else {
165            return Ok(self.durable.scoring_params.read().get(name).cloned());
166        };
167        let value = catalog
168            .load_scoring_params(name)
169            .map_err(|err| SQLError::Internal(format!("catalog load_scoring_params: {err}")))?;
170        let mut cache = self.durable.scoring_params.write();
171        match value.as_ref() {
172            Some(json) => {
173                cache.insert(name.to_string(), json.clone());
174            }
175            None => {
176                cache.remove(name);
177            }
178        }
179        Ok(value)
180    }
181
182    /// Fallible scoring-parameter read. Persistent sessions consult their
183    /// own catalog connection instead of trusting an engine-local snapshot;
184    /// this preserves transaction isolation and makes commits from sibling
185    /// sessions visible without reopening the engine.
186    pub fn try_load_scoring_params(&self, name: &str) -> Result<Option<String>, SQLError> {
187        self.load_scoring_params(name)
188    }
189
190    /// Snapshot every persisted `(name, params_json)` pair.
191    pub fn load_all_scoring_params(&self) -> Result<Vec<(String, String)>, SQLError> {
192        let mut out = if let Some(catalog) = self.storage.catalog.as_ref() {
193            let rows = catalog.load_all_scoring_params().map_err(|err| {
194                SQLError::Internal(format!("catalog load_all_scoring_params: {err}"))
195            })?;
196            let mut cache = self.durable.scoring_params.write();
197            cache.clear();
198            cache.extend(rows.iter().cloned());
199            rows
200        } else {
201            self.durable
202                .scoring_params
203                .read()
204                .iter()
205                .map(|(name, json)| (name.clone(), json.clone()))
206                .collect()
207        };
208        out.sort_by(|a, b| a.0.cmp(&b.0));
209        Ok(out)
210    }
211
212    /// Drop persisted scoring parameters for a single signal. Returns
213    /// `true` when something was removed.
214    pub fn drop_scoring_params(&self, name: &str) -> Result<bool, SQLError> {
215        self.with_implicit_transaction(|engine| engine.drop_scoring_params_inner(name))
216    }
217
218    /// Persist a typed vector-calibration model, including the corpus, index,
219    /// embedding-model, candidate-K, and version provenance that constrains
220    /// its safe reuse.
221    pub fn save_vector_calibration_model(
222        &self,
223        name: &str,
224        model: &uqa_scoring::VectorCalibrationModel,
225    ) -> Result<(), SQLError> {
226        let key = vector_calibration_model_key(name)?;
227        let json = model
228            .to_json()
229            .map_err(|error| SQLError::TypeMismatch(error.to_string()))?;
230        self.save_scoring_params(&key, &json)
231    }
232
233    /// Load and validate a persisted vector-calibration model. Unsupported
234    /// schema versions and invalid numeric parameters are errors rather than
235    /// silently falling back to query-pool calibration.
236    pub fn load_vector_calibration_model(
237        &self,
238        name: &str,
239    ) -> Result<Option<uqa_scoring::VectorCalibrationModel>, SQLError> {
240        let key = vector_calibration_model_key(name)?;
241        self.load_scoring_params(&key)?
242            .as_deref()
243            .map(uqa_scoring::VectorCalibrationModel::from_json)
244            .transpose()
245            .map_err(|error| SQLError::TypeMismatch(error.to_string()))
246    }
247
248    pub fn drop_vector_calibration_model(&self, name: &str) -> Result<bool, SQLError> {
249        let key = vector_calibration_model_key(name)?;
250        self.drop_scoring_params(&key)
251    }
252
253    fn drop_scoring_params_inner(&self, name: &str) -> Result<bool, SQLError> {
254        self.lock_scoring_parameter_write(name)?;
255        if self.load_scoring_params(name)?.is_none() {
256            return Ok(false);
257        }
258        let mut scoring_params = self.durable.scoring_params.write();
259        if let Some(catalog) = self.storage.catalog.as_ref() {
260            catalog
261                .drop_scoring_params(name)
262                .map_err(|err| SQLError::Internal(format!("catalog drop_scoring_params: {err}")))?;
263        }
264        scoring_params.remove(name);
265        drop(scoring_params);
266        self.note_catalog_registry_changed();
267        Ok(true)
268    }
269
270    /// Run saved-model inference through the shared operator optimizer and
271    /// plan executor. A missing model retains the public API's `None`
272    /// contract; the physical driver only receives known models.
273    pub fn deep_predict(&self, name: &str) -> Result<Option<Vec<(DocId, f64)>>, SQLError> {
274        self.with_direct_read_snapshot(|engine| {
275            if engine.load_model_in_execution(name)?.is_none() {
276                return Ok(None);
277            }
278            let tree = uqa_operators::OperatorTree::DeepPredict {
279                model: name.to_string(),
280            };
281            let entries =
282                crate::operator_tree_bridge::execute_scored_tree(engine, "", "", &[], &tree)?;
283            Ok(Some(
284                entries
285                    .into_iter()
286                    .map(|entry| (entry.doc_id, entry.score))
287                    .collect(),
288            ))
289        })
290    }
291
292    pub fn deep_predict_features(
293        &self,
294        name: &str,
295        examples: &[(DocId, Vec<f64>)],
296    ) -> Result<Vec<(DocId, f64)>, SQLError> {
297        self.with_direct_read_snapshot(|engine| {
298            let model = engine
299                .load_model_in_execution(name)?
300                .ok_or_else(|| SQLError::Unsupported(format!("unknown model {name:?}")))?;
301            let (scores, _) = model
302                .predict_features(examples)
303                .map_err(|e| SQLError::Unsupported(format!("deep_predict: {e}")))?;
304            Ok(scores)
305        })
306    }
307}