Skip to main content

uqa_engine/search/
search_api.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Public text search and calibration reporting.
8
9use uqa_storage::InvertedIndex;
10
11use super::{
12    storage_sql_error, Arc, BayesianBM25Scorer, CalibrationMetrics, CalibrationReport, DocId,
13    Engine, Instant, OperatorTree, RawBm25Score, SQLError, ScoredEntry, ScoringMode,
14    TextScoringMode, TextSearchProfile,
15};
16
17impl Engine {
18    /// Run a text query through the shared operator optimizer and executor.
19    pub fn search(
20        &self,
21        table: &str,
22        field: &str,
23        query: &str,
24        mode: &ScoringMode,
25        top_k: usize,
26    ) -> Result<Vec<ScoredEntry>, SQLError> {
27        self.with_direct_table_read(table, |engine, name, _| {
28            let scoring = match mode {
29                ScoringMode::BM25(params) => TextScoringMode::CustomBM25(*params),
30                ScoringMode::BayesianBM25(params) => TextScoringMode::CustomBayesianBM25(*params),
31            };
32            let tree = engine.plan_text_top_k_tree(name, field, query, scoring, top_k)?;
33            let entries =
34                crate::operator_tree_bridge::execute_scored_tree(engine, name, table, &[], &tree)?;
35            Ok(uqa_scoring::rank_scored_entries_top_k(entries, top_k))
36        })
37    }
38
39    /// Run the same planner-selected physical text path as [`Self::search`]
40    /// while returning exact candidate/skip counters for benchmarks and
41    /// production diagnostics.
42    pub fn search_profiled(
43        &self,
44        table: &str,
45        field: &str,
46        query: &str,
47        mode: &ScoringMode,
48        top_k: usize,
49    ) -> Result<TextSearchProfile, SQLError> {
50        let started = Instant::now();
51        let mut profile = self.with_direct_table_read(table, |engine, name, _| {
52            let scoring = match mode {
53                ScoringMode::BM25(params) => TextScoringMode::CustomBM25(*params),
54                ScoringMode::BayesianBM25(params) => TextScoringMode::CustomBayesianBM25(*params),
55            };
56            let tree = engine.plan_text_top_k_tree(name, field, query, scoring, top_k)?;
57            let physical_top_k = match tree {
58                OperatorTree::Term { top_k, .. } => top_k,
59                _ => None,
60            };
61            engine.search_leaf_profiled(name, field, query, mode, top_k, physical_top_k)
62        })?;
63        profile.elapsed_ms = started.elapsed().as_secs_f64() * 1000.0;
64        Ok(profile)
65    }
66
67    /// Compute calibration diagnostics for a Bayesian BM25 query
68    /// against every document in `table`, aligned to `labels` in
69    /// ascending document-id order.
70    pub fn calibration_report(
71        &self,
72        table: &str,
73        field: &str,
74        query: &str,
75        labels: &[u8],
76    ) -> Result<CalibrationReport, SQLError> {
77        self.with_direct_table_query(table, false, |engine, name, _| {
78            engine.calibration_report_in_transaction(name, table, field, query, labels)
79        })
80    }
81
82    fn calibration_report_in_transaction(
83        &self,
84        table: &str,
85        signal_table: &str,
86        field: &str,
87        query: &str,
88        labels: &[u8],
89    ) -> Result<CalibrationReport, SQLError> {
90        let table_state = self.require_query_table(table)?;
91        let doc_ids = self.query_table_doc_ids(table)?;
92        if labels.len() != doc_ids.len() {
93            return Err(SQLError::TypeMismatch(format!(
94                "labels length ({}) must match document count ({})",
95                labels.len(),
96                doc_ids.len()
97            )));
98        }
99        if labels.iter().any(|label| *label > 1) {
100            return Err(SQLError::TypeMismatch(
101                "labels must contain only 0 or 1".into(),
102            ));
103        }
104
105        let params = self.bayesian_params_for_signal(table, signal_table, field)?;
106        let (query_term_count, stats) = {
107            let index = table_state.inverted_index.read();
108            let index = uqa_execution::serializable::text::ObservedTextIndex::new(
109                index.as_ref(),
110                self.serializable_table_read(table)?,
111                table_state.columns.snapshot(),
112            );
113            let query_term_count = index
114                .search_analyzer_revision(field)
115                .map_err(|error| storage_sql_error("resolve calibration analyzer revision", error))?
116                .analyze_tokens(query)
117                .map_err(|error| storage_sql_error("analyze calibration query", error))?
118                .tokens()
119                .len();
120            let stats = Arc::new(
121                index
122                    .field_stats(field)
123                    .map_err(|error| storage_sql_error("read calibration field stats", error))?,
124            );
125            (query_term_count, stats)
126        };
127        let scaled_params = params.scaled_for_query_terms(query_term_count);
128        let non_match_probability = BayesianBM25Scorer::new(scaled_params, stats)
129            .map_err(|error| {
130                SQLError::Internal(format!("build calibration-report scorer: {error}"))
131            })?
132            .calibrate_raw_score(
133                RawBm25Score::new(0.0).expect("the raw BM25 score for a non-match is finite"),
134            )
135            .value();
136
137        let mode = ScoringMode::BayesianBM25(params);
138        let score_map: std::collections::BTreeMap<DocId, f64> = self
139            .search(table, field, query, &mode, usize::MAX)?
140            .into_iter()
141            .map(|entry| (entry.doc_id, entry.score))
142            .collect();
143        let probabilities: Vec<f64> = doc_ids
144            .iter()
145            .map(|doc_id| {
146                score_map
147                    .get(doc_id)
148                    .copied()
149                    .unwrap_or(non_match_probability)
150            })
151            .collect();
152        CalibrationMetrics::report(&probabilities, labels, 10)
153            .map_err(|error| SQLError::Internal(format!("compute calibration report: {error}")))
154    }
155}