Skip to main content

uqa_scoring/
text.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Physical lexical scoring independent of Engine sessions, catalogs, and SQL.
8
9use crate::{rank_scored_entries_top_k, BM25Scorer, ScoringError, ScoringMode};
10use exhaustive::{score_multiple_text_terms, score_single_text_term};
11use statistics::{block_max_scorer_fingerprint, raw_bm25_params, search_stats_for_terms};
12use std::{sync::Arc, time::Instant};
13use uqa_core::ScoredEntry;
14use uqa_storage::{
15    inverted_index::analyze_query_terms, InvertedIndex, StorageBackendError, TokenTermKey,
16};
17
18mod candidates;
19mod exhaustive;
20mod statistics;
21mod top_k;
22
23pub use candidates::TextCandidateScorer;
24
25/// Algorithm that actually produced a text-search result.
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum TextSearchAlgorithm {
28    Exhaustive,
29    Wand,
30    BlockMaxWand,
31}
32
33/// Observable work counters for one text top-k execution.
34#[derive(Debug, Clone)]
35pub struct TextSearchProfile {
36    pub entries: Vec<ScoredEntry>,
37    pub algorithm: TextSearchAlgorithm,
38    pub scored_candidates: u64,
39    /// Exact distinct candidates for exhaustive/materialized execution; for
40    /// score-cursor WAND/BMW this is the sum of term document frequencies, a
41    /// no-prescan upper bound on the distinct candidate count.
42    pub total_candidates: u64,
43    pub cursor_advances: u64,
44    pub skip_rate: f64,
45    pub elapsed_ms: f64,
46}
47
48/// Preserve parameter errors separately from backend and index-integrity failures.
49#[derive(Debug, thiserror::Error)]
50pub enum TextSearchError {
51    #[error("{0}")]
52    Memory(#[from] uqa_core::memory::MemoryError),
53    #[error("{0}")]
54    Cancelled(#[from] uqa_core::QueryCancelled),
55    #[error("{0}")]
56    Parameters(#[from] ScoringError),
57    #[error("{action}: {source}")]
58    Storage {
59        action: &'static str,
60        #[source]
61        source: StorageBackendError,
62    },
63    #[error("{0}")]
64    InvalidIndex(String),
65}
66
67fn storage_error(action: &'static str, error: impl Into<StorageBackendError>) -> TextSearchError {
68    TextSearchError::Storage {
69        action,
70        source: error.into(),
71    }
72}
73
74/// Analyze the complete query with the retained field revision and execute its physical scoring strategy.
75pub fn score_text_query(
76    index: &dyn InvertedIndex,
77    table: &str,
78    field: &str,
79    query: &str,
80    mode: &ScoringMode,
81    top_k: usize,
82    strategy: TextSearchAlgorithm,
83) -> Result<TextSearchProfile, TextSearchError> {
84    let started = Instant::now();
85    let analyzer = index
86        .search_analyzer_revision(field)
87        .map_err(|error| storage_error("resolve text analyzer revision", error))?;
88    let terms = analyze_query_terms(&analyzer, query)
89        .map_err(|error| storage_error("analyze text query", error))?;
90    let mut profile = score_text_terms(index, table, field, &terms, mode, top_k, strategy)?;
91    profile.elapsed_ms = started.elapsed().as_secs_f64() * 1000.0;
92    Ok(profile)
93}
94
95/// Visit complete exhaustive scores in ascending document identity order, without retaining the result population. Repeated query terms and calibration use the same scorer and evaluation order as [`score_text_query`]. A false callback result stops immediately; the return value distinguishes that stop from complete traversal.
96pub fn visit_text_query(
97    index: &dyn InvertedIndex,
98    field: &str,
99    query: &str,
100    mode: &ScoringMode,
101    visitor: &mut dyn FnMut(ScoredEntry) -> bool,
102) -> Result<bool, TextSearchError> {
103    let analyzer = index
104        .search_analyzer_revision(field)
105        .map_err(|error| storage_error("resolve text analyzer revision", error))?;
106    let terms = analyze_query_terms(&analyzer, query)
107        .map_err(|error| storage_error("analyze text query", error))?;
108    exhaustive::visit_text_terms(index, field, &terms, mode, visitor)
109}
110
111/// Visit already-analyzed term occurrences using the exhaustive scorer, preserving their original order and duplicate contributions.
112pub fn visit_text_terms(
113    index: &dyn InvertedIndex,
114    field: &str,
115    terms: &[TokenTermKey],
116    mode: &ScoringMode,
117    visitor: &mut dyn FnMut(ScoredEntry) -> bool,
118) -> Result<bool, TextSearchError> {
119    exhaustive::visit_text_terms(index, field, terms, mode, visitor)
120}
121
122/// Score ordered lossless term occurrences; repeated query terms remain independent scoring contributions.
123pub fn score_text_terms(
124    index: &dyn InvertedIndex,
125    table: &str,
126    field: &str,
127    analyzed_terms: &[TokenTermKey],
128    mode: &ScoringMode,
129    top_k: usize,
130    strategy: TextSearchAlgorithm,
131) -> Result<TextSearchProfile, TextSearchError> {
132    let started = Instant::now();
133    if !analyzed_terms.is_empty() && strategy != TextSearchAlgorithm::Exhaustive {
134        let (entries, stats, algorithm) =
135            top_k::score_text_top_k(index, table, field, analyzed_terms, mode, top_k, strategy)?;
136        return Ok(TextSearchProfile {
137            entries,
138            algorithm,
139            scored_candidates: stats.scored,
140            total_candidates: stats.total_candidates,
141            cursor_advances: stats.cursor_advances,
142            skip_rate: stats.skip_rate(),
143            elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
144        });
145    }
146    let entries = match analyzed_terms.len() {
147        0 => Vec::new(),
148        1 => score_single_text_term(index, field, analyzed_terms, mode)?,
149        _ => score_multiple_text_terms(index, field, analyzed_terms, mode)?,
150    };
151    let total_candidates = u64::try_from(entries.len())
152        .map_err(|_| TextSearchError::InvalidIndex("text candidate count exceeds u64".into()))?;
153    Ok(TextSearchProfile {
154        entries: rank_scored_entries_top_k(entries, top_k),
155        algorithm: TextSearchAlgorithm::Exhaustive,
156        scored_candidates: total_candidates,
157        total_candidates,
158        cursor_advances: 0,
159        skip_rate: 0.0,
160        elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
161    })
162}
163
164/// Materialize block bounds with the exact corpus statistics and scorer identity used during query execution.
165pub fn rebuild_text_block_max(
166    index: &mut dyn InvertedIndex,
167    field: &str,
168    mode: &ScoringMode,
169) -> Result<bool, TextSearchError> {
170    let stats = Arc::new(
171        index
172            .field_stats_scalar(field)
173            .map_err(|error| storage_error("read field statistics", error))?,
174    );
175    let params = raw_bm25_params(mode);
176    params.validate()?;
177    let fingerprint = block_max_scorer_fingerprint(params, stats.as_ref());
178    let scorer = BM25Scorer::new(params, stats);
179    index
180        .rebuild_persisted_block_max(field, &scorer, &fingerprint)
181        .map_err(|error| storage_error("rebuild persisted block-max scores", error))
182}