uqa_scoring/
text.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Physical lexical scoring independent of Engine sessions, catalogs, and SQL.
8
9use crate::{rank_scored_entries_top_k, BM25Scorer, ScoringError, ScoringMode};
10use exhaustive::{score_multiple_text_terms, score_single_text_term};
11use statistics::{block_max_scorer_fingerprint, raw_bm25_params, search_stats_for_terms};
12use std::{sync::Arc, time::Instant};
13use uqa_core::ScoredEntry;
14use uqa_storage::{
15    inverted_index::analyze_query_terms, InvertedIndex, StorageBackendError, TokenTermKey,
16};
17
18mod candidates;
19mod exhaustive;
20mod statistics;
21mod top_k;
22
23pub use candidates::TextCandidateScorer;
24
25/// Algorithm that actually produced a text-search result.
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum TextSearchAlgorithm {
28    Exhaustive,
29    Wand,
30    BlockMaxWand,
31}
32
33/// Observable work counters for one text top-k execution.
34#[derive(Debug, Clone)]
35pub struct TextSearchProfile {
36    pub entries: Vec<ScoredEntry>,
37    pub algorithm: TextSearchAlgorithm,
38    pub scored_candidates: u64,
39    /// Exact distinct candidates for exhaustive/materialized execution; for
40    /// score-cursor WAND/BMW this is the sum of term document frequencies, a
41    /// no-prescan upper bound on the distinct candidate count.
42    pub total_candidates: u64,
43    pub cursor_advances: u64,
44    pub skip_rate: f64,
45    pub elapsed_ms: f64,
46}
47
48/// Preserve parameter errors separately from backend and index-integrity failures.
49#[derive(Debug, thiserror::Error)]
50pub enum TextSearchError {
51    #[error("{0}")]
52    Memory(#[from] uqa_core::memory::MemoryError),
53    #[error("{0}")]
54    Cancelled(#[from] uqa_core::QueryCancelled),
55    #[error("{0}")]
56    Parameters(#[from] ScoringError),
57    #[error("{action}: {source}")]
58    Storage {
59        action: &'static str,
60        #[source]
61        source: StorageBackendError,
62    },
63    #[error("{0}")]
64    InvalidIndex(String),
65}
66
67fn storage_error(action: &'static str, error: impl Into<StorageBackendError>) -> TextSearchError {
68    TextSearchError::Storage {
69        action,
70        source: error.into(),
71    }
72}
73
74/// Analyze the complete query with the retained field revision and execute its physical scoring strategy.
75pub fn score_text_query(
76    index: &dyn InvertedIndex,
77    table: &str,
78    field: &str,
79    query: &str,
80    mode: &ScoringMode,
81    top_k: usize,
82    strategy: TextSearchAlgorithm,
83) -> Result<TextSearchProfile, TextSearchError> {
84    let started = Instant::now();
85    let analyzer = index
86        .search_analyzer_revision(field)
87        .map_err(|error| storage_error("resolve text analyzer revision", error))?;
88    let terms = analyze_query_terms(&analyzer, query)
89        .map_err(|error| storage_error("analyze text query", error))?;
90    let mut profile = score_text_terms(index, table, field, &terms, mode, top_k, strategy)?;
91    profile.elapsed_ms = started.elapsed().as_secs_f64() * 1000.0;
92    Ok(profile)
93}
94
95/// Score ordered lossless term occurrences; repeated query terms remain independent scoring contributions.
96pub fn score_text_terms(
97    index: &dyn InvertedIndex,
98    table: &str,
99    field: &str,
100    analyzed_terms: &[TokenTermKey],
101    mode: &ScoringMode,
102    top_k: usize,
103    strategy: TextSearchAlgorithm,
104) -> Result<TextSearchProfile, TextSearchError> {
105    let started = Instant::now();
106    if !analyzed_terms.is_empty() && strategy != TextSearchAlgorithm::Exhaustive {
107        let (entries, stats, algorithm) =
108            top_k::score_text_top_k(index, table, field, analyzed_terms, mode, top_k, strategy)?;
109        return Ok(TextSearchProfile {
110            entries,
111            algorithm,
112            scored_candidates: stats.scored,
113            total_candidates: stats.total_candidates,
114            cursor_advances: stats.cursor_advances,
115            skip_rate: stats.skip_rate(),
116            elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
117        });
118    }
119    let entries = match analyzed_terms.len() {
120        0 => Vec::new(),
121        1 => score_single_text_term(index, field, analyzed_terms, mode)?,
122        _ => score_multiple_text_terms(index, field, analyzed_terms, mode)?,
123    };
124    let total_candidates = u64::try_from(entries.len())
125        .map_err(|_| TextSearchError::InvalidIndex("text candidate count exceeds u64".into()))?;
126    Ok(TextSearchProfile {
127        entries: rank_scored_entries_top_k(entries, top_k),
128        algorithm: TextSearchAlgorithm::Exhaustive,
129        scored_candidates: total_candidates,
130        total_candidates,
131        cursor_advances: 0,
132        skip_rate: 0.0,
133        elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
134    })
135}
136
137/// Materialize block bounds with the exact corpus statistics and scorer identity used during query execution.
138pub fn rebuild_text_block_max(
139    index: &mut dyn InvertedIndex,
140    field: &str,
141    mode: &ScoringMode,
142) -> Result<bool, TextSearchError> {
143    let stats = Arc::new(
144        index
145            .field_stats_scalar(field)
146            .map_err(|error| storage_error("read field statistics", error))?,
147    );
148    let params = raw_bm25_params(mode);
149    params.validate()?;
150    let fingerprint = block_max_scorer_fingerprint(params, stats.as_ref());
151    let scorer = BM25Scorer::new(params, stats);
152    index
153        .rebuild_persisted_block_max(field, &scorer, &fingerprint)
154        .map_err(|error| storage_error("rebuild persisted block-max scores", error))
155}
uqa_scoring/text.rs

uqa_scoring/
text.rs
