Skip to main content

uqa_fusion/
query_features.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Query-level features for attention-based fusion (Section 8, Paper 4).
8//!
9//! Six features in this fixed order:
10//!   `[mean_idf, max_idf, min_idf, coverage_ratio, query_length,
11//!     vocab_overlap_ratio]`.
12
13use uqa_core::IndexStats;
14
15pub const N_QUERY_FEATURES: usize = 6;
16
17#[derive(Debug, Clone)]
18pub struct QueryFeatureExtractor {
19    stats: IndexStats,
20    field: Option<String>,
21}
22
23impl QueryFeatureExtractor {
24    pub fn new(stats: IndexStats) -> Self {
25        Self { stats, field: None }
26    }
27
28    pub fn with_field(mut self, field: impl Into<String>) -> Self {
29        self.field = Some(field.into());
30        self
31    }
32
33    pub fn n_features(&self) -> usize {
34        N_QUERY_FEATURES
35    }
36
37    pub fn extract<S: AsRef<str>>(&self, query_terms: &[S]) -> [f64; N_QUERY_FEATURES] {
38        let terms: Vec<String> = query_terms
39            .iter()
40            .map(|term| term.as_ref().to_string())
41            .collect();
42        extract_query_features(&self.stats, &terms, self.field.as_deref())
43    }
44}
45
46/// Compute the query feature vector against the supplied
47/// [`IndexStats`]. `field` is the IDF field to consult; pass `None`
48/// for the catch-all `_default` slot.
49pub fn extract_query_features(
50    stats: &IndexStats,
51    query_terms: &[String],
52    field: Option<&str>,
53) -> [f64; N_QUERY_FEATURES] {
54    extract_features(
55        stats,
56        query_terms.len(),
57        query_terms
58            .iter()
59            .map(|term| stats.doc_freq(field.unwrap_or("_default"), term)),
60    )
61}
62
63/// Extract the same query features from exact UTF-16 terms, including isolated surrogate units.
64pub fn extract_query_features_utf16(
65    stats: &IndexStats,
66    query_terms: &[Vec<u16>],
67    field: Option<&str>,
68) -> [f64; N_QUERY_FEATURES] {
69    extract_features(
70        stats,
71        query_terms.len(),
72        query_terms
73            .iter()
74            .map(|term| stats.doc_freq_utf16(field.unwrap_or("_default"), term)),
75    )
76}
77
78fn extract_features(
79    stats: &IndexStats,
80    term_count: usize,
81    frequencies: impl Iterator<Item = u64>,
82) -> [f64; N_QUERY_FEATURES] {
83    let n_docs = stats.total_docs;
84    if n_docs == 0 {
85        return [0.0; N_QUERY_FEATURES];
86    }
87    let mut idfs: Vec<f64> = Vec::with_capacity(term_count);
88    let mut vocab_hits: usize = 0;
89    for df in frequencies {
90        if df > 0 {
91            vocab_hits += 1;
92            let idf = (((n_docs - df) as f64 + 0.5) / (df as f64 + 0.5) + 1.0).ln();
93            idfs.push(idf);
94        }
95    }
96    if idfs.is_empty() {
97        return [0.0, 0.0, 0.0, 0.0, term_count as f64, 0.0];
98    }
99    let mean_idf: f64 = idfs.iter().sum::<f64>() / idfs.len() as f64;
100    let max_idf: f64 = idfs.iter().copied().fold(f64::NEG_INFINITY, f64::max);
101    let min_idf: f64 = idfs.iter().copied().fold(f64::INFINITY, f64::min);
102    let coverage_ratio = idfs.len() as f64 / n_docs.max(1) as f64;
103    let query_length = term_count as f64;
104    let vocab_overlap = vocab_hits as f64 / term_count.max(1) as f64;
105    [
106        mean_idf,
107        max_idf,
108        min_idf,
109        coverage_ratio,
110        query_length,
111        vocab_overlap,
112    ]
113}
114
115#[cfg(test)]
116mod tests {
117    use super::*;
118
119    fn stats_with(field: &str, terms: &[(&str, u64)]) -> IndexStats {
120        let mut s = IndexStats::default();
121        s.total_docs = 1000;
122        s.avg_doc_length = 100.0;
123        for (term, df) in terms {
124            s.set_doc_freq(field, *term, *df);
125        }
126        s
127    }
128
129    #[test]
130    fn raw_utf16_features_keep_repetition_and_replacement_characters_distinct() {
131        let mut stats = stats_with("body", &[("�", 200)]);
132        stats.set_doc_freq_utf16("body", vec![0xd83d], 5);
133        let features = extract_query_features_utf16(
134            &stats,
135            &[vec![0xd83d], vec![0xfffd], vec![0xd83d]],
136            Some("body"),
137        );
138        let scalar = stats_with("body", &[("raw", 5), ("replacement", 200)]);
139        assert_eq!(
140            features,
141            extract_query_features(
142                &scalar,
143                &["raw".into(), "replacement".into(), "raw".into()],
144                Some("body")
145            )
146        );
147    }
148
149    #[test]
150    fn empty_index_returns_zeros() {
151        let stats = IndexStats::default();
152        let feats = extract_query_features(&stats, &["foo".into()], None);
153        assert_eq!(feats, [0.0; N_QUERY_FEATURES]);
154    }
155
156    #[test]
157    fn unknown_terms_yield_idf_zeros_with_query_length() {
158        let stats = stats_with("body", &[]);
159        let feats = extract_query_features(&stats, &["unseen".into()], Some("body"));
160        // mean/max/min/coverage = 0, query length = 1, vocab overlap = 0.
161        assert_eq!(feats[0], 0.0);
162        assert_eq!(feats[4], 1.0);
163        assert_eq!(feats[5], 0.0);
164    }
165
166    #[test]
167    fn known_terms_produce_positive_idf() {
168        let stats = stats_with("body", &[("rust", 5), ("python", 200)]);
169        let feats = extract_query_features(&stats, &["rust".into(), "python".into()], Some("body"));
170        assert!(feats[0] > 0.0);
171        assert!(feats[1] >= feats[0]);
172        assert!(feats[2] <= feats[0]);
173        assert!((feats[5] - 1.0).abs() < 1e-9);
174    }
175}