uqa_fusion/
query_features.rs1use uqa_core::IndexStats;
14
15pub const N_QUERY_FEATURES: usize = 6;
16
17#[derive(Debug, Clone)]
18pub struct QueryFeatureExtractor {
19 stats: IndexStats,
20 field: Option<String>,
21}
22
23impl QueryFeatureExtractor {
24 pub fn new(stats: IndexStats) -> Self {
25 Self { stats, field: None }
26 }
27
28 pub fn with_field(mut self, field: impl Into<String>) -> Self {
29 self.field = Some(field.into());
30 self
31 }
32
33 pub fn n_features(&self) -> usize {
34 N_QUERY_FEATURES
35 }
36
37 pub fn extract<S: AsRef<str>>(&self, query_terms: &[S]) -> [f64; N_QUERY_FEATURES] {
38 let terms: Vec<String> = query_terms
39 .iter()
40 .map(|term| term.as_ref().to_string())
41 .collect();
42 extract_query_features(&self.stats, &terms, self.field.as_deref())
43 }
44}
45
46pub fn extract_query_features(
50 stats: &IndexStats,
51 query_terms: &[String],
52 field: Option<&str>,
53) -> [f64; N_QUERY_FEATURES] {
54 extract_features(
55 stats,
56 query_terms.len(),
57 query_terms
58 .iter()
59 .map(|term| stats.doc_freq(field.unwrap_or("_default"), term)),
60 )
61}
62
63pub fn extract_query_features_utf16(
65 stats: &IndexStats,
66 query_terms: &[Vec<u16>],
67 field: Option<&str>,
68) -> [f64; N_QUERY_FEATURES] {
69 extract_features(
70 stats,
71 query_terms.len(),
72 query_terms
73 .iter()
74 .map(|term| stats.doc_freq_utf16(field.unwrap_or("_default"), term)),
75 )
76}
77
78fn extract_features(
79 stats: &IndexStats,
80 term_count: usize,
81 frequencies: impl Iterator<Item = u64>,
82) -> [f64; N_QUERY_FEATURES] {
83 let n_docs = stats.total_docs;
84 if n_docs == 0 {
85 return [0.0; N_QUERY_FEATURES];
86 }
87 let mut idfs: Vec<f64> = Vec::with_capacity(term_count);
88 let mut vocab_hits: usize = 0;
89 for df in frequencies {
90 if df > 0 {
91 vocab_hits += 1;
92 let idf = (((n_docs - df) as f64 + 0.5) / (df as f64 + 0.5) + 1.0).ln();
93 idfs.push(idf);
94 }
95 }
96 if idfs.is_empty() {
97 return [0.0, 0.0, 0.0, 0.0, term_count as f64, 0.0];
98 }
99 let mean_idf: f64 = idfs.iter().sum::<f64>() / idfs.len() as f64;
100 let max_idf: f64 = idfs.iter().copied().fold(f64::NEG_INFINITY, f64::max);
101 let min_idf: f64 = idfs.iter().copied().fold(f64::INFINITY, f64::min);
102 let coverage_ratio = idfs.len() as f64 / n_docs.max(1) as f64;
103 let query_length = term_count as f64;
104 let vocab_overlap = vocab_hits as f64 / term_count.max(1) as f64;
105 [
106 mean_idf,
107 max_idf,
108 min_idf,
109 coverage_ratio,
110 query_length,
111 vocab_overlap,
112 ]
113}
114
115#[cfg(test)]
116mod tests {
117 use super::*;
118
119 fn stats_with(field: &str, terms: &[(&str, u64)]) -> IndexStats {
120 let mut s = IndexStats::default();
121 s.total_docs = 1000;
122 s.avg_doc_length = 100.0;
123 for (term, df) in terms {
124 s.set_doc_freq(field, *term, *df);
125 }
126 s
127 }
128
129 #[test]
130 fn raw_utf16_features_keep_repetition_and_replacement_characters_distinct() {
131 let mut stats = stats_with("body", &[("�", 200)]);
132 stats.set_doc_freq_utf16("body", vec![0xd83d], 5);
133 let features = extract_query_features_utf16(
134 &stats,
135 &[vec![0xd83d], vec![0xfffd], vec![0xd83d]],
136 Some("body"),
137 );
138 let scalar = stats_with("body", &[("raw", 5), ("replacement", 200)]);
139 assert_eq!(
140 features,
141 extract_query_features(
142 &scalar,
143 &["raw".into(), "replacement".into(), "raw".into()],
144 Some("body")
145 )
146 );
147 }
148
149 #[test]
150 fn empty_index_returns_zeros() {
151 let stats = IndexStats::default();
152 let feats = extract_query_features(&stats, &["foo".into()], None);
153 assert_eq!(feats, [0.0; N_QUERY_FEATURES]);
154 }
155
156 #[test]
157 fn unknown_terms_yield_idf_zeros_with_query_length() {
158 let stats = stats_with("body", &[]);
159 let feats = extract_query_features(&stats, &["unseen".into()], Some("body"));
160 assert_eq!(feats[0], 0.0);
162 assert_eq!(feats[4], 1.0);
163 assert_eq!(feats[5], 0.0);
164 }
165
166 #[test]
167 fn known_terms_produce_positive_idf() {
168 let stats = stats_with("body", &[("rust", 5), ("python", 200)]);
169 let feats = extract_query_features(&stats, &["rust".into(), "python".into()], Some("body"));
170 assert!(feats[0] > 0.0);
171 assert!(feats[1] >= feats[0]);
172 assert!(feats[2] <= feats[0]);
173 assert!((feats[5] - 1.0).abs() < 1e-9);
174 }
175}