Skip to main content

fetchium_core/query/
complexity.rs

1//! Query Complexity Estimator (QCE) — estimates query difficulty to adjust search strategy.
2//!
3//! Simple factual queries ("capital of France") get fast single-backend lookups.
4//! Complex queries ("compare Rust async frameworks performance benchmarks 2024")
5//! trigger multi-backend, expansion, and deeper ranking.
6//!
7//! Scoring signals:
8//! 1. Term count — more terms = more complex
9//! 2. Operator presence — quotes, site:, filetype:, boolean operators
10//! 3. Question words — "how", "why", "compare" indicate depth
11//! 4. Entity count — named entities increase complexity
12//! 5. Temporal markers — "latest", "2024", "recent" add complexity
13
14use crate::rank::fusion::QueryIntent;
15
16/// Configuration for the complexity estimator.
17#[derive(Debug, Clone)]
18pub struct ComplexityConfig {
19    /// Below this threshold: simple query (single backend, no expansion).
20    pub simple_threshold: f64,
21    /// Above this threshold: complex query (multi-backend, expansion, deep ranking).
22    pub complex_threshold: f64,
23}
24
25impl Default for ComplexityConfig {
26    fn default() -> Self {
27        Self {
28            simple_threshold: 0.3,
29            complex_threshold: 0.7,
30        }
31    }
32}
33
34/// Complexity assessment result.
35#[derive(Debug, Clone)]
36pub struct ComplexityAssessment {
37    /// Overall complexity score in [0.0, 1.0].
38    pub score: f64,
39    /// Human-readable complexity level.
40    pub level: ComplexityLevel,
41    /// Recommended number of backends to query.
42    pub recommended_backends: usize,
43    /// Whether query expansion should be applied.
44    pub should_expand: bool,
45    /// Whether deep ranking (HyperFusion) is worthwhile.
46    pub needs_deep_ranking: bool,
47    /// Individual signal scores for debugging.
48    pub signals: ComplexitySignals,
49}
50
51/// Complexity level categories.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum ComplexityLevel {
54    /// Quick factual lookup — 1 backend, no expansion.
55    Simple,
56    /// Moderate query — 2-3 backends, optional expansion.
57    Moderate,
58    /// Deep analysis — all backends, expansion, full ranking.
59    Complex,
60}
61
62/// Individual signal contributions.
63#[derive(Debug, Clone)]
64pub struct ComplexitySignals {
65    pub term_count_signal: f64,
66    pub operator_signal: f64,
67    pub question_depth_signal: f64,
68    pub entity_signal: f64,
69    pub temporal_signal: f64,
70}
71
72/// Estimate the complexity of a search query.
73pub fn estimate_complexity(query: &str, config: &ComplexityConfig) -> ComplexityAssessment {
74    let signals = compute_signals(query);
75
76    // Weighted combination
77    let score = (signals.term_count_signal * 0.25
78        + signals.operator_signal * 0.25
79        + signals.question_depth_signal * 0.25
80        + signals.entity_signal * 0.15
81        + signals.temporal_signal * 0.10)
82        .clamp(0.0, 1.0);
83
84    let level = if score < config.simple_threshold {
85        ComplexityLevel::Simple
86    } else if score < config.complex_threshold {
87        ComplexityLevel::Moderate
88    } else {
89        ComplexityLevel::Complex
90    };
91
92    let recommended_backends = match level {
93        ComplexityLevel::Simple => 1,
94        ComplexityLevel::Moderate => 3,
95        ComplexityLevel::Complex => 5,
96    };
97
98    ComplexityAssessment {
99        score,
100        level,
101        recommended_backends,
102        should_expand: score >= config.simple_threshold,
103        needs_deep_ranking: score >= config.complex_threshold,
104        signals,
105    }
106}
107
108/// Suggest a query intent based on complexity signals.
109pub fn suggest_intent(query: &str) -> QueryIntent {
110    let lower = query.to_lowercase();
111    let words: Vec<&str> = lower.split_whitespace().collect();
112
113    if words.iter().any(|w| COMPARISON_WORDS.contains(w)) || lower.contains(" vs ") {
114        return QueryIntent::Comparison;
115    }
116    if words.iter().any(|w| TEMPORAL_WORDS.contains(w)) || has_year(&lower) {
117        return QueryIntent::CurrentEvents;
118    }
119    if words.first().is_some_and(|w| HOW_WORDS.contains(w)) {
120        return QueryIntent::HowTo;
121    }
122    if words
123        .first()
124        .is_some_and(|w| *w == "is" || *w == "are" || *w == "does" || *w == "do")
125    {
126        return QueryIntent::Verification;
127    }
128    if words.iter().any(|w| CODE_WORDS.contains(w)) {
129        return QueryIntent::Code;
130    }
131    if words.iter().any(|w| ACADEMIC_WORDS.contains(w)) {
132        return QueryIntent::Academic;
133    }
134    if words.iter().any(|w| OPINION_WORDS.contains(w)) {
135        return QueryIntent::Opinion;
136    }
137    if words.iter().any(|w| DATA_WORDS.contains(w)) {
138        return QueryIntent::Data;
139    }
140    if words.len() > 6 {
141        return QueryIntent::DeepAnalysis;
142    }
143    QueryIntent::Factual
144}
145
146// ─── Signal computation ──────────────────────────────────────
147
148fn compute_signals(query: &str) -> ComplexitySignals {
149    let lower = query.to_lowercase();
150    let words: Vec<&str> = lower.split_whitespace().collect();
151
152    ComplexitySignals {
153        term_count_signal: term_count_signal(words.len()),
154        operator_signal: operator_signal(&lower),
155        question_depth_signal: question_depth_signal(&words),
156        entity_signal: entity_signal(&words),
157        temporal_signal: temporal_signal(&lower),
158    }
159}
160
161/// More terms → higher complexity. Sigmoid-like curve: 1 word = 0.0, 8+ words = 1.0.
162fn term_count_signal(count: usize) -> f64 {
163    match count {
164        0..=1 => 0.0,
165        2 => 0.15,
166        3 => 0.3,
167        4 => 0.45,
168        5 => 0.6,
169        6 => 0.75,
170        7 => 0.9,
171        _ => 1.0,
172    }
173}
174
175/// Detect advanced search operators: quotes, site:, filetype:, boolean.
176fn operator_signal(query: &str) -> f64 {
177    let mut score: f64 = 0.0;
178    if query.contains('"') {
179        score += 0.3;
180    }
181    if query.contains("site:") {
182        score += 0.3;
183    }
184    if query.contains("filetype:") || query.contains("type:") {
185        score += 0.2;
186    }
187    if query.contains(" and ") || query.contains(" or ") || query.contains(" not ") {
188        score += 0.2;
189    }
190    if query.contains('(') && query.contains(')') {
191        score += 0.1;
192    }
193    score.min(1.0)
194}
195
196/// Question words signal depth. "how to" < "why does" < "compare X and Y".
197fn question_depth_signal(words: &[&str]) -> f64 {
198    if words.is_empty() {
199        return 0.0;
200    }
201
202    let mut score: f64 = 0.0;
203
204    // Deep question words
205    if words.iter().any(|w| COMPARISON_WORDS.contains(w)) {
206        score += 0.5;
207    }
208    if words.iter().any(|w| DEEP_WORDS.contains(w)) {
209        score += 0.4;
210    }
211    if words.first().is_some_and(|w| HOW_WORDS.contains(w)) {
212        score += 0.2;
213    }
214    if words.first().is_some_and(|w| BASIC_QUESTION.contains(w)) {
215        score += 0.1;
216    }
217
218    score.min(1.0)
219}
220
221/// Capitalized words (potential named entities) increase complexity.
222fn entity_signal(words: &[&str]) -> f64 {
223    // Count words that look like they could be entities
224    // Since we lowercased, check for multi-word terms and proper-noun patterns
225    let entity_indicators: usize = words
226        .iter()
227        .filter(|w| w.len() > 1 && !is_common_word(w))
228        .count();
229
230    match entity_indicators {
231        0..=1 => 0.0,
232        2 => 0.2,
233        3 => 0.4,
234        4 => 0.6,
235        5 => 0.8,
236        _ => 1.0,
237    }
238}
239
240/// Temporal markers increase complexity (need fresh results).
241fn temporal_signal(query: &str) -> f64 {
242    let mut score: f64 = 0.0;
243    if has_year(query) {
244        score += 0.5;
245    }
246    let words: Vec<&str> = query.split_whitespace().collect();
247    if words.iter().any(|w| TEMPORAL_WORDS.contains(w)) {
248        score += 0.5;
249    }
250    score.min(1.0)
251}
252
253fn has_year(query: &str) -> bool {
254    query.split(|c: char| !c.is_ascii_digit()).any(|seg| {
255        seg.len() == 4
256            && seg.starts_with("20")
257            && seg.parse::<u32>().is_ok_and(|y| (2000..=2030).contains(&y))
258    })
259}
260
261fn is_common_word(word: &str) -> bool {
262    const COMMON: &[&str] = &[
263        "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had",
264        "do", "does", "did", "will", "would", "could", "should", "may", "might", "can", "shall",
265        "must", "need", "to", "of", "in", "for", "on", "with", "at", "by", "from", "as", "into",
266        "about", "like", "through", "after", "over", "between", "out", "up", "not", "no", "but",
267        "or", "and", "if", "then", "so", "than", "too", "very", "just", "how", "what", "when",
268        "where", "why", "who", "which", "that", "this", "it", "its", "my", "your", "his", "her",
269        "their", "our",
270    ];
271    COMMON.contains(&word)
272}
273
274const COMPARISON_WORDS: &[&str] = &[
275    "compare",
276    "comparing",
277    "versus",
278    "vs",
279    "difference",
280    "differences",
281    "better",
282];
283const DEEP_WORDS: &[&str] = &[
284    "why",
285    "explain",
286    "analysis",
287    "analyze",
288    "evaluate",
289    "pros",
290    "cons",
291    "tradeoff",
292    "tradeoffs",
293];
294const HOW_WORDS: &[&str] = &["how"];
295const BASIC_QUESTION: &[&str] = &["what", "who", "when", "where"];
296const TEMPORAL_WORDS: &[&str] = &[
297    "latest", "recent", "new", "newest", "current", "today", "updated",
298];
299const CODE_WORDS: &[&str] = &[
300    "code",
301    "function",
302    "error",
303    "bug",
304    "api",
305    "implementation",
306    "library",
307    "framework",
308    "syntax",
309];
310const ACADEMIC_WORDS: &[&str] = &[
311    "paper",
312    "research",
313    "study",
314    "journal",
315    "thesis",
316    "hypothesis",
317    "methodology",
318];
319const OPINION_WORDS: &[&str] = &[
320    "best",
321    "worst",
322    "opinion",
323    "review",
324    "recommend",
325    "recommendation",
326    "favorite",
327];
328const DATA_WORDS: &[&str] = &[
329    "data",
330    "dataset",
331    "statistics",
332    "benchmark",
333    "metrics",
334    "numbers",
335    "chart",
336];
337
338#[cfg(test)]
339mod tests {
340    use super::*;
341
342    #[test]
343    fn simple_factual_query() {
344        let config = ComplexityConfig::default();
345        let result = estimate_complexity("capital of France", &config);
346        assert_eq!(result.level, ComplexityLevel::Simple);
347        assert_eq!(result.recommended_backends, 1);
348        assert!(!result.should_expand);
349    }
350
351    #[test]
352    fn moderate_query() {
353        let config = ComplexityConfig::default();
354        let result = estimate_complexity("how to implement binary search in Rust", &config);
355        assert!(result.score >= config.simple_threshold);
356        assert!(result.recommended_backends >= 2);
357    }
358
359    #[test]
360    fn complex_query_with_operators() {
361        let config = ComplexityConfig::default();
362        let result = estimate_complexity(
363            "compare \"Rust async\" vs \"Go goroutines\" performance benchmarks site:github.com 2024",
364            &config,
365        );
366        assert_eq!(result.level, ComplexityLevel::Complex);
367        assert!(result.needs_deep_ranking);
368        assert_eq!(result.recommended_backends, 5);
369    }
370
371    #[test]
372    fn empty_query_is_simple() {
373        let config = ComplexityConfig::default();
374        let result = estimate_complexity("", &config);
375        assert_eq!(result.level, ComplexityLevel::Simple);
376        assert_eq!(result.score, 0.0);
377    }
378
379    #[test]
380    fn single_word_is_simple() {
381        let config = ComplexityConfig::default();
382        let result = estimate_complexity("rust", &config);
383        assert_eq!(result.level, ComplexityLevel::Simple);
384    }
385
386    #[test]
387    fn temporal_markers_increase_complexity() {
388        let config = ComplexityConfig::default();
389        let base = estimate_complexity("rust frameworks", &config);
390        let temporal = estimate_complexity("latest rust frameworks 2024", &config);
391        assert!(temporal.score > base.score);
392    }
393
394    #[test]
395    fn comparison_query_detected() {
396        let result = suggest_intent("compare React vs Vue");
397        assert_eq!(result, QueryIntent::Comparison);
398    }
399
400    #[test]
401    fn how_to_query_detected() {
402        let result = suggest_intent("how to deploy kubernetes");
403        assert_eq!(result, QueryIntent::HowTo);
404    }
405
406    #[test]
407    fn code_query_detected() {
408        let result = suggest_intent("python function syntax error");
409        assert_eq!(result, QueryIntent::Code);
410    }
411
412    #[test]
413    fn academic_query_detected() {
414        let result = suggest_intent("research paper on transformer models");
415        assert_eq!(result, QueryIntent::Academic);
416    }
417
418    #[test]
419    fn verification_query_detected() {
420        let result = suggest_intent("is rust memory safe");
421        assert_eq!(result, QueryIntent::Verification);
422    }
423
424    #[test]
425    fn signals_are_bounded() {
426        let config = ComplexityConfig::default();
427        let result = estimate_complexity(
428            "compare analyze why explain latest 2024 new site:x.com \"quoted\" and or not data benchmark metrics review best code api",
429            &config,
430        );
431        assert!(result.score <= 1.0);
432        assert!(result.signals.term_count_signal <= 1.0);
433        assert!(result.signals.operator_signal <= 1.0);
434        assert!(result.signals.question_depth_signal <= 1.0);
435        assert!(result.signals.entity_signal <= 1.0);
436        assert!(result.signals.temporal_signal <= 1.0);
437    }
438}