Skip to main content

fetchium_core/query/
expansion.rs

1//! Query Expansion Engine (QXE) — automatic synonym, acronym, and perspective expansion.
2//!
3//! Two expansion modes:
4//! 1. **Lexical** (`expand_query`): Synonym/acronym substitution for technical terms.
5//!    "JS frameworks" → "JavaScript frameworks", "k8s" → "kubernetes"
6//! 2. **Perspective** (`ai_perspective_expand`): AI-generated domain-specific sub-queries.
7//!    "who created the world" → ["Big Bang cosmology universe origin", "religious creation myths", ...]
8//!
9//! Perspective expansion is the high-impact path for ambiguous/multi-domain queries.
10//! It ensures all relevant angles (scientific, religious, philosophical, historical) are covered.
11
12use crate::ai::provider_client::chat_with_fallback;
13use crate::ai::types::{AiConfig, ChatMessage};
14
15/// An expanded query with its variants.
16#[derive(Debug, Clone)]
17pub struct ExpandedQuery {
18    /// The original query, unchanged.
19    pub original: String,
20    /// Expanded variants to also search for.
21    pub expansions: Vec<QueryVariant>,
22}
23
24/// A single query variant produced by expansion.
25#[derive(Debug, Clone)]
26pub struct QueryVariant {
27    /// The expanded query text.
28    pub text: String,
29    /// Confidence that this expansion is relevant (0.0-1.0).
30    pub confidence: f64,
31    /// What type of expansion produced this variant.
32    pub expansion_type: ExpansionType,
33}
34
35/// Types of query expansion.
36#[derive(Debug, Clone, PartialEq, Eq)]
37pub enum ExpansionType {
38    /// Acronym → full form ("JS" → "JavaScript")
39    AcronymExpansion,
40    /// Full form → acronym ("JavaScript" → "JS")
41    AcronymContraction,
42    /// Synonym substitution ("fast" → "quick")
43    Synonym,
44    /// Related term addition
45    Related,
46}
47
48/// Acronym dictionary — maps abbreviations to full forms.
49const ACRONYMS: &[(&str, &str)] = &[
50    ("js", "javascript"),
51    ("ts", "typescript"),
52    ("py", "python"),
53    ("rb", "ruby"),
54    ("rs", "rust"),
55    ("ml", "machine learning"),
56    ("ai", "artificial intelligence"),
57    ("dl", "deep learning"),
58    ("nlp", "natural language processing"),
59    ("cv", "computer vision"),
60    ("llm", "large language model"),
61    ("gpt", "generative pre-trained transformer"),
62    ("rl", "reinforcement learning"),
63    ("nn", "neural network"),
64    ("cnn", "convolutional neural network"),
65    ("rnn", "recurrent neural network"),
66    ("api", "application programming interface"),
67    ("sdk", "software development kit"),
68    ("cli", "command line interface"),
69    ("gui", "graphical user interface"),
70    ("db", "database"),
71    ("sql", "structured query language"),
72    ("nosql", "not only sql"),
73    ("orm", "object relational mapping"),
74    ("css", "cascading style sheets"),
75    ("html", "hypertext markup language"),
76    ("http", "hypertext transfer protocol"),
77    ("rest", "representational state transfer"),
78    ("grpc", "remote procedure call"),
79    ("ci", "continuous integration"),
80    ("cd", "continuous deployment"),
81    ("k8s", "kubernetes"),
82    ("tf", "terraform"),
83    ("aws", "amazon web services"),
84    ("gcp", "google cloud platform"),
85    ("vm", "virtual machine"),
86    ("os", "operating system"),
87    ("cpu", "central processing unit"),
88    ("gpu", "graphics processing unit"),
89    ("ram", "random access memory"),
90    ("ssd", "solid state drive"),
91    ("ux", "user experience"),
92    ("ui", "user interface"),
93    ("seo", "search engine optimization"),
94    ("mvp", "minimum viable product"),
95    ("saas", "software as a service"),
96    ("iot", "internet of things"),
97    ("tls", "transport layer security"),
98    ("jwt", "json web token"),
99    ("oauth", "open authorization"),
100    ("xss", "cross site scripting"),
101    ("dos", "denial of service"),
102    ("wasm", "webassembly"),
103    ("ssr", "server side rendering"),
104    ("csr", "client side rendering"),
105    ("ssg", "static site generation"),
106];
107
108/// Synonym pairs — bidirectional substitutions.
109const SYNONYMS: &[(&str, &str)] = &[
110    ("fast", "quick"),
111    ("error", "bug"),
112    ("fix", "resolve"),
113    ("issue", "problem"),
114    ("tutorial", "guide"),
115    ("library", "package"),
116    ("framework", "library"),
117    ("install", "setup"),
118    ("config", "configuration"),
119    ("deploy", "deployment"),
120    ("auth", "authentication"),
121    ("async", "asynchronous"),
122    ("sync", "synchronous"),
123    ("perf", "performance"),
124    ("benchmark", "performance test"),
125    ("regex", "regular expression"),
126];
127
128/// Expand a query with synonyms and acronyms.
129///
130/// Returns the original query plus expanded variants, each with a
131/// confidence score. Higher confidence = more likely to be helpful.
132pub fn expand_query(query: &str) -> ExpandedQuery {
133    let original = query.to_string();
134    let lower = query.to_lowercase();
135    let tokens: Vec<&str> = lower.split_whitespace().collect();
136    let mut expansions = Vec::new();
137
138    // 1. Acronym expansion: replace abbreviations with full forms
139    for (abbrev, full) in ACRONYMS {
140        if tokens
141            .iter()
142            .any(|t| t.trim_matches(|c: char| c.is_ascii_punctuation()) == *abbrev)
143        {
144            let expanded = lower.replace(abbrev, full);
145            if expanded != lower {
146                expansions.push(QueryVariant {
147                    text: expanded,
148                    confidence: 0.9,
149                    expansion_type: ExpansionType::AcronymExpansion,
150                });
151            }
152        }
153    }
154
155    // 2. Reverse acronym: if query has full form, also try abbreviation
156    for (abbrev, full) in ACRONYMS {
157        if lower.contains(full) {
158            let contracted = lower.replace(full, abbrev);
159            if contracted != lower {
160                expansions.push(QueryVariant {
161                    text: contracted,
162                    confidence: 0.7,
163                    expansion_type: ExpansionType::AcronymContraction,
164                });
165            }
166        }
167    }
168
169    // 3. Synonym substitution
170    for (a, b) in SYNONYMS {
171        if tokens
172            .iter()
173            .any(|t| t.trim_matches(|c: char| c.is_ascii_punctuation()) == *a)
174        {
175            let expanded = lower.replace(a, b);
176            if expanded != lower {
177                expansions.push(QueryVariant {
178                    text: expanded,
179                    confidence: 0.6,
180                    expansion_type: ExpansionType::Synonym,
181                });
182            }
183        }
184        // Bidirectional
185        if tokens
186            .iter()
187            .any(|t| t.trim_matches(|c: char| c.is_ascii_punctuation()) == *b)
188        {
189            let expanded = lower.replace(b, a);
190            if expanded != lower {
191                expansions.push(QueryVariant {
192                    text: expanded,
193                    confidence: 0.6,
194                    expansion_type: ExpansionType::Synonym,
195                });
196            }
197        }
198    }
199
200    // Deduplicate expansions
201    let mut seen = std::collections::HashSet::new();
202    seen.insert(lower.clone());
203    expansions.retain(|v| seen.insert(v.text.clone()));
204
205    // Sort by confidence descending
206    expansions.sort_by(|a, b| {
207        b.confidence
208            .partial_cmp(&a.confidence)
209            .unwrap_or(std::cmp::Ordering::Equal)
210    });
211
212    // Limit to top 3 expansions to prevent query drift
213    expansions.truncate(3);
214
215    ExpandedQuery {
216        original,
217        expansions,
218    }
219}
220
221/// Check if a query would benefit from expansion.
222///
223/// Returns true if the query contains any known acronyms or synonyms.
224pub fn should_expand(query: &str) -> bool {
225    let lower = query.to_lowercase();
226    let tokens: Vec<&str> = lower.split_whitespace().collect();
227
228    // Check acronyms
229    for (abbrev, _) in ACRONYMS {
230        if tokens
231            .iter()
232            .any(|t| t.trim_matches(|c: char| c.is_ascii_punctuation()) == *abbrev)
233        {
234            return true;
235        }
236    }
237
238    // Check synonyms
239    for (a, b) in SYNONYMS {
240        if tokens.iter().any(|t| {
241            let clean = t.trim_matches(|c: char| c.is_ascii_punctuation());
242            clean == *a || clean == *b
243        }) {
244            return true;
245        }
246    }
247
248    false
249}
250
251/// Generate perspective-aware sub-queries using AI.
252///
253/// For ambiguous or multi-domain queries, generates 3-4 domain-specific sub-queries
254/// that cover the main interpretive angles (scientific, religious, philosophical,
255/// historical, technical, etc.). Returns the original query if AI is unavailable.
256///
257/// ## Example
258/// Query: "who created the world"
259/// Perspectives: ["Big Bang cosmology universe origin scientific",
260///                "Earth geological formation history",
261///                "religious creation myths major world religions",
262///                "philosophical cosmological argument origin creation"]
263pub async fn ai_perspective_expand(query: &str, ai_config: &AiConfig) -> Vec<String> {
264    let system = "You are a search query expansion expert. Given a user query, output 3 specific sub-queries covering different perspectives or domains. Rules: each sub-query covers a DIFFERENT angle (scientific, religious, philosophical, technical, historical, etc.); use domain-specific terminology; output ONLY the sub-queries, one per line, no numbering or labels; do NOT repeat the original query; keep each under 10 words.";
265    let messages = vec![
266        ChatMessage {
267            role: "system".into(),
268            content: system.to_string(),
269        },
270        ChatMessage {
271            role: "user".into(),
272            content: format!("Query: {query}\nSub-queries:"),
273        },
274    ];
275
276    let providers = ai_config.providers.clone();
277    let ai_cfg = ai_config.clone();
278
279    // Thread isolation: ai_config contains non-Send elements via dyn FnMut
280    let (tx, rx) = tokio::sync::oneshot::channel::<Vec<String>>();
281    std::thread::spawn(move || {
282        let rt = tokio::runtime::Builder::new_current_thread()
283            .enable_all()
284            .build()
285            .unwrap();
286        let result = rt.block_on(async move {
287            let mut noop = |_: &str| {};
288            match tokio::time::timeout(
289                std::time::Duration::from_secs(6),
290                chat_with_fallback(&messages, None, &ai_cfg, &providers, false, &mut noop),
291            )
292            .await
293            {
294                Ok(Ok(resp)) => resp
295                    .content
296                    .lines()
297                    .map(|l| l.trim().to_string())
298                    .filter(|l| !l.is_empty() && l.len() > 5)
299                    .take(4)
300                    .collect::<Vec<_>>(),
301                _ => vec![],
302            }
303        });
304        let _ = tx.send(result);
305    });
306
307    rx.await.unwrap_or_default()
308}
309
310#[cfg(test)]
311mod tests {
312    use super::*;
313
314    #[test]
315    fn js_expands_to_javascript() {
316        let expanded = expand_query("JS frameworks comparison");
317        assert!(
318            expanded
319                .expansions
320                .iter()
321                .any(|v| v.text.contains("javascript")),
322            "JS should expand to JavaScript, got: {:?}",
323            expanded.expansions
324        );
325    }
326
327    #[test]
328    fn ml_expands_to_machine_learning() {
329        let expanded = expand_query("ML models for text");
330        assert!(
331            expanded
332                .expansions
333                .iter()
334                .any(|v| v.text.contains("machine learning")),
335            "ML should expand to machine learning"
336        );
337    }
338
339    #[test]
340    fn k8s_expands_to_kubernetes() {
341        let expanded = expand_query("k8s deployment guide");
342        assert!(
343            expanded
344                .expansions
345                .iter()
346                .any(|v| v.text.contains("kubernetes")),
347            "k8s should expand to kubernetes"
348        );
349    }
350
351    #[test]
352    fn synonym_expansion() {
353        let expanded = expand_query("fast database library");
354        let has_quick = expanded.expansions.iter().any(|v| v.text.contains("quick"));
355        let has_package = expanded
356            .expansions
357            .iter()
358            .any(|v| v.text.contains("package"));
359        assert!(
360            has_quick || has_package,
361            "Should expand synonyms: {:?}",
362            expanded.expansions
363        );
364    }
365
366    #[test]
367    fn no_expansion_for_unknown() {
368        let expanded = expand_query("quantum entanglement theory");
369        assert!(
370            expanded.expansions.is_empty(),
371            "Unknown terms should not expand"
372        );
373    }
374
375    #[test]
376    fn preserves_original() {
377        let expanded = expand_query("JS tutorial");
378        assert_eq!(expanded.original, "JS tutorial");
379    }
380
381    #[test]
382    fn max_three_expansions() {
383        let expanded = expand_query("fast ML api tutorial");
384        assert!(
385            expanded.expansions.len() <= 3,
386            "Should limit to 3 expansions, got {}",
387            expanded.expansions.len()
388        );
389    }
390
391    #[test]
392    fn should_expand_detects_acronyms() {
393        assert!(should_expand("JS frameworks"));
394        assert!(should_expand("deploy ML model"));
395        assert!(!should_expand("quantum physics"));
396    }
397
398    #[test]
399    fn deduplicates_expansions() {
400        let expanded = expand_query("JS JS tutorial");
401        let texts: Vec<&str> = expanded
402            .expansions
403            .iter()
404            .map(|v| v.text.as_str())
405            .collect();
406        let unique: std::collections::HashSet<&&str> = texts.iter().collect();
407        assert_eq!(
408            texts.len(),
409            unique.len(),
410            "Should not have duplicate expansions"
411        );
412    }
413
414    #[test]
415    fn confidence_ordering() {
416        let expanded = expand_query("fast JS api");
417        for i in 1..expanded.expansions.len() {
418            assert!(
419                expanded.expansions[i - 1].confidence >= expanded.expansions[i].confidence,
420                "Expansions should be sorted by confidence"
421            );
422        }
423    }
424}