Skip to main content

fetchium_core/
api_facade.rs

1use crate::cache::MemoryCache;
2use crate::config::FetchiumConfig;
3use crate::error::FetchiumError;
4use crate::extract::pipeline::extract as cep_extract;
5use crate::http::client::HttpClient;
6use crate::rank::fusion::detect_intent;
7use crate::rank::{assess_quality, rerank, ConfidenceLevel};
8use crate::search::orchestrator::{OrchestratorConfig, SearchOrchestrator};
9use serde_json::{json, Value};
10use std::collections::HashSet;
11use std::time::{Duration, Instant};
12use uuid::Uuid;
13
14// ─── Helpers ──────────────────────────────────────────────────────────────────
15
16/// Truncate a string at a UTF-8 character boundary (safe for multibyte text like Spanish/French/CJK).
17/// Always returns a valid `&str` slice, never panics.
18#[inline]
19fn safe_trunc(s: &str, max_bytes: usize) -> &str {
20    if s.len() <= max_bytes {
21        return s;
22    }
23    // Walk back from max_bytes to the nearest char boundary
24    let mut end = max_bytes;
25    while end > 0 && !s.is_char_boundary(end) {
26        end -= 1;
27    }
28    &s[..end]
29}
30
31/// Inline cosine similarity (avoids feature-gated embeddings module dependency).
32fn cosine_sim(a: &[f32], b: &[f32]) -> f32 {
33    if a.len() != b.len() || a.is_empty() {
34        return 0.0;
35    }
36    let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
37    let na: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
38    let nb: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
39    if na == 0.0 || nb == 0.0 {
40        0.0
41    } else {
42        dot / (na * nb)
43    }
44}
45
46/// Strip markdown code fences and find JSON content in LLM output.
47fn extract_json_from_text(text: &str) -> String {
48    let text = text.trim();
49    if let Some(start) = text.find("```json") {
50        let after = &text[start + 7..];
51        if let Some(end) = after.find("```") {
52            return after[..end].trim().to_string();
53        }
54    }
55    if let Some(start) = text.find("```") {
56        let after = &text[start + 3..];
57        if let Some(end) = after.find("```") {
58            return after[..end].trim().to_string();
59        }
60    }
61    if let Some(pos) = text.find('{').or_else(|| text.find('[')) {
62        return text[pos..].to_string();
63    }
64    text.to_string()
65}
66
67/// Clean a search snippet: strip markdown, HTML tags, metadata noise.
68fn clean_snippet(raw: &str) -> String {
69    let mut s = raw.to_string();
70    // Strip bold/italic markdown
71    s = s
72        .replace("**", "")
73        .replace("__", "")
74        .replace('*', "")
75        .replace('_', " ");
76    // Strip HTML tags
77    let mut out = String::with_capacity(s.len());
78    let mut in_tag = false;
79    for c in s.chars() {
80        match c {
81            '<' => in_tag = true,
82            '>' => in_tag = false,
83            _ if !in_tag => out.push(c),
84            _ => {}
85        }
86    }
87    // Strip [text](url) links → keep text only
88    let mut result = String::new();
89    let mut chars = out.chars().peekable();
90    while let Some(c) = chars.next() {
91        if c == '[' {
92            let mut link_text = String::new();
93            while let Some(&nc) = chars.peek() {
94                chars.next();
95                if nc == ']' {
96                    break;
97                }
98                link_text.push(nc);
99            }
100            // Skip (url) part
101            if chars.peek() == Some(&'(') {
102                chars.next();
103                while let Some(&nc) = chars.peek() {
104                    chars.next();
105                    if nc == ')' {
106                        break;
107                    }
108                }
109            }
110            result.push_str(&link_text);
111        } else {
112            result.push(c);
113        }
114    }
115    // Normalize whitespace
116    result.split_whitespace().collect::<Vec<_>>().join(" ")
117}
118
119// ─── Semantic Reranking ───────────────────────────────────────────────────────
120
121/// Semantically rerank results: blend HyperFusion score (70%) + nomic-embed-text cosine (30%).
122/// Falls back to original order if Ollama is unavailable (500ms timeout).
123async fn semantic_rerank(
124    query: &str,
125    mut results: Vec<crate::types::ResultItem>,
126) -> Vec<crate::types::ResultItem> {
127    if results.len() <= 1 || is_health_sensitive_query(query) {
128        return results;
129    }
130
131    let ollama_url =
132        std::env::var("OLLAMA_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
133
134    let client = match reqwest::Client::builder()
135        .timeout(Duration::from_millis(500))
136        .build()
137    {
138        Ok(c) => c,
139        Err(_) => return results,
140    };
141
142    // Fast check if Ollama is even there
143    if tokio::time::timeout(Duration::from_millis(100), client.get(&ollama_url).send())
144        .await
145        .is_err()
146    {
147        return results;
148    }
149
150    let snippet_texts: Vec<String> = results
151        .iter()
152        .map(|r| format!("{} {}", r.title, r.snippet))
153        .collect();
154    let mut all_texts: Vec<&str> = vec![query];
155    all_texts.extend(snippet_texts.iter().map(|s| s.as_str()));
156
157    let body = serde_json::json!({ "model": "nomic-embed-text", "input": all_texts });
158
159    let resp = match tokio::time::timeout(
160        Duration::from_millis(800),
161        client
162            .post(format!("{ollama_url}/api/embed"))
163            .json(&body)
164            .send(),
165    )
166    .await
167    {
168        Ok(Ok(r)) => r,
169        _ => {
170            tracing::debug!("Semantic rerank: Ollama timed out, keeping HyperFusion order");
171            return results;
172        }
173    };
174
175    let data: serde_json::Value = match resp.json().await {
176        Ok(d) => d,
177        Err(_) => return results,
178    };
179
180    let embeddings: Vec<Vec<f32>> = match data["embeddings"].as_array() {
181        Some(arr) if arr.len() == all_texts.len() => arr
182            .iter()
183            .map(|emb| {
184                emb.as_array()
185                    .unwrap_or(&vec![])
186                    .iter()
187                    .map(|v| v.as_f64().unwrap_or(0.0) as f32)
188                    .collect()
189            })
190            .collect(),
191        _ => return results,
192    };
193
194    let query_emb = &embeddings[0];
195    for (i, result) in results.iter_mut().enumerate() {
196        let sem_sim = cosine_sim(query_emb, &embeddings[i + 1]) as f64;
197        let fusion_score = result.score.unwrap_or(0.5);
198        result.score = Some(0.7 * fusion_score + 0.3 * sem_sim);
199    }
200
201    results.sort_by(|a, b| {
202        b.score
203            .unwrap_or(0.0)
204            .partial_cmp(&a.score.unwrap_or(0.0))
205            .unwrap_or(std::cmp::Ordering::Equal)
206    });
207    tracing::debug!("Semantic rerank: reranked {} results", results.len());
208    results
209}
210
211fn is_health_sensitive_query(query: &str) -> bool {
212    let lower = query.to_lowercase();
213    [
214        "medication",
215        "medicine",
216        "drug",
217        "drugs",
218        "dose",
219        "dosage",
220        "side effect",
221        "side effects",
222        "symptom",
223        "symptoms",
224        "treatment",
225        "treatments",
226        "diagnosis",
227        "disease",
228        "blood pressure",
229        "hypertension",
230        "diabetes",
231        "vaccine",
232        "vaccination",
233        "therapy",
234        "pain",
235        "cancer",
236    ]
237    .iter()
238    .any(|pattern| lower.contains(pattern))
239}
240
241// ─── Semantic Fact Fusion ────────────────────────────────────────────────────
242
243/// Extract verified facts from top snippets via Qwen + multi-source keyword matching.
244/// Returns facts with confidence: 0.6 (1 source), 0.8 (2 sources), 0.95 (3+ sources).
245async fn fact_fusion(query: &str, snippets: &[(usize, &str)], http: &HttpClient) -> Vec<Value> {
246    if snippets.len() < 2 {
247        return Vec::new();
248    }
249
250    let combined: String = snippets
251        .iter()
252        .map(|(i, text)| format!("[Source {}]: {}", i, safe_trunc(text, 400)))
253        .collect::<Vec<_>>()
254        .join("\n\n");
255
256    let prompt = format!(
257        "Extract 3-5 key verifiable facts about '{}' from these snippets.\n\
258         Return ONLY a JSON array: [{{'fact': '...', 'keywords': ['word1','word2']}}]\n\n\
259         Snippets:\n{}\n\nJSON:",
260        query,
261        safe_trunc(&combined, 2000)
262    );
263
264    let body = serde_json::json!({
265        "model": "qwen3.5:2b",
266        "messages": [{"role": "user", "content": prompt}],
267        "stream": false,
268        "options": {"temperature": 0.1, "num_predict": 256}
269    });
270
271    let body_str = match serde_json::to_string(&body) {
272        Ok(s) => s,
273        Err(_) => return Vec::new(),
274    };
275
276    let resp_text = match tokio::time::timeout(
277        Duration::from_millis(1500),
278        http.post_json("http://localhost:11434/api/chat", &body_str),
279    )
280    .await
281    {
282        Ok(Ok(t)) => t,
283        _ => return Vec::new(),
284    };
285
286    let content = match serde_json::from_str::<Value>(&resp_text)
287        .ok()
288        .and_then(|v| {
289            v.get("message")?
290                .get("content")?
291                .as_str()
292                .map(|s| s.to_string())
293        }) {
294        Some(s) => s,
295        None => return Vec::new(),
296    };
297
298    let facts: Vec<Value> = match serde_json::from_str(&extract_json_from_text(&content)) {
299        Ok(v) => v,
300        Err(_) => return Vec::new(),
301    };
302
303    facts
304        .into_iter()
305        .filter_map(|fact_obj| {
306            let keywords: Vec<String> = fact_obj
307                .get("keywords")
308                .and_then(|k| k.as_array())
309                .map(|arr| {
310                    arr.iter()
311                        .filter_map(|v| v.as_str().map(|s| s.to_lowercase()))
312                        .collect()
313                })
314                .unwrap_or_default();
315
316            if keywords.is_empty() {
317                return None;
318            }
319
320            let source_indices: Vec<usize> = snippets
321                .iter()
322                .filter_map(|(idx, text)| {
323                    let text_lower = text.to_lowercase();
324                    let hits = keywords
325                        .iter()
326                        .filter(|kw| text_lower.contains(kw.as_str()))
327                        .count();
328                    if hits >= 1 {
329                        Some(*idx)
330                    } else {
331                        None
332                    }
333                })
334                .collect();
335
336            let confidence: f64 = match source_indices.len() {
337                0 => return None,
338                1 => 0.6,
339                2 => 0.8,
340                _ => 0.95,
341            };
342
343            Some(json!({
344                "fact": fact_obj.get("fact").unwrap_or(&Value::Null),
345                "confidence": confidence,
346                "sources": source_indices,
347            }))
348        })
349        .collect()
350}
351
352// ─── Structured Extraction ────────────────────────────────────────────────────
353
354/// Extract structured data from content using local Qwen model via Ollama.
355async fn extract_structured(content: &str, schema: &Value, http: &HttpClient) -> Option<Value> {
356    let excerpt: String = content.chars().take(12000).collect();
357    let prompt = format!(
358        "Extract structured data per this JSON schema.\nReturn ONLY valid JSON.\n\nSchema: {schema}\n\nContent:\n{excerpt}\n\nJSON:"
359    );
360
361    let body = serde_json::json!({
362        "model": "qwen3.5:2b",
363        "messages": [{"role": "user", "content": prompt}],
364        "stream": false,
365        "options": {"temperature": 0.1, "num_predict": 1024}
366    });
367
368    let body_str = serde_json::to_string(&body).ok()?;
369    let resp_text = tokio::time::timeout(
370        Duration::from_secs(30),
371        http.post_json("http://localhost:11434/api/chat", &body_str),
372    )
373    .await
374    .ok()?
375    .ok()?;
376
377    let content_str = serde_json::from_str::<Value>(&resp_text)
378        .ok()?
379        .get("message")?
380        .get("content")?
381        .as_str()?
382        .to_string();
383
384    serde_json::from_str(&extract_json_from_text(&content_str)).ok()
385}
386
387use crate::intelligence::acs::AdversarialContentShield;
388use crate::intelligence::crp::{
389    resolve as crp_resolve, shared_word_ratio, CrpContradiction, Severity,
390};
391
392// ─── Search Pipeline ──────────────────────────────────────────────────────────
393
394/// Execute a search pipeline: Orchestrator → Semantic Rerank → Fact Fusion → Cache
395///
396/// Fast path (key_facts/summary): uses search snippets directly.
397/// Rich path (detailed/complete/include_content): fetches URLs in parallel.
398pub struct SearchRequest<'a> {
399    pub query: &'a str,
400    pub max_sources: u32,
401    pub tier: &'a str,
402    pub token_budget: usize,
403    pub include_content: bool,
404}
405
406pub async fn search(
407    request: SearchRequest<'_>,
408    config: &FetchiumConfig,
409    http: &HttpClient,
410    cache: Option<&MemoryCache>,
411) -> Result<Value, FetchiumError> {
412    let SearchRequest {
413        query,
414        max_sources,
415        tier,
416        token_budget,
417        include_content,
418    } = request;
419
420    // Check cache first for high-throughput repeated queries
421    let cache_key = format!(
422        "search:{}:{}:{}:{}:{}",
423        query, max_sources, tier, token_budget, include_content
424    );
425    if let Some(c) = cache {
426        if let Some(cached) = c.get::<Value>(&cache_key).await {
427            tracing::debug!("Search cache hit: {}", query);
428            let mut response = cached;
429            if let Some(meta) = response.get_mut("meta") {
430                meta["from_cache"] = json!(true);
431            }
432            return Ok(response);
433        }
434    }
435
436    let start = Instant::now();
437    let retrieval_max_sources = if needs_deeper_recall_query(query) {
438        max_sources.max(10)
439    } else {
440        max_sources
441    };
442
443    let orch_config = OrchestratorConfig::from_fetchium_config(config, retrieval_max_sources);
444    let orchestrator = SearchOrchestrator::new(http.clone(), orch_config);
445    let mut results = orchestrator
446        .search(query, Some(retrieval_max_sources))
447        .await?;
448
449    let unique_domains = count_unique_domains(&results);
450    let multilingual_query = query.chars().any(|c| c.is_alphabetic() && !c.is_ascii());
451    let strong_recall = results.len() >= max_sources as usize;
452    let adequate_recall_diversity = results.len() >= 6 && unique_domains >= 4;
453    let sparse_recall = results.len() < 3 || unique_domains < 2;
454    let quality = assess_quality(&results, query);
455    if matches!(
456        quality.confidence,
457        ConfidenceLevel::Low | ConfidenceLevel::VeryLow
458    ) && sparse_recall
459        && !is_expensive_comparison_query(query)
460        && !(strong_recall
461            || adequate_recall_diversity
462            || (multilingual_query && results.len() >= 8))
463    {
464        let mut seen_urls: HashSet<String> = results.iter().map(|r| r.url.clone()).collect();
465        for corrective_query in generate_corrective_queries(query).into_iter().take(1) {
466            let corrective_results = orchestrator
467                .search(&corrective_query, Some(retrieval_max_sources.min(6)))
468                .await
469                .unwrap_or_default();
470            for mut result in corrective_results {
471                if seen_urls.insert(result.url.clone()) {
472                    result.backend = crate::types::BackendId::Searxng;
473                    results.push(result);
474                }
475            }
476        }
477        results = rerank(results, query);
478        results.truncate(retrieval_max_sources as usize);
479        for (idx, result) in results.iter_mut().enumerate() {
480            result.rank = (idx + 1) as u32;
481        }
482    }
483
484    let result_id = Uuid::new_v4().to_string();
485    let needs_extraction = include_content || matches!(tier, "detailed" | "complete");
486    let max_content_chars = if include_content {
487        token_budget * 4
488    } else {
489        800
490    };
491
492    // Parallelize reranking, fact fusion, and ACS analysis
493    let results_for_rerank = results.clone();
494    let query_for_rerank = query.to_string();
495    let rerank_handle =
496        tokio::spawn(async move { semantic_rerank(&query_for_rerank, results_for_rerank).await });
497
498    let top_snippets: Vec<(usize, String)> = results
499        .iter()
500        .take(5)
501        .enumerate()
502        .map(|(i, r)| (i + 1, r.snippet.clone()))
503        .collect();
504    let http_ff = http.clone();
505    let query_owned = query.to_string();
506    let fact_handle = tokio::spawn(async move {
507        let refs: Vec<(usize, &str)> = top_snippets.iter().map(|(i, s)| (*i, s.as_str())).collect();
508        fact_fusion(&query_owned, &refs, &http_ff).await
509    });
510
511    // Initialize ACS
512    let acs = AdversarialContentShield::new();
513
514    let mut items: Vec<Value> = if needs_extraction {
515        // Parallel URL fetching + CEP extraction (starts immediately, doesn't wait for rerank)
516        let mut handles = std::collections::HashMap::with_capacity(results.len());
517        for r in &results {
518            let http2 = http.clone();
519            let url = r.url.clone();
520            let fallback = r.snippet.clone();
521            let handle = tokio::spawn(async move {
522                match tokio::time::timeout(Duration::from_secs(5), http2.fetch_text(&url)).await {
523                    Ok(Ok(html)) if !html.is_empty() => {
524                        let ext = cep_extract(&html, &url);
525                        let extracted: String = ext.text.chars().take(max_content_chars).collect();
526                        (fallback, Some(extracted))
527                    }
528                    _ => (fallback, None),
529                }
530            });
531            handles.insert(r.url.clone(), handle);
532        }
533
534        // Wait for rerank result to know the final order
535        let reranked_results = rerank_handle.await.unwrap_or_else(|_| results.clone());
536
537        let mut items = Vec::with_capacity(reranked_results.len());
538        for (idx, r) in reranked_results.iter().enumerate() {
539            // Take the handle for this URL
540            let handle = handles.remove(&r.url);
541
542            // Wait for extraction if needed (but it was already running!)
543            let (snippet, extracted) = if let Some(h) = handle {
544                h.await.unwrap_or_else(|_| (r.snippet.clone(), None))
545            } else {
546                (r.snippet.clone(), None)
547            };
548
549            let clean = clean_snippet(&snippet);
550            let domain = HttpClient::extract_domain(&r.url);
551            let acs_result = acs.analyze(extracted.as_deref().unwrap_or(&clean), &domain);
552
553            let mut item = if include_content {
554                json!({
555                    "title": r.title,
556                    "url": r.url,
557                    "snippet": clean,
558                    "score": r.score,
559                    "content": extracted,
560                    "source_index": idx + 1,
561                    "trust_score": acs_result.trust_score,
562                    "acs_flags": acs_result.flags,
563                })
564            } else {
565                json!({
566                    "title": r.title,
567                    "url": r.url,
568                    "snippet": extracted.unwrap_or(clean),
569                    "score": r.score,
570                    "source_index": idx + 1,
571                    "trust_score": acs_result.trust_score,
572                    "acs_flags": acs_result.flags,
573                })
574            };
575            if let Some(ref date) = r.published_date {
576                item["published_date"] = json!(date);
577            }
578            items.push(item);
579        }
580        items
581    } else {
582        // Fast path: wait for rerank then build items
583        let reranked_results = rerank_handle.await.unwrap_or(results.clone());
584        reranked_results
585            .iter()
586            .enumerate()
587            .map(|(idx, r)| {
588                let domain = HttpClient::extract_domain(&r.url);
589                let acs_result = acs.analyze(&r.snippet, &domain);
590                let mut item = json!({
591                    "title": r.title,
592                    "url": r.url,
593                    "snippet": clean_snippet(&r.snippet),
594                    "score": r.score,
595                    "source_index": idx + 1,
596                    "trust_score": acs_result.trust_score,
597                    "acs_flags": acs_result.flags,
598                });
599                if let Some(ref date) = r.published_date {
600                    item["published_date"] = json!(date);
601                }
602                item
603            })
604            .collect()
605    };
606
607    if items.len() > max_sources as usize {
608        items.truncate(max_sources as usize);
609        for (idx, item) in items.iter_mut().enumerate() {
610            item["source_index"] = json!(idx + 1);
611        }
612    }
613
614    // --- CRP: Contradiction Resolution ---
615    // If we have at least 2 top results, check for simple contradictions
616    let mut contradictions = Vec::new();
617    if items.len() >= 2 {
618        let claim_a = items[0]["snippet"].as_str().unwrap_or("");
619        let claim_b = items[1]["snippet"].as_str().unwrap_or("");
620
621        // Simple heuristic: if they share keywords but have different numbers or negations
622        let share_keywords = shared_word_ratio(claim_a, claim_b) > 0.3;
623        let contains_negation = (claim_a.contains(" not ") || claim_a.contains(" no "))
624            != (claim_b.contains(" not ") || claim_b.contains(" no "));
625
626        if share_keywords && contains_negation {
627            let c = CrpContradiction {
628                claim_a: claim_a.to_string(),
629                source_a_domain: HttpClient::extract_domain(items[0]["url"].as_str().unwrap_or("")),
630                source_a_trust: items[0]["trust_score"].as_f64().unwrap_or(0.5),
631                source_a_date: items[0]
632                    .get("published_date")
633                    .and_then(|v| v.as_str())
634                    .map(|s| s.to_string()),
635                claim_b: claim_b.to_string(),
636                source_b_domain: HttpClient::extract_domain(items[1]["url"].as_str().unwrap_or("")),
637                source_b_trust: items[1]["trust_score"].as_f64().unwrap_or(0.5),
638                source_b_date: items[1]
639                    .get("published_date")
640                    .and_then(|v| v.as_str())
641                    .map(|s| s.to_string()),
642                severity: Severity::Medium,
643            };
644            if let Ok(res) = crp_resolve(&c, |_| 0.5) {
645                contradictions.push(res);
646            }
647        }
648    }
649
650    // Predictive pre-fetch: fire-and-forget background fetch of top-3 URLs
651    if !needs_extraction {
652        let top3: Vec<String> = results.iter().take(3).map(|r| r.url.clone()).collect();
653        if let Some(c) = cache {
654            for url in top3 {
655                let http2 = http.clone();
656                let c2 = c.clone();
657                tokio::spawn(async move {
658                    tokio::time::sleep(Duration::from_millis(200)).await;
659                    if let Ok(Ok(html)) =
660                        tokio::time::timeout(Duration::from_secs(8), http2.fetch_text(&url)).await
661                    {
662                        if html.len() > 500 {
663                            let ext = cep_extract(&html, &url);
664                            let data = json!({
665                                "url": &url,
666                                "title": ext.title,
667                                "content": safe_trunc(&ext.text, 8000),
668                                "prefetched": true,
669                            });
670                            c2.set(&format!("prefetch:{url}"), &data).await;
671                        }
672                    }
673                });
674            }
675        }
676    }
677
678    // Collect fact fusion result (was running concurrently with item-building)
679    let verified_facts = fact_handle.await.unwrap_or_default();
680
681    // Citations list for every result
682    let citations: Vec<Value> = items
683        .iter()
684        .enumerate()
685        .map(|(i, item)| {
686            json!({
687                "index": i + 1,
688                "title": item.get("title").and_then(|v| v.as_str()).unwrap_or(""),
689                "url":   item.get("url").and_then(|v| v.as_str()).unwrap_or(""),
690            })
691        })
692        .collect();
693
694    let duration_ms = start.elapsed().as_millis() as u64;
695    let intent = detect_intent(query);
696    let intent_str = format!("{:?}", intent).to_lowercase();
697
698    let response = json!({
699        "meta": {
700            "query": query,
701            "tier": tier,
702            "intent": intent_str,
703            "tokens_used": token_budget,
704            "sources_count": items.len(),
705            "duration_ms": duration_ms,
706            "result_id": result_id,
707            "credits_used": 1,
708            "citations": citations,
709            "verified_facts": verified_facts,
710            "contradictions": contradictions,
711        },
712        "results": items,
713    });
714
715    if let Some(c) = cache {
716        c.set(&format!("expand:{result_id}"), &response).await;
717        c.set(&cache_key, &response).await;
718    }
719
720    Ok(response)
721}
722
723fn needs_deeper_recall_query(query: &str) -> bool {
724    let lower = query.to_lowercase();
725    lower.contains(" vs ")
726        || lower.contains("compare")
727        || lower.contains("comparison")
728        || lower.contains("versus")
729        || lower.contains("status code")
730        || lower.contains("http ")
731        || lower.contains("rfc ")
732        || lower.contains("error code")
733        || lower.contains("medication")
734        || lower.contains("medicine")
735        || lower.contains("drug")
736        || lower.contains("treatment")
737        || lower.contains("diagnosis")
738        || lower.contains("symptoms")
739        || lower.contains("blood pressure")
740        || lower.contains("hypertension")
741        || lower.contains("vaccine")
742        || query.split_whitespace().count() >= 8
743}
744
745fn is_expensive_comparison_query(query: &str) -> bool {
746    let lower = query.to_lowercase();
747    lower.contains(" vs ")
748        || lower.contains("versus")
749        || lower.contains("comparison")
750        || lower.contains("compare")
751}
752
753fn generate_corrective_queries(query: &str) -> Vec<String> {
754    let q = query.trim();
755    let lower = q.to_lowercase();
756    let mut out = Vec::new();
757
758    if !lower.contains("overview") {
759        out.push(format!("{q} overview"));
760    }
761    if !lower.contains("explained") {
762        out.push(format!("{q} explained"));
763    }
764
765    let stripped = q
766        .split_whitespace()
767        .filter(|word| {
768            let lower = word.to_lowercase();
769            !matches!(
770                lower.as_str(),
771                "latest" | "recent" | "today" | "this" | "year" | "month" | "week"
772            ) && !lower.chars().all(|c| c.is_ascii_digit())
773        })
774        .collect::<Vec<_>>()
775        .join(" ");
776    if !stripped.is_empty() && stripped != q {
777        out.push(stripped);
778    }
779
780    out
781}
782
783fn count_unique_domains(items: &[crate::types::ResultItem]) -> usize {
784    items
785        .iter()
786        .filter_map(|item| item.url.split('/').nth(2))
787        .collect::<HashSet<_>>()
788        .len()
789}
790
791// ─── Fetch Pipeline ───────────────────────────────────────────────────────────
792
793/// Execute a fetch pipeline: Fetch → CEP Extract → (optional Qwen schema extraction) → Cache
794pub async fn fetch(
795    url: &str,
796    token_budget: usize,
797    format: &str,
798    http: &HttpClient,
799    cache: Option<&MemoryCache>,
800    schema: Option<&Value>,
801) -> Result<Value, FetchiumError> {
802    // Check prefetch cache first
803    if let Some(c) = cache {
804        if let Some(cached) = c.get::<Value>(&format!("prefetch:{url}")).await {
805            if let Some(content) = cached.get("content").and_then(|v| v.as_str()) {
806                let tokens = crate::extract::layer1::estimate_tokens(content) as usize;
807                let result_id = Uuid::new_v4().to_string();
808                return Ok(json!({
809                    "url": url,
810                    "title": cached.get("title"),
811                    "content": content,
812                    "tokens": tokens,
813                    "format": format,
814                    "result_id": result_id,
815                    "from_cache": true,
816                }));
817            }
818        }
819    }
820
821    let html = http.fetch_text(url).await?;
822    let html = if html.is_empty() {
823        return Err(FetchiumError::Internal("Empty response from URL".into()));
824    } else {
825        html
826    };
827
828    let ext = cep_extract(&html, url);
829    let max_chars = token_budget * 4;
830    let content = if ext.text.len() > max_chars {
831        ext.text[..max_chars].to_string()
832    } else {
833        ext.text
834    };
835
836    let tokens = crate::extract::layer1::estimate_tokens(&content) as usize;
837    let result_id = Uuid::new_v4().to_string();
838
839    let structured = if let Some(s) = schema {
840        extract_structured(&content, s, http).await
841    } else {
842        None
843    };
844
845    let response = json!({
846        "url": url,
847        "title": if ext.title.is_empty() { Value::Null } else { json!(ext.title) },
848        "content": content,
849        "tokens": tokens,
850        "format": format,
851        "result_id": result_id,
852        "structured": structured,
853    });
854
855    if let Some(c) = cache {
856        c.set(&format!("expand:{result_id}"), &response).await;
857    }
858
859    Ok(response)
860}
861
862// ─── Expand ───────────────────────────────────────────────────────────────────
863
864/// Expand a previous result from the session cache.
865pub async fn expand(
866    result_id: &str,
867    tier: &str,
868    cache: Option<&MemoryCache>,
869) -> Result<Value, FetchiumError> {
870    if let Some(c) = cache {
871        if let Some(cached_data) = c.get::<Value>(&format!("expand:{result_id}")).await {
872            let mut expanded_data = cached_data;
873            if let Some(meta) = expanded_data.get_mut("meta") {
874                meta["tier"] = json!(tier);
875            }
876            return Ok(expanded_data);
877        }
878    }
879    Err(FetchiumError::Internal(
880        "Cache miss or cache not configured for session".into(),
881    ))
882}