1use crate::cache::MemoryCache;
2use crate::config::FetchiumConfig;
3use crate::error::FetchiumError;
4use crate::extract::pipeline::extract as cep_extract;
5use crate::http::client::HttpClient;
6use crate::rank::fusion::detect_intent;
7use crate::rank::{assess_quality, rerank, ConfidenceLevel};
8use crate::search::orchestrator::{OrchestratorConfig, SearchOrchestrator};
9use serde_json::{json, Value};
10use std::collections::HashSet;
11use std::time::{Duration, Instant};
12use uuid::Uuid;
13
14#[inline]
19fn safe_trunc(s: &str, max_bytes: usize) -> &str {
20 if s.len() <= max_bytes {
21 return s;
22 }
23 let mut end = max_bytes;
25 while end > 0 && !s.is_char_boundary(end) {
26 end -= 1;
27 }
28 &s[..end]
29}
30
31fn cosine_sim(a: &[f32], b: &[f32]) -> f32 {
33 if a.len() != b.len() || a.is_empty() {
34 return 0.0;
35 }
36 let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
37 let na: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
38 let nb: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
39 if na == 0.0 || nb == 0.0 {
40 0.0
41 } else {
42 dot / (na * nb)
43 }
44}
45
46fn extract_json_from_text(text: &str) -> String {
48 let text = text.trim();
49 if let Some(start) = text.find("```json") {
50 let after = &text[start + 7..];
51 if let Some(end) = after.find("```") {
52 return after[..end].trim().to_string();
53 }
54 }
55 if let Some(start) = text.find("```") {
56 let after = &text[start + 3..];
57 if let Some(end) = after.find("```") {
58 return after[..end].trim().to_string();
59 }
60 }
61 if let Some(pos) = text.find('{').or_else(|| text.find('[')) {
62 return text[pos..].to_string();
63 }
64 text.to_string()
65}
66
67fn clean_snippet(raw: &str) -> String {
69 let mut s = raw.to_string();
70 s = s
72 .replace("**", "")
73 .replace("__", "")
74 .replace('*', "")
75 .replace('_', " ");
76 let mut out = String::with_capacity(s.len());
78 let mut in_tag = false;
79 for c in s.chars() {
80 match c {
81 '<' => in_tag = true,
82 '>' => in_tag = false,
83 _ if !in_tag => out.push(c),
84 _ => {}
85 }
86 }
87 let mut result = String::new();
89 let mut chars = out.chars().peekable();
90 while let Some(c) = chars.next() {
91 if c == '[' {
92 let mut link_text = String::new();
93 while let Some(&nc) = chars.peek() {
94 chars.next();
95 if nc == ']' {
96 break;
97 }
98 link_text.push(nc);
99 }
100 if chars.peek() == Some(&'(') {
102 chars.next();
103 while let Some(&nc) = chars.peek() {
104 chars.next();
105 if nc == ')' {
106 break;
107 }
108 }
109 }
110 result.push_str(&link_text);
111 } else {
112 result.push(c);
113 }
114 }
115 result.split_whitespace().collect::<Vec<_>>().join(" ")
117}
118
119async fn semantic_rerank(
124 query: &str,
125 mut results: Vec<crate::types::ResultItem>,
126) -> Vec<crate::types::ResultItem> {
127 if results.len() <= 1 || is_health_sensitive_query(query) {
128 return results;
129 }
130
131 let ollama_url =
132 std::env::var("OLLAMA_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
133
134 let client = match reqwest::Client::builder()
135 .timeout(Duration::from_millis(500))
136 .build()
137 {
138 Ok(c) => c,
139 Err(_) => return results,
140 };
141
142 if tokio::time::timeout(Duration::from_millis(100), client.get(&ollama_url).send())
144 .await
145 .is_err()
146 {
147 return results;
148 }
149
150 let snippet_texts: Vec<String> = results
151 .iter()
152 .map(|r| format!("{} {}", r.title, r.snippet))
153 .collect();
154 let mut all_texts: Vec<&str> = vec![query];
155 all_texts.extend(snippet_texts.iter().map(|s| s.as_str()));
156
157 let body = serde_json::json!({ "model": "nomic-embed-text", "input": all_texts });
158
159 let resp = match tokio::time::timeout(
160 Duration::from_millis(800),
161 client
162 .post(format!("{ollama_url}/api/embed"))
163 .json(&body)
164 .send(),
165 )
166 .await
167 {
168 Ok(Ok(r)) => r,
169 _ => {
170 tracing::debug!("Semantic rerank: Ollama timed out, keeping HyperFusion order");
171 return results;
172 }
173 };
174
175 let data: serde_json::Value = match resp.json().await {
176 Ok(d) => d,
177 Err(_) => return results,
178 };
179
180 let embeddings: Vec<Vec<f32>> = match data["embeddings"].as_array() {
181 Some(arr) if arr.len() == all_texts.len() => arr
182 .iter()
183 .map(|emb| {
184 emb.as_array()
185 .unwrap_or(&vec![])
186 .iter()
187 .map(|v| v.as_f64().unwrap_or(0.0) as f32)
188 .collect()
189 })
190 .collect(),
191 _ => return results,
192 };
193
194 let query_emb = &embeddings[0];
195 for (i, result) in results.iter_mut().enumerate() {
196 let sem_sim = cosine_sim(query_emb, &embeddings[i + 1]) as f64;
197 let fusion_score = result.score.unwrap_or(0.5);
198 result.score = Some(0.7 * fusion_score + 0.3 * sem_sim);
199 }
200
201 results.sort_by(|a, b| {
202 b.score
203 .unwrap_or(0.0)
204 .partial_cmp(&a.score.unwrap_or(0.0))
205 .unwrap_or(std::cmp::Ordering::Equal)
206 });
207 tracing::debug!("Semantic rerank: reranked {} results", results.len());
208 results
209}
210
211fn is_health_sensitive_query(query: &str) -> bool {
212 let lower = query.to_lowercase();
213 [
214 "medication",
215 "medicine",
216 "drug",
217 "drugs",
218 "dose",
219 "dosage",
220 "side effect",
221 "side effects",
222 "symptom",
223 "symptoms",
224 "treatment",
225 "treatments",
226 "diagnosis",
227 "disease",
228 "blood pressure",
229 "hypertension",
230 "diabetes",
231 "vaccine",
232 "vaccination",
233 "therapy",
234 "pain",
235 "cancer",
236 ]
237 .iter()
238 .any(|pattern| lower.contains(pattern))
239}
240
241async fn fact_fusion(query: &str, snippets: &[(usize, &str)], http: &HttpClient) -> Vec<Value> {
246 if snippets.len() < 2 {
247 return Vec::new();
248 }
249
250 let combined: String = snippets
251 .iter()
252 .map(|(i, text)| format!("[Source {}]: {}", i, safe_trunc(text, 400)))
253 .collect::<Vec<_>>()
254 .join("\n\n");
255
256 let prompt = format!(
257 "Extract 3-5 key verifiable facts about '{}' from these snippets.\n\
258 Return ONLY a JSON array: [{{'fact': '...', 'keywords': ['word1','word2']}}]\n\n\
259 Snippets:\n{}\n\nJSON:",
260 query,
261 safe_trunc(&combined, 2000)
262 );
263
264 let body = serde_json::json!({
265 "model": "qwen3.5:2b",
266 "messages": [{"role": "user", "content": prompt}],
267 "stream": false,
268 "options": {"temperature": 0.1, "num_predict": 256}
269 });
270
271 let body_str = match serde_json::to_string(&body) {
272 Ok(s) => s,
273 Err(_) => return Vec::new(),
274 };
275
276 let resp_text = match tokio::time::timeout(
277 Duration::from_millis(1500),
278 http.post_json("http://localhost:11434/api/chat", &body_str),
279 )
280 .await
281 {
282 Ok(Ok(t)) => t,
283 _ => return Vec::new(),
284 };
285
286 let content = match serde_json::from_str::<Value>(&resp_text)
287 .ok()
288 .and_then(|v| {
289 v.get("message")?
290 .get("content")?
291 .as_str()
292 .map(|s| s.to_string())
293 }) {
294 Some(s) => s,
295 None => return Vec::new(),
296 };
297
298 let facts: Vec<Value> = match serde_json::from_str(&extract_json_from_text(&content)) {
299 Ok(v) => v,
300 Err(_) => return Vec::new(),
301 };
302
303 facts
304 .into_iter()
305 .filter_map(|fact_obj| {
306 let keywords: Vec<String> = fact_obj
307 .get("keywords")
308 .and_then(|k| k.as_array())
309 .map(|arr| {
310 arr.iter()
311 .filter_map(|v| v.as_str().map(|s| s.to_lowercase()))
312 .collect()
313 })
314 .unwrap_or_default();
315
316 if keywords.is_empty() {
317 return None;
318 }
319
320 let source_indices: Vec<usize> = snippets
321 .iter()
322 .filter_map(|(idx, text)| {
323 let text_lower = text.to_lowercase();
324 let hits = keywords
325 .iter()
326 .filter(|kw| text_lower.contains(kw.as_str()))
327 .count();
328 if hits >= 1 {
329 Some(*idx)
330 } else {
331 None
332 }
333 })
334 .collect();
335
336 let confidence: f64 = match source_indices.len() {
337 0 => return None,
338 1 => 0.6,
339 2 => 0.8,
340 _ => 0.95,
341 };
342
343 Some(json!({
344 "fact": fact_obj.get("fact").unwrap_or(&Value::Null),
345 "confidence": confidence,
346 "sources": source_indices,
347 }))
348 })
349 .collect()
350}
351
352async fn extract_structured(content: &str, schema: &Value, http: &HttpClient) -> Option<Value> {
356 let excerpt: String = content.chars().take(12000).collect();
357 let prompt = format!(
358 "Extract structured data per this JSON schema.\nReturn ONLY valid JSON.\n\nSchema: {schema}\n\nContent:\n{excerpt}\n\nJSON:"
359 );
360
361 let body = serde_json::json!({
362 "model": "qwen3.5:2b",
363 "messages": [{"role": "user", "content": prompt}],
364 "stream": false,
365 "options": {"temperature": 0.1, "num_predict": 1024}
366 });
367
368 let body_str = serde_json::to_string(&body).ok()?;
369 let resp_text = tokio::time::timeout(
370 Duration::from_secs(30),
371 http.post_json("http://localhost:11434/api/chat", &body_str),
372 )
373 .await
374 .ok()?
375 .ok()?;
376
377 let content_str = serde_json::from_str::<Value>(&resp_text)
378 .ok()?
379 .get("message")?
380 .get("content")?
381 .as_str()?
382 .to_string();
383
384 serde_json::from_str(&extract_json_from_text(&content_str)).ok()
385}
386
387use crate::intelligence::acs::AdversarialContentShield;
388use crate::intelligence::crp::{
389 resolve as crp_resolve, shared_word_ratio, CrpContradiction, Severity,
390};
391
392pub struct SearchRequest<'a> {
399 pub query: &'a str,
400 pub max_sources: u32,
401 pub tier: &'a str,
402 pub token_budget: usize,
403 pub include_content: bool,
404}
405
406pub async fn search(
407 request: SearchRequest<'_>,
408 config: &FetchiumConfig,
409 http: &HttpClient,
410 cache: Option<&MemoryCache>,
411) -> Result<Value, FetchiumError> {
412 let SearchRequest {
413 query,
414 max_sources,
415 tier,
416 token_budget,
417 include_content,
418 } = request;
419
420 let cache_key = format!(
422 "search:{}:{}:{}:{}:{}",
423 query, max_sources, tier, token_budget, include_content
424 );
425 if let Some(c) = cache {
426 if let Some(cached) = c.get::<Value>(&cache_key).await {
427 tracing::debug!("Search cache hit: {}", query);
428 let mut response = cached;
429 if let Some(meta) = response.get_mut("meta") {
430 meta["from_cache"] = json!(true);
431 }
432 return Ok(response);
433 }
434 }
435
436 let start = Instant::now();
437 let retrieval_max_sources = if needs_deeper_recall_query(query) {
438 max_sources.max(10)
439 } else {
440 max_sources
441 };
442
443 let orch_config = OrchestratorConfig::from_fetchium_config(config, retrieval_max_sources);
444 let orchestrator = SearchOrchestrator::new(http.clone(), orch_config);
445 let mut results = orchestrator
446 .search(query, Some(retrieval_max_sources))
447 .await?;
448
449 let unique_domains = count_unique_domains(&results);
450 let multilingual_query = query.chars().any(|c| c.is_alphabetic() && !c.is_ascii());
451 let strong_recall = results.len() >= max_sources as usize;
452 let adequate_recall_diversity = results.len() >= 6 && unique_domains >= 4;
453 let sparse_recall = results.len() < 3 || unique_domains < 2;
454 let quality = assess_quality(&results, query);
455 if matches!(
456 quality.confidence,
457 ConfidenceLevel::Low | ConfidenceLevel::VeryLow
458 ) && sparse_recall
459 && !is_expensive_comparison_query(query)
460 && !(strong_recall
461 || adequate_recall_diversity
462 || (multilingual_query && results.len() >= 8))
463 {
464 let mut seen_urls: HashSet<String> = results.iter().map(|r| r.url.clone()).collect();
465 for corrective_query in generate_corrective_queries(query).into_iter().take(1) {
466 let corrective_results = orchestrator
467 .search(&corrective_query, Some(retrieval_max_sources.min(6)))
468 .await
469 .unwrap_or_default();
470 for mut result in corrective_results {
471 if seen_urls.insert(result.url.clone()) {
472 result.backend = crate::types::BackendId::Searxng;
473 results.push(result);
474 }
475 }
476 }
477 results = rerank(results, query);
478 results.truncate(retrieval_max_sources as usize);
479 for (idx, result) in results.iter_mut().enumerate() {
480 result.rank = (idx + 1) as u32;
481 }
482 }
483
484 let result_id = Uuid::new_v4().to_string();
485 let needs_extraction = include_content || matches!(tier, "detailed" | "complete");
486 let max_content_chars = if include_content {
487 token_budget * 4
488 } else {
489 800
490 };
491
492 let results_for_rerank = results.clone();
494 let query_for_rerank = query.to_string();
495 let rerank_handle =
496 tokio::spawn(async move { semantic_rerank(&query_for_rerank, results_for_rerank).await });
497
498 let top_snippets: Vec<(usize, String)> = results
499 .iter()
500 .take(5)
501 .enumerate()
502 .map(|(i, r)| (i + 1, r.snippet.clone()))
503 .collect();
504 let http_ff = http.clone();
505 let query_owned = query.to_string();
506 let fact_handle = tokio::spawn(async move {
507 let refs: Vec<(usize, &str)> = top_snippets.iter().map(|(i, s)| (*i, s.as_str())).collect();
508 fact_fusion(&query_owned, &refs, &http_ff).await
509 });
510
511 let acs = AdversarialContentShield::new();
513
514 let mut items: Vec<Value> = if needs_extraction {
515 let mut handles = std::collections::HashMap::with_capacity(results.len());
517 for r in &results {
518 let http2 = http.clone();
519 let url = r.url.clone();
520 let fallback = r.snippet.clone();
521 let handle = tokio::spawn(async move {
522 match tokio::time::timeout(Duration::from_secs(5), http2.fetch_text(&url)).await {
523 Ok(Ok(html)) if !html.is_empty() => {
524 let ext = cep_extract(&html, &url);
525 let extracted: String = ext.text.chars().take(max_content_chars).collect();
526 (fallback, Some(extracted))
527 }
528 _ => (fallback, None),
529 }
530 });
531 handles.insert(r.url.clone(), handle);
532 }
533
534 let reranked_results = rerank_handle.await.unwrap_or_else(|_| results.clone());
536
537 let mut items = Vec::with_capacity(reranked_results.len());
538 for (idx, r) in reranked_results.iter().enumerate() {
539 let handle = handles.remove(&r.url);
541
542 let (snippet, extracted) = if let Some(h) = handle {
544 h.await.unwrap_or_else(|_| (r.snippet.clone(), None))
545 } else {
546 (r.snippet.clone(), None)
547 };
548
549 let clean = clean_snippet(&snippet);
550 let domain = HttpClient::extract_domain(&r.url);
551 let acs_result = acs.analyze(extracted.as_deref().unwrap_or(&clean), &domain);
552
553 let mut item = if include_content {
554 json!({
555 "title": r.title,
556 "url": r.url,
557 "snippet": clean,
558 "score": r.score,
559 "content": extracted,
560 "source_index": idx + 1,
561 "trust_score": acs_result.trust_score,
562 "acs_flags": acs_result.flags,
563 })
564 } else {
565 json!({
566 "title": r.title,
567 "url": r.url,
568 "snippet": extracted.unwrap_or(clean),
569 "score": r.score,
570 "source_index": idx + 1,
571 "trust_score": acs_result.trust_score,
572 "acs_flags": acs_result.flags,
573 })
574 };
575 if let Some(ref date) = r.published_date {
576 item["published_date"] = json!(date);
577 }
578 items.push(item);
579 }
580 items
581 } else {
582 let reranked_results = rerank_handle.await.unwrap_or(results.clone());
584 reranked_results
585 .iter()
586 .enumerate()
587 .map(|(idx, r)| {
588 let domain = HttpClient::extract_domain(&r.url);
589 let acs_result = acs.analyze(&r.snippet, &domain);
590 let mut item = json!({
591 "title": r.title,
592 "url": r.url,
593 "snippet": clean_snippet(&r.snippet),
594 "score": r.score,
595 "source_index": idx + 1,
596 "trust_score": acs_result.trust_score,
597 "acs_flags": acs_result.flags,
598 });
599 if let Some(ref date) = r.published_date {
600 item["published_date"] = json!(date);
601 }
602 item
603 })
604 .collect()
605 };
606
607 if items.len() > max_sources as usize {
608 items.truncate(max_sources as usize);
609 for (idx, item) in items.iter_mut().enumerate() {
610 item["source_index"] = json!(idx + 1);
611 }
612 }
613
614 let mut contradictions = Vec::new();
617 if items.len() >= 2 {
618 let claim_a = items[0]["snippet"].as_str().unwrap_or("");
619 let claim_b = items[1]["snippet"].as_str().unwrap_or("");
620
621 let share_keywords = shared_word_ratio(claim_a, claim_b) > 0.3;
623 let contains_negation = (claim_a.contains(" not ") || claim_a.contains(" no "))
624 != (claim_b.contains(" not ") || claim_b.contains(" no "));
625
626 if share_keywords && contains_negation {
627 let c = CrpContradiction {
628 claim_a: claim_a.to_string(),
629 source_a_domain: HttpClient::extract_domain(items[0]["url"].as_str().unwrap_or("")),
630 source_a_trust: items[0]["trust_score"].as_f64().unwrap_or(0.5),
631 source_a_date: items[0]
632 .get("published_date")
633 .and_then(|v| v.as_str())
634 .map(|s| s.to_string()),
635 claim_b: claim_b.to_string(),
636 source_b_domain: HttpClient::extract_domain(items[1]["url"].as_str().unwrap_or("")),
637 source_b_trust: items[1]["trust_score"].as_f64().unwrap_or(0.5),
638 source_b_date: items[1]
639 .get("published_date")
640 .and_then(|v| v.as_str())
641 .map(|s| s.to_string()),
642 severity: Severity::Medium,
643 };
644 if let Ok(res) = crp_resolve(&c, |_| 0.5) {
645 contradictions.push(res);
646 }
647 }
648 }
649
650 if !needs_extraction {
652 let top3: Vec<String> = results.iter().take(3).map(|r| r.url.clone()).collect();
653 if let Some(c) = cache {
654 for url in top3 {
655 let http2 = http.clone();
656 let c2 = c.clone();
657 tokio::spawn(async move {
658 tokio::time::sleep(Duration::from_millis(200)).await;
659 if let Ok(Ok(html)) =
660 tokio::time::timeout(Duration::from_secs(8), http2.fetch_text(&url)).await
661 {
662 if html.len() > 500 {
663 let ext = cep_extract(&html, &url);
664 let data = json!({
665 "url": &url,
666 "title": ext.title,
667 "content": safe_trunc(&ext.text, 8000),
668 "prefetched": true,
669 });
670 c2.set(&format!("prefetch:{url}"), &data).await;
671 }
672 }
673 });
674 }
675 }
676 }
677
678 let verified_facts = fact_handle.await.unwrap_or_default();
680
681 let citations: Vec<Value> = items
683 .iter()
684 .enumerate()
685 .map(|(i, item)| {
686 json!({
687 "index": i + 1,
688 "title": item.get("title").and_then(|v| v.as_str()).unwrap_or(""),
689 "url": item.get("url").and_then(|v| v.as_str()).unwrap_or(""),
690 })
691 })
692 .collect();
693
694 let duration_ms = start.elapsed().as_millis() as u64;
695 let intent = detect_intent(query);
696 let intent_str = format!("{:?}", intent).to_lowercase();
697
698 let response = json!({
699 "meta": {
700 "query": query,
701 "tier": tier,
702 "intent": intent_str,
703 "tokens_used": token_budget,
704 "sources_count": items.len(),
705 "duration_ms": duration_ms,
706 "result_id": result_id,
707 "credits_used": 1,
708 "citations": citations,
709 "verified_facts": verified_facts,
710 "contradictions": contradictions,
711 },
712 "results": items,
713 });
714
715 if let Some(c) = cache {
716 c.set(&format!("expand:{result_id}"), &response).await;
717 c.set(&cache_key, &response).await;
718 }
719
720 Ok(response)
721}
722
723fn needs_deeper_recall_query(query: &str) -> bool {
724 let lower = query.to_lowercase();
725 lower.contains(" vs ")
726 || lower.contains("compare")
727 || lower.contains("comparison")
728 || lower.contains("versus")
729 || lower.contains("status code")
730 || lower.contains("http ")
731 || lower.contains("rfc ")
732 || lower.contains("error code")
733 || lower.contains("medication")
734 || lower.contains("medicine")
735 || lower.contains("drug")
736 || lower.contains("treatment")
737 || lower.contains("diagnosis")
738 || lower.contains("symptoms")
739 || lower.contains("blood pressure")
740 || lower.contains("hypertension")
741 || lower.contains("vaccine")
742 || query.split_whitespace().count() >= 8
743}
744
745fn is_expensive_comparison_query(query: &str) -> bool {
746 let lower = query.to_lowercase();
747 lower.contains(" vs ")
748 || lower.contains("versus")
749 || lower.contains("comparison")
750 || lower.contains("compare")
751}
752
753fn generate_corrective_queries(query: &str) -> Vec<String> {
754 let q = query.trim();
755 let lower = q.to_lowercase();
756 let mut out = Vec::new();
757
758 if !lower.contains("overview") {
759 out.push(format!("{q} overview"));
760 }
761 if !lower.contains("explained") {
762 out.push(format!("{q} explained"));
763 }
764
765 let stripped = q
766 .split_whitespace()
767 .filter(|word| {
768 let lower = word.to_lowercase();
769 !matches!(
770 lower.as_str(),
771 "latest" | "recent" | "today" | "this" | "year" | "month" | "week"
772 ) && !lower.chars().all(|c| c.is_ascii_digit())
773 })
774 .collect::<Vec<_>>()
775 .join(" ");
776 if !stripped.is_empty() && stripped != q {
777 out.push(stripped);
778 }
779
780 out
781}
782
783fn count_unique_domains(items: &[crate::types::ResultItem]) -> usize {
784 items
785 .iter()
786 .filter_map(|item| item.url.split('/').nth(2))
787 .collect::<HashSet<_>>()
788 .len()
789}
790
791pub async fn fetch(
795 url: &str,
796 token_budget: usize,
797 format: &str,
798 http: &HttpClient,
799 cache: Option<&MemoryCache>,
800 schema: Option<&Value>,
801) -> Result<Value, FetchiumError> {
802 if let Some(c) = cache {
804 if let Some(cached) = c.get::<Value>(&format!("prefetch:{url}")).await {
805 if let Some(content) = cached.get("content").and_then(|v| v.as_str()) {
806 let tokens = crate::extract::layer1::estimate_tokens(content) as usize;
807 let result_id = Uuid::new_v4().to_string();
808 return Ok(json!({
809 "url": url,
810 "title": cached.get("title"),
811 "content": content,
812 "tokens": tokens,
813 "format": format,
814 "result_id": result_id,
815 "from_cache": true,
816 }));
817 }
818 }
819 }
820
821 let html = http.fetch_text(url).await?;
822 let html = if html.is_empty() {
823 return Err(FetchiumError::Internal("Empty response from URL".into()));
824 } else {
825 html
826 };
827
828 let ext = cep_extract(&html, url);
829 let max_chars = token_budget * 4;
830 let content = if ext.text.len() > max_chars {
831 ext.text[..max_chars].to_string()
832 } else {
833 ext.text
834 };
835
836 let tokens = crate::extract::layer1::estimate_tokens(&content) as usize;
837 let result_id = Uuid::new_v4().to_string();
838
839 let structured = if let Some(s) = schema {
840 extract_structured(&content, s, http).await
841 } else {
842 None
843 };
844
845 let response = json!({
846 "url": url,
847 "title": if ext.title.is_empty() { Value::Null } else { json!(ext.title) },
848 "content": content,
849 "tokens": tokens,
850 "format": format,
851 "result_id": result_id,
852 "structured": structured,
853 });
854
855 if let Some(c) = cache {
856 c.set(&format!("expand:{result_id}"), &response).await;
857 }
858
859 Ok(response)
860}
861
862pub async fn expand(
866 result_id: &str,
867 tier: &str,
868 cache: Option<&MemoryCache>,
869) -> Result<Value, FetchiumError> {
870 if let Some(c) = cache {
871 if let Some(cached_data) = c.get::<Value>(&format!("expand:{result_id}")).await {
872 let mut expanded_data = cached_data;
873 if let Some(meta) = expanded_data.get_mut("meta") {
874 meta["tier"] = json!(tier);
875 }
876 return Ok(expanded_data);
877 }
878 }
879 Err(FetchiumError::Internal(
880 "Cache miss or cache not configured for session".into(),
881 ))
882}