Skip to main content

fetchium_core/summarize/
mod.rs

1//! AI-powered URL/text summarization pipeline.
2//!
3//! Pipeline: `input → detect_type → fetch (if URL) → CEP extract → QATBE → AI summarize`
4//!
5//! Used by `fetchium summarize` CLI command and the REST API.
6
7use crate::ai::prompt::summarize_prompt;
8use crate::ai::provider_client::chat_with_fallback;
9use crate::ai::providers::ProviderKind;
10use crate::ai::types::{AiConfig, ChatMessage};
11use crate::config::FetchiumConfig;
12use crate::error::{ErrorKind, FetchiumError};
13use crate::http::client::HttpClient;
14use serde::{Deserialize, Serialize};
15use std::time::Instant;
16
17/// Desired summary length.
18#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
19pub enum SummaryLength {
20    /// ~100 words
21    Short,
22    /// ~300 words
23    #[default]
24    Medium,
25    /// ~700 words
26    Long,
27}
28
29impl SummaryLength {
30    fn description(self) -> &'static str {
31        match self {
32            Self::Short => "short (~100 words)",
33            Self::Medium => "medium (~300 words)",
34            Self::Long => "long (~700 words)",
35        }
36    }
37
38    /// Number of sentences for heuristic fallback.
39    fn fallback_sentences(self) -> usize {
40        match self {
41            Self::Short => 3,
42            Self::Medium => 7,
43            Self::Long => 15,
44        }
45    }
46
47    /// Max characters sent to the LLM for synthesis (latency/cost guardrail).
48    fn max_input_chars(self) -> usize {
49        match self {
50            Self::Short => 8_000,
51            Self::Medium => 16_000,
52            Self::Long => 28_000,
53        }
54    }
55}
56
57/// Configuration for the summarize pipeline.
58#[derive(Debug, Clone)]
59pub struct SummarizeConfig {
60    /// Desired summary length.
61    pub length: SummaryLength,
62    /// Optional model override for AI synthesis.
63    pub model: Option<String>,
64}
65
66impl Default for SummarizeConfig {
67    fn default() -> Self {
68        Self {
69            length: SummaryLength::Medium,
70            model: None,
71        }
72    }
73}
74
75/// Result of summarization.
76#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct SummaryResult {
78    /// The generated summary text.
79    pub summary: String,
80    /// Source URL if input was a URL.
81    pub source_url: Option<String>,
82    /// Title extracted from the source (if URL).
83    pub source_title: Option<String>,
84    /// Whether AI was used (false = heuristic fallback).
85    pub ai_used: bool,
86    /// Duration in milliseconds.
87    pub duration_ms: u64,
88}
89
90/// Summarize a URL or raw text.
91///
92/// If `input` starts with `http://` or `https://`, it is fetched and extracted first.
93/// Otherwise it is treated as raw text.
94pub async fn summarize(
95    input: &str,
96    config: &SummarizeConfig,
97    fetchium_config: &FetchiumConfig,
98) -> Result<SummaryResult, FetchiumError> {
99    let start = Instant::now();
100
101    let is_url = input.starts_with("http://") || input.starts_with("https://");
102
103    let (content, source_url, source_title) = if is_url {
104        let http = HttpClient::new(fetchium_config)?;
105        // YouTube URLs: extract transcript instead of HTML for much better summaries
106        let is_youtube = input.contains("youtube.com/watch")
107            || input.contains("youtu.be/")
108            || input.contains("youtube.com/shorts/");
109
110        if is_youtube {
111            let video_id = crate::multimodal::video::extract_video_id(input).ok();
112            let metadata_fut = async {
113                if let Some(ref vid) = video_id {
114                    crate::youtube::metadata::fetch_metadata(vid, &http, fetchium_config)
115                        .await
116                        .ok()
117                } else {
118                    None
119                }
120            };
121            let (transcript_result, metadata_opt) = tokio::join!(
122                crate::youtube::universal::fetch_universal_transcript(
123                    input,
124                    &http,
125                    fetchium_config
126                ),
127                metadata_fut
128            );
129
130            match transcript_result {
131                Ok(transcript) if !transcript.full_text.trim().is_empty() => {
132                    let video_id = transcript.video_id.clone();
133                    let text = transcript.full_text.clone();
134                    let title = metadata_opt
135                        .as_ref()
136                        .map(|m| m.title.clone())
137                        .unwrap_or_else(|| format!("YouTube Video ({})", video_id));
138                    tracing::info!(
139                        "YouTube transcript extracted: {} chars for video {}",
140                        text.len(),
141                        video_id
142                    );
143                    (text, Some(input.to_string()), Some(title))
144                }
145                Ok(transcript) => {
146                    tracing::warn!(
147                        "YouTube transcript was empty for video {}, falling back to metadata",
148                        transcript.video_id
149                    );
150                    let title = metadata_opt
151                        .as_ref()
152                        .map(|m| m.title.clone())
153                        .unwrap_or_else(|| format!("YouTube Video ({})", transcript.video_id));
154                    let context = metadata_opt
155                        .as_ref()
156                        .map(|m| {
157                            format!(
158                                "Title: {}\nChannel: {}\nDescription: {}",
159                                m.title,
160                                m.channel.name,
161                                trim_to_chars(&m.description, 900),
162                            )
163                        })
164                        .unwrap_or_else(|| "No metadata available.".to_string());
165                    (
166                        format!(
167                            "This YouTube video's transcript could not be extracted.\n\n{context}"
168                        ),
169                        Some(input.to_string()),
170                        Some(title),
171                    )
172                }
173                Err(e) => {
174                    tracing::warn!(
175                        "YouTube transcript fetch failed: {}. Falling back to HTML.",
176                        e
177                    );
178                    // Fall back to regular HTML fetch if transcript unavailable
179                    let html = fetch_html_with_block_fallback(&http, input).await?;
180                    let extracted = crate::extract::pipeline::extract(&html, input);
181                    let title = if extracted.title.is_empty() {
182                        None
183                    } else {
184                        Some(extracted.title)
185                    };
186                    (extracted.text, Some(input.to_string()), title)
187                }
188            }
189        } else {
190            let html = fetch_html_with_block_fallback(&http, input).await?;
191            let extracted = crate::extract::pipeline::extract(&html, input);
192            let title = if extracted.title.is_empty() {
193                None
194            } else {
195                Some(extracted.title)
196            };
197            (extracted.text, Some(input.to_string()), title)
198        }
199    } else {
200        (input.to_string(), None, None)
201    };
202
203    if content.trim().is_empty() {
204        return Ok(SummaryResult {
205            summary: String::new(),
206            source_url,
207            source_title,
208            ai_used: false,
209            duration_ms: start.elapsed().as_millis() as u64,
210        });
211    }
212
213    // Try AI synthesis
214    let ai_config = AiConfig::from_fetchium_config(fetchium_config);
215    let has_ai = has_reachable_ai_provider(&ai_config, fetchium_config).await;
216
217    if has_ai {
218        let system = summarize_prompt(config.length.description());
219        let content_for_ai = prepare_summary_input(&content, config.length);
220        let messages = vec![
221            ChatMessage {
222                role: "system".into(),
223                content: system,
224            },
225            ChatMessage {
226                role: "user".into(),
227                content: content_for_ai,
228            },
229        ];
230
231        let providers = ai_config.providers.clone();
232        let model_override = config.model.clone();
233
234        // Use thread isolation to keep future Send-safe (required for axum handlers).
235        let (tx, rx) = tokio::sync::oneshot::channel::<Option<String>>();
236        std::thread::spawn(move || {
237            let rt = tokio::runtime::Builder::new_current_thread()
238                .enable_all()
239                .build()
240                .unwrap();
241            let result = rt.block_on(async move {
242                let mut noop = |_: &str| {};
243                match tokio::time::timeout(
244                    std::time::Duration::from_secs(12),
245                    chat_with_fallback(
246                        &messages,
247                        model_override.as_deref(),
248                        &ai_config,
249                        &providers,
250                        false,
251                        &mut noop,
252                    ),
253                )
254                .await
255                {
256                    Ok(Ok(result)) => Some(result.content),
257                    _ => None,
258                }
259            });
260            let _ = tx.send(result);
261        });
262
263        if let Ok(Some(summary)) = rx.await {
264            if !summary.is_empty() && grounding_score(&summary, &content) >= 0.12 {
265                return Ok(SummaryResult {
266                    summary,
267                    source_url,
268                    source_title,
269                    ai_used: true,
270                    duration_ms: start.elapsed().as_millis() as u64,
271                });
272            }
273        }
274    }
275
276    // Heuristic fallback: take first N sentences
277    let sentences: Vec<&str> = content.split(". ").collect();
278    let n = config.length.fallback_sentences().min(sentences.len());
279    let summary = sentences[..n]
280        .iter()
281        .map(|s| format!("{}.", s.trim()))
282        .collect::<Vec<_>>()
283        .join(" ");
284
285    Ok(SummaryResult {
286        summary,
287        source_url,
288        source_title,
289        ai_used: false,
290        duration_ms: start.elapsed().as_millis() as u64,
291    })
292}
293
294async fn has_reachable_ai_provider(ai_config: &AiConfig, fetchium_config: &FetchiumConfig) -> bool {
295    let configured = ai_config.providers.configured_providers();
296    if configured.is_empty() {
297        return false;
298    }
299
300    // Non-local network providers are considered reachable if configured.
301    if configured
302        .iter()
303        .any(|k| *k != ProviderKind::Ollama && *k != ProviderKind::GeminiCli)
304    {
305        return true;
306    }
307
308    // Local-only providers require local availability checks.
309    let has_ollama = configured.contains(&ProviderKind::Ollama);
310    if !has_ollama {
311        return false;
312    }
313
314    // Ollama is configured: check if local server is actually reachable.
315    if let Ok(http) = HttpClient::new(fetchium_config) {
316        let url = format!(
317            "{}:{}/api/tags",
318            ai_config.ollama_host, ai_config.ollama_port
319        );
320        return tokio::time::timeout(std::time::Duration::from_millis(1200), async {
321            http.fetch_text_once(&url).await.is_ok()
322        })
323        .await
324        .unwrap_or(false);
325    }
326    false
327}
328
329async fn fetch_html_with_block_fallback(
330    http: &HttpClient,
331    url: &str,
332) -> Result<String, FetchiumError> {
333    match http.fetch_text(url).await {
334        Ok(html) => Ok(html),
335        Err(err) => {
336            if !is_block_like_fetch_error(&err) {
337                return Err(err);
338            }
339
340            let normalized = url
341                .trim_start_matches("https://")
342                .trim_start_matches("http://");
343            let mirror_url = format!("https://r.jina.ai/http://{normalized}");
344
345            match tokio::time::timeout(
346                std::time::Duration::from_secs(8),
347                http.fetch_text_once(&mirror_url),
348            )
349            .await
350            {
351                Ok(Ok(body)) if !body.trim().is_empty() => {
352                    tracing::warn!("Using reader fallback for blocked URL: {}", url);
353                    Ok(body)
354                }
355                _ => Err(err),
356            }
357        }
358    }
359}
360
361fn is_block_like_fetch_error(err: &FetchiumError) -> bool {
362    match err {
363        FetchiumError::Structured(se) => {
364            if matches!(
365                se.kind,
366                ErrorKind::Http403
367                    | ErrorKind::Http429
368                    | ErrorKind::Http5xx
369                    | ErrorKind::AntiBot
370                    | ErrorKind::Paywall
371            ) {
372                return true;
373            }
374
375            // Some JS-heavy sites return synthetic 404/403 variants through edge layers.
376            se.message.contains("HTTP 404")
377                || se.message.contains("HTTP 403")
378                || se.message.contains("HTTP 429")
379        }
380        _ => false,
381    }
382}
383
384fn prepare_summary_input(content: &str, length: SummaryLength) -> String {
385    let max_chars = length.max_input_chars();
386    let content_chars = content.chars().count();
387    if content_chars <= max_chars {
388        return content.to_string();
389    }
390    let head_chars = (max_chars * 7) / 10;
391    let tail_chars = max_chars.saturating_sub(head_chars);
392    let head = trim_to_chars(content, head_chars);
393    let tail = trim_from_end(content, tail_chars);
394    format!("{head}\n\n[... middle omitted for brevity ...]\n\n{tail}")
395}
396
397fn trim_to_chars(s: &str, max_chars: usize) -> String {
398    s.chars().take(max_chars).collect()
399}
400
401fn trim_from_end(s: &str, max_chars: usize) -> String {
402    let count = s.chars().count();
403    if count <= max_chars {
404        return s.to_string();
405    }
406    s.chars().skip(count - max_chars).collect()
407}
408
409fn grounding_score(answer: &str, source: &str) -> f64 {
410    let src = source.to_lowercase();
411    let mut total = 0usize;
412    let mut supported = 0usize;
413
414    for sentence in answer.split(['.', '!', '?']) {
415        let terms = sentence
416            .split(|c: char| !c.is_alphanumeric())
417            .filter(|t| t.len() >= 4)
418            .map(|t| t.to_lowercase())
419            .collect::<Vec<_>>();
420        if terms.is_empty() {
421            continue;
422        }
423        total += 1;
424        let hits = terms.iter().filter(|t| src.contains(t.as_str())).count();
425        if hits >= 2 {
426            supported += 1;
427        }
428    }
429
430    if total == 0 {
431        0.0
432    } else {
433        supported as f64 / total as f64
434    }
435}
436
437#[cfg(test)]
438mod tests {
439    use super::*;
440
441    #[test]
442    fn summary_length_descriptions() {
443        assert!(SummaryLength::Short.description().contains("100"));
444        assert!(SummaryLength::Medium.description().contains("300"));
445        assert!(SummaryLength::Long.description().contains("700"));
446    }
447
448    #[test]
449    fn default_config() {
450        let config = SummarizeConfig::default();
451        assert_eq!(config.length, SummaryLength::Medium);
452        assert!(config.model.is_none());
453    }
454
455    #[tokio::test]
456    async fn summarize_raw_text() {
457        let text = "Rust is a systems language. It ensures memory safety. It has zero-cost abstractions. It supports async/await. It has great tooling.";
458        let config = SummarizeConfig {
459            length: SummaryLength::Short,
460            model: None,
461        };
462        let fetchium_config = FetchiumConfig::default();
463        let result = summarize(text, &config, &fetchium_config).await.unwrap();
464        assert!(!result.summary.is_empty());
465        assert!(result.source_url.is_none());
466    }
467
468    #[tokio::test]
469    async fn summarize_empty_text() {
470        let config = SummarizeConfig::default();
471        let fetchium_config = FetchiumConfig::default();
472        let result = summarize("", &config, &fetchium_config).await.unwrap();
473        assert!(result.summary.is_empty());
474        assert!(!result.ai_used);
475    }
476
477    #[test]
478    fn prepare_summary_input_truncates_long_content() {
479        let long = "a".repeat(40_000);
480        let out = prepare_summary_input(&long, SummaryLength::Medium);
481        assert!(out.len() < long.len());
482        assert!(out.contains("[... middle omitted for brevity ...]"));
483    }
484
485    #[test]
486    fn grounding_score_prefers_supported_text() {
487        let src = "Rust has ownership and borrowing for memory safety.";
488        let good = "Rust uses ownership and borrowing for memory safety.";
489        let bad = "Rust runs on quantum crystals.";
490        assert!(grounding_score(good, src) > grounding_score(bad, src));
491    }
492
493    #[test]
494    fn is_block_like_fetch_error_detects_403() {
495        let err = FetchiumError::Structured(crate::error::StructuredError {
496            kind: ErrorKind::Http403,
497            retryable: false,
498            message: "blocked".into(),
499            source_url: Some("https://example.com".into()),
500            suggested_action: "fallback".into(),
501            alternatives: vec![],
502        });
503        assert!(is_block_like_fetch_error(&err));
504    }
505
506    #[test]
507    fn is_block_like_fetch_error_detects_unknown_404() {
508        let err = FetchiumError::Structured(crate::error::StructuredError {
509            kind: ErrorKind::Unknown,
510            retryable: false,
511            message: "HTTP 404 Not Found from https://example.com".into(),
512            source_url: Some("https://example.com".into()),
513            suggested_action: "fallback".into(),
514            alternatives: vec![],
515        });
516        assert!(is_block_like_fetch_error(&err));
517    }
518
519    #[tokio::test]
520    async fn has_reachable_ai_provider_false_when_unconfigured() {
521        let fetchium_config = FetchiumConfig::default();
522        let mut ai = AiConfig::from_fetchium_config(&fetchium_config);
523        ai.providers.fallback_chain.clear();
524        ai.providers.ollama.enabled = false;
525        ai.providers.openai.enabled = false;
526        ai.providers.anthropic.enabled = false;
527        ai.providers.gemini.enabled = false;
528        ai.providers.gemini_cli.enabled = false;
529        ai.providers.openrouter.enabled = false;
530        ai.providers.antigravity.enabled = false;
531        assert!(!has_reachable_ai_provider(&ai, &fetchium_config).await);
532    }
533}