1use crate::ai::prompt::summarize_prompt;
8use crate::ai::provider_client::chat_with_fallback;
9use crate::ai::providers::ProviderKind;
10use crate::ai::types::{AiConfig, ChatMessage};
11use crate::config::FetchiumConfig;
12use crate::error::{ErrorKind, FetchiumError};
13use crate::http::client::HttpClient;
14use serde::{Deserialize, Serialize};
15use std::time::Instant;
16
17#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
19pub enum SummaryLength {
20 Short,
22 #[default]
24 Medium,
25 Long,
27}
28
29impl SummaryLength {
30 fn description(self) -> &'static str {
31 match self {
32 Self::Short => "short (~100 words)",
33 Self::Medium => "medium (~300 words)",
34 Self::Long => "long (~700 words)",
35 }
36 }
37
38 fn fallback_sentences(self) -> usize {
40 match self {
41 Self::Short => 3,
42 Self::Medium => 7,
43 Self::Long => 15,
44 }
45 }
46
47 fn max_input_chars(self) -> usize {
49 match self {
50 Self::Short => 8_000,
51 Self::Medium => 16_000,
52 Self::Long => 28_000,
53 }
54 }
55}
56
57#[derive(Debug, Clone)]
59pub struct SummarizeConfig {
60 pub length: SummaryLength,
62 pub model: Option<String>,
64}
65
66impl Default for SummarizeConfig {
67 fn default() -> Self {
68 Self {
69 length: SummaryLength::Medium,
70 model: None,
71 }
72 }
73}
74
75#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct SummaryResult {
78 pub summary: String,
80 pub source_url: Option<String>,
82 pub source_title: Option<String>,
84 pub ai_used: bool,
86 pub duration_ms: u64,
88}
89
90pub async fn summarize(
95 input: &str,
96 config: &SummarizeConfig,
97 fetchium_config: &FetchiumConfig,
98) -> Result<SummaryResult, FetchiumError> {
99 let start = Instant::now();
100
101 let is_url = input.starts_with("http://") || input.starts_with("https://");
102
103 let (content, source_url, source_title) = if is_url {
104 let http = HttpClient::new(fetchium_config)?;
105 let is_youtube = input.contains("youtube.com/watch")
107 || input.contains("youtu.be/")
108 || input.contains("youtube.com/shorts/");
109
110 if is_youtube {
111 let video_id = crate::multimodal::video::extract_video_id(input).ok();
112 let metadata_fut = async {
113 if let Some(ref vid) = video_id {
114 crate::youtube::metadata::fetch_metadata(vid, &http, fetchium_config)
115 .await
116 .ok()
117 } else {
118 None
119 }
120 };
121 let (transcript_result, metadata_opt) = tokio::join!(
122 crate::youtube::universal::fetch_universal_transcript(
123 input,
124 &http,
125 fetchium_config
126 ),
127 metadata_fut
128 );
129
130 match transcript_result {
131 Ok(transcript) if !transcript.full_text.trim().is_empty() => {
132 let video_id = transcript.video_id.clone();
133 let text = transcript.full_text.clone();
134 let title = metadata_opt
135 .as_ref()
136 .map(|m| m.title.clone())
137 .unwrap_or_else(|| format!("YouTube Video ({})", video_id));
138 tracing::info!(
139 "YouTube transcript extracted: {} chars for video {}",
140 text.len(),
141 video_id
142 );
143 (text, Some(input.to_string()), Some(title))
144 }
145 Ok(transcript) => {
146 tracing::warn!(
147 "YouTube transcript was empty for video {}, falling back to metadata",
148 transcript.video_id
149 );
150 let title = metadata_opt
151 .as_ref()
152 .map(|m| m.title.clone())
153 .unwrap_or_else(|| format!("YouTube Video ({})", transcript.video_id));
154 let context = metadata_opt
155 .as_ref()
156 .map(|m| {
157 format!(
158 "Title: {}\nChannel: {}\nDescription: {}",
159 m.title,
160 m.channel.name,
161 trim_to_chars(&m.description, 900),
162 )
163 })
164 .unwrap_or_else(|| "No metadata available.".to_string());
165 (
166 format!(
167 "This YouTube video's transcript could not be extracted.\n\n{context}"
168 ),
169 Some(input.to_string()),
170 Some(title),
171 )
172 }
173 Err(e) => {
174 tracing::warn!(
175 "YouTube transcript fetch failed: {}. Falling back to HTML.",
176 e
177 );
178 let html = fetch_html_with_block_fallback(&http, input).await?;
180 let extracted = crate::extract::pipeline::extract(&html, input);
181 let title = if extracted.title.is_empty() {
182 None
183 } else {
184 Some(extracted.title)
185 };
186 (extracted.text, Some(input.to_string()), title)
187 }
188 }
189 } else {
190 let html = fetch_html_with_block_fallback(&http, input).await?;
191 let extracted = crate::extract::pipeline::extract(&html, input);
192 let title = if extracted.title.is_empty() {
193 None
194 } else {
195 Some(extracted.title)
196 };
197 (extracted.text, Some(input.to_string()), title)
198 }
199 } else {
200 (input.to_string(), None, None)
201 };
202
203 if content.trim().is_empty() {
204 return Ok(SummaryResult {
205 summary: String::new(),
206 source_url,
207 source_title,
208 ai_used: false,
209 duration_ms: start.elapsed().as_millis() as u64,
210 });
211 }
212
213 let ai_config = AiConfig::from_fetchium_config(fetchium_config);
215 let has_ai = has_reachable_ai_provider(&ai_config, fetchium_config).await;
216
217 if has_ai {
218 let system = summarize_prompt(config.length.description());
219 let content_for_ai = prepare_summary_input(&content, config.length);
220 let messages = vec![
221 ChatMessage {
222 role: "system".into(),
223 content: system,
224 },
225 ChatMessage {
226 role: "user".into(),
227 content: content_for_ai,
228 },
229 ];
230
231 let providers = ai_config.providers.clone();
232 let model_override = config.model.clone();
233
234 let (tx, rx) = tokio::sync::oneshot::channel::<Option<String>>();
236 std::thread::spawn(move || {
237 let rt = tokio::runtime::Builder::new_current_thread()
238 .enable_all()
239 .build()
240 .unwrap();
241 let result = rt.block_on(async move {
242 let mut noop = |_: &str| {};
243 match tokio::time::timeout(
244 std::time::Duration::from_secs(12),
245 chat_with_fallback(
246 &messages,
247 model_override.as_deref(),
248 &ai_config,
249 &providers,
250 false,
251 &mut noop,
252 ),
253 )
254 .await
255 {
256 Ok(Ok(result)) => Some(result.content),
257 _ => None,
258 }
259 });
260 let _ = tx.send(result);
261 });
262
263 if let Ok(Some(summary)) = rx.await {
264 if !summary.is_empty() && grounding_score(&summary, &content) >= 0.12 {
265 return Ok(SummaryResult {
266 summary,
267 source_url,
268 source_title,
269 ai_used: true,
270 duration_ms: start.elapsed().as_millis() as u64,
271 });
272 }
273 }
274 }
275
276 let sentences: Vec<&str> = content.split(". ").collect();
278 let n = config.length.fallback_sentences().min(sentences.len());
279 let summary = sentences[..n]
280 .iter()
281 .map(|s| format!("{}.", s.trim()))
282 .collect::<Vec<_>>()
283 .join(" ");
284
285 Ok(SummaryResult {
286 summary,
287 source_url,
288 source_title,
289 ai_used: false,
290 duration_ms: start.elapsed().as_millis() as u64,
291 })
292}
293
294async fn has_reachable_ai_provider(ai_config: &AiConfig, fetchium_config: &FetchiumConfig) -> bool {
295 let configured = ai_config.providers.configured_providers();
296 if configured.is_empty() {
297 return false;
298 }
299
300 if configured
302 .iter()
303 .any(|k| *k != ProviderKind::Ollama && *k != ProviderKind::GeminiCli)
304 {
305 return true;
306 }
307
308 let has_ollama = configured.contains(&ProviderKind::Ollama);
310 if !has_ollama {
311 return false;
312 }
313
314 if let Ok(http) = HttpClient::new(fetchium_config) {
316 let url = format!(
317 "{}:{}/api/tags",
318 ai_config.ollama_host, ai_config.ollama_port
319 );
320 return tokio::time::timeout(std::time::Duration::from_millis(1200), async {
321 http.fetch_text_once(&url).await.is_ok()
322 })
323 .await
324 .unwrap_or(false);
325 }
326 false
327}
328
329async fn fetch_html_with_block_fallback(
330 http: &HttpClient,
331 url: &str,
332) -> Result<String, FetchiumError> {
333 match http.fetch_text(url).await {
334 Ok(html) => Ok(html),
335 Err(err) => {
336 if !is_block_like_fetch_error(&err) {
337 return Err(err);
338 }
339
340 let normalized = url
341 .trim_start_matches("https://")
342 .trim_start_matches("http://");
343 let mirror_url = format!("https://r.jina.ai/http://{normalized}");
344
345 match tokio::time::timeout(
346 std::time::Duration::from_secs(8),
347 http.fetch_text_once(&mirror_url),
348 )
349 .await
350 {
351 Ok(Ok(body)) if !body.trim().is_empty() => {
352 tracing::warn!("Using reader fallback for blocked URL: {}", url);
353 Ok(body)
354 }
355 _ => Err(err),
356 }
357 }
358 }
359}
360
361fn is_block_like_fetch_error(err: &FetchiumError) -> bool {
362 match err {
363 FetchiumError::Structured(se) => {
364 if matches!(
365 se.kind,
366 ErrorKind::Http403
367 | ErrorKind::Http429
368 | ErrorKind::Http5xx
369 | ErrorKind::AntiBot
370 | ErrorKind::Paywall
371 ) {
372 return true;
373 }
374
375 se.message.contains("HTTP 404")
377 || se.message.contains("HTTP 403")
378 || se.message.contains("HTTP 429")
379 }
380 _ => false,
381 }
382}
383
384fn prepare_summary_input(content: &str, length: SummaryLength) -> String {
385 let max_chars = length.max_input_chars();
386 let content_chars = content.chars().count();
387 if content_chars <= max_chars {
388 return content.to_string();
389 }
390 let head_chars = (max_chars * 7) / 10;
391 let tail_chars = max_chars.saturating_sub(head_chars);
392 let head = trim_to_chars(content, head_chars);
393 let tail = trim_from_end(content, tail_chars);
394 format!("{head}\n\n[... middle omitted for brevity ...]\n\n{tail}")
395}
396
397fn trim_to_chars(s: &str, max_chars: usize) -> String {
398 s.chars().take(max_chars).collect()
399}
400
401fn trim_from_end(s: &str, max_chars: usize) -> String {
402 let count = s.chars().count();
403 if count <= max_chars {
404 return s.to_string();
405 }
406 s.chars().skip(count - max_chars).collect()
407}
408
409fn grounding_score(answer: &str, source: &str) -> f64 {
410 let src = source.to_lowercase();
411 let mut total = 0usize;
412 let mut supported = 0usize;
413
414 for sentence in answer.split(['.', '!', '?']) {
415 let terms = sentence
416 .split(|c: char| !c.is_alphanumeric())
417 .filter(|t| t.len() >= 4)
418 .map(|t| t.to_lowercase())
419 .collect::<Vec<_>>();
420 if terms.is_empty() {
421 continue;
422 }
423 total += 1;
424 let hits = terms.iter().filter(|t| src.contains(t.as_str())).count();
425 if hits >= 2 {
426 supported += 1;
427 }
428 }
429
430 if total == 0 {
431 0.0
432 } else {
433 supported as f64 / total as f64
434 }
435}
436
437#[cfg(test)]
438mod tests {
439 use super::*;
440
441 #[test]
442 fn summary_length_descriptions() {
443 assert!(SummaryLength::Short.description().contains("100"));
444 assert!(SummaryLength::Medium.description().contains("300"));
445 assert!(SummaryLength::Long.description().contains("700"));
446 }
447
448 #[test]
449 fn default_config() {
450 let config = SummarizeConfig::default();
451 assert_eq!(config.length, SummaryLength::Medium);
452 assert!(config.model.is_none());
453 }
454
455 #[tokio::test]
456 async fn summarize_raw_text() {
457 let text = "Rust is a systems language. It ensures memory safety. It has zero-cost abstractions. It supports async/await. It has great tooling.";
458 let config = SummarizeConfig {
459 length: SummaryLength::Short,
460 model: None,
461 };
462 let fetchium_config = FetchiumConfig::default();
463 let result = summarize(text, &config, &fetchium_config).await.unwrap();
464 assert!(!result.summary.is_empty());
465 assert!(result.source_url.is_none());
466 }
467
468 #[tokio::test]
469 async fn summarize_empty_text() {
470 let config = SummarizeConfig::default();
471 let fetchium_config = FetchiumConfig::default();
472 let result = summarize("", &config, &fetchium_config).await.unwrap();
473 assert!(result.summary.is_empty());
474 assert!(!result.ai_used);
475 }
476
477 #[test]
478 fn prepare_summary_input_truncates_long_content() {
479 let long = "a".repeat(40_000);
480 let out = prepare_summary_input(&long, SummaryLength::Medium);
481 assert!(out.len() < long.len());
482 assert!(out.contains("[... middle omitted for brevity ...]"));
483 }
484
485 #[test]
486 fn grounding_score_prefers_supported_text() {
487 let src = "Rust has ownership and borrowing for memory safety.";
488 let good = "Rust uses ownership and borrowing for memory safety.";
489 let bad = "Rust runs on quantum crystals.";
490 assert!(grounding_score(good, src) > grounding_score(bad, src));
491 }
492
493 #[test]
494 fn is_block_like_fetch_error_detects_403() {
495 let err = FetchiumError::Structured(crate::error::StructuredError {
496 kind: ErrorKind::Http403,
497 retryable: false,
498 message: "blocked".into(),
499 source_url: Some("https://example.com".into()),
500 suggested_action: "fallback".into(),
501 alternatives: vec![],
502 });
503 assert!(is_block_like_fetch_error(&err));
504 }
505
506 #[test]
507 fn is_block_like_fetch_error_detects_unknown_404() {
508 let err = FetchiumError::Structured(crate::error::StructuredError {
509 kind: ErrorKind::Unknown,
510 retryable: false,
511 message: "HTTP 404 Not Found from https://example.com".into(),
512 source_url: Some("https://example.com".into()),
513 suggested_action: "fallback".into(),
514 alternatives: vec![],
515 });
516 assert!(is_block_like_fetch_error(&err));
517 }
518
519 #[tokio::test]
520 async fn has_reachable_ai_provider_false_when_unconfigured() {
521 let fetchium_config = FetchiumConfig::default();
522 let mut ai = AiConfig::from_fetchium_config(&fetchium_config);
523 ai.providers.fallback_chain.clear();
524 ai.providers.ollama.enabled = false;
525 ai.providers.openai.enabled = false;
526 ai.providers.anthropic.enabled = false;
527 ai.providers.gemini.enabled = false;
528 ai.providers.gemini_cli.enabled = false;
529 ai.providers.openrouter.enabled = false;
530 ai.providers.antigravity.enabled = false;
531 assert!(!has_reachable_ai_provider(&ai, &fetchium_config).await);
532 }
533}