Skip to main content

fetchium_core/youtube/
universal.rs

1//! Universal Video Transcript Protocol (UVTP) — transcribe any public video URL.
2//!
3//! Supports 1000+ platforms via yt-dlp subtitle extraction:
4//! YouTube, Vimeo, TikTok, Twitter/X, Instagram, Dailymotion, Twitch, and more.
5//!
6//! ## Algorithm
7//!
8//! 1. **YouTube fast-path**: Innertube API — parallel racing, ~1-2s, no subprocess
9//! 2. **Universal path**: `yt-dlp --write-subs --write-auto-subs --skip-download`
10//!    downloads only the subtitle file (not the video), typically ~2-5s
11//! 3. Parse VTT / SRT / JSON3 subtitle files into structured `TranscriptEntry` vecs
12//! 4. Enhance with gap-based speaker detection and key moment tagging
13//!
14//! ## Supported Platforms (via yt-dlp)
15//!
16//! YouTube, Vimeo, TikTok, Twitter/X, Facebook, Instagram, Reddit (v.redd.it),
17//! Twitch, Dailymotion, Bilibili, Rumble, Odysee, PeerTube, and ~1000 more.
18
19use crate::config::FetchiumConfig;
20use crate::error::{FetchiumError, FetchiumResult};
21use crate::http::client::HttpClient;
22use crate::youtube::types::*;
23use std::time::Duration;
24
25/// Hard timeout for yt-dlp subtitle extraction (subtitles only, not video).
26const YTDLP_SUBTITLE_TIMEOUT_SECS: u64 = 15;
27
28// ─── Public API ────────────────────────────────────────────────
29
30/// Extract transcript from ANY public video URL.
31///
32/// Routes automatically to the optimal extraction method:
33/// - YouTube URLs → Innertube + timedtext parallel racing (~1-2s)
34/// - All other URLs → yt-dlp subtitle extraction (~2-5s if captions exist)
35///
36/// Returns an `EnhancedTranscript` with speaker detection and key moments,
37/// compatible with all downstream YouTube Intelligence analysis.
38pub async fn fetch_universal_transcript(
39    url: &str,
40    http: &HttpClient,
41    config: &FetchiumConfig,
42) -> FetchiumResult<EnhancedTranscript> {
43    if is_youtube_url(url) {
44        let video_id = crate::multimodal::video::extract_video_id(url)?;
45        return crate::youtube::transcript::fetch_transcript(&video_id, http, config).await;
46    }
47    fetch_via_ytdlp_universal(url).await
48}
49
50/// Returns `true` if the URL points to a YouTube video (any format).
51pub fn is_youtube_url(url: &str) -> bool {
52    let lower = url.to_lowercase();
53    lower.contains("youtube.com/watch")
54        || lower.contains("youtu.be/")
55        || lower.contains("youtube.com/shorts/")
56        || lower.contains("youtube.com/embed/")
57        || lower.contains("m.youtube.com/watch")
58        || lower.contains("music.youtube.com/watch")
59}
60
61/// Detect the platform name from a URL (for display purposes).
62pub fn detect_platform(url: &str) -> &'static str {
63    let lower_url = url.to_lowercase();
64    if lower_url.contains("youtube.com") || lower_url.contains("youtu.be") {
65        "YouTube"
66    } else if lower_url.contains("vimeo.com") {
67        "Vimeo"
68    } else if lower_url.contains("tiktok.com") {
69        "TikTok"
70    } else if lower_url.contains("twitter.com") || lower_url.contains("x.com") {
71        "Twitter/X"
72    } else if lower_url.contains("instagram.com") {
73        "Instagram"
74    } else if lower_url.contains("twitch.tv") {
75        "Twitch"
76    } else if lower_url.contains("dailymotion.com") {
77        "Dailymotion"
78    } else if lower_url.contains("facebook.com") || lower_url.contains("fb.watch") {
79        "Facebook"
80    } else if lower_url.contains("reddit.com") || lower_url.contains("v.redd.it") {
81        "Reddit"
82    } else if lower_url.contains("rumble.com") {
83        "Rumble"
84    } else if lower_url.contains("odysee.com") || lower_url.contains("lbry.tv") {
85        "Odysee"
86    } else if lower_url.contains("bilibili.com") {
87        "Bilibili"
88    } else {
89        "Video"
90    }
91}
92
93// ─── yt-dlp Subtitle Extraction ────────────────────────────────
94
95/// Extract transcript via yt-dlp for any yt-dlp-supported platform.
96///
97/// Races manual subtitles against auto-generated captions simultaneously;
98/// returns the first non-empty result. Cleans up temp files afterward.
99async fn fetch_via_ytdlp_universal(url: &str) -> FetchiumResult<EnhancedTranscript> {
100    let hash = url_hash(url);
101    let tmp_dir = std::env::temp_dir().join(format!("fetchium_subs_{hash}"));
102    let _ = tokio::fs::create_dir_all(&tmp_dir).await;
103    let output_template = format!("{}/%(id)s", tmp_dir.display());
104
105    // Race manual subs against auto-generated captions in parallel.
106    // Most platforms with captions respond in 2-5s; timeout caps at 15s.
107    let (manual_result, auto_result) = tokio::join!(
108        run_ytdlp_subs(url, &output_template, false),
109        run_ytdlp_subs(url, &output_template, true),
110    );
111
112    let _ = tokio::fs::remove_dir_all(&tmp_dir).await;
113
114    // Prefer manual subtitles (higher quality), fall back to auto-generated.
115    let entries = manual_result
116        .ok()
117        .filter(|e: &Vec<TranscriptEntry>| !e.is_empty())
118        .or_else(|| auto_result.ok().filter(|e| !e.is_empty()))
119        .ok_or_else(|| {
120            FetchiumError::YouTube(format!(
121                "No captions/subtitles found for '{url}'. \
122                 Make sure yt-dlp is installed (`pip install yt-dlp`) \
123                 and the video has captions enabled."
124            ))
125        })?;
126
127    Ok(build_enhanced_transcript(
128        url,
129        entries,
130        TranscriptSource::YtDlp,
131    ))
132}
133
134/// Invoke yt-dlp to download subtitle files (no video download).
135///
136/// With `auto_subs = false` downloads manual/community captions.
137/// With `auto_subs = true` downloads auto-generated captions (ASR).
138async fn run_ytdlp_subs(
139    url: &str,
140    output_template: &str,
141    auto_subs: bool,
142) -> FetchiumResult<Vec<TranscriptEntry>> {
143    let sub_flag = if auto_subs {
144        "--write-auto-subs"
145    } else {
146        "--write-subs"
147    };
148
149    // Extract parent directory from the template for file scanning.
150    let tmp_dir = std::path::Path::new(output_template)
151        .parent()
152        .map(|p| p.to_path_buf())
153        .unwrap_or_else(std::env::temp_dir);
154
155    let output = tokio::time::timeout(
156        Duration::from_secs(YTDLP_SUBTITLE_TIMEOUT_SECS),
157        tokio::process::Command::new("yt-dlp")
158            .args([
159                sub_flag,
160                "--sub-lang",
161                "en.*,bn.*,hi.*,*",
162                "--sub-format",
163                "json3/vtt/srt/best",
164                "--skip-download",
165                "--no-playlist",
166                "--no-warnings",
167                "--quiet",
168                "--output",
169                output_template,
170                url,
171            ])
172            .output(),
173    )
174    .await
175    .map_err(|_| FetchiumError::YouTube("yt-dlp subtitle extraction timed out (15s)".into()))?
176    .map_err(|e| {
177        FetchiumError::YouTube(format!(
178            "yt-dlp not found — install via `pip install yt-dlp`: {e}"
179        ))
180    })?;
181
182    if !output.status.success() {
183        let stderr = String::from_utf8_lossy(&output.stderr);
184        return Err(FetchiumError::YouTube(format!("yt-dlp error: {stderr}")));
185    }
186
187    // Scan temp dir for subtitle file written by yt-dlp.
188    // Files are named: `{id}.en.vtt`, `{id}.en-orig.vtt`, `{id}.en.srt`, etc.
189    let mut read_dir = tokio::fs::read_dir(&tmp_dir)
190        .await
191        .map_err(|e| FetchiumError::YouTube(format!("temp dir read error: {e}")))?;
192
193    while let Ok(Some(entry)) = read_dir.next_entry().await {
194        let path = entry.path();
195        let ext = path
196            .extension()
197            .and_then(|e| e.to_str())
198            .unwrap_or("")
199            .to_lowercase();
200
201        let is_subtitle = matches!(ext.as_str(), "vtt" | "srt" | "json3");
202        if !is_subtitle {
203            continue;
204        }
205
206        if let Ok(content) = tokio::fs::read_to_string(&path).await {
207            let _ = tokio::fs::remove_file(&path).await;
208
209            let entries = match ext.as_str() {
210                "vtt" => parse_vtt(&content),
211                "srt" => parse_srt(&content),
212                "json3" => parse_json3_subtitle(&content),
213                _ => continue,
214            };
215
216            if !entries.is_empty() {
217                return Ok(entries);
218            }
219        }
220    }
221
222    Err(FetchiumError::YouTube(
223        "yt-dlp wrote no subtitle file".into(),
224    ))
225}
226
227// ─── Subtitle Format Parsers ───────────────────────────────────
228
229/// Parse WebVTT subtitle format into `TranscriptEntry` vec.
230///
231/// Handles all standard VTT features:
232/// - `HH:MM:SS.mmm --> HH:MM:SS.mmm` timestamps
233/// - Positioning tags stripped (`align:start`, `position:0%`)
234/// - Inline tags stripped (`<c>`, `<b>`, timestamp cues like `<00:01.234>`)
235/// - HTML entities decoded
236pub fn parse_vtt(content: &str) -> Vec<TranscriptEntry> {
237    let mut entries = Vec::new();
238    let lines: Vec<&str> = content.lines().collect();
239    let mut i = 0;
240
241    // Skip WEBVTT header and metadata blocks
242    while i < lines.len() && !lines[i].contains("-->") {
243        i += 1;
244    }
245
246    while i < lines.len() {
247        let line = lines[i];
248
249        if !line.contains("-->") {
250            i += 1;
251            continue;
252        }
253
254        // Timestamp line: "00:00:01.000 --> 00:00:04.000 [optional positioning]"
255        // Split on "-->" and take only the timestamp portions
256        let parts: Vec<&str> = line.splitn(2, "-->").collect();
257        if parts.len() < 2 {
258            i += 1;
259            continue;
260        }
261
262        let start_ms = match parse_vtt_timestamp(parts[0].trim()) {
263            Some(ms) => ms,
264            None => {
265                i += 1;
266                continue;
267            }
268        };
269        // End timestamp: strip any trailing positioning info
270        let end_str = parts[1].split_whitespace().next().unwrap_or("");
271        let end_ms = parse_vtt_timestamp(end_str);
272
273        i += 1;
274
275        // Collect text lines until blank line or next cue header
276        let mut text_parts: Vec<String> = Vec::new();
277        while i < lines.len() {
278            let tl = lines[i];
279            if tl.trim().is_empty() {
280                i += 1;
281                break;
282            }
283            // Skip cue identifier lines (pure digits like "1", "2")
284            if tl.trim().chars().all(|c| c.is_ascii_digit()) {
285                i += 1;
286                continue;
287            }
288            let cleaned = strip_vtt_tags(tl);
289            if !cleaned.trim().is_empty() {
290                text_parts.push(cleaned);
291            }
292            i += 1;
293        }
294
295        let text = text_parts.join(" ");
296        let text = text.trim().to_string();
297        if text.is_empty() {
298            continue;
299        }
300
301        let dur_ms = end_ms.map(|e| e.saturating_sub(start_ms)).unwrap_or(2000);
302
303        entries.push(TranscriptEntry {
304            start_ms: start_ms as u32,
305            duration_ms: dur_ms as u32,
306            text: decode_subtitle_text(&text),
307            speaker_id: None,
308        });
309    }
310
311    // Deduplicate consecutive identical entries (common in yt-dlp VTT output)
312    entries.dedup_by(|a, b| a.text == b.text && a.start_ms.abs_diff(b.start_ms) < 500);
313
314    entries
315}
316
317/// Parse SRT subtitle format into `TranscriptEntry` vec.
318///
319/// SRT uses commas as millisecond separators: `00:00:01,000 --> 00:00:04,000`.
320pub fn parse_srt(content: &str) -> Vec<TranscriptEntry> {
321    let mut entries = Vec::new();
322    let mut lines = content.lines().peekable();
323
324    loop {
325        // Find next timestamp line
326        let mut ts_line = String::new();
327        let mut found = false;
328        for line in lines.by_ref() {
329            if line.contains("-->") {
330                ts_line = line.to_string();
331                found = true;
332                break;
333            }
334        }
335        if !found {
336            break;
337        }
338
339        // SRT uses comma for ms: "00:00:01,000 --> 00:00:04,000"
340        let normalized = ts_line.replace(',', ".");
341        let parts: Vec<&str> = normalized.splitn(2, "-->").collect();
342        if parts.len() < 2 {
343            continue;
344        }
345
346        let start_ms = match parse_vtt_timestamp(parts[0].trim()) {
347            Some(ms) => ms,
348            None => continue,
349        };
350        let end_str = parts[1].split_whitespace().next().unwrap_or("");
351        let end_ms = parse_vtt_timestamp(end_str);
352
353        // Collect text until blank line
354        let mut text_parts = Vec::new();
355        for line in lines.by_ref() {
356            if line.trim().is_empty() {
357                break;
358            }
359            let cleaned = strip_vtt_tags(line);
360            if !cleaned.trim().is_empty() {
361                text_parts.push(cleaned);
362            }
363        }
364
365        let text = text_parts.join(" ");
366        let text = text.trim().to_string();
367        if text.is_empty() {
368            continue;
369        }
370
371        let dur_ms = end_ms.map(|e| e.saturating_sub(start_ms)).unwrap_or(2000);
372
373        entries.push(TranscriptEntry {
374            start_ms: start_ms as u32,
375            duration_ms: dur_ms as u32,
376            text: decode_subtitle_text(&text),
377            speaker_id: None,
378        });
379    }
380
381    entries
382}
383
384/// Parse YouTube's JSON3 subtitle format.
385///
386/// Used when yt-dlp requests `--sub-format json3`.
387fn parse_json3_subtitle(content: &str) -> Vec<TranscriptEntry> {
388    let v: serde_json::Value = match serde_json::from_str(content) {
389        Ok(v) => v,
390        Err(_) => return Vec::new(),
391    };
392
393    v["events"]
394        .as_array()
395        .map(|events| {
396            events
397                .iter()
398                .filter_map(|ev| {
399                    let start_ms = ev["tStartMs"].as_u64()? as u32;
400                    let dur_ms = ev["dDurationMs"].as_u64().unwrap_or(2000) as u32;
401                    let segs = ev["segs"].as_array()?;
402                    let text: String = segs
403                        .iter()
404                        .filter_map(|s| s["utf8"].as_str())
405                        .collect::<Vec<_>>()
406                        .join("")
407                        .trim()
408                        .to_string();
409                    if text.is_empty() {
410                        return None;
411                    }
412                    Some(TranscriptEntry {
413                        start_ms,
414                        duration_ms: dur_ms,
415                        text: decode_subtitle_text(&text),
416                        speaker_id: None,
417                    })
418                })
419                .collect()
420        })
421        .unwrap_or_default()
422}
423
424// ─── Timestamp Parsing ─────────────────────────────────────────
425
426/// Parse a VTT/SRT timestamp into milliseconds.
427///
428/// Accepts: `HH:MM:SS.mmm`, `MM:SS.mmm`, `HH:MM:SS,mmm` (SRT with comma).
429pub fn parse_vtt_timestamp(s: &str) -> Option<u64> {
430    // Strip any trailing positioning info
431    let s = s.split_whitespace().next()?;
432    // Normalise comma (SRT) to dot
433    let s = &s.replace(',', ".");
434    let parts: Vec<&str> = s.split(':').collect();
435
436    let (hours, minutes, seconds_str) = match parts.as_slice() {
437        [m, s] => (0u64, m.parse::<u64>().ok()?, *s),
438        [h, m, s] => (h.parse::<u64>().ok()?, m.parse::<u64>().ok()?, *s),
439        _ => return None,
440    };
441
442    let sec_parts: Vec<&str> = seconds_str.split('.').collect();
443    let secs: u64 = sec_parts.first()?.parse().ok()?;
444    let millis: u64 = sec_parts
445        .get(1)
446        .map(|ms| {
447            let padded = format!("{:0<3}", &ms[..ms.len().min(3)]);
448            padded.parse().unwrap_or(0)
449        })
450        .unwrap_or(0);
451
452    Some((hours * 3600 + minutes * 60 + secs) * 1000 + millis)
453}
454
455// ─── Text Cleaning ─────────────────────────────────────────────
456
457/// Strip VTT/SRT inline tags: `<c>`, `</c>`, timestamp cues, `<b>`, `<i>`, etc.
458fn strip_vtt_tags(s: &str) -> String {
459    let mut out = String::with_capacity(s.len());
460    let mut in_tag = false;
461    for ch in s.chars() {
462        match ch {
463            '<' => in_tag = true,
464            '>' => in_tag = false,
465            _ if !in_tag => out.push(ch),
466            _ => {}
467        }
468    }
469    out
470}
471
472/// Decode HTML entities and normalise whitespace for subtitle text.
473fn decode_subtitle_text(s: &str) -> String {
474    s.replace("&amp;", "&")
475        .replace("&lt;", "<")
476        .replace("&gt;", ">")
477        .replace("&quot;", "\"")
478        .replace("&#39;", "'")
479        .replace("&apos;", "'")
480        .replace("&nbsp;", " ")
481        .replace('\n', " ")
482}
483
484// ─── Enhancement ───────────────────────────────────────────────
485
486/// Build `EnhancedTranscript` from raw subtitle entries.
487///
488/// Applies gap-based speaker turn detection and key moment tagging —
489/// same quality as the YouTube-specific path.
490fn build_enhanced_transcript(
491    url: &str,
492    mut entries: Vec<TranscriptEntry>,
493    source: TranscriptSource,
494) -> EnhancedTranscript {
495    // Gap-based speaker turn detection (>3s gap = new speaker turn)
496    let mut current_speaker = 0u32;
497    let mut last_end_ms = 0u32;
498    let mut speaker_counts = [0usize; 2];
499
500    for entry in &mut entries {
501        let gap = entry.start_ms.saturating_sub(last_end_ms);
502        if gap > 3000 {
503            current_speaker = 1 - current_speaker;
504        }
505        entry.speaker_id = Some(current_speaker);
506        speaker_counts[current_speaker as usize] += 1;
507        last_end_ms = entry.start_ms + entry.duration_ms;
508    }
509
510    let speakers: Vec<Speaker> = ["Speaker A", "Speaker B"]
511        .iter()
512        .enumerate()
513        .filter(|(i, _)| speaker_counts[*i] > 0)
514        .map(|(i, label)| Speaker {
515            id: i as u32,
516            label: label.to_string(),
517            segment_count: speaker_counts[i],
518        })
519        .collect();
520
521    let key_moments = detect_key_moments_uvtp(&entries);
522
523    let full_text: String = entries
524        .iter()
525        .map(|e| e.text.as_str())
526        .collect::<Vec<_>>()
527        .join(" ");
528    let word_count = full_text.split_whitespace().count();
529
530    // Use a stable ID derived from the URL
531    let video_id = url_hash(url);
532
533    EnhancedTranscript {
534        video_id,
535        language: "en".to_string(),
536        entries,
537        speakers,
538        key_moments,
539        full_text,
540        word_count,
541        source,
542        quality_score: 1.0,
543    }
544}
545
546/// Key moment detection for universal transcripts.
547fn detect_key_moments_uvtp(entries: &[TranscriptEntry]) -> Vec<KeyMoment> {
548    const TRANSITION_PHRASES: &[(&str, MomentType, f64)] = &[
549        ("in summary", MomentType::Conclusion, 0.9),
550        ("in conclusion", MomentType::Conclusion, 0.9),
551        ("to summarize", MomentType::Conclusion, 0.9),
552        ("the key point", MomentType::KeyPoint, 0.85),
553        ("most importantly", MomentType::KeyPoint, 0.85),
554        ("for example", MomentType::Example, 0.7),
555        ("such as", MomentType::Example, 0.6),
556        ("moving on", MomentType::TopicShift, 0.6),
557        ("next", MomentType::TopicShift, 0.5),
558        ("first", MomentType::TopicShift, 0.5),
559        ("finally", MomentType::TopicShift, 0.7),
560        ("that means", MomentType::Definition, 0.75),
561        ("which means", MomentType::Definition, 0.75),
562        ("is defined as", MomentType::Definition, 0.8),
563    ];
564
565    let mut moments = Vec::new();
566    let mut last_ts = 0u32;
567
568    for entry in entries {
569        // Debounce: don't add two moments within 10s
570        if entry.start_ms < last_ts + 10_000 {
571            continue;
572        }
573
574        let lower = entry.text.to_lowercase();
575        for (phrase, moment_type, importance) in TRANSITION_PHRASES {
576            if lower.contains(phrase) {
577                moments.push(KeyMoment {
578                    timestamp_ms: entry.start_ms,
579                    moment_type: *moment_type,
580                    text: entry.text.clone(),
581                    importance: *importance,
582                });
583                last_ts = entry.start_ms;
584                break;
585            }
586        }
587    }
588
589    moments
590}
591
592// ─── Utilities ─────────────────────────────────────────────────
593
594/// Stable 16-hex-char hash of a URL for temp directory naming.
595fn url_hash(url: &str) -> String {
596    use std::collections::hash_map::DefaultHasher;
597    use std::hash::{Hash, Hasher};
598    let mut h = DefaultHasher::new();
599    url.hash(&mut h);
600    format!("{:016x}", h.finish())
601}
602
603// ─── Tests ─────────────────────────────────────────────────────
604
605#[cfg(test)]
606mod tests {
607    use super::*;
608
609    #[test]
610    fn is_youtube_url_variants() {
611        assert!(is_youtube_url("https://www.youtube.com/watch?v=abc123"));
612        assert!(is_youtube_url("https://youtu.be/abc123"));
613        assert!(is_youtube_url("https://m.youtube.com/watch?v=abc123"));
614        assert!(is_youtube_url("https://youtube.com/shorts/abc123"));
615        assert!(!is_youtube_url("https://vimeo.com/123456789"));
616        assert!(!is_youtube_url("https://tiktok.com/@user/video/123"));
617    }
618
619    #[test]
620    fn detect_platform_variants() {
621        assert_eq!(detect_platform("https://vimeo.com/123"), "Vimeo");
622        assert_eq!(
623            detect_platform("https://tiktok.com/@user/video/123"),
624            "TikTok"
625        );
626        assert_eq!(
627            detect_platform("https://twitter.com/user/status/123"),
628            "Twitter/X"
629        );
630        assert_eq!(detect_platform("https://twitch.tv/videos/123"), "Twitch");
631        assert_eq!(detect_platform("https://example.com/video.mp4"), "Video");
632    }
633
634    #[test]
635    fn parse_vtt_timestamp_variants() {
636        assert_eq!(parse_vtt_timestamp("00:00:01.000"), Some(1000));
637        assert_eq!(parse_vtt_timestamp("01:23:45.678"), Some(5025678));
638        assert_eq!(parse_vtt_timestamp("01:30.500"), Some(90500));
639        assert_eq!(parse_vtt_timestamp("00:00:00.000"), Some(0));
640        // SRT comma format
641        assert_eq!(parse_vtt_timestamp("00:00:01,500"), Some(1500));
642        assert_eq!(parse_vtt_timestamp("invalid"), None);
643    }
644
645    #[test]
646    fn parse_vtt_basic() {
647        let vtt = "WEBVTT\n\n\
648                   00:00:01.000 --> 00:00:04.000\n\
649                   Hello, welcome to the video.\n\n\
650                   00:00:04.500 --> 00:00:08.000\n\
651                   Today we cover Rust.\n\n";
652        let entries = parse_vtt(vtt);
653        assert_eq!(entries.len(), 2);
654        assert_eq!(entries[0].text, "Hello, welcome to the video.");
655        assert_eq!(entries[0].start_ms, 1000);
656        assert_eq!(entries[0].duration_ms, 3000);
657        assert_eq!(entries[1].text, "Today we cover Rust.");
658        assert_eq!(entries[1].start_ms, 4500);
659    }
660
661    #[test]
662    fn parse_vtt_with_inline_tags() {
663        let vtt = "WEBVTT\n\n\
664                   00:00:01.000 --> 00:00:04.000\n\
665                   <c>Hello</c> <00:00:01.234><b>world</b>\n\n";
666        let entries = parse_vtt(vtt);
667        assert_eq!(entries.len(), 1);
668        assert_eq!(entries[0].text, "Hello world");
669    }
670
671    #[test]
672    fn parse_vtt_with_positioning() {
673        let vtt = "WEBVTT\n\n\
674                   00:00:01.000 --> 00:00:04.000 align:start position:0%\n\
675                   Caption with positioning.\n\n";
676        let entries = parse_vtt(vtt);
677        assert_eq!(entries.len(), 1);
678        assert_eq!(entries[0].text, "Caption with positioning.");
679    }
680
681    #[test]
682    fn parse_srt_basic() {
683        let srt = "1\n\
684                   00:00:01,000 --> 00:00:04,000\n\
685                   Hello, welcome to the video.\n\n\
686                   2\n\
687                   00:00:04,500 --> 00:00:08,000\n\
688                   Today we cover Rust.\n\n";
689        let entries = parse_srt(srt);
690        assert_eq!(entries.len(), 2);
691        assert_eq!(entries[0].text, "Hello, welcome to the video.");
692        assert_eq!(entries[0].start_ms, 1000);
693        assert_eq!(entries[1].text, "Today we cover Rust.");
694    }
695
696    #[test]
697    fn parse_srt_multiline() {
698        let srt = "1\n\
699                   00:00:01,000 --> 00:00:05,000\n\
700                   Line one of the caption.\n\
701                   Line two continues here.\n\n";
702        let entries = parse_srt(srt);
703        assert_eq!(entries.len(), 1);
704        assert!(entries[0].text.contains("Line one"));
705        assert!(entries[0].text.contains("Line two"));
706    }
707
708    #[test]
709    fn parse_vtt_dedup() {
710        // yt-dlp sometimes outputs duplicate consecutive lines
711        let vtt = "WEBVTT\n\n\
712                   00:00:01.000 --> 00:00:03.000\n\
713                   Duplicate line.\n\n\
714                   00:00:01.200 --> 00:00:03.000\n\
715                   Duplicate line.\n\n\
716                   00:00:05.000 --> 00:00:07.000\n\
717                   Different line.\n\n";
718        let entries = parse_vtt(vtt);
719        // Should deduplicate the close-together identical entries
720        let dup_count = entries
721            .iter()
722            .filter(|e| e.text == "Duplicate line.")
723            .count();
724        assert_eq!(dup_count, 1);
725    }
726
727    #[test]
728    fn parse_json3_subtitle_basic() {
729        let json = serde_json::json!({
730            "events": [
731                { "tStartMs": 1000, "dDurationMs": 3000, "segs": [{"utf8": "Hello world"}] },
732                { "tStartMs": 5000, "dDurationMs": 2000, "segs": [{"utf8": "Rust is fast"}] }
733            ]
734        });
735        let entries = parse_json3_subtitle(&json.to_string());
736        assert_eq!(entries.len(), 2);
737        assert_eq!(entries[0].text, "Hello world");
738        assert_eq!(entries[1].text, "Rust is fast");
739    }
740
741    #[test]
742    fn strip_vtt_tags_variants() {
743        assert_eq!(strip_vtt_tags("<c>Hello</c> <b>world</b>"), "Hello world");
744        assert_eq!(strip_vtt_tags("plain text"), "plain text");
745        assert_eq!(strip_vtt_tags("<00:01.234>text"), "text");
746    }
747
748    #[test]
749    fn decode_subtitle_entities() {
750        assert_eq!(decode_subtitle_text("It&apos;s &amp; good"), "It's & good");
751        assert_eq!(decode_subtitle_text("&lt;tag&gt;"), "<tag>");
752        assert_eq!(decode_subtitle_text("line1\nline2"), "line1 line2");
753    }
754
755    #[test]
756    fn key_moments_detection() {
757        let entries = vec![
758            TranscriptEntry {
759                start_ms: 0,
760                duration_ms: 2000,
761                text: "Hello everyone".into(),
762                speaker_id: None,
763            },
764            TranscriptEntry {
765                start_ms: 15000,
766                duration_ms: 3000,
767                text: "In conclusion, we learned a lot today".into(),
768                speaker_id: None,
769            },
770            TranscriptEntry {
771                start_ms: 30000,
772                duration_ms: 2000,
773                text: "For example, consider this case".into(),
774                speaker_id: None,
775            },
776        ];
777        let moments = detect_key_moments_uvtp(&entries);
778        assert!(moments.len() >= 2);
779        assert!(moments
780            .iter()
781            .any(|m| m.moment_type == MomentType::Conclusion));
782    }
783
784    #[test]
785    fn build_enhanced_transcript_speakers() {
786        let entries = vec![
787            TranscriptEntry {
788                start_ms: 0,
789                duration_ms: 2000,
790                text: "First speaker".into(),
791                speaker_id: None,
792            },
793            TranscriptEntry {
794                start_ms: 10000, // 10s gap → speaker change
795                duration_ms: 2000,
796                text: "Second speaker".into(),
797                speaker_id: None,
798            },
799        ];
800        let t =
801            build_enhanced_transcript("https://vimeo.com/123", entries, TranscriptSource::YtDlp);
802        assert_eq!(t.word_count, 4);
803        assert_eq!(t.source, TranscriptSource::YtDlp);
804        assert_eq!(t.speakers.len(), 2);
805    }
806
807    #[test]
808    fn url_hash_stable() {
809        let h1 = url_hash("https://vimeo.com/123456");
810        let h2 = url_hash("https://vimeo.com/123456");
811        let h3 = url_hash("https://vimeo.com/789012");
812        assert_eq!(h1, h2);
813        assert_ne!(h1, h3);
814        assert_eq!(h1.len(), 16);
815    }
816}