Skip to main content

fetchium_core/youtube/
transcript.rs

1//! Fast parallel transcript extraction — watch-page scraping + Innertube + fallbacks.
2//!
3//! All sources race simultaneously; first valid result wins within 15s.
4//!
5//! ## Source Priority
6//! 1. **Watch-page + ANDROID Innertube** — GET /watch to extract live API key,
7//!    then POST Innertube with ANDROID client (bypasses poToken/exp=xpe). Returns
8//!    XML transcript from clean baseUrl. Same technique as youtube-transcript-api.
9//! 2. **ANDROID Innertube direct** — POST with hardcoded API key (no watch-page
10//!    fetch; faster but may break if key changes). Both Sources 1 & 2 race.
11//! 3. Invidious captions API (community instance)
12//! 4. Piped captions API (community instance)
13
14use crate::error::{FetchiumError, FetchiumResult};
15use crate::http::client::HttpClient;
16use crate::youtube::types::*;
17use std::time::Duration;
18
19/// Global timeout for transcript fetching across all sources.
20const TRANSCRIPT_TIMEOUT_SECS: u64 = 15;
21
22// ─── Public API ────────────────────────────────────────────────
23
24/// Fetch enhanced transcript by racing all sources simultaneously.
25///
26/// Sources (all start in parallel, first non-empty result wins):
27/// 1. **Watch-page scraping** — extracts pre-signed captionTracks baseUrl from HTML
28///    (same technique as youtube-transcript-api; bypasses bot detection)
29/// 2. Innertube API POST (fallback — may fail without visitor cookies)
30/// 3. Invidious captions API
31/// 4. Piped captions API
32///
33/// Global cap: 15s — the pipeline never blocks longer regardless of source health.
34pub async fn fetch_transcript(
35    video_id: &str,
36    http: &HttpClient,
37    config: &crate::config::FetchiumConfig,
38) -> FetchiumResult<EnhancedTranscript> {
39    let (tx, mut rx) =
40        tokio::sync::mpsc::channel::<(Vec<TranscriptEntry>, TranscriptSource, String)>(8);
41
42    let per_src = Duration::from_secs(config.youtube.timeout_secs.max(10));
43
44    // Source 1: Watch-page scraping — MOST RELIABLE.
45    // Fetches /watch page, extracts ytInitialPlayerResponse, uses the pre-signed
46    // captionTracks baseUrl (contains expire + signature tokens already embedded).
47    // This is the approach used by youtube-transcript-api (Python) and works on
48    // any public video regardless of YouTube's bot detection changes.
49    {
50        let tx = tx.clone();
51        let http = http.clone();
52        let vid = video_id.to_string();
53        tokio::spawn(async move {
54            if let Ok((entries, lang)) = fetch_via_watch_page(&vid, &http, per_src).await {
55                if !entries.is_empty() {
56                    let _ = tx
57                        .send((entries, TranscriptSource::YouTubeTimedtext, lang))
58                        .await;
59                }
60            }
61        });
62    }
63
64    // Source 2: ANDROID Innertube direct — no watch-page fetch, uses stable key.
65    // Races with Source 1; faster on good connections, same quality.
66    {
67        let tx = tx.clone();
68        let http = http.clone();
69        let vid = video_id.to_string();
70        tokio::spawn(async move {
71            if let Ok((entries, lang)) = fetch_via_android_innertube(&vid, &http, per_src).await {
72                if !entries.is_empty() {
73                    let _ = tx
74                        .send((entries, TranscriptSource::YouTubeTimedtext, lang))
75                        .await;
76                }
77            }
78        });
79    }
80
81    // Source 3: TVHTML5 (Smart TV) Innertube — different client fingerprint.
82    // Succeeds for some videos where ANDROID returns UNPLAYABLE or times out.
83    {
84        let tx = tx.clone();
85        let http = http.clone();
86        let vid = video_id.to_string();
87        tokio::spawn(async move {
88            if let Ok((entries, lang)) = fetch_via_tvhtml5(&vid, &http, per_src).await {
89                if !entries.is_empty() {
90                    let _ = tx
91                        .send((entries, TranscriptSource::YouTubeTimedtext, lang))
92                        .await;
93                }
94            }
95        });
96    }
97
98    // Source 4+: Invidious captions (race all instances)
99    for instance in &config.youtube.invidious_instances {
100        let tx = tx.clone();
101        let http = http.clone();
102        let vid = video_id.to_string();
103        let inst = instance.clone();
104        tokio::spawn(async move {
105            if let Ok(entries) = fetch_invidious_captions_fast(&vid, &inst, &http, per_src).await {
106                if !entries.is_empty() {
107                    let _ = tx
108                        .send((entries, TranscriptSource::Invidious, "en".to_string()))
109                        .await;
110                }
111            }
112        });
113    }
114
115    // Source 5+: Piped captions (race all instances)
116    for instance in &config.youtube.piped_instances {
117        let tx = tx.clone();
118        let http = http.clone();
119        let vid = video_id.to_string();
120        let inst = instance.clone();
121        tokio::spawn(async move {
122            if let Ok(entries) = fetch_piped_captions_fast(&vid, &inst, &http, per_src).await {
123                if !entries.is_empty() {
124                    let _ = tx
125                        .send((entries, TranscriptSource::Piped, "en".to_string()))
126                        .await;
127                }
128            }
129        });
130    }
131
132    drop(tx);
133
134    // Wait for first result from any parallel source within the global cap.
135    let parallel_result =
136        tokio::time::timeout(Duration::from_secs(TRANSCRIPT_TIMEOUT_SECS), rx.recv()).await;
137
138    if let Ok(Some((entries, source, lang))) = parallel_result {
139        return Ok(enhance_transcript(video_id, entries, source, lang));
140    }
141
142    // ── Whisper ASR fallback ──────────────────────────────────────────────────
143    // All caption sources failed (video has no captions, or network issues).
144    // Try Whisper speech-to-text as a last resort: yt-dlp downloads audio-only,
145    // Whisper transcribes it. Works for ANY public video regardless of captions.
146    // Requires: `yt-dlp` + `whisper` (pip install yt-dlp openai-whisper).
147    if let Ok((entries, lang)) = fetch_via_whisper(video_id).await {
148        if !entries.is_empty() {
149            return Ok(enhance_transcript(
150                video_id,
151                entries,
152                TranscriptSource::YtDlp,
153                lang,
154            ));
155        }
156    }
157
158    Err(FetchiumError::YouTube(format!(
159        "No transcript available for video {video_id}. \
160         Caption sources returned nothing and Whisper ASR fallback also failed. \
161         Ensure yt-dlp and openai-whisper are installed: \
162         `pip install yt-dlp openai-whisper`"
163    )))
164}
165
166// ─── Known Innertube API key (stable, extracted from any YouTube page) ─────
167
168/// YouTube's public Innertube API key — stable since 2020.
169/// Used as fallback; watch-page path extracts the live key for reliability.
170const INNERTUBE_API_KEY: &str = "***REMOVED***";
171
172// ─── Noise Detection & Quality Scoring ────────────────────────
173
174/// Returns `true` if a transcript entry consists entirely of noise annotations.
175///
176/// YouTube auto-captions emit tags like `[Music]`, `[Applause]`, `[foreign]`,
177/// `[Laughter]` for non-speech segments. These are useful for chapter alignment
178/// but should be excluded from readable output and quality scoring.
179fn is_noise_text(text: &str) -> bool {
180    static NOISE_RE: once_cell::sync::Lazy<regex::Regex> =
181        once_cell::sync::Lazy::new(|| regex::Regex::new(r"^(\s*\[[^\]]*\]\s*)+$").unwrap());
182    let t = text.trim();
183    t.is_empty() || NOISE_RE.is_match(t)
184}
185
186/// Score the quality of a transcript to detect wrong-language ASR or garbled output.
187///
188/// Returns 0.0 (garbage) to 1.0 (excellent). Penalises:
189/// - High noise ratio (`[Music]` / `[Applause]` etc.)
190/// - `[foreign]` markers (YouTube's signal that English ASR detected non-English speech)
191/// - Very short average word length (garbled ASR produces truncated tokens like "spee")
192fn score_quality(entries: &[TranscriptEntry]) -> f64 {
193    if entries.is_empty() {
194        return 0.0;
195    }
196
197    let total = entries.len() as f64;
198    let noise_count = entries.iter().filter(|e| is_noise_text(&e.text)).count() as f64;
199    let foreign_count = entries
200        .iter()
201        .filter(|e| {
202            let lower = e.text.to_lowercase();
203            lower.contains("[foreign]") || lower.contains("foreign")
204        })
205        .count() as f64;
206
207    let noise_ratio = noise_count / total;
208    let foreign_ratio = foreign_count / total;
209
210    // Average word length on speech-only entries (garbled ASR → very short words).
211    let speech_words: Vec<usize> = entries
212        .iter()
213        .filter(|e| !is_noise_text(&e.text))
214        .flat_map(|e| {
215            e.text
216                .split_whitespace()
217                .map(|w| w.trim_matches(|c: char| !c.is_alphabetic()).len())
218        })
219        .filter(|&l| l > 0)
220        .collect();
221
222    let avg_word_len = if speech_words.is_empty() {
223        0.0_f64
224    } else {
225        speech_words.iter().sum::<usize>() as f64 / speech_words.len() as f64
226    };
227
228    let mut score = (1.0 - noise_ratio).max(0.0);
229
230    // [foreign] = YouTube explicitly detected non-English speech in English ASR — heavy penalty.
231    if foreign_ratio > 0.05 {
232        score *= 0.25;
233    }
234
235    // Garbled ASR word length heuristic.
236    match avg_word_len as u32 {
237        0 => score = 0.0,
238        1..=2 => score *= 0.2,
239        3 => score *= 0.5,
240        _ => {}
241    }
242
243    score
244}
245
246// ─── Watch-Page + ANDROID Innertube (Primary Source) ──────────
247
248/// Fetch transcript via watch-page scraping + ANDROID Innertube API.
249///
250/// ## Algorithm (mirrors youtube-transcript-api Python library)
251/// 1. GET https://www.youtube.com/watch?v={id} with browser User-Agent
252/// 2. Extract live `INNERTUBE_API_KEY` from the page HTML
253/// 3. POST to Innertube with **ANDROID client context** — bypasses poToken/exp=xpe
254/// 4. Auto-quality: if English ASR is garbled, retry with native language track
255async fn fetch_via_watch_page(
256    video_id: &str,
257    http: &HttpClient,
258    timeout: Duration,
259) -> FetchiumResult<(Vec<TranscriptEntry>, String)> {
260    let url = format!("https://www.youtube.com/watch?v={video_id}&hl=en");
261
262    let html = tokio::time::timeout(timeout, async {
263        http.client()
264            .get(&url)
265            .header(
266                "User-Agent",
267                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
268                 AppleWebKit/537.36 (KHTML, like Gecko) \
269                 Chrome/120.0.0.0 Safari/537.36",
270            )
271            .header("Accept-Language", "en-US,en;q=0.9")
272            .send()
273            .await
274            .map_err(FetchiumError::Network)?
275            .text()
276            .await
277            .map_err(FetchiumError::Network)
278    })
279    .await
280    .map_err(|_| FetchiumError::YouTube("Watch page request timed out".into()))??;
281
282    let api_key = extract_innertube_api_key(&html).unwrap_or_else(|| INNERTUBE_API_KEY.to_string());
283
284    fetch_innertube_with_client(
285        video_id,
286        http,
287        &api_key,
288        timeout.min(Duration::from_secs(10)),
289        "ANDROID",
290        "20.10.38",
291        "com.google.android.youtube/20.10.38 (Linux; U; Android 11)",
292    )
293    .await
294}
295
296/// Extract YouTube's Innertube API key from a watch page HTML blob.
297fn extract_innertube_api_key(html: &str) -> Option<String> {
298    let marker = "\"INNERTUBE_API_KEY\":\"";
299    let start = html.find(marker)? + marker.len();
300    let end = html[start..].find('"')? + start;
301    Some(html[start..end].to_string())
302}
303
304// ─── ANDROID Innertube (Fast Fallback) ────────────────────────
305
306/// ANDROID Innertube — hardcoded stable API key, no watch-page fetch required.
307/// Races with `fetch_via_watch_page`; whichever responds first wins.
308async fn fetch_via_android_innertube(
309    video_id: &str,
310    http: &HttpClient,
311    timeout: Duration,
312) -> FetchiumResult<(Vec<TranscriptEntry>, String)> {
313    fetch_innertube_with_client(
314        video_id,
315        http,
316        INNERTUBE_API_KEY,
317        timeout,
318        "ANDROID",
319        "20.10.38",
320        "com.google.android.youtube/20.10.38 (Linux; U; Android 11)",
321    )
322    .await
323}
324
325// ─── TVHTML5 Innertube (Timeout Fallback) ─────────────────────
326
327/// TVHTML5 (Smart TV) Innertube — different client context than ANDROID.
328///
329/// Some videos that time out or return `UNPLAYABLE` for the ANDROID client
330/// succeed with the TV client, which has a different bot-detection surface.
331/// Runs in parallel with the other sources.
332async fn fetch_via_tvhtml5(
333    video_id: &str,
334    http: &HttpClient,
335    timeout: Duration,
336) -> FetchiumResult<(Vec<TranscriptEntry>, String)> {
337    fetch_innertube_with_client(
338        video_id,
339        http,
340        INNERTUBE_API_KEY,
341        timeout,
342        "TVHTML5",
343        "7.20241201.11.00",
344        "Mozilla/5.0 (SMART-TV; LINUX; Tizen 6.0) AppleWebKit/538.1 \
345         (KHTML, like Gecko) Version/6.0 TV Safari/538.1",
346    )
347    .await
348}
349
350// ─── Generic Innertube Core ────────────────────────────────────
351
352/// Core Innertube transcript extraction — parameterised by client type.
353///
354/// Supports any Innertube client (ANDROID, TVHTML5, IOS, …).
355///
356/// ## Quality-Aware Language Selection
357/// After fetching the English track, we score transcript quality. If the score
358/// is below 0.4 — indicating garbled ASR (e.g. English ASR on a Bengali video
359/// producing "[foreign] spee speee") — we automatically retry with the best
360/// available manual caption track in the video's native language.
361///
362/// Flow: POST /youtubei/v1/player → captionTracks → pick best track →
363///       fetch XML timedtext → quality check → optional native-lang retry → parse.
364async fn fetch_innertube_with_client(
365    video_id: &str,
366    http: &HttpClient,
367    api_key: &str,
368    timeout: Duration,
369    client_name: &str,
370    client_version: &str,
371    user_agent: &str,
372) -> FetchiumResult<(Vec<TranscriptEntry>, String)> {
373    let innertube_url =
374        format!("https://www.youtube.com/youtubei/v1/player?key={api_key}&prettyPrint=false");
375    let body = serde_json::json!({
376        "context": {
377            "client": {
378                "clientName": client_name,
379                "clientVersion": client_version,
380                "hl": "en",
381                "gl": "US"
382            }
383        },
384        "videoId": video_id
385    })
386    .to_string();
387
388    let response_text = tokio::time::timeout(timeout, async {
389        http.client()
390            .post(&innertube_url)
391            .header("Content-Type", "application/json")
392            .header("User-Agent", user_agent)
393            .body(body)
394            .send()
395            .await
396            .map_err(FetchiumError::Network)?
397            .text()
398            .await
399            .map_err(FetchiumError::Network)
400    })
401    .await
402    .map_err(|_| {
403        FetchiumError::YouTube(format!("Innertube ({client_name}) request timed out"))
404    })??;
405
406    let v: serde_json::Value = serde_json::from_str(&response_text)
407        .map_err(|e| FetchiumError::YouTube(format!("Innertube JSON parse: {e}")))?;
408
409    let tracks = v["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]
410        .as_array()
411        .ok_or_else(|| {
412            FetchiumError::YouTube(
413                "No caption tracks in Innertube response (video may have no captions)".into(),
414            )
415        })?;
416
417    // ── Track selection ────────────────────────────────────────────────────────
418    // Priority: manual-en → ASR-en → manual-any → ASR-any
419    // If we later detect the English track is garbage, we retry with native-any.
420    let (language_code, caption_url) = {
421        let selected = tracks
422            .iter()
423            .find(|t| {
424                t["languageCode"]
425                    .as_str()
426                    .map(|c| c.starts_with("en"))
427                    .unwrap_or(false)
428                    && t["kind"].as_str().map(|k| k != "asr").unwrap_or(true)
429            })
430            .or_else(|| {
431                tracks.iter().find(|t| {
432                    t["languageCode"]
433                        .as_str()
434                        .map(|c| c.starts_with("en"))
435                        .unwrap_or(false)
436                })
437            })
438            .or_else(|| {
439                tracks.iter().find(|t| {
440                    t["kind"].as_str().map(|k| k != "asr").unwrap_or(true)
441                        && t["baseUrl"].as_str().is_some()
442                })
443            })
444            .or_else(|| tracks.first())
445            .ok_or_else(|| FetchiumError::YouTube("No caption tracks available".into()))?;
446
447        let lang = selected["languageCode"]
448            .as_str()
449            .unwrap_or("und")
450            .to_string();
451        let url = selected["baseUrl"]
452            .as_str()
453            .ok_or_else(|| FetchiumError::YouTube("No baseUrl in caption track".into()))?
454            .replace("&fmt=srv3", ""); // ANDROID/TVHTML5 URLs are XML — don't add &fmt=json3
455        (lang, url)
456    }; // `selected` borrow dropped here so we can iterate `tracks` again below
457
458    let cap_timeout = timeout.min(Duration::from_secs(8));
459    let xml_body = fetch_caption_xml(&caption_url, http, cap_timeout).await?;
460    let entries = parse_timedtext_xml(&xml_body)?;
461
462    // ── Quality check: auto-retry with native language ──────────────────────
463    // Detect garbled ASR (English ASR on non-English video). Hallmarks:
464    //   - Many [foreign] markers (YouTube's own signal)
465    //   - Very short average word length (truncated/garbage tokens)
466    //   - High noise ratio ([Music] drowning out speech)
467    let quality = score_quality(&entries);
468    if quality < 0.4 && language_code.starts_with("en") {
469        // Try to find a manual caption track in the video's native language.
470        let native = tracks.iter().find(|t| {
471            let is_non_english = t["languageCode"]
472                .as_str()
473                .map(|c| !c.starts_with("en"))
474                .unwrap_or(false);
475            let is_manual = t["kind"].as_str().map(|k| k != "asr").unwrap_or(true);
476            is_non_english && is_manual && t["baseUrl"].as_str().is_some()
477        });
478
479        if let Some(native_track) = native {
480            let native_lang = native_track["languageCode"]
481                .as_str()
482                .unwrap_or("und")
483                .to_string();
484            let native_url = native_track["baseUrl"]
485                .as_str()
486                .unwrap_or("")
487                .replace("&fmt=srv3", "");
488
489            let retry_timeout = Duration::from_secs(6);
490            if let Ok(native_xml) = fetch_caption_xml(&native_url, http, retry_timeout).await {
491                if let Ok(native_entries) = parse_timedtext_xml(&native_xml) {
492                    let native_quality = score_quality(&native_entries);
493                    if native_quality > quality || !native_entries.is_empty() {
494                        return Ok((native_entries, native_lang));
495                    }
496                }
497            }
498        }
499
500        // If no manual native track, also try ASR in any other language.
501        let asr_native = tracks.iter().find(|t| {
502            t["languageCode"]
503                .as_str()
504                .map(|c| !c.starts_with("en"))
505                .unwrap_or(false)
506                && t["baseUrl"].as_str().is_some()
507        });
508        if let Some(asr_track) = asr_native {
509            let asr_lang = asr_track["languageCode"]
510                .as_str()
511                .unwrap_or("und")
512                .to_string();
513            let asr_url = asr_track["baseUrl"]
514                .as_str()
515                .unwrap_or("")
516                .replace("&fmt=srv3", "");
517            let retry_timeout = Duration::from_secs(6);
518            if let Ok(asr_xml) = fetch_caption_xml(&asr_url, http, retry_timeout).await {
519                if let Ok(asr_entries) = parse_timedtext_xml(&asr_xml) {
520                    let asr_quality = score_quality(&asr_entries);
521                    if asr_quality > quality {
522                        return Ok((asr_entries, asr_lang));
523                    }
524                }
525            }
526        }
527    }
528
529    Ok((entries, language_code))
530}
531
532/// Fetch raw caption XML from a URL with a timeout.
533async fn fetch_caption_xml(
534    url: &str,
535    http: &HttpClient,
536    timeout: Duration,
537) -> FetchiumResult<String> {
538    tokio::time::timeout(timeout, async {
539        http.client()
540            .get(url)
541            .header(
542                "User-Agent",
543                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
544            )
545            .send()
546            .await
547            .map_err(FetchiumError::Network)?
548            .text()
549            .await
550            .map_err(FetchiumError::Network)
551    })
552    .await
553    .map_err(|_| FetchiumError::YouTube("Caption XML fetch timed out".into()))?
554}
555
556// ─── Fallback Sources ──────────────────────────────────────────
557
558/// Invidious captions — uses fetch_text_once (no retries) for both requests.
559///
560/// Connection errors on Invidious instances are expected; fail fast rather than
561/// burning 3.5s on retry sleeps when Innertube and timedtext are already racing.
562async fn fetch_invidious_captions_fast(
563    video_id: &str,
564    instance: &str,
565    http: &HttpClient,
566    timeout: Duration,
567) -> FetchiumResult<Vec<TranscriptEntry>> {
568    let url = format!("{instance}/api/v1/captions/{video_id}");
569    let body = tokio::time::timeout(timeout, http.fetch_text_once(&url))
570        .await
571        .map_err(|_| FetchiumError::YouTube("Invidious captions timeout".into()))??;
572
573    let v: serde_json::Value = serde_json::from_str(&body)
574        .map_err(|e| FetchiumError::YouTube(format!("Invidious captions parse: {e}")))?;
575
576    let captions = v["captions"]
577        .as_array()
578        .ok_or_else(|| FetchiumError::YouTube("No captions".into()))?;
579
580    // Find English caption URL
581    let en_url_str = captions
582        .iter()
583        .find(|c| {
584            c["language_code"]
585                .as_str()
586                .map(|l| l.starts_with("en"))
587                .unwrap_or(false)
588        })
589        .and_then(|c| c["url"].as_str())
590        .ok_or_else(|| FetchiumError::YouTube("No English captions in Invidious".into()))?
591        .to_string();
592
593    let caption_url = if en_url_str.starts_with("http") {
594        en_url_str
595    } else {
596        format!("{instance}{en_url_str}")
597    };
598
599    let caption_body = tokio::time::timeout(timeout, http.fetch_text_once(&caption_url))
600        .await
601        .map_err(|_| FetchiumError::YouTube("Invidious caption body timeout".into()))??;
602
603    parse_timedtext_xml(&caption_body)
604}
605
606/// Piped captions — uses fetch_text_once (no retries) for both requests.
607async fn fetch_piped_captions_fast(
608    video_id: &str,
609    instance: &str,
610    http: &HttpClient,
611    timeout: Duration,
612) -> FetchiumResult<Vec<TranscriptEntry>> {
613    let url = format!("{instance}/streams/{video_id}");
614    let body = tokio::time::timeout(timeout, http.fetch_text_once(&url))
615        .await
616        .map_err(|_| FetchiumError::YouTube("Piped stream timeout".into()))??;
617
618    let v: serde_json::Value = serde_json::from_str(&body)
619        .map_err(|e| FetchiumError::YouTube(format!("Piped parse: {e}")))?;
620
621    let subtitles = v["subtitles"]
622        .as_array()
623        .ok_or_else(|| FetchiumError::YouTube("No subtitles in Piped".into()))?;
624
625    let en_url = subtitles
626        .iter()
627        .find(|s| {
628            s["code"]
629                .as_str()
630                .map(|c| c.starts_with("en"))
631                .unwrap_or(false)
632        })
633        .and_then(|s| s["url"].as_str())
634        .ok_or_else(|| FetchiumError::YouTube("No English subtitles in Piped".into()))?
635        .to_string();
636
637    let sub_body = tokio::time::timeout(timeout, http.fetch_text_once(&en_url))
638        .await
639        .map_err(|_| FetchiumError::YouTube("Piped subtitle body timeout".into()))??;
640
641    parse_timedtext_xml(&sub_body)
642}
643
644// ─── Whisper ASR Fallback (videos with no captions) ───────────
645
646/// Transcribe a YouTube video using Whisper speech-to-text via yt-dlp audio extraction.
647///
648/// ## Pipeline
649/// 1. `yt-dlp -x --audio-format wav -q` — download audio-only (~30s for a 10-min video)
650/// 2. Select Whisper model based on available disk RAM:
651///    - `tiny`  (~39 MB):  40× realtime, ~5.7% WER  — fast machines/short videos
652///    - `base`  (~74 MB):  30× realtime, ~4.9% WER  — default
653///    - `small` (~244 MB): 15× realtime, ~3.4% WER  — better accuracy
654/// 3. `whisper --model base --output_format json --fp16 False` — transcribe
655/// 4. Parse JSON output → `Vec<TranscriptEntry>` with timestamps
656///
657/// Requires: `pip install yt-dlp openai-whisper` (or `faster-whisper` for 2× speed).
658/// Returns `Err` if yt-dlp or whisper are not installed — caller falls back gracefully.
659async fn fetch_via_whisper(video_id: &str) -> FetchiumResult<(Vec<TranscriptEntry>, String)> {
660    use tokio::process::Command;
661
662    // Check that both tools are available before starting heavy work.
663    let ytdlp_ok = Command::new("yt-dlp")
664        .arg("--version")
665        .output()
666        .await
667        .is_ok();
668    let whisper_ok = Command::new("whisper").arg("--help").output().await.is_ok();
669
670    if !ytdlp_ok || !whisper_ok {
671        return Err(FetchiumError::YouTube(
672            "Whisper ASR requires yt-dlp and openai-whisper: \
673             `pip install yt-dlp openai-whisper`"
674                .into(),
675        ));
676    }
677
678    let tmp_dir = std::env::temp_dir().join(format!("fetchium_whisper_{video_id}"));
679    let _ = tokio::fs::create_dir_all(&tmp_dir).await;
680    let audio_template = format!("{}/audio.%(ext)s", tmp_dir.display());
681    // Step 1: Download best audio only (no video) without forced WAV conversion.
682    // Avoiding WAV post-processing significantly reduces fallback latency.
683    let yt_out = tokio::time::timeout(
684        Duration::from_secs(120),
685        Command::new("yt-dlp")
686            .args([
687                "-f",
688                "bestaudio/best",
689                "--no-playlist",
690                "--quiet",
691                "--output",
692                &audio_template,
693                &format!("https://www.youtube.com/watch?v={video_id}"),
694            ])
695            .output(),
696    )
697    .await
698    .map_err(|_| FetchiumError::YouTube("yt-dlp audio download timed out (120s)".into()))?
699    .map_err(|e| FetchiumError::YouTube(format!("yt-dlp failed: {e}")))?;
700
701    if !yt_out.status.success() {
702        let _ = tokio::fs::remove_dir_all(&tmp_dir).await;
703        return Err(FetchiumError::YouTube(format!(
704            "yt-dlp audio download failed: {}",
705            String::from_utf8_lossy(&yt_out.stderr)
706        )));
707    }
708
709    // Resolve downloaded audio file path (`audio.m4a`, `audio.webm`, ...).
710    let mut audio_path: Option<std::path::PathBuf> = None;
711    let mut rd = tokio::fs::read_dir(&tmp_dir)
712        .await
713        .map_err(|e| FetchiumError::YouTube(format!("failed to read ASR temp dir: {e}")))?;
714    while let Ok(Some(entry)) = rd.next_entry().await {
715        let p = entry.path();
716        if p.is_file() {
717            let ext = p
718                .extension()
719                .and_then(|e| e.to_str())
720                .unwrap_or("")
721                .to_ascii_lowercase();
722            if matches!(
723                ext.as_str(),
724                "m4a" | "webm" | "opus" | "mp3" | "aac" | "wav" | "ogg" | "flac"
725            ) {
726                audio_path = Some(p);
727                break;
728            }
729        }
730    }
731    let audio_path = audio_path.ok_or_else(|| {
732        FetchiumError::YouTube("yt-dlp did not produce an audio file for Whisper".into())
733    })?;
734
735    // Step 2: Run Whisper transcription.
736    // Default to `tiny` for low-latency fallback. Override with FETCHIUM_WHISPER_MODEL.
737    // `--fp16 False` ensures CPU compatibility (avoids GPU-only FP16 error).
738    let model = whisper_model_from_env();
739    let whisper_timeout_secs = match model.as_str() {
740        "tiny" | "base" | "turbo" => 240,
741        "small" => 420,
742        _ => 600,
743    };
744    let mut whisper_cmd = Command::new("whisper");
745    whisper_cmd
746        .arg(audio_path.to_str().unwrap_or("audio"))
747        .arg("--model")
748        .arg(&model)
749        .arg("--output_format")
750        .arg("json")
751        .arg("--output_dir")
752        .arg(tmp_dir.to_str().unwrap_or("/tmp"))
753        .arg("--fp16")
754        .arg("False")
755        .arg("--temperature")
756        .arg("0")
757        .arg("--best_of")
758        .arg("1")
759        .arg("--beam_size")
760        .arg("1");
761    if let Ok(threads) = std::env::var("FETCHIUM_WHISPER_THREADS") {
762        let valid_threads = threads.parse::<u32>().ok().filter(|t| *t > 0).is_some();
763        if valid_threads {
764            whisper_cmd.arg("--threads").arg(threads);
765        }
766    }
767    let whisper_out = tokio::time::timeout(
768        Duration::from_secs(whisper_timeout_secs),
769        whisper_cmd.output(),
770    )
771    .await
772    .map_err(|_| FetchiumError::YouTube(format!("Whisper timed out ({whisper_timeout_secs}s)")))?
773    .map_err(|e| FetchiumError::YouTube(format!("Whisper failed: {e}")))?;
774
775    let _ = tokio::fs::remove_file(&audio_path).await;
776
777    if !whisper_out.status.success() {
778        let _ = tokio::fs::remove_dir_all(&tmp_dir).await;
779        return Err(FetchiumError::YouTube(format!(
780            "Whisper failed: {}",
781            String::from_utf8_lossy(&whisper_out.stderr)
782        )));
783    }
784
785    // Step 3: Find the JSON output file and parse it.
786    let json_path = tmp_dir.join("audio.json");
787    let json_content = tokio::fs::read_to_string(&json_path)
788        .await
789        .map_err(|e| FetchiumError::YouTube(format!("Whisper JSON output not found: {e}")))?;
790
791    let _ = tokio::fs::remove_dir_all(&tmp_dir).await;
792
793    parse_whisper_json(&json_content)
794}
795
796fn whisper_model_from_env() -> String {
797    let raw = std::env::var("FETCHIUM_WHISPER_MODEL")
798        .unwrap_or_else(|_| "tiny".to_string())
799        .to_lowercase();
800    match raw.as_str() {
801        "tiny" | "base" | "small" | "medium" | "large" | "turbo" => raw,
802        _ => "tiny".to_string(),
803    }
804}
805
806/// Parse Whisper's JSON output into transcript entries plus detected language.
807///
808/// Whisper JSON format: `{"language": "english", "segments": [{"start": 0.0, "end": 2.5, "text": "..."}]}`
809/// Returns `(entries, language_code)` where language_code is an ISO 639-1 code (e.g. "en", "bn").
810fn parse_whisper_json(json: &str) -> FetchiumResult<(Vec<TranscriptEntry>, String)> {
811    let v: serde_json::Value = serde_json::from_str(json)
812        .map_err(|e| FetchiumError::YouTube(format!("Whisper JSON parse: {e}")))?;
813
814    // Whisper outputs the language name in English (e.g. "english", "bengali").
815    // Map common names to ISO 639-1 codes; default to "und" (undetermined) if unknown.
816    let lang_name = v["language"].as_str().unwrap_or("").to_lowercase();
817    let language_code = whisper_lang_to_iso(&lang_name).to_string();
818
819    let segments = v["segments"]
820        .as_array()
821        .ok_or_else(|| FetchiumError::YouTube("No segments in Whisper output".into()))?;
822
823    let entries: Vec<TranscriptEntry> = segments
824        .iter()
825        .filter_map(|seg| {
826            let start_secs = seg["start"].as_f64()?;
827            let end_secs = seg["end"].as_f64().unwrap_or(start_secs + 2.0);
828            let text = seg["text"].as_str()?.trim().to_string();
829            if text.is_empty() {
830                return None;
831            }
832            Some(TranscriptEntry {
833                start_ms: (start_secs * 1000.0) as u32,
834                duration_ms: ((end_secs - start_secs) * 1000.0) as u32,
835                text: decode_html_entities(&text),
836                speaker_id: None,
837            })
838        })
839        .collect();
840
841    if entries.is_empty() {
842        return Err(FetchiumError::YouTube(
843            "Whisper produced no transcript segments".into(),
844        ));
845    }
846
847    Ok((entries, language_code))
848}
849
850/// Map Whisper's English language name to ISO 639-1 code.
851fn whisper_lang_to_iso(name: &str) -> &'static str {
852    match name {
853        "english" => "en",
854        "bengali" => "bn",
855        "hindi" => "hi",
856        "spanish" => "es",
857        "french" => "fr",
858        "german" => "de",
859        "chinese" => "zh",
860        "japanese" => "ja",
861        "korean" => "ko",
862        "arabic" => "ar",
863        "portuguese" => "pt",
864        "russian" => "ru",
865        "italian" => "it",
866        "dutch" => "nl",
867        "turkish" => "tr",
868        "polish" => "pl",
869        "swedish" => "sv",
870        "urdu" => "ur",
871        "indonesian" => "id",
872        "thai" => "th",
873        "vietnamese" => "vi",
874        _ => "und",
875    }
876}
877
878// ─── XML Parser (fallback for non-json3 sources) ───────────────
879
880/// Parse YouTube timedtext XML into transcript entries.
881fn parse_timedtext_xml(xml: &str) -> FetchiumResult<Vec<TranscriptEntry>> {
882    let re = once_cell::sync::Lazy::new(|| {
883        regex::Regex::new(
884            r#"<text[^>]*start="([0-9.]+)"[^>]*(?:dur="([0-9.]+)")?[^>]*>([^<]*)</text>"#,
885        )
886        .unwrap()
887    });
888
889    let mut entries = Vec::new();
890    for cap in re.captures_iter(xml) {
891        let start_secs: f64 = cap[1].parse().unwrap_or(0.0);
892        let dur_secs: f64 = cap
893            .get(2)
894            .and_then(|m| m.as_str().parse().ok())
895            .unwrap_or(2.0);
896        let raw_text = cap[3].trim().to_string();
897        let text = decode_html_entities(&raw_text);
898
899        if !text.is_empty() {
900            entries.push(TranscriptEntry {
901                start_ms: (start_secs * 1000.0) as u32,
902                duration_ms: (dur_secs * 1000.0) as u32,
903                text,
904                speaker_id: None,
905            });
906        }
907    }
908
909    if entries.is_empty() {
910        return Err(FetchiumError::YouTube(
911            "No transcript entries parsed from XML".into(),
912        ));
913    }
914
915    Ok(entries)
916}
917
918/// Decode basic HTML entities in transcript text.
919fn decode_html_entities(s: &str) -> String {
920    s.replace("&amp;", "&")
921        .replace("&lt;", "<")
922        .replace("&gt;", ">")
923        .replace("&quot;", "\"")
924        .replace("&#39;", "'")
925        .replace("&apos;", "'")
926        .replace('\n', " ")
927}
928
929// ─── Enhancement (speaker detection + key moments) ─────────────
930
931/// Enhance raw transcript entries with speaker detection and key moments.
932///
933/// Noise entries (`[Music]`, `[Applause]` etc.) are preserved in `entries`
934/// for chapter alignment purposes but excluded from `full_text` / `word_count`.
935fn enhance_transcript(
936    video_id: &str,
937    mut entries: Vec<TranscriptEntry>,
938    source: TranscriptSource,
939    language: String,
940) -> EnhancedTranscript {
941    let quality_score = score_quality(&entries);
942    let speakers = detect_speakers(&mut entries);
943    let key_moments = detect_key_moments(&entries);
944
945    // Build full_text from speech-only entries — skip pure [Music]/[Applause] etc.
946    let full_text: String = entries
947        .iter()
948        .filter(|e| !is_noise_text(&e.text))
949        .map(|e| e.text.as_str())
950        .collect::<Vec<_>>()
951        .join(" ");
952    let word_count = full_text.split_whitespace().count();
953
954    EnhancedTranscript {
955        video_id: video_id.to_string(),
956        language,
957        entries,
958        speakers,
959        key_moments,
960        full_text,
961        word_count,
962        source,
963        quality_score,
964    }
965}
966
967// ─── Speaker Detection ─────────────────────────────────────────
968
969/// Detect speakers using caption cues, pause gaps, and style shifts.
970fn detect_speakers(entries: &mut [TranscriptEntry]) -> Vec<Speaker> {
971    let speaker_re = once_cell::sync::Lazy::new(|| {
972        regex::Regex::new(r"^(?:\[([^\]]+)\]|>>?\s*([A-Z][A-Z\s]+):)\s*").unwrap()
973    });
974
975    let mut speaker_map: std::collections::HashMap<String, u32> = std::collections::HashMap::new();
976    let mut speaker_counts: std::collections::HashMap<u32, usize> =
977        std::collections::HashMap::new();
978    let mut next_id = 0u32;
979
980    for entry in entries.iter_mut() {
981        if let Some(cap) = speaker_re.captures(&entry.text) {
982            let name = cap
983                .get(1)
984                .or(cap.get(2))
985                .map(|m| m.as_str().trim().to_string());
986            if let Some(name) = name {
987                let id = *speaker_map.entry(name).or_insert_with(|| {
988                    let id = next_id;
989                    next_id += 1;
990                    id
991                });
992                entry.speaker_id = Some(id);
993                *speaker_counts.entry(id).or_insert(0) += 1;
994            }
995        }
996    }
997
998    // Detect speaker changes via long pause gaps (>3s) when no explicit labels
999    if speaker_map.is_empty() {
1000        let mut current_speaker = 0u32;
1001        let mut last_end_ms = 0u32;
1002        for entry in entries.iter_mut() {
1003            let gap = entry.start_ms.saturating_sub(last_end_ms);
1004            if gap > 3000 {
1005                current_speaker = (current_speaker + 1) % 2;
1006            }
1007            entry.speaker_id = Some(current_speaker);
1008            *speaker_counts.entry(current_speaker).or_insert(0) += 1;
1009            last_end_ms = entry.start_ms + entry.duration_ms;
1010        }
1011        speaker_map.insert("Speaker A".into(), 0);
1012        speaker_map.insert("Speaker B".into(), 1);
1013    }
1014
1015    speaker_map
1016        .into_iter()
1017        .map(|(name, id)| Speaker {
1018            id,
1019            label: name,
1020            segment_count: speaker_counts.get(&id).copied().unwrap_or(0),
1021        })
1022        .collect()
1023}
1024
1025// ─── Key Moment Detection ──────────────────────────────────────
1026
1027/// Detect key moments in a transcript using transitional phrases.
1028fn detect_key_moments(entries: &[TranscriptEntry]) -> Vec<KeyMoment> {
1029    let mut moments = Vec::new();
1030
1031    for entry in entries {
1032        let lower = entry.text.to_lowercase();
1033
1034        for phrase in TRANSITION_PHRASES {
1035            if lower.contains(phrase) {
1036                let moment_type = classify_moment(phrase);
1037                moments.push(KeyMoment {
1038                    timestamp_ms: entry.start_ms,
1039                    moment_type,
1040                    text: entry.text.clone(),
1041                    importance: moment_importance(&moment_type),
1042                });
1043                break;
1044            }
1045        }
1046
1047        for pattern in DEFINITION_PATTERNS {
1048            if lower.contains(pattern) {
1049                moments.push(KeyMoment {
1050                    timestamp_ms: entry.start_ms,
1051                    moment_type: MomentType::Definition,
1052                    text: entry.text.clone(),
1053                    importance: 0.8,
1054                });
1055                break;
1056            }
1057        }
1058    }
1059
1060    moments.dedup_by(|a, b| {
1061        a.timestamp_ms.abs_diff(b.timestamp_ms) < 5000 && a.moment_type == b.moment_type
1062    });
1063
1064    moments
1065}
1066
1067fn classify_moment(phrase: &str) -> MomentType {
1068    if phrase.contains("summary") || phrase.contains("conclusion") || phrase.contains("bottom line")
1069    {
1070        MomentType::Conclusion
1071    } else if phrase.contains("example") || phrase.contains("look at") {
1072        MomentType::Example
1073    } else if phrase.contains("explain") || phrase.contains("key") || phrase.contains("important") {
1074        MomentType::KeyPoint
1075    } else if phrase.contains("moving on")
1076        || phrase.contains("next")
1077        || phrase.contains("first")
1078        || phrase.contains("secondly")
1079        || phrase.contains("finally")
1080    {
1081        MomentType::TopicShift
1082    } else {
1083        MomentType::KeyPoint
1084    }
1085}
1086
1087fn moment_importance(mt: &MomentType) -> f64 {
1088    match mt {
1089        MomentType::Conclusion => 0.9,
1090        MomentType::KeyPoint => 0.8,
1091        MomentType::Definition => 0.8,
1092        MomentType::Example => 0.6,
1093        MomentType::TopicShift => 0.5,
1094    }
1095}
1096
1097/// Align transcript entries to video chapters.
1098pub fn align_to_chapters(
1099    entries: &[TranscriptEntry],
1100    chapters: &[crate::youtube::types::Chapter],
1101) -> Vec<(String, Vec<TranscriptEntry>)> {
1102    let mut result = Vec::new();
1103
1104    for chapter in chapters {
1105        let start_ms = (chapter.start_secs * 1000) as u32;
1106        let end_ms = chapter
1107            .end_secs
1108            .map(|e| (e * 1000) as u32)
1109            .unwrap_or(u32::MAX);
1110
1111        let chapter_entries: Vec<TranscriptEntry> = entries
1112            .iter()
1113            .filter(|e| e.start_ms >= start_ms && e.start_ms < end_ms)
1114            .cloned()
1115            .collect();
1116
1117        result.push((chapter.title.clone(), chapter_entries));
1118    }
1119
1120    result
1121}
1122
1123#[cfg(test)]
1124mod tests {
1125    use super::*;
1126
1127    #[test]
1128    fn parse_timedtext_basic() {
1129        let xml = r#"<transcript>
1130            <text start="0.5" dur="2.0">Hello world</text>
1131            <text start="2.5" dur="1.5">How are you</text>
1132        </transcript>"#;
1133        let entries = parse_timedtext_xml(xml).unwrap();
1134        assert_eq!(entries.len(), 2);
1135        assert_eq!(entries[0].text, "Hello world");
1136        assert_eq!(entries[0].start_ms, 500);
1137        assert_eq!(entries[0].duration_ms, 2000);
1138    }
1139
1140    #[test]
1141    fn parse_timedtext_entities() {
1142        let xml = r#"<transcript><text start="0" dur="1">It&apos;s &amp; good &lt;test&gt;</text></transcript>"#;
1143        let entries = parse_timedtext_xml(xml).unwrap();
1144        assert_eq!(entries[0].text, "It's & good <test>");
1145    }
1146
1147    #[test]
1148    fn extract_innertube_api_key_basic() {
1149        let html = r#"<script>var _yt_cfg = {"INNERTUBE_API_KEY":"***REMOVED***"};</script>"#;
1150        let key = extract_innertube_api_key(html);
1151        assert_eq!(
1152            key.as_deref(),
1153            Some("***REMOVED***")
1154        );
1155    }
1156
1157    #[test]
1158    fn extract_innertube_api_key_not_found() {
1159        let html = "<html>no key here</html>";
1160        assert!(extract_innertube_api_key(html).is_none());
1161    }
1162
1163    #[test]
1164    fn parse_whisper_json_basic() {
1165        let json = serde_json::json!({
1166            "language": "english",
1167            "segments": [
1168                {"start": 0.0, "end": 2.5, "text": " Hello world"},
1169                {"start": 3.0, "end": 6.0, "text": " Rust is fast"}
1170            ]
1171        });
1172        let (entries, lang) = parse_whisper_json(&json.to_string()).unwrap();
1173        assert_eq!(lang, "en");
1174        assert_eq!(entries.len(), 2);
1175        assert_eq!(entries[0].text, "Hello world");
1176        assert_eq!(entries[0].start_ms, 0);
1177        assert_eq!(entries[0].duration_ms, 2500);
1178        assert_eq!(entries[1].text, "Rust is fast");
1179        assert_eq!(entries[1].start_ms, 3000);
1180    }
1181
1182    #[test]
1183    fn parse_whisper_json_bengali() {
1184        let json = serde_json::json!({
1185            "language": "bengali",
1186            "segments": [
1187                {"start": 0.0, "end": 3.0, "text": " আমার সোনার বাংলা"}
1188            ]
1189        });
1190        let (entries, lang) = parse_whisper_json(&json.to_string()).unwrap();
1191        assert_eq!(lang, "bn");
1192        assert_eq!(entries.len(), 1);
1193    }
1194
1195    #[test]
1196    fn detect_key_moments_basic() {
1197        let entries = vec![
1198            TranscriptEntry {
1199                start_ms: 0,
1200                duration_ms: 2000,
1201                text: "Hello everyone".into(),
1202                speaker_id: None,
1203            },
1204            TranscriptEntry {
1205                start_ms: 5000,
1206                duration_ms: 3000,
1207                text: "The key takeaway is this".into(),
1208                speaker_id: None,
1209            },
1210            TranscriptEntry {
1211                start_ms: 20000,
1212                duration_ms: 2000,
1213                text: "In conclusion we learned".into(),
1214                speaker_id: None,
1215            },
1216        ];
1217        let moments = detect_key_moments(&entries);
1218        assert!(moments.len() >= 2);
1219    }
1220
1221    #[test]
1222    fn enhance_transcript_basic() {
1223        let entries = vec![
1224            TranscriptEntry {
1225                start_ms: 0,
1226                duration_ms: 2000,
1227                text: "Hello".into(),
1228                speaker_id: None,
1229            },
1230            TranscriptEntry {
1231                start_ms: 5000,
1232                duration_ms: 2000,
1233                text: "World".into(),
1234                speaker_id: None,
1235            },
1236        ];
1237        let transcript = enhance_transcript(
1238            "test123",
1239            entries,
1240            TranscriptSource::YouTubeTimedtext,
1241            "en".to_string(),
1242        );
1243        assert_eq!(transcript.video_id, "test123");
1244        assert_eq!(transcript.language, "en");
1245        assert_eq!(transcript.word_count, 2);
1246        assert_eq!(transcript.full_text, "Hello World");
1247    }
1248
1249    #[test]
1250    fn align_to_chapters_basic() {
1251        let entries = vec![
1252            TranscriptEntry {
1253                start_ms: 0,
1254                duration_ms: 1000,
1255                text: "Intro text".into(),
1256                speaker_id: None,
1257            },
1258            TranscriptEntry {
1259                start_ms: 60000,
1260                duration_ms: 1000,
1261                text: "Main text".into(),
1262                speaker_id: None,
1263            },
1264        ];
1265        let chapters = vec![
1266            Chapter {
1267                title: "Intro".into(),
1268                start_secs: 0,
1269                end_secs: Some(60),
1270            },
1271            Chapter {
1272                title: "Main".into(),
1273                start_secs: 60,
1274                end_secs: None,
1275            },
1276        ];
1277        let aligned = align_to_chapters(&entries, &chapters);
1278        assert_eq!(aligned.len(), 2);
1279        assert_eq!(aligned[0].0, "Intro");
1280        assert_eq!(aligned[1].1[0].text, "Main text");
1281    }
1282
1283    #[test]
1284    fn detect_speakers_with_labels() {
1285        let mut entries = vec![
1286            TranscriptEntry {
1287                start_ms: 0,
1288                duration_ms: 2000,
1289                text: "[HOST] Welcome everyone".into(),
1290                speaker_id: None,
1291            },
1292            TranscriptEntry {
1293                start_ms: 5000,
1294                duration_ms: 2000,
1295                text: "[GUEST] Thanks for having me".into(),
1296                speaker_id: None,
1297            },
1298        ];
1299        let speakers = detect_speakers(&mut entries);
1300        assert!(speakers.len() >= 2);
1301        assert!(entries[0].speaker_id.is_some());
1302    }
1303}