omni_dev/transcript/sources/
youtube.rs1use std::time::Duration;
11
12use async_trait::async_trait;
13
14use crate::transcript::error::Result;
15use crate::transcript::source::{FetchOpts, LanguageInfo, MediaInfo, Transcript, TranscriptSource};
16
17pub mod channel;
18pub mod innertube;
19pub mod player_response;
20pub mod timedtext;
21pub mod url;
22pub mod watch_page;
23
24pub use channel::VideoEntry;
25
26pub use player_response::{
27 check_playability, extract_media_info, list_languages, parse as parse_player_response,
28 select_track, CaptionTrack, PlayerResponse, SelectedTrack,
29};
30pub use timedtext::parse as parse_timedtext;
31pub use url::extract_video_id;
32
33const DEFAULT_BASE_URL: &str = "https://www.youtube.com";
36
37const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
40
41const USER_AGENT: &str = "com.google.android.apps.youtube.vr.oculus/1.62.27 \
50 (Linux; U; Android 12; Quest 3) gzip";
51
52pub fn matches_url(input: &str) -> bool {
57 extract_video_id(input).is_ok()
58}
59
60#[derive(Debug, Clone)]
71pub struct Youtube {
72 http: reqwest::Client,
73 base_url: String,
74 visitor_data: tokio::sync::OnceCell<String>,
75}
76
77impl Youtube {
78 pub fn new() -> Result<Self> {
81 let http = reqwest::Client::builder()
82 .timeout(REQUEST_TIMEOUT)
83 .user_agent(USER_AGENT)
84 .build()?;
85 Ok(Self {
86 http,
87 base_url: DEFAULT_BASE_URL.to_string(),
88 visitor_data: tokio::sync::OnceCell::new(),
89 })
90 }
91
92 pub fn with_base_url(base_url: impl Into<String>) -> Result<Self> {
97 let http = reqwest::Client::builder()
98 .timeout(REQUEST_TIMEOUT)
99 .user_agent(USER_AGENT)
100 .build()?;
101 Ok(Self {
102 http,
103 base_url: base_url.into(),
104 visitor_data: tokio::sync::OnceCell::new(),
105 })
106 }
107
108 async fn visitor_data(&self) -> Result<&str> {
112 self.visitor_data
113 .get_or_try_init(|| watch_page::fetch_visitor_data(&self.http, &self.base_url))
114 .await
115 .map(String::as_str)
116 }
117
118 async fn load_player_response(&self, locator: &str) -> Result<PlayerResponse> {
125 let video_id = extract_video_id(locator)?;
126 let visitor_data = self.visitor_data().await?;
127 let raw =
128 innertube::fetch_player_response(&self.http, &self.base_url, &video_id, visitor_data)
129 .await?;
130 let response = parse_player_response(&raw)?;
131 check_playability(&response)?;
132 Ok(response)
133 }
134
135 pub async fn resolve_channel_id(&self, input: &str) -> Result<String> {
139 channel::resolve_channel_id(&self.http, &self.base_url, input).await
140 }
141
142 pub async fn recent_channel_videos(&self, channel_id: &str) -> Result<Vec<VideoEntry>> {
145 channel::fetch_recent_videos(&self.http, &self.base_url, channel_id).await
146 }
147
148 pub async fn all_channel_video_ids(&self, channel_id: &str) -> Result<Vec<String>> {
153 channel::fetch_all_video_ids(&self.http, &self.base_url, channel_id).await
154 }
155}
156
157#[async_trait]
158impl TranscriptSource for Youtube {
159 fn name(&self) -> &'static str {
160 "youtube"
161 }
162
163 fn matches(url: &str) -> bool {
164 matches_url(url)
165 }
166
167 async fn fetch(&self, locator: &str, opts: &FetchOpts) -> Result<Transcript> {
168 let response = self.load_player_response(locator).await?;
169 let selected = select_track(&response, opts)?;
170 let body = timedtext::fetch(&self.http, &selected.fetch_url).await?;
171 let cues = timedtext::parse(&body)?;
172 let locator_id = response
173 .video_details
174 .as_ref()
175 .map(|d| d.video_id.clone())
176 .unwrap_or_default();
177 Ok(Transcript {
178 source: self.name().to_string(),
179 locator_id,
180 language: selected.language.clone(),
181 kind: selected.kind,
182 cues,
183 })
184 }
185
186 async fn list_languages(&self, locator: &str) -> Result<Vec<LanguageInfo>> {
187 let response = self.load_player_response(locator).await?;
188 Ok(list_languages(&response))
189 }
190
191 async fn info(&self, locator: &str) -> Result<MediaInfo> {
192 let response = self.load_player_response(locator).await?;
193 Ok(extract_media_info(&response))
194 }
195}
196
197#[cfg(test)]
198#[allow(clippy::unwrap_used, clippy::expect_used)]
199mod tests {
200 use super::*;
213 use crate::transcript::error::TranscriptError;
214 use crate::transcript::format::srt;
215 use crate::transcript::source::{FetchOpts, TrackKind};
216 use serde_json::Value;
217 use wiremock::matchers::{method, path};
218 use wiremock::{Mock, MockServer, ResponseTemplate};
219
220 const PLAYER_RESPONSE: &str = include_str!("youtube/fixtures/player_response_basic.json");
221 const PLAYER_RESPONSE_AGE_GATED: &str =
222 include_str!("youtube/fixtures/player_response_age_gated.json");
223 const TIMEDTEXT: &str = include_str!("youtube/fixtures/timedtext_basic.json");
224 const EXPECTED_SRT: &str = include_str!("youtube/fixtures/expected_basic.srt");
225
226 const VIDEO_ID: &str = "dQw4w9WgXcQ";
227
228 #[test]
231 fn matches_url_accepts_canonical_forms() {
232 assert!(matches_url("https://www.youtube.com/watch?v=dQw4w9WgXcQ"));
233 assert!(matches_url("https://youtu.be/dQw4w9WgXcQ"));
234 }
235
236 #[test]
237 fn matches_url_rejects_other_hosts() {
238 assert!(!matches_url("https://vimeo.com/123456"));
239 assert!(!matches_url("not a url"));
240 }
241
242 #[test]
243 fn matches_url_accepts_bare_video_id() {
244 assert!(matches_url(VIDEO_ID));
245 }
246
247 #[test]
248 fn end_to_end_player_response_to_srt() {
249 let response = parse_player_response(PLAYER_RESPONSE).unwrap();
250 check_playability(&response).unwrap();
251
252 let opts = FetchOpts::new("en-US");
253 let selected = select_track(&response, &opts).unwrap();
254 assert_eq!(selected.kind, TrackKind::Manual);
255 assert_eq!(selected.language, "en-US");
256
257 let cues = parse_timedtext(TIMEDTEXT).unwrap();
258 assert_eq!(cues.len(), 3);
259
260 let video_id = response
261 .video_details
262 .as_ref()
263 .map(|d| d.video_id.clone())
264 .unwrap_or_default();
265 let transcript = Transcript {
266 source: "youtube".to_string(),
267 locator_id: video_id,
268 language: selected.language.clone(),
269 kind: selected.kind,
270 cues,
271 };
272 let rendered = srt::render(&transcript.cues);
273 assert_eq!(rendered, EXPECTED_SRT);
274 }
275
276 #[test]
277 fn end_to_end_translation_path_picks_target_language() {
278 let response = parse_player_response(PLAYER_RESPONSE).unwrap();
279 let mut opts = FetchOpts::new("ja");
280 opts.translate_to = Some("fr".into());
281 let selected = select_track(&response, &opts).unwrap();
282 assert_eq!(selected.kind, TrackKind::Translated);
283 assert_eq!(selected.language, "fr");
284 assert!(selected.fetch_url.contains("tlang=fr"));
285 }
286
287 fn fixture_with_rewritten_caption_urls(mock_uri: &str) -> String {
294 let mut value: Value = serde_json::from_str(PLAYER_RESPONSE).unwrap();
295 let tracks = value["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]
296 .as_array_mut()
297 .unwrap();
298 for track in tracks {
299 let lang = track["languageCode"].as_str().unwrap().to_string();
300 track["baseUrl"] = Value::String(format!("{mock_uri}/api/timedtext?lang={lang}"));
301 }
302 serde_json::to_string(&value).unwrap()
303 }
304
305 const WATCH_PAGE: &str = include_str!("youtube/fixtures/watch_page_with_visitor_data.html");
310
311 async fn mount_watch_page(server: &MockServer) {
316 Mock::given(method("GET"))
317 .and(path("/watch"))
318 .respond_with(ResponseTemplate::new(200).set_body_string(WATCH_PAGE))
319 .mount(server)
320 .await;
321 }
322
323 async fn mock_server_with_basic_video() -> MockServer {
324 let server = MockServer::start().await;
325 let player_response = fixture_with_rewritten_caption_urls(&server.uri());
326
327 mount_watch_page(&server).await;
328
329 Mock::given(method("POST"))
330 .and(path(innertube::PLAYER_PATH))
331 .respond_with(ResponseTemplate::new(200).set_body_string(player_response))
332 .mount(&server)
333 .await;
334
335 Mock::given(method("GET"))
336 .and(path("/api/timedtext"))
337 .respond_with(ResponseTemplate::new(200).set_body_string(TIMEDTEXT))
338 .mount(&server)
339 .await;
340
341 server
342 }
343
344 #[tokio::test]
345 async fn fetch_returns_transcript_assembled_from_both_endpoints() {
346 let server = mock_server_with_basic_video().await;
347 let yt = Youtube::with_base_url(server.uri()).unwrap();
348 let opts = FetchOpts::new("en-US");
349
350 let transcript = yt
351 .fetch(
352 &format!("https://www.youtube.com/watch?v={VIDEO_ID}"),
353 &opts,
354 )
355 .await
356 .unwrap();
357
358 assert_eq!(transcript.source, "youtube");
359 assert_eq!(transcript.locator_id, VIDEO_ID);
360 assert_eq!(transcript.language, "en-US");
361 assert_eq!(transcript.kind, TrackKind::Manual);
362 assert_eq!(transcript.cues.len(), 3);
363 assert_eq!(srt::render(&transcript.cues), EXPECTED_SRT);
366 }
367
368 #[tokio::test]
369 async fn fetch_accepts_bare_video_id_as_locator() {
370 let server = mock_server_with_basic_video().await;
371 let yt = Youtube::with_base_url(server.uri()).unwrap();
372 let opts = FetchOpts::new("en-US");
373
374 let transcript = yt.fetch(VIDEO_ID, &opts).await.unwrap();
375 assert_eq!(transcript.locator_id, VIDEO_ID);
376 }
377
378 #[tokio::test]
379 async fn fetch_propagates_language_not_found() {
380 let server = mock_server_with_basic_video().await;
381 let yt = Youtube::with_base_url(server.uri()).unwrap();
382 let opts = FetchOpts::new("zz");
383
384 let err = yt.fetch(VIDEO_ID, &opts).await.unwrap_err();
385 assert!(matches!(err, TranscriptError::LanguageNotFound { .. }));
386 }
387
388 #[tokio::test]
389 async fn fetch_surfaces_age_gated_as_playability_refused() {
390 let server = MockServer::start().await;
391 mount_watch_page(&server).await;
392 Mock::given(method("POST"))
393 .and(path(innertube::PLAYER_PATH))
394 .respond_with(ResponseTemplate::new(200).set_body_string(PLAYER_RESPONSE_AGE_GATED))
395 .mount(&server)
396 .await;
397
398 let yt = Youtube::with_base_url(server.uri()).unwrap();
399 let err = yt.fetch(VIDEO_ID, &FetchOpts::new("en")).await.unwrap_err();
400 match err {
401 TranscriptError::PlayabilityRefused { status, .. } => {
402 assert_eq!(status, "LOGIN_REQUIRED");
403 }
404 other => panic!("wrong variant: {other:?}"),
405 }
406 }
407
408 #[tokio::test]
409 async fn fetch_invalid_locator_short_circuits_before_http() {
410 let yt = Youtube::with_base_url("http://127.0.0.1:1").unwrap();
412 let err = yt
413 .fetch("not-a-url", &FetchOpts::new("en"))
414 .await
415 .unwrap_err();
416 assert!(matches!(err, TranscriptError::InvalidLocator(_)));
417 }
418
419 #[tokio::test]
420 async fn fetch_surfaces_innertube_500_as_http_error() {
421 let server = MockServer::start().await;
422 mount_watch_page(&server).await;
423 Mock::given(method("POST"))
424 .and(path(innertube::PLAYER_PATH))
425 .respond_with(ResponseTemplate::new(500))
426 .mount(&server)
427 .await;
428
429 let yt = Youtube::with_base_url(server.uri()).unwrap();
430 let err = yt.fetch(VIDEO_ID, &FetchOpts::new("en")).await.unwrap_err();
431 assert!(matches!(err, TranscriptError::Http(_)));
432 }
433
434 #[tokio::test]
435 async fn list_languages_projects_caption_tracks() {
436 let server = mock_server_with_basic_video().await;
437 let yt = Youtube::with_base_url(server.uri()).unwrap();
438
439 let langs = yt.list_languages(VIDEO_ID).await.unwrap();
440 let codes: Vec<_> = langs.iter().map(|l| l.code.as_str()).collect();
441 assert!(codes.contains(&"en-US"));
442 assert!(codes.contains(&"es"));
443 assert!(codes.contains(&"en"));
444 }
445
446 #[tokio::test]
447 async fn info_returns_video_metadata() {
448 let server = mock_server_with_basic_video().await;
449 let yt = Youtube::with_base_url(server.uri()).unwrap();
450
451 let info = yt.info(VIDEO_ID).await.unwrap();
452 assert_eq!(info.source, "youtube");
453 assert_eq!(info.locator_id, VIDEO_ID);
454 assert_eq!(info.title, "Sample Video");
455 assert_eq!(info.duration_ms, Some(212_000));
456 assert_eq!(info.languages.len(), 3);
457 }
458
459 #[tokio::test]
460 async fn matches_static_dispatch_through_trait() {
461 assert!(<Youtube as TranscriptSource>::matches(
463 "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
464 ));
465 assert!(!<Youtube as TranscriptSource>::matches(
466 "https://vimeo.com/1"
467 ));
468 }
469
470 #[tokio::test]
471 async fn name_is_lowercase_youtube() {
472 let server = mock_server_with_basic_video().await;
473 let yt = Youtube::with_base_url(server.uri()).unwrap();
474 assert_eq!(yt.name(), "youtube");
475 }
476
477 #[test]
478 fn new_constructs_default_client() {
479 let yt = Youtube::new().unwrap();
482 assert_eq!(yt.base_url, DEFAULT_BASE_URL);
483 }
484
485 #[tokio::test]
486 async fn fetch_threads_visitor_data_into_innertube_body() {
487 const EXPECTED_TOKEN: &str = "CgtkUTQyOFR3aV9NSSjFoYvBBjIKCgJVUxIEGgAgPg%3D%3D";
492
493 let server = MockServer::start().await;
494 mount_watch_page(&server).await;
495 let player_response = fixture_with_rewritten_caption_urls(&server.uri());
496
497 Mock::given(method("POST"))
498 .and(path(innertube::PLAYER_PATH))
499 .respond_with(move |req: &wiremock::Request| {
500 let parsed: Value = serde_json::from_slice(&req.body).unwrap();
501 assert_eq!(
502 parsed["context"]["client"]["visitorData"],
503 Value::String(EXPECTED_TOKEN.to_string()),
504 );
505 ResponseTemplate::new(200).set_body_string(player_response.clone())
506 })
507 .expect(1)
508 .mount(&server)
509 .await;
510
511 Mock::given(method("GET"))
512 .and(path("/api/timedtext"))
513 .respond_with(ResponseTemplate::new(200).set_body_string(TIMEDTEXT))
514 .mount(&server)
515 .await;
516
517 let yt = Youtube::with_base_url(server.uri()).unwrap();
518 let _ = yt.fetch(VIDEO_ID, &FetchOpts::new("en-US")).await.unwrap();
519 }
520
521 #[tokio::test]
522 async fn visitor_data_fetched_only_once_for_repeated_calls() {
523 let server = MockServer::start().await;
526 let player_response = fixture_with_rewritten_caption_urls(&server.uri());
527
528 Mock::given(method("GET"))
529 .and(path("/watch"))
530 .respond_with(ResponseTemplate::new(200).set_body_string(WATCH_PAGE))
531 .expect(1)
532 .mount(&server)
533 .await;
534
535 Mock::given(method("POST"))
536 .and(path(innertube::PLAYER_PATH))
537 .respond_with(ResponseTemplate::new(200).set_body_string(player_response))
538 .mount(&server)
539 .await;
540
541 Mock::given(method("GET"))
542 .and(path("/api/timedtext"))
543 .respond_with(ResponseTemplate::new(200).set_body_string(TIMEDTEXT))
544 .mount(&server)
545 .await;
546
547 let yt = Youtube::with_base_url(server.uri()).unwrap();
548 let _ = yt.fetch(VIDEO_ID, &FetchOpts::new("en-US")).await.unwrap();
549 let _ = yt.fetch(VIDEO_ID, &FetchOpts::new("en-US")).await.unwrap();
550 }
552
553 #[tokio::test]
554 async fn visitor_data_fetched_only_once_under_concurrency() {
555 let server = MockServer::start().await;
560 let player_response = fixture_with_rewritten_caption_urls(&server.uri());
561
562 Mock::given(method("GET"))
563 .and(path("/watch"))
564 .respond_with(ResponseTemplate::new(200).set_body_string(WATCH_PAGE))
565 .expect(1)
566 .mount(&server)
567 .await;
568
569 Mock::given(method("POST"))
570 .and(path(innertube::PLAYER_PATH))
571 .respond_with(ResponseTemplate::new(200).set_body_string(player_response))
572 .mount(&server)
573 .await;
574
575 Mock::given(method("GET"))
576 .and(path("/api/timedtext"))
577 .respond_with(ResponseTemplate::new(200).set_body_string(TIMEDTEXT))
578 .mount(&server)
579 .await;
580
581 let yt = Youtube::with_base_url(server.uri()).unwrap();
582 let opts = FetchOpts::new("en-US");
583 let (a, b, c) = tokio::join!(
584 yt.fetch(VIDEO_ID, &opts),
585 yt.fetch(VIDEO_ID, &opts),
586 yt.fetch(VIDEO_ID, &opts),
587 );
588 a.unwrap();
589 b.unwrap();
590 c.unwrap();
591 }
593
594 #[tokio::test]
595 async fn fetch_surfaces_missing_visitor_data_as_typed_error() {
596 let server = MockServer::start().await;
600 Mock::given(method("GET"))
601 .and(path("/watch"))
602 .respond_with(
603 ResponseTemplate::new(200).set_body_string("<html><body>no token</body></html>"),
604 )
605 .mount(&server)
606 .await;
607
608 let yt = Youtube::with_base_url(server.uri()).unwrap();
609 let err = yt.fetch(VIDEO_ID, &FetchOpts::new("en")).await.unwrap_err();
610 assert!(matches!(err, TranscriptError::MissingVisitorData { .. }));
611 }
612
613 #[tokio::test]
614 async fn fetch_surfaces_malformed_innertube_json_as_parse_error() {
615 let server = MockServer::start().await;
616 mount_watch_page(&server).await;
617 Mock::given(method("POST"))
618 .and(path(innertube::PLAYER_PATH))
619 .respond_with(ResponseTemplate::new(200).set_body_string("{ not json"))
620 .mount(&server)
621 .await;
622
623 let yt = Youtube::with_base_url(server.uri()).unwrap();
624 let err = yt.fetch(VIDEO_ID, &FetchOpts::new("en")).await.unwrap_err();
625 assert!(matches!(err, TranscriptError::ParseError(_)));
626 }
627
628 #[cfg(online_tests)]
639 #[tokio::test]
640 async fn online_fetch_against_public_video() {
641 const STABLE_VIDEO_ID: &str = "jNQXAC9IVRw";
643 let yt = Youtube::new().unwrap();
644 let opts = FetchOpts::new("en");
645 let transcript = yt.fetch(STABLE_VIDEO_ID, &opts).await.unwrap();
646 assert_eq!(transcript.source, "youtube");
647 assert_eq!(transcript.locator_id, STABLE_VIDEO_ID);
648 assert!(!transcript.cues.is_empty());
649 }
650}