rsynko_yt/
extraction.rs

1use crate::*;
2use alux_ext::ext;
3use rsynko_media::*;
4use std::fmt::{Display, Formatter, Result as FormatResult};
5use thiserror::Error;
6
7/// Derives Youtube extraction from request, response, and domain-data algebras.
8#[ext(name = YoutubeExtractionExt)]
9pub impl<This, RequestError, ResponseError, ChallengeError> This
10where
11    This: YoutubeRequestAlg
12        + YoutubeClientAlg
13        + YoutubeProgramAlg
14        + YoutubeRequestBytesAlg<Error = RequestError>
15        + YoutubeResponseAlg<Error = ResponseError>
16        + YoutubeChallengeAlg<Error = ChallengeError>
17        + YoutubeSolutionAlg
18        + YoutubeUrlAlg
19        + MetadataAlg
20        + FormatAlg
21        + ExtractionAlg,
22{
23    /// Extracts one Youtube watch URL into a preference-ordered media catalog.
24    ///
25    /// # Errors
26    ///
27    /// Returns URL, request, response, availability, challenge, or empty-catalog failure.
28    fn extract_youtube(
29        &self,
30        url: &str,
31    ) -> Result<This::Extraction, YoutubeError<RequestError, ResponseError, ChallengeError>> {
32        let id = youtube_id(url).ok_or_else(|| YoutubeError::Url(url.to_owned()))?;
33        let watch_request = self.watch_request(url);
34        let page_bytes = self.youtube_request_bytes(&watch_request).map_err(YoutubeError::Request)?;
35        let page = self.decode_youtube_watch(&page_bytes).map_err(YoutubeError::Response)?;
36        if !page.has_player_response {
37            return Err(YoutubeError::MissingPlayerResponse);
38        }
39        let api_key = page.api_key.ok_or(YoutubeError::MissingApiKey)?;
40        // A challenge is posed by one player program, and Youtube grants a catalog matching the
41        // program the client states it runs, so the program is retrieved before the catalog.
42        let program = match page.player_url {
43            Some(url) => {
44                let request = self.player_program_request(url);
45                let bytes = self.youtube_request_bytes(&request).map_err(YoutubeError::Request)?;
46                String::from_utf8_lossy(&bytes).into_owned()
47            }
48            None => String::new(),
49        };
50        let player = self.granted_catalog(&id, &api_key, page.visitor_data.as_deref(), &program)?;
51        if player.status.as_deref() != Some("OK") {
52            return Err(YoutubeError::Unavailable(player.reason.unwrap_or_else(|| "video unavailable".to_owned())));
53        }
54        let unreadable = player.unreadable;
55        let mut ranked = player.formats;
56        ranked.sort_by_key(|format| (format.height.unwrap_or(0), format.bitrate.unwrap_or(0)));
57        if ranked.is_empty() {
58            return Err(YoutubeError::NoDirectFormat);
59        }
60        let grants = self.grant_formats(&program, &ranked).map_err(YoutubeError::Challenge)?;
61        let mut formats = Vec::new();
62        let mut withheld = None;
63        let mut withheld_count = 0_usize;
64        let mut throttled_count = 0_usize;
65        for (format, grant) in ranked.iter().zip(grants) {
66            match grant {
67                // A throttled representation is retrievable, only slowly, so it stays in the
68                // catalog rather than disappearing from it.
69                YoutubeGrant::Granted { url, .. } => {
70                    formats.push(self.youtube_format(format.clone(), url));
71                }
72                YoutubeGrant::Throttled { url, .. } => {
73                    throttled_count += 1;
74                    formats.push(self.youtube_format(format.clone(), url));
75                }
76                YoutubeGrant::Withheld { challenge, .. } => {
77                    withheld_count += 1;
78                    withheld = Some(withheld_reason(&challenge).max(withheld.unwrap_or(WithheldReason::Throttle)));
79                }
80            }
81        }
82        if formats.is_empty() {
83            return Err(YoutubeError::Withheld(withheld.unwrap_or(WithheldReason::Throttle)));
84        }
85        let title = player.title.or(page.title);
86        let mut fields = vec![
87            (YOUTUBE_DESCRIBED.to_owned(), self.integer_metadata(i64::try_from(ranked.len()).unwrap_or(i64::MAX))),
88            (YOUTUBE_WITHHELD.to_owned(), self.integer_metadata(i64::try_from(withheld_count).unwrap_or(i64::MAX))),
89            (YOUTUBE_THROTTLED.to_owned(), self.integer_metadata(i64::try_from(throttled_count).unwrap_or(i64::MAX))),
90            (YOUTUBE_UNREADABLE.to_owned(), self.integer_metadata(i64::try_from(unreadable).unwrap_or(i64::MAX))),
91        ];
92        if let Some(title) = title {
93            fields.push(("title".to_owned(), self.string_metadata(title)));
94        }
95        Ok(self.media(id, self.metadata(fields), formats))
96    }
97
98    /// Observes the union of the catalogs this interpreter's clients are granted, in claim order.
99    ///
100    /// Each client is granted a different catalog, and neither is the whole truth: one states a
101    /// muxed representation carrying both streams, another states every adaptive representation on
102    /// its own. A client refused states nothing about the others.
103    ///
104    /// # Errors
105    ///
106    /// Returns request or response failure.
107    fn granted_catalog(
108        &self,
109        id: &str,
110        api_key: &str,
111        visitor: Option<&str>,
112        program: &str,
113    ) -> Result<YoutubePlayer, YoutubeError<RequestError, ResponseError, ChallengeError>> {
114        let mut catalog = YoutubePlayer { status: None, reason: None, title: None, formats: Vec::new(), unreadable: 0 };
115        let clients: Vec<String> = self.player_clients().map(str::to_owned).collect();
116        for client in clients {
117            let claim =
118                PlayerClaim { client, visitor: visitor.map(str::to_owned), timestamp: self.program_timestamp(program) };
119            let request = self.player_request(id, api_key, &claim);
120            let bytes = self.youtube_request_bytes(&request).map_err(YoutubeError::Request)?;
121            let granted = self.decode_youtube_player(&bytes).map_err(YoutubeError::Response)?;
122            if granted.status.as_deref() != Some("OK") {
123                catalog.status = catalog.status.or(granted.status);
124                catalog.reason = catalog.reason.or(granted.reason);
125                continue;
126            }
127            catalog.status = Some("OK".to_owned());
128            catalog.reason = None;
129            catalog.title = catalog.title.or(granted.title);
130            catalog.unreadable += granted.unreadable;
131            for format in granted.formats {
132                if !catalog.formats.iter().any(|held| held.id == format.id) {
133                    catalog.formats.push(format);
134                }
135            }
136        }
137        Ok(catalog)
138    }
139
140    /// Defines one generic format from Youtube player observations and its granted URL.
141    fn youtube_format(&self, format: YoutubeFormat, url: String) -> This::Format {
142        let mut fields = vec![
143            (FORMAT_SOURCE.to_owned(), self.string_metadata(url)),
144            (FORMAT_HAS_AUDIO.to_owned(), self.boolean_metadata(format.has_audio)),
145            (FORMAT_HAS_VIDEO.to_owned(), self.boolean_metadata(format.has_video)),
146        ];
147        if let Some(value) = format.extension {
148            fields.push((FORMAT_EXTENSION.to_owned(), self.string_metadata(value)));
149        }
150        if let Some(value) = format.quality {
151            fields.push(("quality".to_owned(), self.string_metadata(value)));
152        }
153        for (key, value) in [
154            ("width", format.width),
155            ("height", format.height),
156            ("bitrate", format.bitrate),
157            ("content_length", format.content_length),
158        ] {
159            if let Some(value) = value {
160                fields.push((key.to_owned(), self.integer_metadata(value)));
161            }
162        }
163        if let Some(value) = format.codecs {
164            fields.push(("codecs".to_owned(), self.string_metadata(value)));
165        }
166        self.format(format.id, self.metadata(fields))
167    }
168}
169
170/// Denotes failure to interpret one Youtube watch URL.
171#[derive(Debug, Error)]
172pub enum YoutubeError<RequestError, ResponseError, ChallengeError> {
173    /// Denotes a malformed or unsupported Youtube URL.
174    #[error("malformed Youtube URL: {0}")]
175    Url(String),
176    /// Denotes request execution failure.
177    #[error("Youtube request failed: {0}")]
178    Request(RequestError),
179    /// Denotes response decoding failure.
180    #[error("Youtube response decoding failed: {0}")]
181    Response(ResponseError),
182    /// Denotes absence of a parseable player response.
183    #[error("the Youtube page contains no parseable initial player response")]
184    MissingPlayerResponse,
185    /// Denotes absence of the player API key in the watch-page configuration.
186    #[error("the Youtube page contains no InnerTube API key")]
187    MissingApiKey,
188    /// Denotes an unavailable video according to the player response.
189    #[error("Youtube reports: {0}")]
190    Unavailable(String),
191    /// Denotes absence of any described format.
192    #[error("the player response describes no audio or video format")]
193    NoDirectFormat,
194    /// Denotes failure to resolve the challenges the player response poses.
195    #[error("Youtube challenge resolution failed: {0}")]
196    Challenge(ChallengeError),
197    /// Denotes that every described format stayed guarded by an unresolved challenge.
198    #[error("every described format is withheld behind an unresolved {0} challenge")]
199    Withheld(WithheldReason),
200}
201
202/// Names the kind of unresolved challenge withholding an entire format catalog.
203#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
204pub enum WithheldReason {
205    /// Withholds because throttling parameters governing the formats stayed unresolved.
206    Throttle,
207    /// Withholds because signatures guarding the formats stayed unresolved.
208    Signature,
209}
210
211impl Display for WithheldReason {
212    fn fmt(&self, formatter: &mut Formatter<'_>) -> FormatResult {
213        formatter.write_str(match self {
214            Self::Signature => "signature",
215            Self::Throttle => "throttling",
216        })
217    }
218}
219
220/// Names why one challenge withheld a format.
221fn withheld_reason(challenge: &YoutubeChallenge) -> WithheldReason {
222    match challenge {
223        YoutubeChallenge::Signature(_) => WithheldReason::Signature,
224        YoutubeChallenge::Throttle(_) => WithheldReason::Throttle,
225    }
226}
227
228/// Names the observation counting the formats the player response described.
229pub const YOUTUBE_DESCRIBED: &str = "youtube_described";
230
231/// Names the observation counting the formats an unresolved signature withheld.
232pub const YOUTUBE_WITHHELD: &str = "youtube_withheld";
233
234/// Names the observation counting the formats an unresolved throttling parameter governs.
235pub const YOUTUBE_THROTTLED: &str = "youtube_throttled";
236
237/// Names the observation counting the described formats the interpreter could not read.
238pub const YOUTUBE_UNREADABLE: &str = "youtube_unreadable";
239
240/// Derives what a reader is told about extraction and about the requests that follow it.
241#[ext(name = YoutubeNotesExt)]
242pub impl<This> This
243where
244    This: MetadataViewAlg,
245{
246    /// States what extraction observed about granting, in the words a reader reads.
247    ///
248    /// A player response describes formats this interpreter may or may not be able to read: some
249    /// state no location, some are held behind an unanswered signature, and some are served
250    /// slowly on purpose. What is said here is what was counted, and nothing is said about a
251    /// count the extractor did not state.
252    fn granting_notes(&self, metadata: &This::Metadata) -> Vec<String> {
253        let counted = |key| self.metadata_number(metadata, key);
254        let mut notes = Vec::new();
255        if let Some(described) = counted(YOUTUBE_DESCRIBED) {
256            notes.push(format!("the player response described {described} formats"));
257        }
258        if let Some(unreadable) = counted(YOUTUBE_UNREADABLE).filter(|count| *count > 0) {
259            notes.push(format!("{unreadable} described formats state no location this interpreter can read"));
260        }
261        if let Some(withheld) = counted(YOUTUBE_WITHHELD) {
262            notes.push(if withheld == 0 {
263                "every signature was answered by the player program".to_owned()
264            } else {
265                format!("{withheld} formats withheld behind an unanswered signature")
266            });
267        }
268        if let Some(throttled) = counted(YOUTUBE_THROTTLED).filter(|count| *count > 0) {
269            notes.push(format!("{throttled} formats served at a throttled rate"));
270        }
271        notes
272    }
273}
274
275/// Names what a media URL carries that a reader must never be shown.
276const SIGNED_URL_MARK: &str = " for url (";
277
278/// Names the rejection every reader of withheld media eventually meets.
279const FORBIDDEN: &str = "403 Forbidden";
280
281/// States what a reader is told about one failed media request.
282///
283/// A media URL is signed, which makes it a credential rather than an address: it is never
284/// repeated, however the failure is worded. What is left is stated first in words a reader knows,
285/// and the diagnostic follows for whoever goes looking.
286#[must_use]
287pub fn media_failure(detail: &str) -> String {
288    let stated = detail
289        .find(SIGNED_URL_MARK)
290        .map_or_else(|| detail.to_owned(), |index| format!("{}; signed media URL redacted", &detail[..index]));
291    if stated.contains(FORBIDDEN) {
292        return format!("media request rejected: HTTP 403 Forbidden\n{stated}");
293    }
294    stated
295}
296
297/// Observes the video identity denoted by a supported Youtube URL.
298#[must_use]
299pub fn youtube_id(input: &str) -> Option<String> {
300    let without_scheme = input.strip_prefix("https://").or_else(|| input.strip_prefix("http://"))?;
301    let (authority, suffix) =
302        without_scheme.split_once('/').map_or((without_scheme, ""), |(authority, suffix)| (authority, suffix));
303    let host = authority.split('@').next_back()?.split(':').next()?;
304    match host {
305        "youtu.be" | "www.youtu.be" => {
306            suffix.split(['?', '#', '/']).next().filter(|id| !id.is_empty()).map(str::to_owned)
307        }
308        "youtube.com" | "www.youtube.com" | "m.youtube.com" => {
309            if let Some(query) = suffix.strip_prefix("watch?") {
310                query.split('&').find_map(|pair| {
311                    let (key, value) = pair.split_once('=')?;
312                    (key == "v" && !value.is_empty()).then(|| value.to_owned())
313                })
314            } else {
315                suffix.split(['?', '#']).next()?.split('/').nth(1).filter(|id| !id.is_empty()).map(str::to_owned)
316            }
317        }
318        _ => None,
319    }
320}
rsynko_yt/extraction.rs

rsynko_yt/
extraction.rs
