youtube-legend-cli 0.4.0

//! The watch-page probe consulted once, after the chain is exhausted.
//!
//! One reason to change: what the watch page can prove about a failure
//! the providers could not explain.

use std::time::Duration;

use crate::error::{AppError, AppResult};

/// Origin the watch-page probe reads. Compiled rather than configured:
/// pointing the probe at another host would not make it a probe of
/// something else, it would make it read a page this crate cannot
/// parse. `ProviderChain::with_watch_probe_base` overrides it for the
/// tests, which is the only caller that has a reason to.
///
/// Deliberately NOT an intra-doc link. That method is `#[cfg(test)]`,
/// and `cargo doc` builds without `cfg(test)`, so the item does not
/// exist in the documentation build and no path can reach it. MEASURED
/// on 2026-09-01: rustdoc resolved the struct and then reported it has
/// no associated item by that name, which is exactly what a
/// test-only item looks like from inside a doc build.
const WATCH_PAGE_ORIGIN: &str = "https://www.youtube.com";

/// Compiled default behind `net.watch_probe_timeout_secs`.
///
/// The probe runs after every provider has already failed, so it is
/// spending time the request has in principle already lost. The budget
/// is therefore short: a page that has not arrived in this many seconds
/// is an inconclusive probe, and an inconclusive probe adds nothing.
pub const DEFAULT_WATCH_PROBE_TIMEOUT_SECS: u64 = 20;

/// Wall-clock ceiling for the single watch-page request.
///
/// Resolves `net.watch_probe_timeout_secs`. Zero is refused because it
/// would abort the request before it could answer.
#[must_use]
pub fn watch_probe_timeout() -> Duration {
    Duration::from_secs(crate::config::tuning_u64_in_range(
        "net.watch_probe_timeout_secs",
        DEFAULT_WATCH_PROBE_TIMEOUT_SECS,
        1,
        3_600,
    ))
}

/// Reads the watch page once, after the provider chain is exhausted.
///
/// The probe is a *classifier* and never a download path: it hands the
/// body to [`crate::parse::player_response`], which reads `baseUrl` for
/// identity only. A direct `GET` on that URL was measured answering
/// `HTTP 200` with a zero-byte body, so fetching it would manufacture a
/// false success.
#[derive(Debug, Clone)]
pub(super) struct WatchProbe {
    /// Origin the watch page is read from.
    pub(super) base: String,
    /// Ceiling for the single request.
    pub(super) timeout: Duration,
}

impl WatchProbe {
    /// The probe as production runs it.
    pub(super) fn live() -> Self {
        Self {
            base: WATCH_PAGE_ORIGIN.to_string(),
            timeout: watch_probe_timeout(),
        }
    }

    /// Fetch the complete watch-page body.
    ///
    /// The whole body is required: the `captionTracks` key was measured
    /// starting at byte 737 363 of a 1 314 762-byte page, so a
    /// truncated read answers "no captions" with apparent success.
    pub(super) async fn watch_page(&self, video_id: &str) -> AppResult<String> {
        let client = crate::net::session::chrome_client(self.timeout)?;
        let url = format!("{}/watch?v={video_id}", self.base.trim_end_matches('/'));
        client
            .get(url)
            .send()
            .await
            .map_err(AppError::Http)?
            .error_for_status()
            .map_err(AppError::Http)?
            .text()
            .await
            .map_err(AppError::Http)
    }
}

/// The language tags a watch page publishes, or `None` when the page
/// could not be read.
///
/// `classify_watch_page` answers the question "does this page justify an
/// error", which is the wrong question on a run that SUCCEEDED. This one
/// answers "what does the page publish", which is what lets a delivered
/// body be attributed to a track: a page publishing exactly one language
/// leaves no room for the delivered body to be in another.
///
/// An empty vector is a real answer and means the video publishes no
/// caption track at all; `None` means the page did not parse.
pub(super) fn published_languages(html: &str) -> Option<Vec<String>> {
    use crate::parse::player_response::{available_languages, caption_tracks};

    let tracks = caption_tracks(html).ok()?;
    Some(available_languages(&tracks))
}

/// Turn a watch page into the error it justifies, or `None` when it
/// justifies nothing better than what the chain already reported.
///
/// `None` is the honest answer for every page this crate could not
/// read: an inconclusive probe must never replace a decisive chain
/// error, because that would trade a real cause for a guess.
pub(super) fn classify_watch_page(html: &str, language: &str) -> Option<AppError> {
    use crate::parse::player_response::{available_languages, caption_tracks, classify};

    let tracks = caption_tracks(html).ok()?;

    // 1. The video publishes nothing. No provider could have helped.
    if tracks.is_empty() {
        return Some(AppError::NoSubtitle(
            crate::error::NoSubtitleReason::NotPublished,
        ));
    }

    match classify(&tracks, language) {
        // 2. Tracks exist and none of them is the requested language.
        //    This is what finally fills `available_languages` in the
        //    error envelope, which the schema had declared and nothing
        //    had ever populated.
        Err(err @ AppError::LanguageUnavailable { .. }) => Some(err),
        // Any other classification failure is a fact about the parse,
        // not about the video.
        Err(_) => None,
        Ok(_) => {
            // 3. The requested language is published and every track on
            //    the page is machine-generated. The condition is
            //    composed on purpose: it is only reached once every
            //    provider has failed, and no provider refuses an ASR
            //    track by decision of its own code. Asserting this from
            //    the ASR flag alone would make the `kind` lie about the
            //    cause.
            if tracks
                .iter()
                .all(crate::parse::player_response::CaptionTrack::is_asr)
            {
                Some(AppError::CaptionsAsrOnly {
                    asr_languages: available_languages(&tracks),
                })
            } else {
                // 4. Nothing the probe saw explains the failure better
                //    than the chain already did.
                None
            }
        }
    }
}

/// Whether the chain was turned away before any provider could reach a
/// track at all.
///
/// Cases 1 and 2 of [`classify_watch_page`] are facts about the video:
/// they hold whatever the providers did, so they may replace anything.
/// Case 3 is different in kind. `CaptionsAsrOnly` does not describe the
/// video, it claims no provider could *serve* the machine-generated
/// track — and a rate-limited chain never got far enough to find out.
///
/// The bar is deliberately narrow. A provider that failed to render is
/// ambiguous: it may well have choked on the ASR track itself, which is
/// the reading the surrounding design already chose. HTTP 429 admits no
/// such reading. The upstream refused at the door, before any track was
/// requested, so the ASR verdict has nothing to stand on.
///
/// MEASURED on 2026-09-04 against `zHGqpjCV6Tg`, which publishes a
/// single ASR track in `pt`. Pinning `provider-decopy` and
/// `provider-noiz` produced `HTTP 429` on an exhausted quota, and the
/// probe reported "only machine-generated captions, none deliverable"
/// for both. `provider-noiz` had delivered that exact track minutes
/// earlier, byte for byte, so the substituted message was false — and
/// it aimed the operator at the video when the fix was to wait for the
/// quota to reset, which was measured resetting nine minutes later.
/// MEASURED on 2026-09-04 against getsubs.cc, which now serves its own
/// checkbox-grid challenge in place of the track list. The provider
/// classified it correctly and reported `CaptchaChallenge`, whose own
/// documentation states the case in as many words: no retry and no
/// sibling provider can clear it, and the operator has to be told a
/// human is required. The probe then replaced that with the ASR verdict
/// and the operator was told to give up on the video, when the actual
/// state of the world was a wall a person can walk through in seconds.
///
/// A challenge is the same shape as a 429 for this purpose: the
/// upstream refused at the door and no track was ever requested, so the
/// claim that none could be served has nothing to stand on.
///
/// `ProviderUnavailable` is deliberately NOT on this list. A provider
/// that failed to render is genuinely ambiguous — it may have choked on
/// the ASR track itself — and the surrounding design already chose that
/// reading, in a test that pins it. Widening this predicate to cover it
/// would reverse a decision rather than complete one.
pub(super) fn chain_never_reached_a_track(err: &AppError) -> bool {
    matches!(
        err,
        AppError::RateLimited { .. } | AppError::CaptchaChallenge { .. }
    )
}

/// The watch-page probe at the exhaustion point of the chain.
///
/// Every test here is offline: the classifier is exercised as a pure
/// function, and the two that go through `fetch_subtitle` serve the
/// synthetic page from a local mock. A test that needed the real origin
/// would be an observation, not a gate.
#[cfg(test)]
mod watch_probe_tests {
    use super::*;
    use crate::error::AppResult;
    use crate::error::NoSubtitleReason;
    use crate::parse::player_response::test_pages::{
        watch_page_with, ASR_ONLY_BLOCK, CAPTIONS_BLOCK, NO_CAPTIONS_BLOCK, PT_ONLY_BLOCK,
    };
    use crate::provider::chain::ledger::AttemptOutcome;
    use crate::provider::{Format, Provider, ProviderChain, SubtitleInfo};
    use async_trait::async_trait;

    /// A provider that always fails transiently, so the chain always
    /// reaches the point where the probe is consulted.
    struct AlwaysUnavailable;

    #[async_trait]
    impl Provider for AlwaysUnavailable {
        fn name(&self) -> &'static str {
            "provider-test"
        }
        async fn fetch_subtitle(
            &self,
            _video_id: &str,
            _language: &str,
            _format: Format,
        ) -> AppResult<SubtitleInfo> {
            Err(AppError::ProviderUnavailable {
                provider: "provider-test",
            })
        }
        async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
            Err(AppError::ProviderUnavailable {
                provider: "provider-test",
            })
        }
    }

    /// A provider the upstream turned away with HTTP 429, so it never
    /// reached a track to form an opinion about one.
    struct AlwaysRateLimited;

    #[async_trait]
    impl Provider for AlwaysRateLimited {
        fn name(&self) -> &'static str {
            "provider-test"
        }
        async fn fetch_subtitle(
            &self,
            _video_id: &str,
            _language: &str,
            _format: Format,
        ) -> AppResult<SubtitleInfo> {
            Err(AppError::RateLimited {
                provider: "provider-test",
                retry_after_secs: None,
            })
        }
        async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
            Err(AppError::RateLimited {
                provider: "provider-test",
                retry_after_secs: None,
            })
        }
    }

    /// A provider stopped at a challenge wall, which is what getsubs.cc
    /// serves as of 2026-09-04.
    struct AlwaysChallenged;

    #[async_trait]
    impl Provider for AlwaysChallenged {
        fn name(&self) -> &'static str {
            "provider-test"
        }
        async fn fetch_subtitle(
            &self,
            _video_id: &str,
            _language: &str,
            _format: Format,
        ) -> AppResult<SubtitleInfo> {
            Err(AppError::CaptchaChallenge {
                provider: "provider-test",
                kind: "checkbox-grid",
            })
        }
        async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
            Err(AppError::CaptchaChallenge {
                provider: "provider-test",
                kind: "checkbox-grid",
            })
        }
    }

    /// A challenge is the one outcome whose own documentation says a
    /// human is required and no sibling provider can help. Replacing it
    /// with the ASR verdict tells the operator to give up on the video
    /// when the real obstacle is a wall a person clears in seconds.
    ///
    /// MEASURED on 2026-09-04: getsubs.cc classified its checkbox grid
    /// correctly and the probe overwrote the result anyway.
    #[tokio::test]
    async fn a_challenged_chain_keeps_its_own_cause_over_the_asr_verdict() {
        let server = watch_server(200, &watch_page_with(ASR_ONLY_BLOCK)).await;
        let chain = ProviderChain::with_min_interval(
            vec![Box::new(AlwaysChallenged)],
            std::time::Duration::ZERO,
        )
        .with_watch_probe_base(server.uri());
        let err = chain
            .fetch_subtitle("Ze0i7zxpyrw", "pt", Format::Srt)
            .await
            .expect_err("every provider was challenged");
        assert!(
            matches!(err, AppError::CaptchaChallenge { kind, .. } if kind == "checkbox-grid"),
            "the challenge must survive the probe, got {err:?}"
        );
    }

    /// A provider that always hands over a body without ever naming the
    /// track it came from, which is what every surviving provider does.
    struct AlwaysDelivers;

    #[async_trait]
    impl Provider for AlwaysDelivers {
        fn name(&self) -> &'static str {
            "provider-test"
        }
        async fn fetch_subtitle(
            &self,
            video_id: &str,
            language: &str,
            format: Format,
        ) -> AppResult<SubtitleInfo> {
            Ok(SubtitleInfo {
                video_id: video_id.to_string(),
                // The request echoed back, exactly as `noiz` does it.
                language: language.to_string(),
                // The unknown stays declared: this provider never reads
                // a track identity out of the upstream response.
                delivered_language: None,
                format,
                source_url: "https://example.invalid/track".to_string(),
                byte_size: 0,
                format_hint: crate::provider::SubtitleFormat::Srt,
                provider: "provider-test",
            })
        }
        async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
            Ok(b"1\n00:00:01,000 --> 00:00:02,000\nola\n".to_vec())
        }
    }

    /// A body in the wrong language is worse than no body, because the
    /// caller cannot tell it is wrong.
    ///
    /// MEASURED on 2026-09-04 against a live video that publishes only
    /// `pt`: `--lang en` returned exit 0, `"language":"en"` and 241304
    /// bytes whose function words were Portuguese 2361 times and
    /// English zero times. Nothing in the envelope disagreed, because
    /// `language` is the request echoed back. The evidence needed to
    /// catch it was one GET away and the success path never asked.
    #[tokio::test]
    async fn a_delivered_body_is_refused_when_the_page_lacks_the_requested_language() {
        let server = watch_server(200, &watch_page_with(PT_ONLY_BLOCK)).await;
        let chain = ProviderChain::with_min_interval(
            vec![Box::new(AlwaysDelivers)],
            std::time::Duration::ZERO,
        )
        .with_watch_probe_base(server.uri());

        let err = chain
            .fetch_subtitle("Ze0i7zxpyrw", "en", Format::Srt)
            .await
            .expect_err("the page publishes pt only, so en was never deliverable");

        match err {
            AppError::LanguageUnavailable { available } => {
                assert_eq!(available, vec!["pt".to_string()]);
            }
            other => panic!("expected LanguageUnavailable, got {other:?}"),
        }
    }

    /// The refusal has to travel with the evidence that produced it, or
    /// the envelope publishes a verdict from a source it never shows.
    #[tokio::test]
    async fn the_refusal_records_the_watch_page_as_its_own_evidence() {
        let server = watch_server(200, &watch_page_with(PT_ONLY_BLOCK)).await;
        let chain = ProviderChain::with_min_interval(
            vec![Box::new(AlwaysDelivers)],
            std::time::Duration::ZERO,
        )
        .with_watch_probe_base(server.uri());

        let (result, attempts) = chain
            .fetch_subtitle_traced("Ze0i7zxpyrw", "en", Format::Srt)
            .await;
        assert!(
            result.is_err(),
            "en was not deliverable from a pt-only page"
        );

        let probe = attempts
            .iter()
            .find(|a| a.provider == super::super::WATCH_PAGE_SOURCE)
            .expect("the source that decided the outcome must appear in attempts");
        assert_eq!(probe.outcome, AttemptOutcome::LanguageUnavailable);
        assert!(
            attempts.iter().any(|a| a.provider == "provider-test"),
            "the provider that delivered must still be recorded"
        );
    }

    /// One published language leaves no room for the body to be another,
    /// so this is the one case where the track can be named.
    #[tokio::test]
    async fn a_single_language_page_names_the_delivered_track() {
        let server = watch_server(200, &watch_page_with(PT_ONLY_BLOCK)).await;
        let chain = ProviderChain::with_min_interval(
            vec![Box::new(AlwaysDelivers)],
            std::time::Duration::ZERO,
        )
        .with_watch_probe_base(server.uri());

        let (info, _body) = chain
            .fetch_subtitle("Ze0i7zxpyrw", "pt", Format::Srt)
            .await
            .expect("pt is exactly what the page publishes");

        assert_eq!(
            info.delivered_language,
            Some("pt".to_string()),
            "the observation was available on the page and had to reach the envelope"
        );
    }

    /// A page this crate could not read must never cancel a delivery:
    /// the provider observed something, the probe observed nothing.
    #[tokio::test]
    async fn an_unreadable_page_leaves_a_successful_delivery_alone() {
        let server = watch_server(500, "not a watch page").await;
        let chain = ProviderChain::with_min_interval(
            vec![Box::new(AlwaysDelivers)],
            std::time::Duration::ZERO,
        )
        .with_watch_probe_base(server.uri());

        let (info, _body) = chain
            .fetch_subtitle("Ze0i7zxpyrw", "en", Format::Srt)
            .await
            .expect("an inconclusive probe cannot overrule a delivery");

        assert!(
            info.delivered_language.is_none(),
            "an unknown answer stays declared instead of being guessed"
        );
    }

    /// Serve `body` at `/watch` with `status`, and return the origin.
    async fn watch_server(status: u16, body: &str) -> wiremock::MockServer {
        let server = wiremock::MockServer::start().await;
        wiremock::Mock::given(wiremock::matchers::method("GET"))
            .and(wiremock::matchers::path("/watch"))
            .respond_with(wiremock::ResponseTemplate::new(status).set_body_string(body))
            .mount(&server)
            .await;
        server
    }

    /// A chain that always fails, probing the given origin.
    fn probing_chain(base: &str) -> ProviderChain {
        ProviderChain::with_min_interval(
            vec![Box::new(AlwaysUnavailable)],
            std::time::Duration::ZERO,
        )
        .with_watch_probe_base(base)
    }

    // -- the classifier, branch by branch ----------------------------

    #[test]
    fn zero_tracks_classify_as_not_published() {
        let page = watch_page_with(NO_CAPTIONS_BLOCK);
        assert!(matches!(
            classify_watch_page(&page, "pt"),
            Some(AppError::NoSubtitle(NoSubtitleReason::NotPublished))
        ));
    }

    #[test]
    fn tracks_without_the_requested_language_name_the_ones_that_exist() {
        let page = watch_page_with(CAPTIONS_BLOCK);
        match classify_watch_page(&page, "de") {
            Some(AppError::LanguageUnavailable { available }) => {
                assert_eq!(available, vec!["en", "pt"]);
            }
            other => panic!("expected LanguageUnavailable, got {other:?}"),
        }
    }

    #[test]
    fn an_all_asr_page_carrying_the_requested_language_is_the_new_kind() {
        let page = watch_page_with(ASR_ONLY_BLOCK);
        match classify_watch_page(&page, "pt") {
            Some(err @ AppError::CaptionsAsrOnly { .. }) => {
                assert_eq!(err.kind(), "captions_asr_unsupported_by_provider");
                assert_eq!(err.exit_code(), crate::error::sysexits::EX_NOINPUT);
                assert!(!err.retryable(), "the cause is stable");
                let AppError::CaptionsAsrOnly { asr_languages } = err else {
                    unreachable!("matched above")
                };
                assert_eq!(asr_languages, vec!["en", "pt"]);
            }
            other => panic!("expected CaptionsAsrOnly, got {other:?}"),
        }
    }

    /// The control for the branch above: one manual track on the page is
    /// enough for the ASR verdict to be false, even when the requested
    /// language itself is machine-generated. This is what keeps the kind
    /// from being asserted on a simpler signal than the one it names.
    #[test]
    fn a_page_with_one_manual_track_explains_nothing_the_chain_did_not() {
        let page = watch_page_with(CAPTIONS_BLOCK);
        assert!(classify_watch_page(&page, "pt").is_none());
    }

    /// A body this crate cannot read is a statement about the page, not
    /// about the captions.
    #[test]
    fn an_unreadable_page_yields_no_verdict() {
        assert!(classify_watch_page("<html>challenge</html>", "pt").is_none());
    }

    // -- the chain, end to end over a local mock ----------------------

    #[tokio::test]
    async fn the_chain_returns_the_new_kind_for_an_asr_only_video() {
        let server = watch_server(200, &watch_page_with(ASR_ONLY_BLOCK)).await;
        let err = probing_chain(&server.uri())
            .fetch_subtitle("Ze0i7zxpyrw", "pt", Format::Srt)
            .await
            .expect_err("every provider failed");
        assert_eq!(err.kind(), "captions_asr_unsupported_by_provider");
    }

    /// The probe may not convert an exhausted quota into a verdict about
    /// the captions. A chain that was refused with HTTP 429 never asked
    /// for a track, so "none deliverable" is a claim it cannot support —
    /// and the operator who reads it goes looking at the wrong thing.
    ///
    /// The page here is the ASR-only one, so the classifier really does
    /// return `CaptionsAsrOnly`; the guard is what keeps it from
    /// replacing the cause that has a witness.
    #[tokio::test]
    async fn a_rate_limited_chain_keeps_its_own_cause_over_the_asr_verdict() {
        let server = watch_server(200, &watch_page_with(ASR_ONLY_BLOCK)).await;
        let chain = ProviderChain::with_min_interval(
            vec![Box::new(AlwaysRateLimited)],
            std::time::Duration::ZERO,
        )
        .with_watch_probe_base(server.uri());
        let err = chain
            .fetch_subtitle("Ze0i7zxpyrw", "pt", Format::Srt)
            .await
            .expect_err("every provider failed");
        assert!(
            matches!(err, AppError::RateLimited { provider, .. } if provider == "provider-test"),
            "the 429 must survive the probe, got {err:?}"
        );
    }

    #[tokio::test]
    async fn the_chain_reports_the_languages_the_video_does_publish() {
        let server = watch_server(200, &watch_page_with(CAPTIONS_BLOCK)).await;
        let err = probing_chain(&server.uri())
            .fetch_subtitle("Ze0i7zxpyrw", "de", Format::Srt)
            .await
            .expect_err("every provider failed");
        match err {
            AppError::LanguageUnavailable { available } => {
                assert_eq!(available, vec!["en", "pt"]);
            }
            other => panic!("expected LanguageUnavailable, got {other:?}"),
        }
    }

    /// The invariant a future refactor is most likely to break: an
    /// inconclusive probe must leave the chain's own error untouched,
    /// never turn into a failure mode of its own.
    #[tokio::test]
    async fn a_failed_probe_returns_the_original_chain_error() {
        let server = watch_server(500, "upstream is angry").await;
        let err = probing_chain(&server.uri())
            .fetch_subtitle("Ze0i7zxpyrw", "pt", Format::Srt)
            .await
            .expect_err("every provider failed");
        assert!(
            matches!(err, AppError::ProviderUnavailable { provider } if provider == "provider-test"),
            "got {err:?}"
        );
    }

    /// A chain built without a probe behaves exactly as it did before
    /// the probe existed.
    #[tokio::test]
    async fn a_chain_without_a_probe_keeps_its_own_error() {
        let chain = ProviderChain::with_min_interval(
            vec![Box::new(AlwaysUnavailable)],
            std::time::Duration::ZERO,
        );
        let err = chain
            .fetch_subtitle("Ze0i7zxpyrw", "pt", Format::Srt)
            .await
            .expect_err("every provider failed");
        assert!(
            matches!(err, AppError::ProviderUnavailable { provider } if provider == "provider-test"),
            "got {err:?}"
        );
    }
}