youtube-legend-cli 0.4.0

//! Subtitle provider trait, the two upstream implementations, and a
//! throttled provider chain.
//!
//! The implementations are `decopy` and `noiz`. This header named the
//! noteey.com implementation until 2026-09-04, when that provider was
//! removed together with the whole browser subsystem.
//!
//! A `Provider` is the unit of pluggable I/O against one upstream
//! subtitle source. A `ProviderChain` walks its providers in order,
//! honours a one-request-per-second throttle, and aggregates the
//! results into a single `SubtitleInfo` + body pair.

mod chain;
pub mod decopy;
/// Per-provider health that survives the process, so `retryable`
/// can carry observed persistence instead of type alone.
pub(crate) mod health;
pub mod noiz;
pub mod robots;
pub mod stealth;

pub use decopy::ProviderDecopy;
pub use noiz::ProviderNoiz;

pub(crate) use chain::http_failure;
pub use chain::{
    per_host_concurrency, throttle_interval, watch_probe_timeout, AttemptOutcome, ProviderAttempt,
    ProviderChain, ProviderOutcome, DEFAULT_PER_HOST_CONCURRENCY, DEFAULT_THROTTLE_INTERVAL_MS,
    DEFAULT_WATCH_PROBE_TIMEOUT_SECS,
};

use async_trait::async_trait;
use fluent_langneg::negotiate::{negotiate_languages, NegotiationStrategy};
use serde::{Deserialize, Serialize};
use unic_langid::LanguageIdentifier;

use crate::error::{AppError, AppResult};

/// Whether this run refuses every outbound request.
///
/// Resolves the `offline` configuration key, which the `--offline`
/// flag mirrors. Offline mode is what lets an audit job exercise the
/// whole pipeline without touching the network: each provider bails
/// out with [`AppError::ProviderUnavailable`] instead of launching a
/// browser or opening a socket.
///
/// This is configuration, not an environment variable: the crate reads
/// no product environment variable of its own.
#[must_use]
pub fn is_offline() -> bool {
    crate::config::tuning_bool_or("offline", false)
}

/// Whether the caller asked for machine-generated tracks by preference.
///
/// Resolved through the tuning registry for the same reason `--offline`
/// is: negotiation happens far from the parsed `Cli`, and threading a
/// second boolean through every provider signature would put the policy
/// in five places instead of one.
#[must_use]
pub fn prefers_asr() -> bool {
    crate::config::tuning_bool_or("asr", false)
}

/// Subtitle delivery format.
///
/// `Srt` preserves the raw `SubRip` text; `Txt` strips timestamps and
/// joins cues with blank lines. The variant is serialised in the
/// `--json` envelope.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub enum Format {
    /// `SubRip` text with timestamps preserved.
    Srt,
    /// Plain text with timestamps removed.
    Txt,
}

impl Format {
    /// Lowercase string identifier (`"srt"` or `"txt"`).
    pub fn as_str(&self) -> &'static str {
        match self {
            Format::Srt => "srt",
            Format::Txt => "txt",
        }
    }

    /// File extension associated with the format. Identical to
    /// [`Format::as_str`] for the current variants, but kept as a
    /// separate method so future variants can diverge.
    pub fn extension(&self) -> &'static str {
        match self {
            Format::Srt => "srt",
            Format::Txt => "txt",
        }
    }
}

/// Metadata for a single subtitle retrieval, returned by
/// [`Provider::fetch_subtitle`] before the body is fetched by
/// [`Provider::fetch_content`].
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct SubtitleInfo {
    /// 11-character `YouTube` video id.
    pub video_id: String,
    /// Language tag this run settled on, as the provider spells it.
    ///
    /// GAP-2026-158: this is NOT evidence of what the upstream
    /// delivered. `noiz` fills it by echoing the request back, and
    /// `decopy` fills it with the literal `und`. Neither reads the
    /// track identity out of the response, so NEITHER surviving
    /// provider can populate this from observation. Read
    /// [`SubtitleInfo::delivered_language`] when the question is which
    /// track actually came back.
    ///
    /// This paragraph cited `provider_noteey`, `ProviderNoteey::tracks_for`
    /// and `getsubs` until 2026-09-04. All three were removed with the
    /// browser subsystem, and with them went the claim that `getsubs`
    /// was the only provider deriving this from a track the upstream
    /// itself named. No provider does that today.
    pub language: String,
    /// Language of the track the upstream actually delivered, when the
    /// upstream named it.
    ///
    /// GAP-2026-158: `None` means "we do not know which track came
    /// back", and NEVER "the requested track came back". Only a
    /// provider that reads the track identity out of the upstream
    /// response may set this; a provider that echoes the request must
    /// leave it `None`, so an unknown answer stays declared instead of
    /// being guessed.
    pub delivered_language: Option<String>,
    /// Delivery format the body will be in.
    pub format: Format,
    /// Provider-supplied URL to the raw subtitle body.
    pub source_url: String,
    /// Body size in bytes, populated by `fetch_content` (zero before).
    pub byte_size: usize,
    /// GAP-AUD-2026-038: discriminator for which parser the body
    /// needs. `Srt` (default) for `SubRip`; `NoteeyTranscript` for the
    /// `MM:SS`-prefixed plain text that noteey.com emits. Consumed
    /// by `commands::convert_format` to pick the right parser.
    pub format_hint: SubtitleFormat,
    /// GAP-AUD-2026-050: stable provider identifier that produced
    /// this `SubtitleInfo`. Mirrors [`Provider::name`] exactly so
    /// downstream consumers can correlate the JSON envelope field
    /// `provider` with tracing events. Populated by every concrete
    /// provider at `fetch_subtitle` time.
    pub provider: &'static str,
}

/// Subtitle body shape returned by a provider.
///
/// Distinct from [`Format`] (the user-requested delivery format):
/// `format_hint` tells the CLI what the body bytes *actually* look
/// like so the right parser is invoked. The user-requested `Format`
/// may still differ — `noteey_to_text` always emits plain text even
/// when the user asked for `--format srt`, in which case the chain
/// returns `AppError::InvalidUsage` rather than fabricating timestamps.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub enum SubtitleFormat {
    /// Body is in `SubRip` (`Srt`) format.
    #[default]
    Srt,
    /// Body is noteey-style transcript: one cue per line with a
    /// leading `MM:SS` (or `HH:MM:SS`) timestamp prefix.
    NoteeyTranscript,
}

impl SubtitleFormat {
    /// Lowercase kebab-case identifier for logs and tracing.
    pub fn as_str(&self) -> &'static str {
        match self {
            SubtitleFormat::Srt => "srt",
            SubtitleFormat::NoteeyTranscript => "noteey-transcript",
        }
    }
}

/// One subtitle track a provider can offer for a video.
///
/// The tag is what negotiation matches on, so it must be the tag the
/// upstream advertises, normalised to BCP 47. Providers that receive a
/// legacy `YouTube` code (`iw`, `in`, `ji`) should hand it to
/// [`crate::cli::LanguageArg::parse`], which folds it onto the modern
/// code, rather than storing the legacy spelling here.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct SubtitleTrack {
    /// BCP 47 tag of the track, for example `pt-BR` or `zh-Hant`.
    pub tag: String,
    /// Human-readable label as the upstream presents it, for example
    /// `Português (Brasil)`.
    pub label: String,
    /// `true` when the track was produced by automatic speech
    /// recognition rather than authored by a human.
    pub auto_generated: bool,
    /// Delivery format the body will arrive in.
    pub format: Format,
}

impl SubtitleTrack {
    /// Build a track from its tag, defaulting the label to the tag.
    #[must_use]
    pub fn new(tag: impl Into<String>, format: Format) -> Self {
        let tag = tag.into();
        Self {
            label: tag.clone(),
            tag,
            auto_generated: false,
            format,
        }
    }

    /// Builder-style: attach a human-readable label.
    #[must_use]
    pub fn with_label(mut self, label: impl Into<String>) -> Self {
        self.label = label.into();
        self
    }

    /// Builder-style: mark the track as automatically generated.
    #[must_use]
    pub fn with_auto_generated(mut self, auto_generated: bool) -> Self {
        self.auto_generated = auto_generated;
        self
    }
}

/// Pick the track that best matches `requested` out of `tracks`.
///
/// Matching is BCP 47 language negotiation, not string equality, using
/// [`NegotiationStrategy::Lookup`] so exactly one candidate comes back.
/// `pt-BR` therefore reaches a `pt` track, `pt` reaches `pt-BR`, and
/// `zh-Hans` never silently resolves to a `zh-Hant` track.
///
/// English is the fallback, matching the CLI default — but only when a
/// track actually offers it. A fallback that is not on the menu is not
/// a match, and the caller is told so.
///
/// # Errors
///
/// Returns [`AppError::NoSubtitle`] carrying
/// [`crate::error::NoSubtitleReason::LanguageUnavailable`] when the
/// video has tracks but none in a language close enough to the
/// request, and when the track list is empty. This is deliberately
/// distinct from `NotPublished`: "the video has no captions" and "the
/// video has captions, just not yours" are different facts and must
/// stay distinguishable downstream.
pub fn negotiate_track(
    requested: &LanguageIdentifier,
    tracks: &[SubtitleTrack],
) -> AppResult<SubtitleTrack> {
    if tracks.is_empty() {
        return Err(AppError::NoSubtitle(
            crate::error::NoSubtitleReason::LanguageUnavailable,
        ));
    }

    let requested_id = crate::i18n::negotiable(&requested.to_string())
        .ok_or_else(|| AppError::LanguageParseError(requested.to_string()))?;

    // Tracks whose tag does not parse are unusable for negotiation;
    // they stay out of the candidate set rather than aborting the whole
    // lookup, so one malformed entry cannot hide the rest.
    let parsed: Vec<(crate::i18n::NegotiableId, &SubtitleTrack)> = tracks
        .iter()
        .filter_map(|track| crate::i18n::negotiable(&track.tag).map(|id| (id, track)))
        .collect();
    let available: Vec<crate::i18n::NegotiableId> =
        parsed.iter().map(|(id, _)| id.clone()).collect();

    let english = crate::i18n::negotiable(crate::cli::LanguageArg::english().as_str())
        .ok_or_else(|| AppError::Internal("english is not a negotiable tag".to_string()))?;
    let matched = negotiate_languages(
        &[requested_id],
        &available,
        Some(&english),
        NegotiationStrategy::Lookup,
    );

    let winner = matched
        .first()
        .ok_or(AppError::NoSubtitle(
            crate::error::NoSubtitleReason::LanguageUnavailable,
        ))?
        .to_string();

    // `Lookup` always yields one element: the match when there is one,
    // the default otherwise. The default only counts as a result when
    // some track really carries it.
    let candidates: Vec<&SubtitleTrack> = parsed
        .iter()
        .filter(|(id, _)| id.to_string() == winner)
        .map(|(_, track)| *track)
        .collect();

    // One language can publish BOTH a human-authored track and a
    // machine-generated one, and until 2026-09-01 the winner was
    // whichever the upstream happened to list first — an arbitrary
    // answer to a question nobody had asked. `auto_generated` was
    // already populated by three of the four providers, and
    // `getsubs` even derives it from the real upstream label, so the
    // fact was present and simply unread.
    //
    // The preference is a TIE-BREAK and never a filter: when the
    // wanted kind is absent, the other kind is served. A caller who
    // asked for a subtitle in a language gets one, and the kind was
    // the second wish.
    pick_by_kind(&candidates, prefers_asr())
        .cloned()
        .ok_or(AppError::NoSubtitle(
            crate::error::NoSubtitleReason::LanguageUnavailable,
        ))
}

/// Choose between tracks of one language by kind, preferring
/// `wants_asr` and falling back to whatever exists.
///
/// Split out of [`negotiate_track`] so the policy can be tested at both
/// values of the preference. The lookup that feeds it reads a
/// process-wide `OnceLock`, which one test binary can only set once, so
/// a policy left inline would have been testable in one direction only
/// — and a rule tested in one direction is half a rule.
fn pick_by_kind<'a>(
    candidates: &[&'a SubtitleTrack],
    wants_asr: bool,
) -> Option<&'a SubtitleTrack> {
    candidates
        .iter()
        .find(|track| track.auto_generated == wants_asr)
        .or_else(|| candidates.first())
        .copied()
}

/// Pluggable subtitle source. Implementations must be `Send + Sync` so
/// they can be stored in a `Box<dyn Provider>` inside a [`ProviderChain`].
#[doc(alias = "Source")]
#[doc(alias = "Upstream")]
#[doc(alias = "Backend")]
#[doc(alias = "pluggable")]
#[doc(alias = "trait")]
#[doc(alias = "async trait")]
#[doc(alias = "subtitle source")]
#[doc(alias = "upstream")]
#[async_trait]
pub trait Provider: Send + Sync {
    /// Short human-readable identifier, used in tracing events.
    fn name(&self) -> &'static str;

    /// Every subtitle track this provider can offer for `video_id`.
    ///
    /// The default returns an empty list, which
    /// [`negotiate_track`] reads as "this provider cannot tell us what
    /// it has". Providers that genuinely enumerate tracks override it;
    /// the rest keep working unchanged.
    ///
    /// # Errors
    ///
    /// - [`AppError::ProviderUnavailable`] on transient upstream
    ///   failure while listing.
    /// - [`AppError::Http`] on transport errors.
    async fn list_tracks(&self, _video_id: &str) -> AppResult<Vec<SubtitleTrack>> {
        Ok(Vec::new())
    }

    /// Resolve the subtitle URL and language match for a given video.
    ///
    /// # Errors
    ///
    /// - [`AppError::NoSubtitle`] when the provider has nothing for the
    ///   request (a structured [`crate::error::NoSubtitleReason`] is
    ///   attached).
    /// - [`AppError::ProviderUnavailable`] on transient upstream
    ///   failure.
    /// - [`AppError::Http`] on transport errors.
    async fn fetch_subtitle(
        &self,
        video_id: &str,
        language: &str,
        format: Format,
    ) -> AppResult<SubtitleInfo>;

    /// Download the body bytes for the given [`SubtitleInfo`].
    ///
    /// # Errors
    ///
    /// - [`AppError::NoSubtitle`] when the body is empty or the URL has
    ///   gone stale.
    /// - [`AppError::ProviderUnavailable`] on transient upstream
    ///   failure.
    async fn fetch_content(&self, info: &SubtitleInfo) -> AppResult<Vec<u8>>;
}

#[cfg(test)]
mod negotiation_tests {
    use super::*;
    use crate::cli::LanguageArg;
    use crate::error::NoSubtitleReason;

    fn track(tag: &str) -> SubtitleTrack {
        SubtitleTrack::new(tag, Format::Txt)
    }

    fn negotiate(requested: &str, tags: &[&str]) -> AppResult<SubtitleTrack> {
        let tracks: Vec<SubtitleTrack> = tags.iter().map(|t| track(t)).collect();
        let requested = LanguageArg::parse(requested).expect("test tag parses");
        negotiate_track(&requested.to_langid(), &tracks)
    }

    #[test]
    fn exact_tag_wins() {
        let hit = negotiate("pt-BR", &["en", "pt-BR", "es"]).expect("pt-BR is on the menu");
        assert_eq!(hit.tag, "pt-BR");
    }

    #[test]
    fn regional_request_reaches_the_bare_language() {
        let hit = negotiate("pt-BR", &["en", "pt"]).expect("pt is close enough");
        assert_eq!(hit.tag, "pt");
    }

    #[test]
    fn bare_request_reaches_a_regional_track() {
        let hit = negotiate("pt", &["en", "pt-BR"]).expect("pt-BR is close enough");
        assert_eq!(hit.tag, "pt-BR");
    }

    /// The regression the old `split('-')` parser could never catch:
    /// Simplified and Traditional Chinese are different tracks.
    #[test]
    fn simplified_chinese_never_resolves_to_traditional() {
        let hit = negotiate("zh-Hans", &["zh-Hant", "zh-Hans"]).expect("zh-Hans is on the menu");
        assert_eq!(hit.tag, "zh-Hans");
        let hit = negotiate("zh-Hant", &["zh-Hant", "zh-Hans"]).expect("zh-Hant is on the menu");
        assert_eq!(hit.tag, "zh-Hant");
    }

    #[test]
    fn english_is_the_fallback_when_the_request_misses() {
        let hit = negotiate("ja", &["en", "pt-BR"]).expect("english backstops the miss");
        assert_eq!(hit.tag, "en");
    }

    #[test]
    fn a_miss_without_english_is_language_unavailable_not_not_published() {
        let err = negotiate("ja", &["pt-BR", "es"]).expect_err("no japanese, no english");
        assert!(
            matches!(
                err,
                AppError::NoSubtitle(NoSubtitleReason::LanguageUnavailable)
            ),
            "expected LanguageUnavailable, got {err:?}"
        );
    }

    /// One language publishing both kinds must not resolve by upstream
    /// order. The TED fixture is the real shape of this: sixty-four
    /// human-authored tracks and exactly one ASR track.
    #[test]
    fn the_kind_preference_decides_between_two_tracks_of_one_language() {
        let manual = track("pt-BR");
        let auto = SubtitleTrack::new("pt-BR", Format::Txt).with_auto_generated(true);

        // Upstream order is deliberately ASR-first in both cases, so a
        // passing result cannot come from the old `find` picking the
        // head of the list.
        let menu = [&auto, &manual];

        let chosen = pick_by_kind(&menu, false).expect("a track matches");
        assert!(
            !chosen.auto_generated,
            "without --asr the human-authored track wins even when listed second"
        );

        let chosen = pick_by_kind(&menu, true).expect("a track matches");
        assert!(
            chosen.auto_generated,
            "with --asr the machine-generated track wins"
        );
    }

    /// The flag is a preference, not a filter: asking for a kind the
    /// language does not publish still answers with the other kind.
    #[test]
    fn a_kind_that_is_absent_falls_back_instead_of_failing() {
        let manual = track("pt-BR");
        let only_manual = [&manual];
        let chosen = pick_by_kind(&only_manual, true).expect("fallback serves the other kind");
        assert!(
            !chosen.auto_generated,
            "--asr on a video with no ASR track must still deliver the manual one"
        );

        let auto = SubtitleTrack::new("pt-BR", Format::Txt).with_auto_generated(true);
        let only_auto = [&auto];
        let chosen = pick_by_kind(&only_auto, false).expect("fallback serves the other kind");
        assert!(
            chosen.auto_generated,
            "without --asr an ASR-only video must still deliver its one track"
        );
    }

    /// Control: an empty candidate set must not resolve to a track.
    #[test]
    fn no_candidate_yields_no_choice() {
        let empty: [&SubtitleTrack; 0] = [];
        assert!(pick_by_kind(&empty, false).is_none());
        assert!(pick_by_kind(&empty, true).is_none());
    }

    #[test]
    fn an_empty_menu_is_language_unavailable() {
        let err = negotiate_track(&LanguageArg::english().to_langid(), &[])
            .expect_err("nothing to negotiate against");
        assert!(matches!(
            err,
            AppError::NoSubtitle(NoSubtitleReason::LanguageUnavailable)
        ));
    }

    #[test]
    fn a_malformed_track_tag_does_not_hide_the_rest() {
        let hit = negotiate("pt-BR", &["!!! not a tag", "pt-BR"]).expect("the good tag survives");
        assert_eq!(hit.tag, "pt-BR");
    }

    #[test]
    fn hebrew_negotiates_through_the_modern_code() {
        // A provider that normalised YouTube's `iw` into `he` before
        // building the menu is matched by a `--lang he` request.
        let hit = negotiate("he", &["en", "he"]).expect("hebrew is on the menu");
        assert_eq!(hit.tag, "he");
    }

    #[test]
    fn track_builder_defaults_label_to_tag_and_is_not_auto_generated() {
        let t = SubtitleTrack::new("pt-BR", Format::Txt);
        assert_eq!(t.label, "pt-BR");
        assert!(!t.auto_generated);
        let t = t.with_label("Português (Brasil)").with_auto_generated(true);
        assert_eq!(t.label, "Português (Brasil)");
        assert!(t.auto_generated);
    }

    /// A provider that does not override `list_tracks` reports an empty
    /// menu rather than failing to compile.
    #[tokio::test]
    async fn default_list_tracks_is_empty() {
        struct Silent;
        #[async_trait]
        impl Provider for Silent {
            fn name(&self) -> &'static str {
                "silent"
            }
            async fn fetch_subtitle(
                &self,
                _video_id: &str,
                _language: &str,
                _format: Format,
            ) -> AppResult<SubtitleInfo> {
                Err(AppError::ProviderUnavailable { provider: "silent" })
            }
            async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
                Err(AppError::ProviderUnavailable { provider: "silent" })
            }
        }
        assert!(Silent
            .list_tracks("dQw4w9WgXcQ")
            .await
            .expect("ok")
            .is_empty());
    }
}