youtube-legend-cli 0.4.0

//! JSON [`Provider`] backed by the decopy.ai transcription endpoint.
//!
//! The upstream is a single synchronous `POST` that returns the whole
//! cue list in one envelope, so no browser and no polling loop are
//! involved. That makes it the cheapest provider in the chain and the
//! right fallback when the browser tier is unavailable.
//!
//! # Language
//!
//! The endpoint exposes **no language parameter**. It returns the
//! video's native track and does not name it. Rather than mislabel the
//! result with whatever the caller asked for, this provider tags the
//! track [`UNDETERMINED_TAG`] (BCP 47 `und`) and logs a warning when the
//! request named a concrete language. [`Provider::list_tracks`] reports
//! the same single `und` entry, which is the honest menu: "one track,
//! language unknown".
//!
//! Deliberately, `fetch_subtitle` does **not** run the track through
//! [`crate::provider::negotiate_track`]. Negotiating `und` against any
//! concrete request always misses, which would make the provider dead
//! weight in the chain instead of the last-resort source it is.
//!
//! # Format
//!
//! Cue bounds arrive as `HH:MM:SS` strings, which is enough to render
//! real `SubRip`. The provider therefore advertises
//! [`SubtitleFormat::Srt`] and `--format srt` works through it.

pub(crate) mod cue;

use std::collections::HashMap;
use std::sync::Mutex;
use std::time::Duration;

use async_trait::async_trait;
use serde::Deserialize;

use super::{Format, Provider, SubtitleFormat, SubtitleInfo, SubtitleTrack};
use crate::error::{AppError, AppResult, NoSubtitleReason};
use crate::provider::stealth::session_rng_fork;
use crate::secret_endpoints::{
    decopy_api_base, decopy_api_host, decopy_create_job_path, decopy_product_code,
};

/// Stable provider identifier. Mirrored into
/// [`SubtitleInfo::provider`] and into every tracing event.
pub const PROVIDER_NAME: &str = "provider-decopy";

/// BCP 47 subtag for "language not determined". Used because the
/// upstream never names the track it returns.
pub const UNDETERMINED_TAG: &str = "und";

/// Wall-clock ceiling for the single upstream request. The endpoint
/// transcribes synchronously, so the budget covers the whole job.
///
/// Compiled default behind `providers.decopy.request_timeout_secs`.
const DEFAULT_DECOPY_REQUEST_TIMEOUT_SECS: u64 = 180;

/// Wall-clock ceiling for the single upstream request.
///
/// Resolves `providers.decopy.request_timeout_secs`.
fn decopy_request_timeout() -> Duration {
    Duration::from_secs(crate::config::tuning_u64_in_range(
        "providers.decopy.request_timeout_secs",
        DEFAULT_DECOPY_REQUEST_TIMEOUT_SECS,
        1,
        3_600,
    ))
}

/// Maximum response body accepted before parsing, as a guard against an
/// upstream that answers with an unbounded stream.
///
/// Compiled default behind `providers.decopy.max_body_bytes`.
const DEFAULT_DECOPY_MAX_BODY_BYTES: usize = 32 * 1024 * 1024;

/// Maximum response body accepted before parsing.
///
/// Resolves `providers.decopy.max_body_bytes`.
fn decopy_max_body_bytes() -> usize {
    crate::config::tuning_usize_in_range(
        "providers.decopy.max_body_bytes",
        DEFAULT_DECOPY_MAX_BODY_BYTES,
        1_024,
        1_073_741_824,
    )
}

/// Number of hex characters in a `Product-Serial`.
///
/// Compiled default behind `providers.decopy.serial_hex_len`.
const DEFAULT_DECOPY_SERIAL_HEX_LEN: usize = 32;

/// Number of hex characters in a `Product-Serial`.
///
/// Resolves `providers.decopy.serial_hex_len`. A zero-length serial
/// would be rejected upstream, so the range starts at one character.
fn decopy_serial_hex_len() -> usize {
    crate::config::tuning_usize_in_range(
        "providers.decopy.serial_hex_len",
        DEFAULT_DECOPY_SERIAL_HEX_LEN,
        1,
        256,
    )
}

/// Envelope code meaning the job completed.
const CODE_OK: i64 = 100_000;

/// Envelope code meaning required parameters were missing. This is a
/// defect on our side, not an upstream outage.
const CODE_MISSING_PARAMS: i64 = 400_301;

/// Envelope code meaning the `Product-Serial` was absent or rejected.
const CODE_BAD_SERIAL: i64 = 400_401;

/// Envelope code meaning the anonymous quota is exhausted. Observed to
/// be scoped per source address rather than per serial.
const CODE_QUOTA_EXHAUSTED: i64 = 210_301;

/// Multipart field name carrying the bare 11-character video id. The
/// endpoint rejects a full watch URL here.
const FIELD_VIDEO_ID: &str = "video_id";

/// Multipart field name for the speaker-identification toggle. The
/// anonymous tier only accepts `false`.
const FIELD_IDENTIFICATION_SWITCH: &str = "identification_switch";

/// Value sent for [`FIELD_IDENTIFICATION_SWITCH`].
const IDENTIFICATION_SWITCH_VALUE: &str = "false";

/// Response envelope. Only the fields the provider reads are modelled;
/// everything else in the payload is ignored so an upstream addition
/// cannot break deserialisation.
#[derive(Debug, Deserialize)]
struct DecopyEnvelope {
    /// Application-level status code. Distinct from the HTTP status:
    /// the endpoint answers `200` even when `code` reports a quota
    /// failure.
    code: i64,
    /// Present only when `code` is [`CODE_OK`].
    #[serde(default)]
    result: Option<DecopyResult>,
    /// Optional human-readable diagnostic.
    ///
    /// Deliberately untyped. The upstream changed this field from a
    /// bare string to a language-keyed map without notice, which is
    /// what broke deserialisation in v0.3.5. Accepting any shape here
    /// is safe because the field is diagnostic only: [`classify_code`]
    /// branches on `code` alone and never on this text.
    #[serde(default)]
    message: Option<serde_json::Value>,
}

/// Extract the human-readable text from a [`DecopyEnvelope::message`].
///
/// Two shapes are known to arrive from the upstream and both are
/// accepted, because the field has already flipped once:
///
/// - a bare string, which is returned as-is;
/// - a map keyed by language tag, as in `{"en": "…", "zh": "…"}`,
///   where the `en` entry wins and any other string value is the
///   fallback.
///
/// Any other shape yields `None`, which degrades the diagnostic text
/// without ever failing the request.
fn message_text(message: Option<&serde_json::Value>) -> Option<String> {
    match message? {
        serde_json::Value::String(text) => Some(text.clone()),
        // `serde_json::Map` without `preserve_order` is backed by a
        // `BTreeMap`, so the fallback pick is deterministic.
        serde_json::Value::Object(map) => map
            .get("en")
            .and_then(serde_json::Value::as_str)
            .or_else(|| map.values().find_map(serde_json::Value::as_str))
            .map(str::to_owned),
        _ => None,
    }
}

/// Payload carried by a successful envelope.
#[derive(Debug, Deserialize)]
struct DecopyResult {
    /// Timed segments, in presentation order.
    #[serde(default)]
    subtitles: Vec<DecopySubtitle>,
}

/// One timed segment as the upstream spells it.
#[derive(Debug, Deserialize)]
struct DecopySubtitle {
    /// `HH:MM:SS` start bound.
    #[serde(default)]
    start: String,
    /// `HH:MM:SS` end bound.
    #[serde(default)]
    end: String,
    /// Segment text.
    #[serde(default)]
    content: String,
}

/// decopy.ai provider. Construct with [`ProviderDecopy::new`].
pub struct ProviderDecopy {
    base_url: String,
    product_serial: String,
    cache: Mutex<HashMap<String, Vec<u8>>>,
}

impl Default for ProviderDecopy {
    fn default() -> Self {
        Self::new()
    }
}

impl ProviderDecopy {
    /// Build a provider against the production endpoint with a freshly
    /// generated `Product-Serial`.
    #[must_use]
    #[tracing::instrument(level = "debug")]
    pub fn new() -> Self {
        Self {
            base_url: decopy_api_base(),
            product_serial: generate_product_serial(),
            cache: Mutex::new(HashMap::new()),
        }
    }

    /// Builder-style: point the provider at a different origin.
    ///
    /// The only production origin is the provider's compiled-in API
    /// base; this exists so
    /// the integration suite can aim the provider at a local mock
    /// server without a network round trip.
    #[must_use]
    pub fn with_base_url(mut self, base_url: impl Into<String>) -> Self {
        self.base_url = base_url.into();
        self
    }

    /// Builder-style: use a `Product-Serial` harvested from a real
    /// browser session instead of a generated one.
    ///
    /// The derivation of a serial from the page's `web-model-uuid` has
    /// not been recovered, so a generated value may be rejected with
    /// the upstream bad-serial error code. An operator who has a
    /// known-good serial can
    /// supply it here; the provider degrades cleanly either way.
    #[must_use]
    pub fn with_product_serial(mut self, serial: impl Into<String>) -> Self {
        self.product_serial = serial.into();
        self
    }

    /// The single-entry menu this provider can honestly offer.
    fn native_track() -> SubtitleTrack {
        SubtitleTrack::new(UNDETERMINED_TAG, Format::Srt)
            .with_label("native track (language undetermined)")
            .with_auto_generated(true)
    }

    /// Absolute URL of the create-job endpoint.
    fn create_job_url(&self) -> String {
        format!(
            "{}{}",
            self.base_url.trim_end_matches('/'),
            decopy_create_job_path()
        )
    }
}

/// Generate a 32-character lowercase hex `Product-Serial`.
///
/// The upstream derivation is unknown (five `MD5` hypotheses over the
/// page's `web-model-uuid` were measured and none reproduced a live
/// serial), so this synthesises a well-formed value of the right shape.
/// A rejection is classified as a degraded failure, never as "the video
/// has no subtitles".
fn generate_product_serial() -> String {
    let mut rng = session_rng_fork();
    let hex_len = decopy_serial_hex_len();
    let mut out = String::with_capacity(hex_len);
    while out.len() < hex_len {
        out.push_str(&format!("{:016x}", rng.next_u64()));
    }
    out.truncate(hex_len);
    out
}

/// Build a `multipart/form-data` body from plain text fields.
///
/// Written by hand rather than through `reqwest`'s `multipart` feature,
/// which this crate does not enable. The two fields carry an id and a
/// boolean literal, so no quoting or transfer encoding is required.
///
/// Returns the body and the `Content-Type` value that describes it.
fn build_multipart(boundary: &str, fields: &[(&str, &str)]) -> (String, String) {
    let mut body = String::with_capacity(fields.len() * 96);
    for (name, value) in fields {
        body.push_str("--");
        body.push_str(boundary);
        body.push_str("\r\n");
        body.push_str("Content-Disposition: form-data; name=\"");
        body.push_str(name);
        body.push_str("\"\r\n\r\n");
        body.push_str(value);
        body.push_str("\r\n");
    }
    body.push_str("--");
    body.push_str(boundary);
    body.push_str("--\r\n");
    let content_type = format!("multipart/form-data; boundary={boundary}");
    (body, content_type)
}

/// Draw a boundary token that cannot collide with the field values.
fn random_boundary() -> String {
    let mut rng = session_rng_fork();
    format!("----youtubelegend{:016x}", rng.next_u64())
}

/// Translate an application-level `code` into the chain's error model.
///
/// The mapping is contractual:
/// - [`CODE_QUOTA_EXHAUSTED`] is a rate limit, which the chain treats as
///   degraded and walks past.
/// - [`CODE_BAD_SERIAL`] is an upstream refusal of our credentials, also
///   degraded — the next provider may still answer.
/// - [`CODE_MISSING_PARAMS`] is our own defect and surfaces as
///   [`AppError::Internal`].
///
/// Nothing here ever produces [`NoSubtitleReason::NotPublished`]: an
/// envelope that failed validation says nothing about whether the video
/// has captions.
fn classify_code(code: i64, message: Option<&str>) -> AppError {
    let detail = message.unwrap_or("no message").to_string();
    match code {
        CODE_QUOTA_EXHAUSTED => {
            tracing::warn!(
                target: "events",
                provider = PROVIDER_NAME,
                code,
                "decopy anonymous quota exhausted; degrading"
            );
            AppError::RateLimited {
                provider: PROVIDER_NAME,
                retry_after_secs: None,
            }
        }
        CODE_BAD_SERIAL => {
            tracing::warn!(
                target: "events",
                provider = PROVIDER_NAME,
                code,
                "decopy rejected the Product-Serial; degrading"
            );
            AppError::ProviderUnavailable {
                provider: PROVIDER_NAME,
            }
        }
        CODE_MISSING_PARAMS => AppError::Internal(format!(
            "{PROVIDER_NAME} sent an incomplete request (code {code}): {detail}"
        )),
        other => {
            tracing::warn!(
                target: "events",
                provider = PROVIDER_NAME,
                code = other,
                detail = %detail,
                "decopy returned an unrecognised code; degrading"
            );
            // The upstream explained itself and, until 2026-09-04, that
            // explanation reached only `stderr`. The error envelope
            // declares a `diagnostic` field for exactly this, and it had
            // no producer left once the browser providers were removed:
            // the datum was produced and thrown away on the way out,
            // which is the GAP-2026-180 class of defect.
            crate::provider::chain::record_upstream_diagnostic(&detail);
            AppError::ProviderUnavailable {
                provider: PROVIDER_NAME,
            }
        }
    }
}

/// Turn a parsed envelope into `SubRip` bytes.
///
/// # Errors
///
/// - [`AppError::NoSubtitle`] carrying [`NoSubtitleReason::NotPublished`]
///   when the job succeeded but carried zero usable cues. That is the
///   one shape in this provider that genuinely means "no captions".
/// - Whatever [`classify_code`] returns for a non-success code.
fn envelope_to_srt(envelope: DecopyEnvelope) -> AppResult<String> {
    if envelope.code != CODE_OK {
        return Err(classify_code(
            envelope.code,
            message_text(envelope.message.as_ref()).as_deref(),
        ));
    }
    let subtitles = envelope.result.map(|r| r.subtitles).unwrap_or_default();
    let cues: Vec<cue::Cue> = subtitles
        .into_iter()
        .filter_map(|s| {
            let start = cue::parse_clock(&s.start)?;
            // A missing end bound is recoverable: `render_srt` widens
            // the cue rather than dropping the text.
            let end = cue::parse_clock(&s.end).unwrap_or(start);
            Some(cue::Cue {
                start_secs: start,
                end_secs: end,
                text: s.content.trim().to_string(),
            })
        })
        .collect();
    let srt = cue::render_srt(&cues);
    if srt.is_empty() {
        return Err(AppError::NoSubtitle(NoSubtitleReason::NotPublished));
    }
    Ok(srt)
}

#[async_trait]
impl Provider for ProviderDecopy {
    fn name(&self) -> &'static str {
        PROVIDER_NAME
    }

    async fn list_tracks(&self, _video_id: &str) -> AppResult<Vec<SubtitleTrack>> {
        Ok(vec![Self::native_track()])
    }

    async fn fetch_subtitle(
        &self,
        video_id: &str,
        language: &str,
        _format: Format,
    ) -> AppResult<SubtitleInfo> {
        if crate::provider::is_offline() {
            return Err(AppError::ProviderUnavailable {
                provider: PROVIDER_NAME,
            });
        }

        // NFR-007: confirm the ruleset permits the endpoint before the
        // first byte leaves the process.
        crate::provider::robots::check_allowed(
            &decopy_api_host(),
            &decopy_create_job_path(),
            &crate::net::user_agent(),
            PROVIDER_NAME,
        )
        .await?;

        if !language.is_empty() && language != UNDETERMINED_TAG {
            tracing::warn!(
                target: "events",
                provider = PROVIDER_NAME,
                requested = language,
                "decopy exposes no language parameter; returning the native track tagged `und`"
            );
        }

        let boundary = random_boundary();
        let (body, content_type) = build_multipart(
            &boundary,
            &[
                (FIELD_VIDEO_ID, video_id),
                (FIELD_IDENTIFICATION_SWITCH, IDENTIFICATION_SWITCH_VALUE),
            ],
        );

        let client = crate::net::session::chrome_client(decopy_request_timeout())?;

        tracing::debug!(
            target: "events",
            provider = PROVIDER_NAME,
            video_id,
            "fetch_subtitle_started"
        );

        let response = client
            .post(self.create_job_url())
            .header(reqwest::header::CONTENT_TYPE, content_type)
            // Measured by OPTIONS preflight: the endpoint requires the
            // product code and a serial, and accepts an EMPTY
            // Authorization header for the anonymous tier. Omitting the
            // header entirely is rejected; sending it empty is not.
            .header("Product-Code", decopy_product_code())
            .header("Product-Serial", self.product_serial.as_str())
            .header(reqwest::header::AUTHORIZATION, "")
            .body(body)
            .send()
            .await
            .map_err(AppError::Http)?;

        let status = response.status();
        if !status.is_success() {
            return Err(super::http_failure(
                status,
                response.headers(),
                PROVIDER_NAME,
            ));
        }

        let raw = response.text().await.map_err(AppError::Http)?;
        if raw.len() > decopy_max_body_bytes() {
            return Err(AppError::SubtitleTooLarge(raw.len()));
        }
        let envelope: DecopyEnvelope =
            serde_json::from_str(&raw).map_err(|e| AppError::ProviderProtocolError {
                provider: PROVIDER_NAME,
                detail: format!("returned a body this crate cannot model: {e}"),
            })?;
        let srt = envelope_to_srt(envelope)?;

        let source_url = format!("decopy://{video_id}/{UNDETERMINED_TAG}/srt");
        self.cache
            .lock()
            .map_err(|_| AppError::Internal("decopy cache poisoned".to_string()))?
            .insert(source_url.clone(), srt.clone().into_bytes());

        tracing::debug!(
            target: "events",
            provider = PROVIDER_NAME,
            video_id,
            body_bytes = srt.len(),
            "fetch_subtitle_completed"
        );

        Ok(SubtitleInfo {
            video_id: video_id.to_string(),
            language: UNDETERMINED_TAG.to_string(),
            // GAP-2026-158: decopy exposes no language parameter and
            // names no track; `und` is the tag it settled on, never an
            // observation of what came back.
            delivered_language: None,
            format: Format::Srt,
            source_url,
            byte_size: srt.len(),
            format_hint: SubtitleFormat::Srt,
            provider: PROVIDER_NAME,
        })
    }

    async fn fetch_content(&self, info: &SubtitleInfo) -> AppResult<Vec<u8>> {
        let bytes = self
            .cache
            .lock()
            .map_err(|_| AppError::Internal("decopy cache poisoned".to_string()))?
            .get(&info.source_url)
            .cloned();
        match bytes {
            Some(b) if !b.is_empty() => Ok(b),
            _ => Err(AppError::NoSubtitle(NoSubtitleReason::NotPublished)),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn provider_name_is_stable() {
        assert_eq!(ProviderDecopy::new().name(), PROVIDER_NAME);
    }

    #[test]
    fn generated_serial_is_32_lowercase_hex() {
        let serial = generate_product_serial();
        assert_eq!(serial.len(), decopy_serial_hex_len());
        assert!(
            serial
                .chars()
                .all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()),
            "serial must be lowercase hex: {serial}"
        );
    }

    #[test]
    fn two_serials_differ() {
        assert_ne!(generate_product_serial(), generate_product_serial());
    }

    #[test]
    fn multipart_body_frames_every_field() {
        let (body, content_type) =
            build_multipart("BOUND", &[("video_id", "dQw4w9WgXcQ"), ("k", "v")]);
        assert_eq!(content_type, "multipart/form-data; boundary=BOUND");
        assert!(body.contains(
            "--BOUND\r\nContent-Disposition: form-data; name=\"video_id\"\r\n\r\ndQw4w9WgXcQ\r\n"
        ));
        assert!(body.ends_with("--BOUND--\r\n"));
    }

    #[test]
    fn create_job_url_does_not_double_the_slash() {
        let p = ProviderDecopy::new().with_base_url("https://example.test/");
        assert_eq!(
            p.create_job_url(),
            format!("https://example.test{}", decopy_create_job_path())
        );
    }

    #[test]
    fn quota_code_is_rate_limited_never_no_subtitle() {
        let err = classify_code(CODE_QUOTA_EXHAUSTED, None);
        assert!(
            matches!(err, AppError::RateLimited { .. }),
            "quota must degrade, got {err:?}"
        );
    }

    #[test]
    fn bad_serial_is_provider_unavailable_never_no_subtitle() {
        let err = classify_code(CODE_BAD_SERIAL, Some("invalid serial"));
        assert!(
            matches!(err, AppError::ProviderUnavailable { .. }),
            "got {err:?}"
        );
    }

    #[test]
    fn missing_params_is_an_internal_defect() {
        let err = classify_code(CODE_MISSING_PARAMS, Some("video_id required"));
        assert!(matches!(err, AppError::Internal(_)), "got {err:?}");
    }

    #[test]
    fn unknown_code_degrades_rather_than_claiming_absence() {
        let err = classify_code(999_999, None);
        assert!(
            matches!(err, AppError::ProviderUnavailable { .. }),
            "got {err:?}"
        );
    }

    /// What the upstream said has to reach the envelope, not only the log.
    ///
    /// The error envelope declares a `diagnostic` field and, between the
    /// removal of the browser providers on 2026-09-04 and this call, it
    /// had NO producer left: the explanation was received, written to
    /// `stderr` and dropped on the way out. That is the GAP-2026-180
    /// class of defect, where a datum is produced and discarded in
    /// transit.
    ///
    /// The assertion is over the CHANNEL and not over the log line: a
    /// `tracing` field proves the operator can read it, and only the
    /// task-local proves an automated caller can.
    #[tokio::test]
    async fn an_unknown_code_carries_the_upstream_words_into_the_envelope() {
        use std::sync::{Arc, Mutex};

        let sink: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
        crate::provider::chain::UPSTREAM_DIAGNOSTIC
            .scope(Arc::clone(&sink), async {
                let _ = classify_code(999_999, Some("quota exhausted for this account"));
            })
            .await;

        let recorded = sink
            .lock()
            .expect("uncontended")
            .take()
            .expect("the upstream explanation must reach the diagnostic channel");
        assert_eq!(recorded, "quota exhausted for this account");
    }

    #[test]
    fn envelope_renders_subrip() {
        let raw = r#"{"code":100000,"result":{"subtitles":[
            {"start":"00:00:00","end":"00:00:02","content":"hello"},
            {"start":"00:00:02","end":"00:00:04","content":"world"}]}}"#;
        let envelope: DecopyEnvelope = serde_json::from_str(raw).expect("fixture parses");
        let srt = envelope_to_srt(envelope).expect("cues render");
        assert!(srt.starts_with("1\n00:00:00,000 --> 00:00:02,000\nhello\n\n"));
        assert!(srt.contains("2\n00:00:02,000 --> 00:00:04,000\nworld"));
    }

    #[test]
    fn empty_subtitle_list_is_not_published() {
        let raw = r#"{"code":100000,"result":{"subtitles":[]}}"#;
        let envelope: DecopyEnvelope = serde_json::from_str(raw).expect("fixture parses");
        let err = envelope_to_srt(envelope).expect_err("no cues");
        assert!(
            matches!(err, AppError::NoSubtitle(NoSubtitleReason::NotPublished)),
            "got {err:?}"
        );
    }

    #[tokio::test]
    async fn list_tracks_reports_one_undetermined_track() {
        let tracks = ProviderDecopy::new()
            .list_tracks("dQw4w9WgXcQ")
            .await
            .expect("menu");
        assert_eq!(tracks.len(), 1);
        assert_eq!(tracks[0].tag, UNDETERMINED_TAG);
        assert!(tracks[0].auto_generated);
    }

    #[tokio::test]
    async fn fetch_content_without_a_prior_fetch_is_not_published() {
        let p = ProviderDecopy::new();
        let info = SubtitleInfo {
            video_id: "dQw4w9WgXcQ".to_string(),
            language: UNDETERMINED_TAG.to_string(),
            // GAP-2026-158: decopy exposes no language parameter and
            // names no track; `und` is the tag it settled on, never an
            // observation of what came back.
            delivered_language: None,
            format: Format::Srt,
            source_url: "decopy://absent".to_string(),
            byte_size: 0,
            format_hint: SubtitleFormat::Srt,
            provider: PROVIDER_NAME,
        };
        assert!(matches!(
            p.fetch_content(&info).await,
            Err(AppError::NoSubtitle(NoSubtitleReason::NotPublished))
        ));
    }
}