mod ledger;
mod watch_probe;
pub(crate) use ledger::record_upstream_diagnostic;
pub(crate) use ledger::UPSTREAM_DIAGNOSTIC;
pub use ledger::{AttemptOutcome, ProviderAttempt};
pub use watch_probe::{watch_probe_timeout, DEFAULT_WATCH_PROBE_TIMEOUT_SECS};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::Semaphore;
use self::ledger::{attempt_outcome, publish, ATTEMPT_LEDGER};
const WATCH_PAGE_SOURCE: &str = "watch-page";
use self::watch_probe::{
chain_never_reached_a_track, classify_watch_page, published_languages, WatchProbe,
};
use super::{health, is_offline, Format, Provider, SubtitleInfo};
use crate::error::{AppError, AppResult};
#[allow(dead_code)]
pub(crate) fn http_failure(
status: reqwest::StatusCode,
headers: &reqwest::header::HeaderMap,
provider: &'static str,
) -> AppError {
if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
let retry_after_secs = headers
.get(reqwest::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())
.and_then(parse_retry_after);
return AppError::RateLimited {
provider,
retry_after_secs,
};
}
AppError::ProviderUnavailable { provider }
}
#[allow(dead_code)]
fn parse_retry_after(raw: &str) -> Option<u64> {
let s = raw.trim();
if let Ok(secs) = s.parse::<u64>() {
return Some(secs);
}
let dt = chrono::DateTime::parse_from_rfc2822(s).ok()?;
let delta = (dt.with_timezone(&chrono::Utc) - chrono::Utc::now()).num_seconds();
Some(delta.max(0) as u64)
}
fn remember_failure(last_err: &mut Option<AppError>, e: AppError) {
let downgrade = matches!(
last_err,
Some(
AppError::RateLimited { .. }
| AppError::BrowserNotFound(_)
| AppError::CaptchaChallenge { .. }
)
) && !matches!(
e,
AppError::RateLimited { .. }
| AppError::BrowserNotFound(_)
| AppError::CaptchaChallenge { .. }
);
if !downgrade {
*last_err = Some(e);
}
}
fn chain_retryable(err: &AppError) -> bool {
match err {
AppError::RateLimited {
retry_after_secs, ..
} => retry_after_secs.is_some(),
other => other.retryable(),
}
}
async fn fetch_subtitle_with_retry(
provider: &dyn Provider,
video_id: &str,
language: &str,
format: Format,
) -> AppResult<SubtitleInfo> {
let nested = crate::retry::retry_with_backoff(
|| async {
match provider.fetch_subtitle(video_id, language, format).await {
Ok(info) => Ok(Ok(info)),
Err(e) if !chain_retryable(&e) => Ok(Err(e)),
Err(e) => Err(e),
}
},
crate::retry::max_attempts(),
)
.await;
match nested {
Ok(inner) => inner,
Err(exhausted) => Err(exhausted),
}
}
#[derive(Debug)]
pub enum ProviderOutcome {
Subtitle(SubtitleInfo, Vec<u8>),
ChainError {
source: &'static str,
error: AppError,
degraded: bool,
},
}
impl ProviderOutcome {
pub fn from_http_status(
source: &'static str,
status: u16,
retry_after_secs: Option<u64>,
) -> Self {
let degraded = matches!(status, 500..=599) || status == 429;
let error = if let Some(reason) = crate::error::NoSubtitleReason::from_status(status) {
AppError::NoSubtitle(reason)
} else if status == 429 {
AppError::RateLimited {
provider: source,
retry_after_secs,
}
} else {
AppError::ProviderUnavailable { provider: source }
};
ProviderOutcome::ChainError {
source,
error,
degraded,
}
}
pub fn chain_error(source: &'static str, error: AppError) -> Self {
ProviderOutcome::ChainError {
source,
error,
degraded: false,
}
}
}
pub struct ProviderChain {
providers: Vec<Box<dyn Provider>>,
min_interval: Duration,
last_call: Mutex<Option<Instant>>,
host_limits: Mutex<HashMap<&'static str, Arc<Semaphore>>>,
per_host: usize,
watch_probe: Option<WatchProbe>,
}
#[must_use]
pub fn per_host_concurrency() -> usize {
crate::config::tuning_usize_in_range(
"net.per_host_concurrency",
DEFAULT_PER_HOST_CONCURRENCY,
1,
64,
)
}
pub const DEFAULT_PER_HOST_CONCURRENCY: usize = 2;
pub const DEFAULT_THROTTLE_INTERVAL_MS: u64 = 1_000;
#[must_use]
pub fn throttle_interval() -> Duration {
Duration::from_millis(crate::config::tuning_u64_in_range(
"net.throttle_interval_ms",
DEFAULT_THROTTLE_INTERVAL_MS,
0,
3_600_000,
))
}
impl ProviderChain {
#[tracing::instrument(level = "debug", skip_all, fields(providers = providers.len()))]
pub fn new(providers: Vec<Box<dyn Provider>>) -> Self {
let mut chain = Self::with_min_interval(providers, throttle_interval());
chain.watch_probe = Some(WatchProbe::live());
chain
}
#[cfg(test)]
#[must_use]
pub(super) fn with_watch_probe_base(mut self, base: impl Into<String>) -> Self {
self.watch_probe = Some(WatchProbe {
base: base.into(),
timeout: watch_probe_timeout(),
});
self
}
async fn refine_with_watch_probe(
&self,
video_id: &str,
language: &str,
original: AppError,
attempts: &mut Vec<ProviderAttempt>,
) -> AppError {
let Some(probe) = self.watch_probe.as_ref() else {
return original;
};
if is_offline() {
return original;
}
let started = std::time::Instant::now();
match probe.watch_page(video_id).await {
Ok(html) => match classify_watch_page(&html, language) {
Some(AppError::CaptionsAsrOnly { .. })
if chain_never_reached_a_track(&original) =>
{
original
}
Some(refined) => {
attempts.push(ProviderAttempt {
provider: WATCH_PAGE_SOURCE,
outcome: attempt_outcome(&refined),
elapsed_ms: u64::try_from(started.elapsed().as_millis()).ok(),
http_status: None,
body_len: None,
diagnostic: None,
});
refined
}
None => original,
},
Err(e) => {
tracing::debug!(
target: "events",
event = "watch_probe_inconclusive",
error = %e,
"watch-page probe failed; keeping the chain error"
);
original
}
}
}
async fn verify_delivered_language(
&self,
video_id: &str,
language: &str,
mut info: crate::provider::SubtitleInfo,
attempts: &mut Vec<ProviderAttempt>,
) -> Result<crate::provider::SubtitleInfo, AppError> {
if info.delivered_language.is_some() {
return Ok(info);
}
if !crate::config::tuning_bool_or("net.verify_delivered_language", true) {
return Ok(info);
}
let Some(probe) = self.watch_probe.as_ref() else {
return Ok(info);
};
if is_offline() {
return Ok(info);
}
let started = std::time::Instant::now();
let Ok(html) = probe.watch_page(video_id).await else {
return Ok(info);
};
if let Some(err @ AppError::LanguageUnavailable { .. }) =
classify_watch_page(&html, language)
{
attempts.push(ProviderAttempt {
provider: WATCH_PAGE_SOURCE,
outcome: attempt_outcome(&err),
elapsed_ms: u64::try_from(started.elapsed().as_millis()).ok(),
http_status: None,
body_len: None,
diagnostic: None,
});
return Err(err);
}
if let Some([only]) = published_languages(&html).as_deref() {
info.delivered_language = Some(only.clone());
}
Ok(info)
}
fn host_permit(&self, name: &'static str) -> Arc<Semaphore> {
let mut guard = self
.host_limits
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Arc::clone(
guard
.entry(name)
.or_insert_with(|| Arc::new(Semaphore::new(self.per_host))),
)
}
#[tracing::instrument(level = "debug", skip_all, fields(min_interval_ms = %min_interval.as_millis()))]
pub fn with_min_interval(providers: Vec<Box<dyn Provider>>, min_interval: Duration) -> Self {
Self {
providers,
min_interval,
last_call: Mutex::new(None),
host_limits: Mutex::new(HashMap::new()),
per_host: per_host_concurrency(),
watch_probe: None,
}
}
#[tracing::instrument(level = "debug", skip(self))]
pub async fn throttle(&self) {
let now = Instant::now();
let wait = {
let guard = self
.last_call
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
guard
.map(|t| {
let elapsed = now.duration_since(t);
if elapsed < self.min_interval {
Some(self.min_interval - elapsed)
} else {
None
}
})
.unwrap_or(None)
};
if let Some(d) = wait {
tokio::time::sleep(d).await;
}
*self
.last_call
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Instant::now());
}
#[tracing::instrument(level = "debug", err, skip(self), fields(video_id, language, format = ?format))]
pub async fn fetch_subtitle(
&self,
video_id: &str,
language: &str,
format: Format,
) -> AppResult<(SubtitleInfo, Vec<u8>)> {
self.fetch_subtitle_traced(video_id, language, format)
.await
.0
}
pub(crate) async fn fetch_subtitle_traced_into(
&self,
video_id: &str,
language: &str,
format: Format,
sink: &Arc<Mutex<Vec<ProviderAttempt>>>,
) -> (AppResult<(SubtitleInfo, Vec<u8>)>, Vec<ProviderAttempt>) {
ATTEMPT_LEDGER
.scope(
Arc::clone(sink),
self.fetch_subtitle_traced(video_id, language, format),
)
.await
}
#[tracing::instrument(level = "debug", skip(self), fields(video_id, language, format = ?format))]
pub async fn fetch_subtitle_traced(
&self,
video_id: &str,
language: &str,
format: Format,
) -> (AppResult<(SubtitleInfo, Vec<u8>)>, Vec<ProviderAttempt>) {
let mut attempts: Vec<ProviderAttempt> = Vec::new();
let mut last_err: Option<AppError> = None;
let mut saw_genuine_no_subtitle = false;
let mut degraded_providers: std::collections::BTreeSet<&'static str> =
std::collections::BTreeSet::new();
let total = self.providers.len();
for (idx, provider) in self.providers.iter().enumerate() {
if degraded_providers.contains(provider.name()) {
tracing::debug!(
target: "events",
provider = provider.name(),
event = "chain_skipping_degraded_provider",
"provider already degraded in this run; not calling it again"
);
publish(
&mut attempts,
ProviderAttempt {
provider: provider.name(),
outcome: AttemptOutcome::SkippedDegraded,
elapsed_ms: None,
http_status: None,
body_len: None,
diagnostic: None,
},
);
continue;
}
tracing::debug!(
target: "events",
chain_index = idx,
chain_total = total,
provider = provider.name(),
event = "chain_attempting_provider",
"chain attempting provider"
);
let _permit = self.host_permit(provider.name()).acquire_owned().await;
self.throttle().await;
let started = Instant::now();
let sink: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
let outcome = UPSTREAM_DIAGNOSTIC
.scope(Arc::clone(&sink), async {
let attempt =
fetch_subtitle_with_retry(provider.as_ref(), video_id, language, format).await;
match attempt {
Ok(info) => match provider.fetch_content(&info).await {
Ok(content) if !content.is_empty() => ProviderOutcome::Subtitle(info, content),
Ok(_) => ProviderOutcome::ChainError {
source: provider.name(),
error: AppError::NoSubtitle(crate::error::NoSubtitleReason::NotPublished),
degraded: false,
},
Err(e) => ProviderOutcome::ChainError {
source: provider.name(),
error: e,
degraded: false,
},
},
Err(AppError::NoSubtitle(reason)) => {
tracing::warn!(target: "events", provider = provider.name(), reason = %reason, "provider returned no subtitle");
ProviderOutcome::ChainError {
source: provider.name(),
error: AppError::NoSubtitle(reason),
degraded: false,
}
}
Err(
e @ (AppError::ProviderUnavailable { .. }
| AppError::RateLimited { .. }
| AppError::CaptchaChallenge { .. }
| AppError::BrowserNotFound(_)),
) => {
ProviderOutcome::ChainError {
source: provider.name(),
error: e,
degraded: true,
}
}
Err(e) => ProviderOutcome::ChainError {
source: provider.name(),
error: e,
degraded: false,
},
}
})
.await;
let elapsed_ms = started.elapsed().as_millis() as u64;
let diagnostic = sink
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
match outcome {
ProviderOutcome::Subtitle(info, content) => {
publish(
&mut attempts,
ProviderAttempt {
provider: provider.name(),
outcome: AttemptOutcome::Delivered,
elapsed_ms: Some(elapsed_ms),
http_status: None,
body_len: Some(content.len()),
diagnostic,
},
);
health::record_success(provider.name());
return match self
.verify_delivered_language(video_id, language, info, &mut attempts)
.await
{
Ok(info) => (Ok((info, content)), attempts),
Err(err) => (Err(err), attempts),
};
}
ProviderOutcome::ChainError {
source,
error,
degraded,
} => {
publish(
&mut attempts,
ProviderAttempt {
provider: source,
outcome: attempt_outcome(&error),
elapsed_ms: Some(elapsed_ms),
http_status: match &error {
AppError::Http(e) => e.status().map(|s| s.as_u16()),
_ => None,
},
body_len: None,
diagnostic,
},
);
if degraded {
tracing::warn!(
target: "events",
provider = source,
degraded = true,
error = %error,
"provider_failed_degraded_skipping"
);
degraded_providers.insert(source);
health::record_failure(source);
remember_failure(&mut last_err, error);
continue;
}
if matches!(error, AppError::NoSubtitle(_)) {
saw_genuine_no_subtitle = true;
}
remember_failure(&mut last_err, error);
}
}
}
let chain_error = match last_err {
Some(err @ AppError::BrowserNotFound(_))
| Some(err @ AppError::CaptchaChallenge { .. })
| Some(err @ AppError::RateLimited { .. }) => err,
_ if saw_genuine_no_subtitle => {
AppError::NoSubtitle(crate::error::NoSubtitleReason::NotPublished)
}
other => other.unwrap_or(AppError::ProviderUnavailable { provider: "auto" }),
};
let refined = self
.refine_with_watch_probe(video_id, language, chain_error, &mut attempts)
.await;
(Err(refined), attempts)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::provider::SubtitleFormat;
use async_trait::async_trait;
#[test]
fn http_failure_maps_429_with_retry_after() {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(reqwest::header::RETRY_AFTER, "7".parse().expect("ascii"));
let err = http_failure(
reqwest::StatusCode::TOO_MANY_REQUESTS,
&headers,
"provider-test",
);
assert!(matches!(
err,
AppError::RateLimited {
retry_after_secs: Some(7),
..
}
));
}
#[test]
fn http_failure_maps_429_without_header() {
let headers = reqwest::header::HeaderMap::new();
let err = http_failure(
reqwest::StatusCode::TOO_MANY_REQUESTS,
&headers,
"provider-test",
);
assert!(matches!(
err,
AppError::RateLimited {
retry_after_secs: None,
..
}
));
}
#[test]
fn http_failure_parses_http_date_retry_after() {
let future = (chrono::Utc::now() + chrono::Duration::seconds(120)).to_rfc2822();
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(reqwest::header::RETRY_AFTER, future.parse().expect("ascii"));
let err = http_failure(
reqwest::StatusCode::TOO_MANY_REQUESTS,
&headers,
"provider-test",
);
match err {
AppError::RateLimited {
retry_after_secs: Some(n),
..
} => assert!((115..=120).contains(&n), "delta out of range: {n}"),
other => panic!("expected RateLimited with seconds, got {other:?}"),
}
}
#[test]
fn http_failure_http_date_in_past_clamps_to_zero() {
let past = (chrono::Utc::now() - chrono::Duration::seconds(3600)).to_rfc2822();
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(reqwest::header::RETRY_AFTER, past.parse().expect("ascii"));
let err = http_failure(
reqwest::StatusCode::TOO_MANY_REQUESTS,
&headers,
"provider-test",
);
assert!(matches!(
err,
AppError::RateLimited {
retry_after_secs: Some(0),
..
}
));
}
#[test]
fn http_failure_garbage_retry_after_is_none() {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::RETRY_AFTER,
"not-a-date-or-number".parse().expect("ascii"),
);
let err = http_failure(
reqwest::StatusCode::TOO_MANY_REQUESTS,
&headers,
"provider-test",
);
assert!(matches!(
err,
AppError::RateLimited {
retry_after_secs: None,
..
}
));
}
#[test]
fn http_failure_maps_other_status_to_unavailable() {
let headers = reqwest::header::HeaderMap::new();
let err = http_failure(
reqwest::StatusCode::SERVICE_UNAVAILABLE,
&headers,
"provider-test",
);
assert!(matches!(err, AppError::ProviderUnavailable { .. }));
}
#[test]
fn rate_limited_survives_later_transient_failure() {
let mut last = Some(AppError::RateLimited {
provider: "provider-noiz",
retry_after_secs: Some(5),
});
remember_failure(
&mut last,
AppError::ProviderUnavailable {
provider: "provider-noiz",
},
);
assert!(matches!(
last,
Some(AppError::RateLimited {
retry_after_secs: Some(5),
..
})
));
remember_failure(
&mut last,
AppError::RateLimited {
provider: "provider-noiz",
retry_after_secs: None,
},
);
assert!(matches!(
last,
Some(AppError::RateLimited {
retry_after_secs: None,
..
})
));
}
#[test]
fn remember_failure_records_first_error() {
let mut last = None;
remember_failure(
&mut last,
AppError::ProviderUnavailable {
provider: "provider-noiz",
},
);
assert!(matches!(last, Some(AppError::ProviderUnavailable { .. })));
}
#[tokio::test]
async fn wiremock_429_delta_seconds_reaches_http_failure() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(429).insert_header("Retry-After", "2"))
.mount(&server)
.await;
let resp = reqwest::Client::new()
.get(server.uri())
.send()
.await
.expect("mock request");
let err = http_failure(resp.status(), resp.headers(), "provider-test");
assert!(matches!(
err,
AppError::RateLimited {
retry_after_secs: Some(2),
..
}
));
}
#[tokio::test]
async fn wiremock_429_http_date_reaches_http_failure() {
let future = (chrono::Utc::now() + chrono::Duration::seconds(90)).to_rfc2822();
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(
wiremock::ResponseTemplate::new(429).insert_header("Retry-After", future.as_str()),
)
.mount(&server)
.await;
let resp = reqwest::Client::new()
.get(server.uri())
.send()
.await
.expect("mock request");
let err = http_failure(resp.status(), resp.headers(), "provider-test");
match err {
AppError::RateLimited {
retry_after_secs: Some(n),
..
} => assert!((85..=90).contains(&n), "delta out of range: {n}"),
other => panic!("expected RateLimited with seconds, got {other:?}"),
}
}
#[tokio::test]
async fn chain_treats_429_and_503_as_degraded_skips_both() {
struct MockStatusProvider {
url: String,
}
#[async_trait]
impl Provider for MockStatusProvider {
fn name(&self) -> &'static str {
"mock-status"
}
async fn fetch_subtitle(
&self,
_video_id: &str,
_language: &str,
_format: Format,
) -> AppResult<SubtitleInfo> {
let resp = reqwest::Client::new()
.get(&self.url)
.send()
.await
.map_err(AppError::Http)?;
Err(http_failure(resp.status(), resp.headers(), "provider-test"))
}
async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
Err(AppError::ProviderUnavailable {
provider: self.name(),
})
}
}
let rate_limited = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(429).insert_header("Retry-After", "3"))
.mount(&rate_limited)
.await;
let unavailable = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(503))
.mount(&unavailable)
.await;
let chain = ProviderChain::with_min_interval(
vec![
Box::new(MockStatusProvider {
url: rate_limited.uri(),
}),
Box::new(MockStatusProvider {
url: unavailable.uri(),
}),
],
Duration::from_millis(1),
);
let err = chain
.fetch_subtitle("dQw4w9WgXcQ", "en", Format::Srt)
.await
.expect_err("both providers fail");
match err {
AppError::RateLimited {
retry_after_secs: Some(n),
..
} => assert!(
(2..=4).contains(&n),
"retry_after out of expected window: {n}"
),
other => panic!("expected RateLimited with retry_after, got {other:?}"),
}
}
#[tokio::test]
async fn chain_records_genuine_no_subtitle_after_degraded_provider() {
struct MockStatusProvider {
url: String,
name: &'static str,
}
#[async_trait]
impl Provider for MockStatusProvider {
fn name(&self) -> &'static str {
self.name
}
async fn fetch_subtitle(
&self,
_video_id: &str,
_language: &str,
_format: Format,
) -> AppResult<SubtitleInfo> {
let resp = reqwest::Client::new()
.get(&self.url)
.send()
.await
.map_err(AppError::Http)?;
let status = resp.status().as_u16();
if let Some(reason) = crate::error::NoSubtitleReason::from_status(status) {
return Err(AppError::NoSubtitle(reason));
}
Err(http_failure(resp.status(), resp.headers(), "provider-test"))
}
async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
Err(AppError::ProviderUnavailable {
provider: self.name(),
})
}
}
let unavailable = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(503))
.mount(&unavailable)
.await;
let not_found = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(404))
.mount(¬_found)
.await;
let chain = ProviderChain::with_min_interval(
vec![
Box::new(MockStatusProvider {
url: unavailable.uri(),
name: "mock-503",
}),
Box::new(MockStatusProvider {
url: not_found.uri(),
name: "mock-404",
}),
],
Duration::from_millis(1),
);
let err = chain
.fetch_subtitle("dQw4w9WgXcQ", "en", Format::Srt)
.await
.expect_err("both providers fail");
assert!(
matches!(
err,
AppError::NoSubtitle(crate::error::NoSubtitleReason::NotPublished)
),
"expected NoSubtitle(NotPublished) (consolidated) after degraded 503, got {err:?}"
);
}
#[tokio::test]
async fn chain_records_genuine_no_subtitle_after_two_degraded_providers() {
struct MockStatusProvider {
url: String,
name: &'static str,
}
#[async_trait]
impl Provider for MockStatusProvider {
fn name(&self) -> &'static str {
self.name
}
async fn fetch_subtitle(
&self,
_video_id: &str,
_language: &str,
_format: Format,
) -> AppResult<SubtitleInfo> {
let resp = reqwest::Client::new()
.get(&self.url)
.send()
.await
.map_err(AppError::Http)?;
let status = resp.status().as_u16();
if let Some(reason) = crate::error::NoSubtitleReason::from_status(status) {
return Err(AppError::NoSubtitle(reason));
}
Err(http_failure(resp.status(), resp.headers(), "provider-test"))
}
async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
Err(AppError::ProviderUnavailable {
provider: self.name(),
})
}
}
let s503 = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(503))
.mount(&s503)
.await;
let s429 = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(429).insert_header("Retry-After", "9"))
.mount(&s429)
.await;
let s404 = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(404))
.mount(&s404)
.await;
let chain = ProviderChain::with_min_interval(
vec![
Box::new(MockStatusProvider {
url: s503.uri(),
name: "mock-503",
}),
Box::new(MockStatusProvider {
url: s429.uri(),
name: "mock-429",
}),
Box::new(MockStatusProvider {
url: s404.uri(),
name: "mock-404",
}),
],
Duration::from_millis(1),
);
let err = chain
.fetch_subtitle("dQw4w9WgXcQ", "en", Format::Srt)
.await
.expect_err("all providers fail");
match err {
AppError::RateLimited {
retry_after_secs: Some(n),
..
} => assert!(
(8..=10).contains(&n),
"retry_after out of expected window: {n}"
),
other => panic!(
"expected RateLimited from the second provider (EC-021 wins over later NoSubtitle), got {other:?}"
),
}
}
#[tokio::test]
async fn a_healthy_provider_is_called_exactly_once_after_a_degraded_one() {
struct MockHttpProvider {
url: String,
name: &'static str,
}
#[async_trait]
impl Provider for MockHttpProvider {
fn name(&self) -> &'static str {
self.name
}
async fn fetch_subtitle(
&self,
video_id: &str,
language: &str,
format: Format,
) -> AppResult<SubtitleInfo> {
let resp = reqwest::Client::new()
.get(&self.url)
.send()
.await
.map_err(AppError::Http)?;
if !resp.status().is_success() {
return Err(http_failure(resp.status(), resp.headers(), "provider-test"));
}
Ok(SubtitleInfo {
video_id: video_id.to_string(),
language: language.to_string(),
delivered_language: None,
format,
source_url: self.url.clone(),
byte_size: 0,
format_hint: SubtitleFormat::Srt,
provider: self.name,
})
}
async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
Ok(b"1\n00:00:00,000 --> 00:00:01,000\nhello\n".to_vec())
}
}
let degraded = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(503))
.mount(°raded)
.await;
let healthy = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_string("ok"))
.mount(&healthy)
.await;
let chain = ProviderChain::with_min_interval(
vec![
Box::new(MockHttpProvider {
url: degraded.uri(),
name: "mock-degraded",
}),
Box::new(MockHttpProvider {
url: healthy.uri(),
name: "mock-healthy",
}),
],
Duration::from_millis(1),
);
let (info, body) = chain
.fetch_subtitle("dQw4w9WgXcQ", "en", Format::Srt)
.await
.expect("the second provider answers");
assert_eq!(info.provider, "mock-healthy");
assert!(!body.is_empty());
let healthy_requests = healthy
.received_requests()
.await
.expect("the mock records its requests");
assert_eq!(
healthy_requests.len(),
1,
"the healthy provider must be asked exactly once; \
a chain-level retry would have re-entered it"
);
let degraded_requests = degraded
.received_requests()
.await
.expect("the mock records its requests");
assert_eq!(
degraded_requests.len(),
usize::from(crate::retry::max_attempts()),
"the retry budget belongs to the failing provider"
);
}
#[tokio::test]
async fn a_429_without_retry_after_is_not_retried() {
struct MockRateLimited {
url: String,
}
#[async_trait]
impl Provider for MockRateLimited {
fn name(&self) -> &'static str {
"mock-429"
}
async fn fetch_subtitle(
&self,
_video_id: &str,
_language: &str,
_format: Format,
) -> AppResult<SubtitleInfo> {
let resp = reqwest::Client::new()
.get(&self.url)
.send()
.await
.map_err(AppError::Http)?;
Err(http_failure(resp.status(), resp.headers(), "provider-test"))
}
async fn fetch_content(&self, _info: &SubtitleInfo) -> AppResult<Vec<u8>> {
Err(AppError::ProviderUnavailable {
provider: self.name(),
})
}
}
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(429))
.mount(&server)
.await;
let chain = ProviderChain::with_min_interval(
vec![Box::new(MockRateLimited { url: server.uri() })],
Duration::from_millis(1),
);
let err = chain
.fetch_subtitle("dQw4w9WgXcQ", "en", Format::Srt)
.await
.expect_err("a spent quota is a failure");
assert!(
matches!(
err,
AppError::RateLimited {
retry_after_secs: None,
..
}
),
"got {err:?}"
);
let requests = server
.received_requests()
.await
.expect("the mock records its requests");
assert_eq!(
requests.len(),
1,
"a headerless 429 must be definitive, not slept on"
);
}
#[test]
fn chain_retryability_subtracts_the_headerless_rate_limit() {
assert!(!chain_retryable(&AppError::RateLimited {
provider: "provider-noiz",
retry_after_secs: None
}));
assert!(chain_retryable(&AppError::RateLimited {
provider: "provider-noiz",
retry_after_secs: Some(5)
}));
assert!(chain_retryable(&AppError::ProviderUnavailable {
provider: "provider-noiz"
}));
assert!(!chain_retryable(&AppError::NoSubtitle(
crate::error::NoSubtitleReason::NotPublished
)));
}
#[tokio::test]
async fn chain_throttles_to_one_per_second() {
let chain = ProviderChain::new(vec![]);
let start = std::time::Instant::now();
for _ in 0..3 {
chain.throttle().await;
}
let elapsed = start.elapsed();
assert!(elapsed >= std::time::Duration::from_millis(1900));
}
#[test]
fn provider_outcome_503_is_degraded_and_unavailable() {
let outcome = ProviderOutcome::from_http_status("provider-a", 503, None);
match outcome {
ProviderOutcome::ChainError {
source,
error,
degraded,
} => {
assert_eq!(source, "provider-a");
assert!(degraded);
assert!(matches!(error, AppError::ProviderUnavailable { .. }));
}
other => panic!("expected ChainError, got {other:?}"),
}
}
#[test]
fn provider_outcome_500_is_degraded_and_unavailable() {
let outcome = ProviderOutcome::from_http_status("provider-a", 500, None);
match outcome {
ProviderOutcome::ChainError {
degraded, error, ..
} => {
assert!(degraded);
assert!(matches!(error, AppError::ProviderUnavailable { .. }));
}
other => panic!("expected ChainError, got {other:?}"),
}
}
#[test]
fn provider_outcome_429_is_degraded_and_rate_limited() {
let outcome = ProviderOutcome::from_http_status("provider-a", 429, Some(120));
match outcome {
ProviderOutcome::ChainError {
degraded, error, ..
} => {
assert!(degraded);
assert!(matches!(
error,
AppError::RateLimited {
retry_after_secs: Some(120),
..
}
));
}
other => panic!("expected ChainError, got {other:?}"),
}
}
#[test]
fn provider_outcome_404_is_genuine_no_subtitle_not_degraded() {
let outcome = ProviderOutcome::from_http_status("provider-a", 404, None);
match outcome {
ProviderOutcome::ChainError {
degraded, error, ..
} => {
assert!(!degraded);
assert!(matches!(
error,
AppError::NoSubtitle(crate::error::NoSubtitleReason::NotFound)
));
}
other => panic!("expected ChainError, got {other:?}"),
}
}
#[test]
fn provider_outcome_400_is_genuine_no_subtitle_not_degraded() {
let outcome = ProviderOutcome::from_http_status("provider-a", 400, None);
match outcome {
ProviderOutcome::ChainError {
degraded, error, ..
} => {
assert!(!degraded);
assert!(matches!(
error,
AppError::NoSubtitle(crate::error::NoSubtitleReason::NotPublished)
));
}
other => panic!("expected ChainError, got {other:?}"),
}
}
#[test]
fn provider_outcome_chain_error_defaults_to_not_degraded() {
let outcome =
ProviderOutcome::chain_error("provider-x", AppError::Internal("synthetic".to_string()));
match outcome {
ProviderOutcome::ChainError {
degraded,
source,
error,
} => {
assert!(!degraded);
assert_eq!(source, "provider-x");
assert!(matches!(error, AppError::Internal(_)));
}
other => panic!("expected ChainError, got {other:?}"),
}
}
}