use std::future::Future;
use std::time::Duration;
use anyhow::{Error, Result};
use crate::config::MAX_NETWORK_DOWNLOAD_RETRIES;
use crate::error_handling::ReqwestErrorExt;
pub(crate) fn is_retriable_error(error: &Error) -> bool {
for cause in error.chain() {
if let Some(reqwest_err) = cause.downcast_ref::<reqwest::Error>() {
return reqwest_err.is_retriable();
}
if cause.downcast_ref::<url::ParseError>().is_some() {
return false;
}
if cause.downcast_ref::<sqlx::Error>().is_some() {
return false;
}
if cause
.downcast_ref::<hickory_resolver::net::NetError>()
.is_some()
{
return true;
}
}
false
}
pub(crate) async fn with_network_download_retry<T, F, Fut>(
label: &str,
base_delay_secs: u64,
mut op: F,
) -> Result<T>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T>>,
{
let mut last_error = None;
for attempt in 1..=MAX_NETWORK_DOWNLOAD_RETRIES {
match op().await {
Ok(value) => return Ok(value),
Err(e) => {
last_error = Some(e);
if attempt < MAX_NETWORK_DOWNLOAD_RETRIES {
log::warn!(
"Failed {label} (attempt {attempt}/{MAX_NETWORK_DOWNLOAD_RETRIES}), retrying..."
);
tokio::time::sleep(Duration::from_secs(base_delay_secs << (attempt - 1))).await;
}
}
}
}
Err(last_error.unwrap_or_else(|| {
anyhow::anyhow!("Failed {label} after {MAX_NETWORK_DOWNLOAD_RETRIES} attempts")
}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_is_retriable_error_timeout_string_only() {
let err = anyhow::anyhow!("Request timeout");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_url_parse() {
let parse_err = url::ParseError::EmptyHost;
let err: anyhow::Error = parse_err.into();
let result = is_retriable_error(&err);
assert!(!result, "URL parse error should not be retriable");
}
#[test]
fn test_is_retriable_error_database() {
let db_err = sqlx::Error::PoolClosed;
let err: anyhow::Error = db_err.into();
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_404() {
let err = anyhow::anyhow!("404 not found");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_403() {
let err = anyhow::anyhow!("403 forbidden");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_401() {
let err = anyhow::anyhow!("401 unauthorized");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_dns_typed() {
let resolve_err = hickory_resolver::net::NetError::from("lookup failed");
let err: anyhow::Error = resolve_err.into();
assert!(is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_dns_string_not_retriable() {
let err = anyhow::anyhow!("DNS lookup failed");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_resolve_string_not_retriable() {
let err = anyhow::anyhow!("Failed to resolve hostname");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_unknown() {
let err = anyhow::anyhow!("Some unknown error");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_empty() {
let err = anyhow::anyhow!("");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_5xx_string_only() {
let err = anyhow::anyhow!("500 internal server error");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_429_string_only() {
let err = anyhow::anyhow!("429 too many requests");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_400() {
let err = anyhow::anyhow!("400 bad request");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_redirect() {
let err = anyhow::anyhow!("Redirect error");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_connection_failure_string() {
let err = anyhow::anyhow!("Connection failed");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_timeout_message_string() {
let err = anyhow::anyhow!("Request timed out");
assert!(!is_retriable_error(&err));
}
#[test]
fn test_is_retriable_error_mixed_chain() {
let db_err = sqlx::Error::PoolClosed;
let err: anyhow::Error = db_err.into();
let wrapped = err.context("Additional context");
assert!(!is_retriable_error(&wrapped));
}
#[test]
fn test_is_retriable_error_chain_order() {
let parse_err = url::ParseError::EmptyHost;
let err: anyhow::Error = parse_err.into();
let wrapped = err.context("Some other context");
assert!(!is_retriable_error(&wrapped));
}
#[test]
fn test_is_retriable_error_message_case_insensitive() {
let err1 = anyhow::anyhow!("404 NOT FOUND");
let err2 = anyhow::anyhow!("Not Found");
let err3 = anyhow::anyhow!("FORBIDDEN");
assert!(!is_retriable_error(&err1));
let _result2 = is_retriable_error(&err2);
let result3 = is_retriable_error(&err3);
assert!(!result3, "FORBIDDEN should be non-retriable");
}
#[test]
fn test_is_retriable_error_partial_message_match() {
let err1 = anyhow::anyhow!("Error: 404 page not found");
let err2 = anyhow::anyhow!("HTTP 500 internal server error occurred");
let err3 = anyhow::anyhow!("DNS resolution failed for domain");
assert!(!is_retriable_error(&err1)); assert!(!is_retriable_error(&err2)); assert!(!is_retriable_error(&err3)); }
#[test]
fn test_is_retriable_error_nested_context_resolve_error() {
let resolve_err = hickory_resolver::net::NetError::from("lookup failed");
let err: anyhow::Error = resolve_err.into();
let err = err.context("First context").context("Second context");
assert!(is_retriable_error(&err));
}
#[tokio::test]
async fn test_is_retriable_error_reqwest_status_429() {
use httptest::{matchers::*, responders::*, Expectation, Server};
let server = Server::run();
server.expect(
Expectation::matching(request::method_path("GET", "/429"))
.respond_with(status_code(429).body("Too Many Requests")),
);
let client = reqwest::Client::new();
let url = server.url("/429").to_string();
let response = client
.get(&url)
.send()
.await
.expect("Failed to create test request");
let reqwest_err = response.error_for_status().unwrap_err();
let error: anyhow::Error = reqwest_err.into();
assert!(is_retriable_error(&error));
}
#[tokio::test]
async fn test_is_retriable_error_reqwest_status_5xx() {
use httptest::{matchers::*, responders::*, Expectation, Server};
let server = Server::run();
server.expect(
Expectation::matching(request::method_path("GET", "/500"))
.respond_with(status_code(500).body("Internal Server Error")),
);
let client = reqwest::Client::new();
let url = server.url("/500").to_string();
let response = client
.get(&url)
.send()
.await
.expect("Failed to create test request");
let reqwest_err = response.error_for_status().unwrap_err();
let error: anyhow::Error = reqwest_err.into();
assert!(is_retriable_error(&error));
}
#[tokio::test]
async fn test_is_retriable_error_reqwest_status_4xx_not_429() {
use httptest::{matchers::*, responders::*, Expectation, Server};
let server = Server::run();
server.expect(
Expectation::matching(request::method_path("GET", "/400"))
.respond_with(status_code(400).body("Bad Request")),
);
let client = reqwest::Client::new();
let url = server.url("/400").to_string();
let response = client
.get(&url)
.send()
.await
.expect("Failed to create test request");
let reqwest_err = response.error_for_status().unwrap_err();
let error: anyhow::Error = reqwest_err.into();
assert!(!is_retriable_error(&error));
}
#[tokio::test]
async fn test_is_retriable_error_reqwest_timeout() {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_millis(1))
.build()
.expect("Failed to create client");
let error = client.get("http://127.0.0.1:1/").send().await.unwrap_err();
let error: anyhow::Error = error.into();
assert!(is_retriable_error(&error));
}
#[test]
fn test_is_retriable_error_message_no_string_matching() {
let dns_error = anyhow::anyhow!("DNS resolution failed");
let resolve_error = anyhow::anyhow!("Failed to resolve hostname");
let lookup_error = anyhow::anyhow!("DNS lookup failed");
assert!(!is_retriable_error(&dns_error));
assert!(!is_retriable_error(&resolve_error));
assert!(!is_retriable_error(&lookup_error));
}
#[test]
fn test_is_retriable_error_unknown_defaults_to_not_retriable() {
let unknown_error = anyhow::anyhow!("Some completely unknown error type");
assert!(!is_retriable_error(&unknown_error));
}
}