use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use bytes::Bytes;
use serde::de::DeserializeOwned;
use crate::error::{is_retryable_status, ApiErrorBody, NganoError, Result};
pub(crate) const DEFAULT_MAX_RETRIES: u32 = 4;
pub(crate) const DEFAULT_BACKOFF: Duration = Duration::from_millis(500);
const MAX_BACKOFF: Duration = Duration::from_secs(30);
static SEED: AtomicU64 = AtomicU64::new(0);
fn unit_random() -> f64 {
let mut s = SEED.load(Ordering::Relaxed);
if s == 0 {
s = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0x9E37_79B9_7F4A_7C15)
| 1;
}
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
SEED.store(s, Ordering::Relaxed);
(s >> 11) as f64 / (1u64 << 53) as f64
}
pub(crate) fn backoff_delay(base: Duration, attempt: u32) -> Duration {
let scaled = base.saturating_mul(1u32 << attempt.min(16));
let capped = scaled.min(MAX_BACKOFF);
capped.mul_f64(unit_random())
}
#[derive(Debug, Clone)]
pub(crate) struct HttpClient {
client: reqwest::Client,
token: Option<String>,
source_name: &'static str,
max_retries: u32,
backoff: Duration,
}
impl HttpClient {
pub(crate) fn new(source_name: &'static str) -> Result<Self> {
let client = reqwest::Client::builder()
.user_agent(concat!("ngano-rs/", env!("CARGO_PKG_VERSION")))
.build()?;
Ok(Self {
client,
token: None,
source_name,
max_retries: DEFAULT_MAX_RETRIES,
backoff: DEFAULT_BACKOFF,
})
}
pub(crate) fn with_token(mut self, token: Option<String>) -> Self {
self.token = token.filter(|t| !t.trim().is_empty());
self
}
pub(crate) fn with_retries(mut self, max_retries: u32, backoff: Duration) -> Self {
self.max_retries = max_retries;
self.backoff = backoff;
self
}
pub(crate) async fn get_json<T: DeserializeOwned>(
&self,
url: &str,
repo: Option<&str>,
) -> Result<T> {
let body = self.get_bytes_inner(url, repo).await?;
serde_json::from_slice(&body).map_err(|e| NganoError::InvalidResponse {
url: url.to_string(),
detail: e.to_string(),
})
}
pub(crate) async fn get_bytes(&self, url: &str, repo: Option<&str>) -> Result<Bytes> {
self.get_bytes_inner(url, repo).await
}
async fn get_bytes_inner(&self, url: &str, repo: Option<&str>) -> Result<Bytes> {
let mut last_status = None;
for attempt in 0..=self.max_retries {
let mut req = self.client.get(url);
if let Some(token) = &self.token {
req = req.bearer_auth(token);
}
match req.send().await {
Ok(resp) => {
let status = resp.status().as_u16();
if resp.status().is_success() {
return Ok(resp.bytes().await?);
}
let retry_after = parse_retry_after(&resp);
let text = resp.text().await.unwrap_or_default();
if is_retryable_status(status) && attempt < self.max_retries {
last_status = Some(status);
let wait = retry_after
.unwrap_or_else(|| backoff_delay(self.backoff, attempt))
.min(MAX_BACKOFF);
tokio::time::sleep(wait).await;
continue;
}
return Err(self.status_error(url, status, &text, repo));
}
Err(e) => {
if attempt < self.max_retries && !e.is_builder() && !e.is_decode() {
tokio::time::sleep(backoff_delay(self.backoff, attempt)).await;
continue;
}
return Err(NganoError::Http(e));
}
}
}
Err(NganoError::RetryLimit {
url: url.to_string(),
attempts: self.max_retries + 1,
status: last_status,
})
}
fn status_error(&self, url: &str, status: u16, body: &str, repo: Option<&str>) -> NganoError {
if matches!(status, 401 | 403) {
if let Some(repo) = repo {
return NganoError::Gated {
repo: repo.to_string(),
url: format!("https://huggingface.co/datasets/{repo}"),
};
}
}
let message = extract_message(body).unwrap_or_else(|| {
let trimmed = body.trim();
if trimmed.is_empty() {
url.to_string()
} else {
trimmed.chars().take(400).collect()
}
});
if status == 404 {
return NganoError::NotFound(format!("{url}: {message}"));
}
if is_retryable_status(status) {
return NganoError::RetryLimit {
url: url.to_string(),
attempts: self.max_retries + 1,
status: Some(status),
};
}
NganoError::Api {
source_name: self.source_name.to_string(),
status,
message,
}
}
}
fn extract_message(body: &str) -> Option<String> {
if let Ok(parsed) = serde_json::from_str::<ApiErrorBody>(body) {
return Some(parsed.error.to_string());
}
let value: serde_json::Value = serde_json::from_str(body).ok()?;
value
.get("error")
.and_then(|e| e.as_str())
.map(str::to_string)
}
fn parse_retry_after(resp: &reqwest::Response) -> Option<Duration> {
let raw = resp.headers().get(reqwest::header::RETRY_AFTER)?;
let secs: u64 = raw.to_str().ok()?.trim().parse().ok()?;
Some(Duration::from_secs(secs))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn backoff_grows_and_stays_within_bounds() {
let base = Duration::from_millis(100);
for attempt in 0..8 {
let ceiling = base.saturating_mul(1 << attempt).min(MAX_BACKOFF);
for _ in 0..64 {
let d = backoff_delay(base, attempt);
assert!(d <= ceiling, "{d:?} exceeded {ceiling:?}");
}
}
}
#[test]
fn backoff_is_jittered() {
let base = Duration::from_millis(1000);
let draws: Vec<Duration> = (0..32).map(|_| backoff_delay(base, 3)).collect();
assert!(
draws.windows(2).any(|w| w[0] != w[1]),
"expected jitter, got a constant delay"
);
}
#[test]
fn backoff_is_capped() {
assert!(backoff_delay(Duration::from_secs(10), 20) <= MAX_BACKOFF);
}
#[test]
fn messages_come_out_of_both_error_shapes() {
assert_eq!(
extract_message(r#"{"error":{"code":"bad_request","message":"nope","status":400}}"#),
Some("nope (bad_request)".to_string())
);
assert_eq!(
extract_message(r#"{"error":"Dataset not found"}"#),
Some("Dataset not found".to_string())
);
assert_eq!(extract_message("not json"), None);
}
}