use std::io::Write;
use anyhow::{Context, Result};
use reqwest::header::HeaderMap;
use super::client::{self, HttpStatusError};
use crate::xet_fetch::{self, XetFileRef};
#[derive(Debug, Default, Clone)]
pub struct FileMetadata {
pub digest: Option<String>,
pub size: i64,
pub xet_hash: Option<String>,
}
const MAX_HOPS: u32 = 5;
pub async fn head_metadata(
client: &reqwest::Client,
mut url: String,
token: Option<&str>,
) -> Result<FileMetadata> {
for _ in 0..MAX_HOPS {
let mut req = client
.head(&url)
.header(reqwest::header::ACCEPT_ENCODING, "identity");
if let Some(t) = token {
req = req.bearer_auth(t);
}
let resp = req.send().await.with_context(|| format!("HEAD {url}"))?;
let status = resp.status();
let is_redirect = status.is_redirection();
if status != reqwest::StatusCode::OK && !is_redirect {
return Err(HttpStatusError::new(
format!("HEAD {url}"),
status.as_u16(),
resp.headers(),
)
.into());
}
let headers = resp.headers().clone();
let x_linked_etag = header_str(&headers, "x-linked-etag");
let x_linked_size = header_str(&headers, "x-linked-size");
let x_xet_hash = header_str(&headers, "x-xet-hash");
if is_redirect && x_linked_etag.is_none() && x_linked_size.is_none() {
let Some(location) = headers
.get(reqwest::header::LOCATION)
.and_then(|v| v.to_str().ok())
else {
anyhow::bail!("HEAD {url}: redirect with no Location header");
};
url = reqwest::Url::parse(&url)?.join(location)?.to_string();
continue;
}
let size = x_linked_size
.as_deref()
.and_then(|s| s.parse().ok())
.or_else(|| {
(status == reqwest::StatusCode::OK)
.then(|| header_str(&headers, "content-length"))
.flatten()
.and_then(|s| s.parse().ok())
})
.unwrap_or(0);
let etag = x_linked_etag.or_else(|| header_str(&headers, "etag"));
let digest = etag
.map(|e| e.trim_start_matches("W/").trim_matches('"').to_lowercase())
.filter(|e| e.len() == 64);
return Ok(FileMetadata {
digest,
size,
xet_hash: x_xet_hash,
});
}
anyhow::bail!("HEAD {url}: gave up after {MAX_HOPS} redirects with no usable metadata");
}
fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
.and_then(|v| v.to_str().ok())
.map(str::to_string)
.filter(|s| !s.is_empty())
}
pub struct FetchRequest<'a> {
pub url: String,
pub token: Option<&'a str>,
pub xet: Option<XetFileRef>,
pub label: String,
}
pub async fn fetch_once(
client: &reqwest::Client,
req: &FetchRequest<'_>,
writer: &mut (impl Write + Send),
) -> Result<u64> {
if let Some(xet) = &req.xet {
let mut counting = CountingWriter {
inner: writer,
total: 0,
};
xet_fetch::stream_to_writer(xet, &mut counting)
.await
.context("xet download")?;
return Ok(counting.total);
}
let mut r = client.get(&req.url);
if let Some(t) = req.token {
r = r.bearer_auth(t);
}
r = r.header(reqwest::header::RANGE, "bytes=0-");
let resp = r.send().await.with_context(|| req.label.clone())?;
let status = resp.status();
if status != reqwest::StatusCode::OK && status != reqwest::StatusCode::PARTIAL_CONTENT {
return Err(
HttpStatusError::new(req.label.clone(), status.as_u16(), resp.headers()).into(),
);
}
client::copy_with_stall_detection(resp.bytes_stream(), |chunk| writer.write_all(chunk)).await
}
pub fn should_retry(err: &anyhow::Error, attempt: u32) -> Option<std::time::Duration> {
if client::is_permanent(err) {
return None;
}
Some(client::retry_after_of(err).unwrap_or_else(|| client::retry_delay(attempt)))
}
struct CountingWriter<'a, W: Write> {
inner: &'a mut W,
total: u64,
}
impl<W: Write> Write for CountingWriter<'_, W> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let n = self.inner.write(buf)?;
self.total += n as u64;
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
self.inner.flush()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn head_metadata_reads_the_intermediate_hop_not_the_final_cdn_response() {
let url = "https://huggingface.co/hf-internal-testing/tiny-random-gpt2/resolve/main/model.safetensors".to_string();
let client = super::super::head_client().unwrap();
let meta = head_metadata(&client, url, None)
.await
.expect("head_metadata should succeed against a real, known-good file");
assert_eq!(meta.digest.as_deref(), Some("8111d5afb0715dbf5a31396d31432cb56370ba23f6650a035ea0fc8a20b4e500"), "digest must be the file's real sha256, bare hex, not the xet hash and not sha256:-prefixed");
assert_eq!(
meta.xet_hash.as_deref(),
Some("f8accece953fd366d4ce30597b97acc1ccedc3c785187a5ef6ecb4a8e1755122"),
"xet_hash must also be read from the same intermediate hop"
);
assert_eq!(meta.size, 453864);
}
}