use http::uri::Uri;
use http::HeaderMap;
#[cfg(any(feature = "tokio", test))]
use std::cmp::Reverse;
#[cfg(any(feature = "tokio", test))]
use std::collections::BinaryHeap;
use std::num::ParseIntError;
use thiserror::Error;
const VERSION: &str = env!("CARGO_PKG_VERSION");
const NAME: &str = env!("CARGO_PKG_NAME");
pub(crate) const LOCATION: &str = "location";
pub(crate) const LINK: &str = "link";
pub(crate) const ETAG: &str = "etag";
pub(crate) const X_LINKED_ETAG: &str = "x-linked-etag";
pub(crate) const X_REPO_COMMIT: &str = "x-repo-commit";
pub(crate) const CONTENT_RANGE: &str = "content-range";
const MAX_REDIRECTS: u8 = 10;
#[derive(Debug, Error, PartialEq)]
pub(crate) enum MetadataError {
#[error("Header {0} is missing")]
MissingHeader(&'static str),
#[error("Header {0} is invalid")]
InvalidHeader(&'static str),
#[error("Too many redirects (max {MAX_REDIRECTS})")]
TooManyRedirects,
#[error("Cannot parse int")]
ParseIntError(#[from] ParseIntError),
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct FileMetadata {
pub(crate) commit_hash: String,
pub(crate) etag: String,
pub(crate) size: usize,
}
#[derive(Debug, PartialEq)]
pub(crate) enum MetadataStep {
FetchNoRedirect { url: String },
Probe { url: String },
Done(FileMetadata),
}
pub(crate) struct MetadataFlow {
current_url: String,
hops: u8,
probing: Option<(String, String)>,
}
impl MetadataFlow {
pub(crate) fn new(url: &str) -> Self {
Self {
current_url: url.to_string(),
hops: 0,
probing: None,
}
}
pub(crate) fn first_step(&self) -> MetadataStep {
MetadataStep::FetchNoRedirect {
url: self.current_url.clone(),
}
}
pub(crate) fn on_response(
&mut self,
status: u16,
headers: &HeaderMap,
) -> Result<MetadataStep, MetadataError> {
if let Some((commit_hash, etag)) = self.probing.take() {
let size = parse_content_range(headers)?;
return Ok(MetadataStep::Done(FileMetadata {
commit_hash,
etag,
size,
}));
}
if is_followable_redirect(status) {
if let Some(location) = get_str(headers, LOCATION)? {
if let Some(next_url) = merge_relative_location(&self.current_url, location)? {
self.hops += 1;
if self.hops > MAX_REDIRECTS {
return Err(MetadataError::TooManyRedirects);
}
self.current_url = next_url.clone();
return Ok(MetadataStep::FetchNoRedirect { url: next_url });
}
}
}
let etag = match get_str(headers, X_LINKED_ETAG)? {
Some(etag) => etag,
None => get_str(headers, ETAG)?.ok_or(MetadataError::MissingHeader(ETAG))?,
}
.replace('"', "");
let commit_hash = get_str(headers, X_REPO_COMMIT)?
.ok_or(MetadataError::MissingHeader(X_REPO_COMMIT))?
.to_string();
if (300..400).contains(&status) {
let url = get_str(headers, LOCATION)?
.ok_or(MetadataError::MissingHeader(LOCATION))?
.to_string();
self.probing = Some((commit_hash, etag));
Ok(MetadataStep::Probe { url })
} else {
let size = parse_content_range(headers)?;
Ok(MetadataStep::Done(FileMetadata {
commit_hash,
etag,
size,
}))
}
}
}
fn is_followable_redirect(status: u16) -> bool {
matches!(status, 301 | 302 | 303 | 307 | 308)
}
fn get_str<'h>(
headers: &'h HeaderMap,
name: &'static str,
) -> Result<Option<&'h str>, MetadataError> {
match headers.get(name) {
None => Ok(None),
Some(value) => Ok(Some(
value
.to_str()
.map_err(|_| MetadataError::InvalidHeader(name))?,
)),
}
}
fn parse_content_range(headers: &HeaderMap) -> Result<usize, MetadataError> {
let content_range =
get_str(headers, CONTENT_RANGE)?.ok_or(MetadataError::MissingHeader(CONTENT_RANGE))?;
Ok(content_range
.split('/')
.next_back()
.ok_or(MetadataError::InvalidHeader(CONTENT_RANGE))?
.parse()?)
}
fn merge_relative_location(base: &str, location: &str) -> Result<Option<String>, MetadataError> {
let uri: Uri = location
.parse()
.map_err(|_| MetadataError::InvalidHeader(LOCATION))?;
if uri.host().is_some() {
return Ok(None);
}
let mut parts = base
.parse::<Uri>()
.map_err(|_| MetadataError::InvalidHeader(LOCATION))?
.into_parts();
parts.path_and_query = uri.into_parts().path_and_query;
let merged = Uri::from_parts(parts).map_err(|_| MetadataError::InvalidHeader(LOCATION))?;
Ok(Some(merged.to_string()))
}
pub(crate) fn file_url(endpoint: &str, repo_id: &str, revision: &str, filename: &str) -> String {
format!("{endpoint}/{repo_id}/resolve/{revision}/{filename}")
}
pub(crate) fn percent_encode_path(path: &str) -> String {
let mut encoded = String::with_capacity(path.len());
for byte in path.bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
encoded.push(byte as char)
}
_ => encoded.push_str(&format!("%{byte:02X}")),
}
}
encoded
}
pub(crate) fn bucket_resolve_url(endpoint: &str, bucket_id: &str, path: &str) -> String {
format!(
"{endpoint}/buckets/{bucket_id}/resolve/{}",
percent_encode_path(path)
)
}
pub(crate) fn bucket_tree_url(
endpoint: &str,
bucket_id: &str,
prefix: Option<&str>,
recursive: bool,
) -> String {
let base = format!("{endpoint}/api/buckets/{bucket_id}/tree");
match prefix {
Some(prefix) => format!(
"{base}/{}?recursive={recursive}",
percent_encode_path(prefix)
),
None => format!("{base}?recursive={recursive}"),
}
}
pub(crate) fn parse_next_link(headers: &HeaderMap) -> Result<Option<String>, MetadataError> {
let Some(link) = get_str(headers, LINK)? else {
return Ok(None);
};
for entry in link.split(',') {
let (url, params) = entry
.split_once(';')
.ok_or(MetadataError::InvalidHeader(LINK))?;
let url = url
.trim()
.strip_prefix('<')
.and_then(|url| url.strip_suffix('>'))
.ok_or(MetadataError::InvalidHeader(LINK))?;
if params
.split(';')
.any(|param| param.trim() == "rel=\"next\"")
{
return Ok(Some(url.to_string()));
}
}
Ok(None)
}
pub(crate) fn default_user_agent() -> Vec<(String, String)> {
vec![
("unknown".to_string(), "None".to_string()),
(NAME.to_string(), VERSION.to_string()),
("rust".to_string(), "unknown".to_string()),
]
}
pub(crate) fn user_agent_string(parts: &[(String, String)]) -> String {
parts
.iter()
.map(|(key, value)| format!("{key}/{value}"))
.collect::<Vec<_>>()
.join("; ")
}
fn jitter() -> usize {
use rand::RngExt;
rand::rng().random_range(0..=500)
}
pub(crate) fn exponential_backoff(base_wait_time: usize, n: usize, max: usize) -> usize {
(base_wait_time + n.pow(2) + jitter()).min(max)
}
#[cfg(any(feature = "tokio", test))]
pub(crate) const TRAILER_LEN: usize = size_of::<u64>();
#[cfg(any(feature = "tokio", test))]
pub(crate) fn part_file_len(content_len: u64) -> u64 {
content_len + TRAILER_LEN as u64
}
#[cfg(any(feature = "tokio", test))]
pub(crate) fn encode_trailer(committed: u64) -> [u8; TRAILER_LEN] {
committed.to_le_bytes()
}
#[cfg(any(feature = "tokio", test))]
pub(crate) fn decode_trailer(buf: [u8; TRAILER_LEN]) -> u64 {
u64::from_le_bytes(buf)
}
#[cfg(any(feature = "ureq", test))]
pub(crate) fn sequential_resume_start(part_len: u64, total_len: u64) -> Option<u64> {
if part_len > total_len {
None
} else {
Some(part_len)
}
}
#[cfg(any(feature = "tokio", test))]
pub(crate) fn chunk_ranges(
start: usize,
length: usize,
chunk_size: usize,
) -> impl Iterator<Item = (usize, usize)> {
(start..length).step_by(chunk_size).map(move |chunk_start| {
let stop = std::cmp::min(chunk_start + chunk_size - 1, length);
(chunk_start, stop)
})
}
#[cfg(any(feature = "tokio", test))]
pub(crate) struct CommitTracker {
committed: u64,
pending: BinaryHeap<Reverse<(usize, usize)>>,
}
#[cfg(any(feature = "tokio", test))]
impl CommitTracker {
pub(crate) fn new(start: u64) -> Self {
Self {
committed: start,
pending: BinaryHeap::new(),
}
}
pub(crate) fn advance(&mut self, start: usize, stop: usize) -> Option<u64> {
self.pending.push(Reverse((start, stop)));
let mut modified = false;
while let Some(Reverse((min, max))) = self.pending.pop() {
if min as u64 == self.committed {
self.committed = max as u64 + 1;
modified = true;
} else {
self.pending.push(Reverse((min, max)));
break;
}
}
modified.then_some(self.committed)
}
}
#[cfg(test)]
mod tests {
use super::*;
use http::header::HeaderValue;
fn headers(pairs: &[(&'static str, &str)]) -> HeaderMap {
let mut map = HeaderMap::new();
for (name, value) in pairs {
map.insert(*name, HeaderValue::from_str(value).unwrap());
}
map
}
const URL: &str = "https://huggingface.co/gpt2/resolve/main/model.safetensors";
#[test]
fn metadata_direct_response() {
let mut flow = MetadataFlow::new(URL);
assert_eq!(
flow.first_step(),
MetadataStep::FetchNoRedirect {
url: URL.to_string()
}
);
let step = flow
.on_response(
206,
&headers(&[
("etag", "\"abc123\""),
("x-repo-commit", "deadbeef"),
("content-range", "bytes 0-0/5342"),
]),
)
.unwrap();
match step {
MetadataStep::Done(metadata) => {
assert_eq!(metadata.etag, "abc123");
assert_eq!(metadata.commit_hash, "deadbeef");
assert_eq!(metadata.size, 5342);
}
step => panic!("unexpected step {step:?}"),
}
}
#[test]
fn metadata_prefers_linked_etag() {
let mut flow = MetadataFlow::new(URL);
let step = flow
.on_response(
200,
&headers(&[
("etag", "\"weak\""),
("x-linked-etag", "\"lfs-sha\""),
("x-repo-commit", "deadbeef"),
("content-range", "bytes 0-0/22"),
]),
)
.unwrap();
match step {
MetadataStep::Done(metadata) => assert_eq!(metadata.etag, "lfs-sha"),
step => panic!("unexpected step {step:?}"),
}
}
#[test]
fn metadata_follows_relative_redirect() {
let mut flow = MetadataFlow::new(URL);
let step = flow
.on_response(
302,
&headers(&[("location", "/gpt2/resolve/abc/model.safetensors")]),
)
.unwrap();
assert_eq!(
step,
MetadataStep::FetchNoRedirect {
url: "https://huggingface.co/gpt2/resolve/abc/model.safetensors".to_string()
}
);
let step = flow
.on_response(
200,
&headers(&[
("etag", "\"abc\""),
("x-repo-commit", "abc"),
("content-range", "bytes 0-0/10"),
]),
)
.unwrap();
assert!(matches!(step, MetadataStep::Done(_)));
}
#[test]
fn metadata_probes_absolute_redirect() {
let mut flow = MetadataFlow::new(URL);
let step = flow
.on_response(
302,
&headers(&[
("location", "https://cdn-lfs.huggingface.co/some/blob"),
("etag", "\"abc\""),
("x-linked-etag", "\"lfs-sha\""),
("x-repo-commit", "deadbeef"),
]),
)
.unwrap();
assert_eq!(
step,
MetadataStep::Probe {
url: "https://cdn-lfs.huggingface.co/some/blob".to_string()
}
);
let step = flow
.on_response(206, &headers(&[("content-range", "bytes 0-0/987654")]))
.unwrap();
match step {
MetadataStep::Done(metadata) => {
assert_eq!(metadata.etag, "lfs-sha");
assert_eq!(metadata.commit_hash, "deadbeef");
assert_eq!(metadata.size, 987654);
}
step => panic!("unexpected step {step:?}"),
}
}
#[test]
fn metadata_redirect_loop_is_capped() {
let mut flow = MetadataFlow::new(URL);
let redirect = headers(&[("location", "/loop")]);
for _ in 0..10 {
let step = flow.on_response(302, &redirect).unwrap();
assert!(matches!(step, MetadataStep::FetchNoRedirect { .. }));
}
assert_eq!(
flow.on_response(302, &redirect),
Err(MetadataError::TooManyRedirects)
);
}
#[test]
fn metadata_missing_headers() {
assert_eq!(
MetadataFlow::new(URL).on_response(200, &headers(&[])),
Err(MetadataError::MissingHeader(ETAG))
);
assert_eq!(
MetadataFlow::new(URL).on_response(200, &headers(&[("etag", "\"x\"")])),
Err(MetadataError::MissingHeader(X_REPO_COMMIT))
);
assert_eq!(
MetadataFlow::new(URL)
.on_response(200, &headers(&[("etag", "\"x\""), ("x-repo-commit", "c")])),
Err(MetadataError::MissingHeader(CONTENT_RANGE))
);
assert_eq!(
MetadataFlow::new(URL)
.on_response(303, &headers(&[("etag", "\"x\""), ("x-repo-commit", "c")])),
Err(MetadataError::MissingHeader(LOCATION))
);
}
#[test]
fn percent_encoding() {
assert_eq!(percent_encode_path("a/b.txt"), "a%2Fb.txt");
assert_eq!(
percent_encode_path("sub dir/héllo.txt"),
"sub%20dir%2Fh%C3%A9llo.txt"
);
assert_eq!(percent_encode_path("AZaz09-_.~"), "AZaz09-_.~");
assert_eq!(percent_encode_path("100%?#"), "100%25%3F%23");
}
#[test]
fn bucket_urls() {
assert_eq!(
bucket_resolve_url("https://huggingface.co", "myorg/mybucket", "data/file.bin"),
"https://huggingface.co/buckets/myorg/mybucket/resolve/data%2Ffile.bin"
);
assert_eq!(
bucket_tree_url("https://huggingface.co", "myorg/mybucket", None, false),
"https://huggingface.co/api/buckets/myorg/mybucket/tree?recursive=false"
);
assert_eq!(
bucket_tree_url(
"https://huggingface.co",
"myorg/mybucket",
Some("data/sub"),
true
),
"https://huggingface.co/api/buckets/myorg/mybucket/tree/data%2Fsub?recursive=true"
);
}
#[test]
fn next_link() {
assert_eq!(parse_next_link(&headers(&[])), Ok(None));
assert_eq!(
parse_next_link(&headers(&[(
"link",
"<https://huggingface.co/api/buckets/a/b/tree?cursor=x>; rel=\"next\""
)])),
Ok(Some(
"https://huggingface.co/api/buckets/a/b/tree?cursor=x".to_string()
))
);
assert_eq!(
parse_next_link(&headers(&[(
"link",
"<https://x/prev>; rel=\"prev\", <https://x/next>; rel=\"next\""
)])),
Ok(Some("https://x/next".to_string()))
);
assert_eq!(
parse_next_link(&headers(&[("link", "<https://x/prev>; rel=\"prev\"")])),
Ok(None)
);
assert_eq!(
parse_next_link(&headers(&[("link", "garbage")])),
Err(MetadataError::InvalidHeader(LINK))
);
}
#[test]
fn content_range_parsing() {
assert_eq!(
parse_content_range(&headers(&[("content-range", "bytes 0-0/123")])),
Ok(123)
);
assert!(matches!(
parse_content_range(&headers(&[("content-range", "bytes 0-0/xyz")])),
Err(MetadataError::ParseIntError(_))
));
}
#[test]
fn relative_location_merge() {
assert_eq!(
merge_relative_location(URL, "/other/path?q=1").unwrap(),
Some("https://huggingface.co/other/path?q=1".to_string())
);
assert_eq!(
merge_relative_location(URL, "https://elsewhere.com/x").unwrap(),
None
);
}
#[test]
fn chunk_ranges_cover_the_file() {
let ranges: Vec<_> = chunk_ranges(0, 30, 10).collect();
assert_eq!(ranges, vec![(0, 9), (10, 19), (20, 29)]);
let ranges: Vec<_> = chunk_ranges(10, 30, 10).collect();
assert_eq!(ranges, vec![(10, 19), (20, 29)]);
let ranges: Vec<_> = chunk_ranges(0, 25, 10).collect();
assert_eq!(ranges, vec![(0, 9), (10, 19), (20, 25)]);
assert_eq!(chunk_ranges(30, 30, 10).count(), 0);
}
#[test]
fn commit_tracker_watermark() {
let mut tracker = CommitTracker::new(0);
assert_eq!(tracker.advance(10, 19), None);
assert_eq!(tracker.advance(20, 29), None);
assert_eq!(tracker.advance(0, 9), Some(30));
}
#[test]
fn commit_tracker_resumed_start() {
let mut tracker = CommitTracker::new(10);
assert_eq!(tracker.advance(10, 19), Some(20));
assert_eq!(tracker.advance(30, 39), None);
assert_eq!(tracker.advance(20, 29), Some(40));
}
#[test]
fn trailer_round_trip() {
assert_eq!(decode_trailer(encode_trailer(0)), 0);
assert_eq!(decode_trailer(encode_trailer(123456789)), 123456789);
assert_eq!(part_file_len(100), 108);
}
#[test]
fn sequential_resume() {
assert_eq!(sequential_resume_start(0, 100), Some(0));
assert_eq!(sequential_resume_start(50, 100), Some(50));
assert_eq!(sequential_resume_start(100, 100), Some(100));
assert_eq!(sequential_resume_start(101, 100), None);
}
#[test]
fn backoff_is_bounded() {
for n in 0..10 {
let wait = exponential_backoff(300, n, 10_000);
assert!(wait >= (300 + n.pow(2)).min(10_000));
assert!(wait <= (300 + n.pow(2) + 500).min(10_000));
}
}
#[test]
fn user_agent() {
let parts = default_user_agent();
assert_eq!(
user_agent_string(&parts),
format!("unknown/None; {NAME}/{VERSION}; rust/unknown")
);
}
}