use super::fs;
use super::proto::{self, FileMetadata, MetadataError, MetadataFlow, MetadataStep};
use super::{BucketEntry, RepoInfo, HF_ENDPOINT};
use crate::api::Progress;
use crate::{Cache, Repo, RepoType};
use indicatif::ProgressBar;
use std::collections::HashMap;
use std::io::Read;
use std::io::Seek;
use std::num::ParseIntError;
use std::path::PathBuf;
use thiserror::Error;
use ureq::config::ConfigBuilder;
use ureq::config::RedirectAuthHeaders;
use ureq::tls::{TlsConfig, TlsProvider};
use ureq::typestate::{AgentScope, WithoutBody};
use ureq::{Agent, RequestBuilder};
const RANGE: &str = "Range";
const USER_AGENT: &str = "User-Agent";
const AUTHORIZATION: &str = "Authorization";
type HeaderMap = HashMap<&'static str, String>;
type HeaderName = &'static str;
const EXTENSION: &str = "part";
struct Wrapper<'a, P: Progress, R: Read> {
progress: &'a mut P,
inner: R,
}
fn wrap_read<P: Progress, R: Read>(inner: R, progress: &mut P) -> Wrapper<'_, P, R> {
Wrapper { inner, progress }
}
impl<P: Progress, R: Read> Read for Wrapper<'_, P, R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let read = self.inner.read(buf)?;
self.progress.update(read);
Ok(read)
}
}
#[derive(Clone, Debug)]
pub struct HeaderAgent {
agent: Agent,
headers: HeaderMap,
}
impl HeaderAgent {
fn new(agent: Agent, headers: HeaderMap) -> Self {
Self { agent, headers }
}
fn get(&self, url: &str) -> RequestBuilder<WithoutBody> {
let mut request = self.agent.get(url);
for (header, value) in &self.headers {
request = request.header(*header, value);
}
request
}
}
struct Handle {
file: std::fs::File,
}
impl Drop for Handle {
fn drop(&mut self) {
fs::unlock(&self.file);
}
}
fn lock_file(mut path: PathBuf) -> Result<Handle, ApiError> {
path.set_extension("lock");
let file = std::fs::File::create(path.clone())?;
let mut res = fs::lock(&file);
for _ in 0..5 {
if res == 0 {
break;
}
std::thread::sleep(std::time::Duration::from_secs(1));
res = fs::lock(&file);
}
if res != 0 {
Err(ApiError::LockAcquisition(path))
} else {
Ok(Handle { file })
}
}
#[derive(Debug, Error)]
pub enum ApiError {
#[error("Header {0} is missing")]
MissingHeader(HeaderName),
#[error("Header {0} is invalid")]
InvalidHeader(HeaderName),
#[error("request error: {0}")]
RequestError(#[from] Box<ureq::Error>),
#[error("Cannot parse int")]
ParseIntError(#[from] ParseIntError),
#[error("I/O error {0}")]
IoError(#[from] std::io::Error),
#[error("Too many retries: {0}")]
TooManyRetries(Box<ApiError>),
#[error("Native tls: {0}")]
#[cfg(feature = "native-tls")]
Native(#[from] native_tls::Error),
#[error("Invalid part file - corrupted file")]
InvalidResume,
#[error("Lock acquisition failed: {0}")]
LockAcquisition(PathBuf),
}
impl From<MetadataError> for ApiError {
fn from(error: MetadataError) -> Self {
match error {
MetadataError::MissingHeader(name) => ApiError::MissingHeader(name),
MetadataError::InvalidHeader(name) => ApiError::InvalidHeader(name),
MetadataError::ParseIntError(parse) => ApiError::ParseIntError(parse),
MetadataError::TooManyRedirects => ApiError::InvalidHeader(proto::LOCATION),
}
}
}
#[derive(Debug)]
pub struct ApiBuilder {
endpoint: String,
cache: Cache,
token: Option<String>,
max_retries: usize,
progress: bool,
user_agent: Vec<(String, String)>,
}
impl Default for ApiBuilder {
fn default() -> Self {
Self::new()
}
}
impl ApiBuilder {
pub fn new() -> Self {
let cache = Cache::default();
Self::from_cache(cache)
}
pub fn from_env() -> Self {
let cache = Cache::from_env();
let mut builder = Self::from_cache(cache);
if let Ok(endpoint) = std::env::var(HF_ENDPOINT) {
builder = builder.with_endpoint(endpoint);
}
builder
}
pub fn from_cache(cache: Cache) -> Self {
let token = cache.token();
let max_retries = 0;
let progress = true;
let endpoint = "https://huggingface.co".to_string();
let user_agent = proto::default_user_agent();
Self {
endpoint,
cache,
token,
max_retries,
progress,
user_agent,
}
}
pub fn with_progress(mut self, progress: bool) -> Self {
self.progress = progress;
self
}
pub fn with_endpoint(mut self, endpoint: String) -> Self {
self.endpoint = endpoint;
self
}
pub fn with_cache_dir(mut self, cache_dir: PathBuf) -> Self {
self.cache = Cache::new(cache_dir);
self
}
pub fn with_token(mut self, token: Option<String>) -> Self {
self.token = token;
self
}
pub fn with_retries(mut self, max_retries: usize) -> Self {
self.max_retries = max_retries;
self
}
pub fn with_user_agent(mut self, key: &str, value: &str) -> Self {
self.user_agent.push((key.to_string(), value.to_string()));
self
}
fn build_headers(&self) -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, proto::user_agent_string(&self.user_agent));
if let Some(token) = &self.token {
headers.insert(AUTHORIZATION, format!("Bearer {token}"));
}
headers
}
pub fn build(self) -> Result<Api, ApiError> {
let headers = self.build_headers();
let agent_builder = builder()?.redirect_auth_headers(RedirectAuthHeaders::SameHost);
let agent: Agent = agent_builder.build().into();
let client = HeaderAgent::new(agent, headers.clone());
let no_redirect_agent: Agent = builder()?.max_redirects(0).build().into();
let no_redirect_client = HeaderAgent::new(no_redirect_agent, headers);
Ok(Api {
endpoint: self.endpoint,
cache: self.cache,
client,
no_redirect_client,
max_retries: self.max_retries,
progress: self.progress,
})
}
}
#[derive(Clone, Debug)]
pub struct Api {
endpoint: String,
cache: Cache,
client: HeaderAgent,
no_redirect_client: HeaderAgent,
max_retries: usize,
progress: bool,
}
impl Api {
pub fn new() -> Result<Self, ApiError> {
ApiBuilder::new().build()
}
pub fn client(&self) -> &HeaderAgent {
&self.client
}
fn metadata(&self, url: &str) -> Result<FileMetadata, ApiError> {
let mut flow = MetadataFlow::new(url);
let mut step = flow.first_step();
loop {
let response = match step {
MetadataStep::Done(metadata) => return Ok(metadata),
MetadataStep::FetchNoRedirect { url } => self
.no_redirect_client
.get(&url)
.header(RANGE, "bytes=0-0")
.call()
.map_err(Box::new)?,
MetadataStep::Probe { url } => self
.client
.get(&url)
.header(RANGE, "bytes=0-0")
.call()
.map_err(Box::new)?,
};
step = flow.on_response(response.status().as_u16(), response.headers())?;
}
}
fn download_tempfile<P: Progress>(
&self,
url: &str,
size: usize,
mut progress: P,
tmp_path: PathBuf,
filename: &str,
) -> Result<PathBuf, ApiError> {
progress.init(size, filename);
let filepath = tmp_path;
let mut file = match std::fs::OpenOptions::new().append(true).open(&filepath) {
Ok(f) => f,
Err(_) => std::fs::File::create(&filepath)?,
};
let start = proto::sequential_resume_start(file.metadata()?.len(), size as u64)
.ok_or(ApiError::InvalidResume)?;
let mut res = self.download_from(url, start, size, &mut file, filename, &mut progress);
if self.max_retries > 0 {
let mut i = 0;
while let Err(dlerr) = res {
let wait_time = proto::exponential_backoff(300, i, 10_000);
std::thread::sleep(std::time::Duration::from_millis(wait_time as u64));
let current = file.stream_position()?;
res = self.download_from(url, current, size, &mut file, filename, &mut progress);
i += 1;
if i > self.max_retries {
return Err(ApiError::TooManyRetries(dlerr.into()));
}
}
}
res?;
Ok(filepath)
}
fn download_from<P>(
&self,
url: &str,
current: u64,
size: usize,
file: &mut std::fs::File,
filename: &str,
progress: &mut P,
) -> Result<(), ApiError>
where
P: Progress,
{
let range = format!("bytes={current}-");
let response = self
.client
.get(url)
.header(RANGE, &range)
.call()
.map_err(Box::new)?;
let (_, body) = response.into_parts();
let reader = body.into_reader();
progress.init(size, filename);
progress.update(current as usize);
let mut reader = Box::new(wrap_read(reader, progress));
std::io::copy(&mut reader, file)?;
progress.finish();
Ok(())
}
pub fn repo(&self, repo: Repo) -> ApiRepo {
ApiRepo::new(self.clone(), repo)
}
pub fn model(&self, model_id: String) -> ApiRepo {
self.repo(Repo::new(model_id, RepoType::Model))
}
pub fn dataset(&self, model_id: String) -> ApiRepo {
self.repo(Repo::new(model_id, RepoType::Dataset))
}
pub fn space(&self, model_id: String) -> ApiRepo {
self.repo(Repo::new(model_id, RepoType::Space))
}
pub fn bucket(&self, bucket_id: String) -> ApiBucket {
ApiBucket {
api: self.clone(),
bucket_id,
}
}
}
#[derive(Debug)]
pub struct ApiRepo {
api: Api,
repo: Repo,
}
impl ApiRepo {
fn new(api: Api, repo: Repo) -> Self {
Self { api, repo }
}
}
#[cfg(feature = "native-tls")]
fn builder() -> Result<ConfigBuilder<AgentScope>, ApiError> {
Ok(Agent::config_builder().tls_config(
TlsConfig::builder()
.provider(TlsProvider::NativeTls)
.build(),
))
}
#[cfg(not(feature = "native-tls"))]
fn builder() -> Result<ConfigBuilder<AgentScope>, ApiError> {
Ok(Agent::config_builder()
.tls_config(TlsConfig::builder().provider(TlsProvider::Rustls).build()))
}
impl ApiRepo {
pub fn url(&self, filename: &str) -> String {
proto::file_url(
&self.api.endpoint,
&self.repo.url(),
&self.repo.url_revision(),
filename,
)
}
pub fn get(&self, filename: &str) -> Result<PathBuf, ApiError> {
if let Some(path) = self.api.cache.repo(self.repo.clone()).get(filename) {
Ok(path)
} else {
self.download(filename)
}
}
pub fn download_with_progress<P: Progress>(
&self,
filename: &str,
progress: P,
) -> Result<PathBuf, ApiError> {
let url = self.url(filename);
let metadata = self.api.metadata(&url)?;
let blob_path = self
.api
.cache
.repo(self.repo.clone())
.blob_path(&metadata.etag);
std::fs::create_dir_all(blob_path.parent().unwrap())?;
let lock = lock_file(blob_path.clone())?;
let mut tmp_path = blob_path.clone();
tmp_path.set_extension(EXTENSION);
let tmp_filename =
self.api
.download_tempfile(&url, metadata.size, progress, tmp_path, filename)?;
std::fs::rename(tmp_filename, &blob_path)?;
drop(lock);
let mut pointer_path = self
.api
.cache
.repo(self.repo.clone())
.pointer_path(&metadata.commit_hash);
pointer_path.push(filename);
std::fs::create_dir_all(pointer_path.parent().unwrap()).ok();
fs::symlink_or_rename(&blob_path, &pointer_path)?;
self.api
.cache
.repo(self.repo.clone())
.create_ref(&metadata.commit_hash)?;
assert!(pointer_path.exists());
Ok(pointer_path)
}
pub fn download(&self, filename: &str) -> Result<PathBuf, ApiError> {
if self.api.progress {
self.download_with_progress(filename, ProgressBar::new(0))
} else {
self.download_with_progress(filename, ())
}
}
pub fn info(&self) -> Result<RepoInfo, ApiError> {
let mut response = self.info_request().call().map_err(Box::new)?;
Ok(response.body_mut().read_json().map_err(Box::new)?)
}
pub fn info_request(&self) -> RequestBuilder<WithoutBody> {
let url = format!("{}/api/{}", self.api.endpoint, self.repo.api_url());
self.api.client.get(&url)
}
}
#[derive(Debug)]
pub struct ApiBucket {
api: Api,
bucket_id: String,
}
impl ApiBucket {
pub fn url(&self, path: &str) -> String {
proto::bucket_resolve_url(&self.api.endpoint, &self.bucket_id, path)
}
pub fn list(&self, prefix: Option<&str>, recursive: bool) -> BucketList {
BucketList {
client: self.api.client.clone(),
next_url: Some(proto::bucket_tree_url(
&self.api.endpoint,
&self.bucket_id,
prefix,
recursive,
)),
page: Vec::new().into_iter(),
}
}
pub fn get(&self, path: &str) -> Result<Vec<u8>, ApiError> {
let url = self.url(path);
let mut buf = Vec::new();
let mut res = self.fetch_into(&url, &mut buf);
if self.api.max_retries > 0 {
let mut i = 0;
while let Err(dlerr) = res {
let wait_time = proto::exponential_backoff(300, i, 10_000);
std::thread::sleep(std::time::Duration::from_millis(wait_time as u64));
res = self.fetch_into(&url, &mut buf);
i += 1;
if i > self.api.max_retries {
return Err(ApiError::TooManyRetries(dlerr.into()));
}
}
}
res?;
Ok(buf)
}
fn fetch_into(&self, url: &str, buf: &mut Vec<u8>) -> Result<(), ApiError> {
let range = format!("bytes={}-", buf.len());
let response = self
.api
.client
.get(url)
.header(RANGE, &range)
.call()
.map_err(Box::new)?;
let (_, body) = response.into_parts();
body.into_reader().read_to_end(buf)?;
Ok(())
}
}
pub struct BucketList {
client: HeaderAgent,
next_url: Option<String>,
page: std::vec::IntoIter<BucketEntry>,
}
impl BucketList {
fn fetch(&mut self, url: &str) -> Result<(), ApiError> {
let mut response = self.client.get(url).call().map_err(Box::new)?;
self.next_url = proto::parse_next_link(response.headers())?;
let page: Vec<BucketEntry> = response.body_mut().read_json().map_err(Box::new)?;
self.page = page.into_iter();
Ok(())
}
}
impl Iterator for BucketList {
type Item = Result<BucketEntry, ApiError>;
fn next(&mut self) -> Option<Self::Item> {
loop {
if let Some(entry) = self.page.next() {
return Some(Ok(entry));
}
let url = self.next_url.take()?;
if let Err(err) = self.fetch(&url) {
return Some(Err(err));
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::api::test_server::{BucketFixture, Fixture, TestServer};
use crate::api::Siblings;
use crate::assert_no_diff;
use rand::{distr::Alphanumeric, RngExt};
use serde_json::{json, Value};
use std::io::{Seek, SeekFrom, Write};
use std::sync::atomic::AtomicBool;
use std::time::Duration;
fn fixture_content(len: usize) -> Vec<u8> {
(1..=255u8).cycle().take(len).collect()
}
struct TempDir {
path: PathBuf,
}
impl TempDir {
pub fn new() -> Self {
let s: String = rand::rng()
.sample_iter(&Alphanumeric)
.take(7)
.map(char::from)
.collect();
let mut path = std::env::temp_dir();
path.push(s);
std::fs::create_dir(&path).unwrap();
Self { path }
}
}
impl Drop for TempDir {
fn drop(&mut self) {
std::fs::remove_dir_all(&self.path).unwrap()
}
}
#[test]
fn simple() {
let content: Vec<u8> = fixture_content(1030);
let server = TestServer::spawn(Fixture {
repo: "local/test",
filename: "model.bin",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 2,
cdn_redirect: false,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(Some("hub_token".to_string()))
.with_cache_dir(tmp.path.clone())
.with_endpoint(server.endpoint())
.build()
.unwrap();
let repo = api.model("local/test".to_string());
let downloaded_path = repo.download("model.bin").unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
assert!(downloaded_path.to_str().unwrap().contains("commitsha456"));
let blob = std::fs::canonicalize(&downloaded_path).unwrap();
assert!(blob.to_str().unwrap().ends_with("etag123abc"));
let log = server.requests();
assert_eq!(log[0].range.as_deref(), Some("bytes=0-0"));
assert_eq!(log[0].authorization.as_deref(), Some("Bearer hub_token"));
assert_eq!(log[1].path, "/_redirect/1");
assert_eq!(log[2].path, "/_redirect/2");
assert_eq!(log.last().unwrap().range.as_deref(), Some("bytes=0-"));
drop(server);
let cached_path = api
.model("local/test".to_string())
.get("model.bin")
.unwrap();
assert_eq!(cached_path, downloaded_path);
}
#[test]
fn resume() {
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "local/test",
filename: "model.bin",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: false,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.build()
.unwrap();
let model_id = "local/test".to_string();
let downloaded_path = api.model(model_id.clone()).download("model.bin").unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
let blob = std::fs::canonicalize(&downloaded_path).unwrap();
let file = std::fs::OpenOptions::new().write(true).open(&blob).unwrap();
let size = file.metadata().unwrap().len();
let truncate: f32 = rand::random();
let new_size = (size as f32 * truncate) as u64;
file.set_len(new_size).unwrap();
let mut blob_part = blob.clone();
blob_part.set_extension("part");
std::fs::rename(blob, &blob_part).unwrap();
std::fs::remove_file(&downloaded_path).unwrap();
assert_eq!(std::fs::read(&blob_part).unwrap().len() as u64, new_size);
let new_downloaded_path = api.model(model_id.clone()).download("model.bin").unwrap();
assert_eq!(downloaded_path, new_downloaded_path);
assert_eq!(std::fs::read(&new_downloaded_path).unwrap(), content);
let blob = std::fs::canonicalize(&downloaded_path).unwrap();
let mut file = std::fs::OpenOptions::new().write(true).open(&blob).unwrap();
let size = file.metadata().unwrap().len();
let truncate: f32 = 0.5;
let new_size = (size as f32 * truncate) as u64;
file.set_len(new_size).unwrap();
file.seek(SeekFrom::Start(new_size - 1)).unwrap();
file.write_all(&[0]).unwrap();
let mut blob_part = blob.clone();
blob_part.set_extension("part");
std::fs::rename(blob, &blob_part).unwrap();
std::fs::remove_file(&downloaded_path).unwrap();
let new_downloaded_path = api.model(model_id.clone()).download("model.bin").unwrap();
assert_eq!(downloaded_path, new_downloaded_path);
let mut expected = content.clone();
expected[new_size as usize - 1] = 0;
assert_eq!(std::fs::read(&new_downloaded_path).unwrap(), expected);
}
#[test]
fn locking() {
use std::sync::{Arc, Mutex};
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "local/test",
filename: "model.bin",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: false,
});
let endpoint = server.endpoint();
let tmp = Arc::new(Mutex::new(TempDir::new()));
let mut handles = vec![];
for _ in 0..5 {
let tmp2 = tmp.clone();
let endpoint = endpoint.clone();
let f = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(rand::random::<u8>().into()));
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(endpoint)
.with_cache_dir(tmp2.lock().unwrap().path.clone())
.build()
.unwrap();
api.model("local/test".to_string())
.download("model.bin")
.unwrap()
});
handles.push(f);
}
while let Some(handle) = handles.pop() {
let downloaded_path = handle.join().unwrap();
assert!(downloaded_path.exists());
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
}
}
#[test]
fn simple_with_retries() {
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "local/test",
filename: "model.bin",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: false,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.with_retries(3)
.build()
.unwrap();
let model_id = "local/test".to_string();
let downloaded_path = api.model(model_id.clone()).download("model.bin").unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
let cache_path = api
.cache
.repo(Repo::new(model_id, RepoType::Model))
.get("model.bin")
.unwrap();
assert_eq!(cache_path, downloaded_path);
}
#[test]
fn dataset_with_revision() {
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "datasets/wikitext",
filename: "wikitext-103-v1/test/0000.parquet",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: true,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.build()
.unwrap();
let repo = Repo::with_revision(
"wikitext".to_string(),
RepoType::Dataset,
"refs/convert/parquet".to_string(),
);
let downloaded_path = api
.repo(repo)
.download("wikitext-103-v1/test/0000.parquet")
.unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
let log = server.requests();
assert_eq!(
log[0].path,
"/datasets/wikitext/resolve/refs%2Fconvert%2Fparquet/wikitext-103-v1/test/0000.parquet"
);
}
#[test]
fn model_with_revision() {
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "BAAI/bGe-reRanker-Base",
filename: "tokenizer.json",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: true,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.build()
.unwrap();
let repo = Repo::with_revision(
"BAAI/bGe-reRanker-Base".to_string(),
RepoType::Model,
"refs/pr/5".to_string(),
);
let downloaded_path = api.repo(repo).download("tokenizer.json").unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
let log = server.requests();
assert_eq!(
log[0].path,
"/BAAI/bGe-reRanker-Base/resolve/refs%2Fpr%2F5/tokenizer.json"
);
}
#[test]
fn info() {
let server = TestServer::spawn(Fixture {
repo: "datasets/wikitext",
filename: "wikitext-103-v1/test/0000.parquet",
content: fixture_content(16),
etag: "etag123abc",
commit: "infosha",
relative_redirects: 0,
cdn_redirect: false,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.build()
.unwrap();
let repo = Repo::with_revision(
"wikitext".to_string(),
RepoType::Dataset,
"refs/convert/parquet".to_string(),
);
let model_info = api.repo(repo.clone()).info().unwrap();
assert_eq!(
model_info,
RepoInfo {
siblings: vec![Siblings {
rfilename: "wikitext-103-v1/test/0000.parquet".to_string()
}],
sha: "infosha".to_string()
}
);
let blobs_info: Value = api
.repo(repo)
.info_request()
.query("blobs", "true")
.call()
.unwrap()
.body_mut()
.read_json()
.unwrap();
assert_no_diff!(
blobs_info,
json!({
"siblings": [{"rfilename": "wikitext-103-v1/test/0000.parquet"}],
"sha": "infosha"
})
);
}
#[test]
fn endpoint() {
let api = ApiBuilder::new().build().unwrap();
assert_eq!(api.endpoint, "https://huggingface.co".to_string());
let fake_endpoint = "https://fake_endpoint.com".to_string();
let api = ApiBuilder::new()
.with_endpoint(fake_endpoint.clone())
.build()
.unwrap();
assert_eq!(api.endpoint, fake_endpoint);
}
#[test]
fn headers_with_token() {
let api = ApiBuilder::new()
.with_token(Some("token".to_string()))
.build()
.unwrap();
let headers = api.client.headers;
assert_eq!(
headers.get("Authorization"),
Some(&"Bearer token".to_string())
);
}
#[test]
fn headers_default() {
let api = ApiBuilder::new().build().unwrap();
let headers = api.client.headers;
assert_eq!(
headers.get(USER_AGENT),
Some(&format!(
"unknown/None; {}/{}; rust/unknown",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
))
);
}
#[test]
fn headers_custom() {
let api = ApiBuilder::new()
.with_user_agent("origin", "custom")
.build()
.unwrap();
let headers = api.client.headers;
assert_eq!(
headers.get(USER_AGENT),
Some(&format!(
"unknown/None; {}/{}; rust/unknown; origin/custom",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
))
);
}
#[test]
fn bucket_list_paginated() {
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![
r#"[{"type":"file","path":"data/a.bin","size":4,"xetHash":"hashaaa","mtime":"2024-01-15T10:30:00.123Z","uploadedAt":"2024-01-16T00:00:00Z"}]"#,
r#"[{"type":"directory","path":"data/sub","uploadedAt":"2024-01-16T00:00:00Z"},{"type":"file","path":"data/b.bin","size":7,"xetHash":"hashbbb","uploadedAt":"2024-01-17T00:00:00Z"}]"#,
],
files: vec![],
truncate_first_cdn_response: AtomicBool::new(false),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(Some("hub_token".to_string()))
.with_endpoint(server.endpoint())
.build()
.unwrap();
let bucket = api.bucket("myorg/mybucket".to_string());
let mut entries = bucket.list(None, true);
assert_eq!(
entries.next().unwrap().unwrap(),
BucketEntry::File {
path: "data/a.bin".to_string(),
size: 4,
xet_hash: "hashaaa".to_string(),
uploaded_at: "2024-01-16T00:00:00Z".to_string(),
}
);
let log = server.requests();
assert_eq!(
log[0].path,
"/api/buckets/myorg/mybucket/tree?recursive=true"
);
assert_eq!(log[0].authorization.as_deref(), Some("Bearer hub_token"));
assert_eq!(log.len(), 1);
let rest: Vec<BucketEntry> = entries.collect::<Result<_, _>>().unwrap();
assert_eq!(
rest,
vec![
BucketEntry::Directory {
path: "data/sub".to_string(),
uploaded_at: "2024-01-16T00:00:00Z".to_string(),
},
BucketEntry::File {
path: "data/b.bin".to_string(),
size: 7,
xet_hash: "hashbbb".to_string(),
uploaded_at: "2024-01-17T00:00:00Z".to_string(),
},
]
);
let log = server.requests();
assert_eq!(log[0].path, "/api/buckets/myorg/mybucket/tree?cursor=1");
assert_eq!(log.len(), 1);
bucket
.list(Some("data/sub"), false)
.collect::<Result<Vec<_>, _>>()
.unwrap();
let log = server.requests();
assert_eq!(
log[0].path,
"/api/buckets/myorg/mybucket/tree/data%2Fsub?recursive=false"
);
}
#[test]
fn bucket_get() {
let content = fixture_content(1030);
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![],
files: vec![("data%2Ffile.bin", content.clone())],
truncate_first_cdn_response: AtomicBool::new(false),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(Some("hub_token".to_string()))
.with_endpoint(server.endpoint())
.build()
.unwrap();
let bytes = api
.bucket("myorg/mybucket".to_string())
.get("data/file.bin")
.unwrap();
assert_eq!(bytes, content);
let log = server.requests();
assert_eq!(
log[0].path,
"/buckets/myorg/mybucket/resolve/data%2Ffile.bin"
);
assert_eq!(log[0].range.as_deref(), Some("bytes=0-"));
assert_eq!(log[0].authorization.as_deref(), Some("Bearer hub_token"));
assert_eq!(log[1].path, "/cdn/bucket/data%2Ffile.bin");
}
#[test]
fn bucket_get_resume() {
let content = fixture_content(4096);
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![],
files: vec![("model.bin", content.clone())],
truncate_first_cdn_response: AtomicBool::new(true),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_retries(3)
.build()
.unwrap();
let bytes = api
.bucket("myorg/mybucket".to_string())
.get("model.bin")
.unwrap();
assert_eq!(bytes, content);
let log = server.requests();
assert_eq!(log.last().unwrap().range.as_deref(), Some("bytes=2048-"));
}
#[test]
fn bucket_unicode_path() {
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![],
files: vec![("sub%20dir%2Fh%C3%A9llo.txt", b"unicode!".to_vec())],
truncate_first_cdn_response: AtomicBool::new(false),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.build()
.unwrap();
let bytes = api
.bucket("myorg/mybucket".to_string())
.get("sub dir/héllo.txt")
.unwrap();
assert_eq!(bytes, b"unicode!");
let log = server.requests();
assert_eq!(
log[0].path,
"/buckets/myorg/mybucket/resolve/sub%20dir%2Fh%C3%A9llo.txt"
);
}
}