use super::fs;
use super::proto::{self, FileMetadata, MetadataError, MetadataFlow, MetadataStep};
use super::Progress as SyncProgress;
use super::{BucketEntry, RepoInfo, HF_ENDPOINT};
use crate::{Cache, Repo, RepoType};
use futures::stream::FuturesUnordered;
use futures::{Stream, StreamExt};
use indicatif::ProgressBar;
use reqwest::{
header::{
HeaderMap, HeaderName, HeaderValue, InvalidHeaderValue, ToStrError, AUTHORIZATION, RANGE,
USER_AGENT,
},
redirect::Policy,
Client, Error as ReqwestError, RequestBuilder,
};
use std::num::ParseIntError;
use std::path::PathBuf;
use std::sync::Arc;
use thiserror::Error;
use tokio::io::AsyncReadExt;
use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
use tokio::sync::{AcquireError, Semaphore, TryAcquireError};
use tokio::task::JoinError;
const EXTENSION: &str = "sync.part";
pub trait Progress {
fn init(&mut self, size: usize, filename: &str)
-> impl std::future::Future<Output = ()> + Send;
fn update(&mut self, size: usize) -> impl std::future::Future<Output = ()> + Send;
fn finish(&mut self) -> impl std::future::Future<Output = ()> + Send;
}
impl Progress for ProgressBar {
async fn init(&mut self, size: usize, filename: &str) {
<ProgressBar as SyncProgress>::init(self, size, filename);
}
async fn finish(&mut self) {
<ProgressBar as SyncProgress>::finish(self);
}
async fn update(&mut self, size: usize) {
<ProgressBar as SyncProgress>::update(self, size);
}
}
impl Progress for () {
async fn init(&mut self, _size: usize, _filename: &str) {}
async fn finish(&mut self) {}
async fn update(&mut self, _size: usize) {}
}
struct Handle {
file: tokio::fs::File,
}
impl Drop for Handle {
fn drop(&mut self) {
fs::unlock(&self.file);
}
}
async fn lock_file(mut path: PathBuf) -> Result<Handle, ApiError> {
path.set_extension("lock");
let file = tokio::fs::File::create(path.clone()).await?;
let mut res = fs::lock(&file);
for _ in 0..5 {
if res == 0 {
break;
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
res = fs::lock(&file);
}
if res != 0 {
Err(ApiError::LockAcquisition(path))
} else {
Ok(Handle { file })
}
}
#[derive(Debug, Error)]
pub enum ApiError {
#[error("Header {0} is missing")]
MissingHeader(HeaderName),
#[error("Header {0} is invalid")]
InvalidHeader(HeaderName),
#[error("Invalid header value {0}")]
InvalidHeaderValue(#[from] InvalidHeaderValue),
#[error("header value is not a string")]
ToStr(#[from] ToStrError),
#[error("request error: {0}")]
RequestError(#[from] ReqwestError),
#[error("Cannot parse int")]
ParseIntError(#[from] ParseIntError),
#[error("I/O error {0}")]
IoError(#[from] std::io::Error),
#[error("Too many retries: {0}")]
TooManyRetries(Box<ApiError>),
#[error("Try acquire: {0}")]
TryAcquireError(#[from] TryAcquireError),
#[error("Acquire: {0}")]
AcquireError(#[from] AcquireError),
#[error("Join: {0}")]
Join(#[from] JoinError),
#[error("Lock acquisition failed: {0}")]
LockAcquisition(PathBuf),
}
impl From<MetadataError> for ApiError {
fn from(error: MetadataError) -> Self {
match error {
MetadataError::MissingHeader(name) => {
ApiError::MissingHeader(HeaderName::from_static(name))
}
MetadataError::InvalidHeader(name) => {
ApiError::InvalidHeader(HeaderName::from_static(name))
}
MetadataError::ParseIntError(parse) => ApiError::ParseIntError(parse),
MetadataError::TooManyRedirects => {
ApiError::InvalidHeader(HeaderName::from_static(proto::LOCATION))
}
}
}
}
#[derive(Debug)]
pub struct ApiBuilder {
endpoint: String,
cache: Cache,
token: Option<String>,
max_files: usize,
chunk_size: Option<usize>,
parallel_failures: usize,
max_retries: usize,
progress: bool,
user_agent: Vec<(String, String)>,
}
impl Default for ApiBuilder {
fn default() -> Self {
Self::new()
}
}
impl ApiBuilder {
pub fn new() -> Self {
let cache = Cache::default();
Self::from_cache(cache)
}
pub fn from_env() -> Self {
let cache = Cache::from_env();
let mut builder = Self::from_cache(cache);
if let Ok(endpoint) = std::env::var(HF_ENDPOINT) {
builder = builder.with_endpoint(endpoint);
}
builder
}
pub fn high(self) -> Self {
self.with_max_files(num_cpus::get())
.with_chunk_size(Some(10_000_000))
}
pub fn from_cache(cache: Cache) -> Self {
let token = cache.token();
let progress = true;
let user_agent = proto::default_user_agent();
Self {
endpoint: "https://huggingface.co".to_string(),
cache,
token,
max_files: 1,
chunk_size: Some(10_000_000),
parallel_failures: 0,
max_retries: 0,
progress,
user_agent,
}
}
pub fn with_progress(mut self, progress: bool) -> Self {
self.progress = progress;
self
}
pub fn with_endpoint(mut self, endpoint: String) -> Self {
self.endpoint = endpoint;
self
}
pub fn with_cache_dir(mut self, cache_dir: PathBuf) -> Self {
self.cache = Cache::new(cache_dir);
self
}
pub fn with_token(mut self, token: Option<String>) -> Self {
self.token = token;
self
}
pub fn with_max_files(mut self, max_files: usize) -> Self {
self.max_files = max_files;
self
}
pub fn with_chunk_size(mut self, chunk_size: Option<usize>) -> Self {
self.chunk_size = chunk_size;
self
}
pub fn with_retries(mut self, max_retries: usize) -> Self {
self.max_retries = max_retries;
self
}
pub fn with_user_agent(mut self, key: &str, value: &str) -> Self {
self.user_agent.push((key.to_string(), value.to_string()));
self
}
fn build_headers(&self) -> Result<HeaderMap, ApiError> {
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_str(&proto::user_agent_string(&self.user_agent))?,
);
if let Some(token) = &self.token {
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {token}"))?,
);
}
Ok(headers)
}
pub fn build(self) -> Result<Api, ApiError> {
let headers = self.build_headers()?;
let client = Client::builder().default_headers(headers.clone()).build()?;
let no_redirect_client = Client::builder()
.redirect(Policy::none())
.default_headers(headers)
.build()?;
Ok(Api {
endpoint: self.endpoint,
cache: self.cache,
client,
no_redirect_client,
max_files: self.max_files,
chunk_size: self.chunk_size,
parallel_failures: self.parallel_failures,
max_retries: self.max_retries,
progress: self.progress,
})
}
}
#[derive(Clone, Debug)]
pub struct Api {
endpoint: String,
cache: Cache,
client: Client,
no_redirect_client: Client,
max_files: usize,
chunk_size: Option<usize>,
parallel_failures: usize,
max_retries: usize,
progress: bool,
}
impl Api {
pub fn new() -> Result<Self, ApiError> {
ApiBuilder::new().build()
}
pub fn client(&self) -> &Client {
&self.client
}
async fn metadata(&self, url: &str) -> Result<FileMetadata, ApiError> {
let mut flow = MetadataFlow::new(url);
let mut step = flow.first_step();
loop {
let (client, url) = match step {
MetadataStep::Done(metadata) => return Ok(metadata),
MetadataStep::FetchNoRedirect { url } => (&self.no_redirect_client, url),
MetadataStep::Probe { url } => (&self.client, url),
};
let response = client
.get(&url)
.header(RANGE, "bytes=0-0")
.send()
.await?
.error_for_status()?;
step = flow.on_response(response.status().as_u16(), response.headers())?;
}
}
pub fn repo(&self, repo: Repo) -> ApiRepo {
ApiRepo::new(self.clone(), repo)
}
pub fn model(&self, model_id: String) -> ApiRepo {
self.repo(Repo::new(model_id, RepoType::Model))
}
pub fn dataset(&self, model_id: String) -> ApiRepo {
self.repo(Repo::new(model_id, RepoType::Dataset))
}
pub fn space(&self, model_id: String) -> ApiRepo {
self.repo(Repo::new(model_id, RepoType::Space))
}
pub fn bucket(&self, bucket_id: String) -> ApiBucket {
ApiBucket {
api: self.clone(),
bucket_id,
}
}
}
#[derive(Debug)]
pub struct ApiRepo {
api: Api,
repo: Repo,
}
impl ApiRepo {
fn new(api: Api, repo: Repo) -> Self {
Self { api, repo }
}
}
impl ApiRepo {
pub fn url(&self, filename: &str) -> String {
proto::file_url(
&self.api.endpoint,
&self.repo.url(),
&self.repo.url_revision(),
filename,
)
}
async fn download_tempfile<P: Progress + Clone + Send + Sync + 'static>(
&self,
url: &str,
length: usize,
filename: PathBuf,
mut progressbar: P,
) -> Result<PathBuf, ApiError> {
let semaphore = Arc::new(Semaphore::new(self.api.max_files));
let parallel_failures_semaphore = Arc::new(Semaphore::new(self.api.parallel_failures));
let start = match tokio::fs::OpenOptions::new()
.read(true)
.open(&filename)
.await
{
Ok(mut f) => {
let len = f.metadata().await?.len();
if len == proto::part_file_len(length as u64) {
f.seek(SeekFrom::Start(length as u64)).await?;
let mut buf = [0u8; proto::TRAILER_LEN];
let n = f.read(buf.as_mut_slice()).await?;
if n == proto::TRAILER_LEN {
proto::decode_trailer(buf) as usize
} else {
0
}
} else {
0
}
}
Err(_err) => {
tokio::fs::File::create(&filename)
.await?
.set_len(proto::part_file_len(length as u64))
.await?;
0
}
};
progressbar.update(start).await;
let chunk_size = self.api.chunk_size.unwrap_or(length);
let n_chunks = length / chunk_size;
let mut handles = Vec::with_capacity(n_chunks);
for (start, stop) in proto::chunk_ranges(start, length, chunk_size) {
let url = url.to_string();
let filename = filename.clone();
let client = self.api.client.clone();
let permit = semaphore.clone();
let parallel_failures = self.api.parallel_failures;
let max_retries = self.api.max_retries;
let parallel_failures_semaphore = parallel_failures_semaphore.clone();
let progress = progressbar.clone();
handles.push(tokio::spawn(async move {
let permit = permit.acquire_owned().await?;
let mut chunk =
Self::download_chunk(&client, &url, &filename, start, stop, progress.clone())
.await;
let mut i = 0;
if parallel_failures > 0 {
while let Err(dlerr) = chunk {
let parallel_failure_permit =
parallel_failures_semaphore.clone().try_acquire_owned()?;
let wait_time = proto::exponential_backoff(300, i, 10_000);
tokio::time::sleep(tokio::time::Duration::from_millis(wait_time as u64))
.await;
chunk = Self::download_chunk(
&client,
&url,
&filename,
start,
stop,
progress.clone(),
)
.await;
i += 1;
if i > max_retries {
return Err(ApiError::TooManyRetries(dlerr.into()));
}
drop(parallel_failure_permit);
}
}
drop(permit);
chunk
}));
}
let mut futures: FuturesUnordered<_> = handles.into_iter().collect();
let mut tracker = proto::CommitTracker::new(start as u64);
while let Some(chunk) = futures.next().await {
let chunk = chunk?;
let (start, stop) = chunk?;
if let Some(committed) = tracker.advance(start, stop) {
let mut file = tokio::fs::OpenOptions::new()
.write(true)
.open(&filename)
.await?;
file.seek(SeekFrom::Start(length as u64)).await?;
file.write_all(&proto::encode_trailer(committed)).await?;
file.flush().await?;
}
}
let mut f = tokio::fs::OpenOptions::new()
.write(true)
.open(&filename)
.await?;
f.set_len(length as u64).await?;
f.flush().await?;
progressbar.finish().await;
Ok(filename)
}
async fn download_chunk<P>(
client: &reqwest::Client,
url: &str,
filename: &PathBuf,
start: usize,
stop: usize,
mut progress: P,
) -> Result<(usize, usize), ApiError>
where
P: Progress,
{
let range = format!("bytes={start}-{stop}");
let response = client
.get(url)
.header(RANGE, range)
.send()
.await?
.error_for_status()?;
let mut byte_stream = response.bytes_stream();
let mut buf: Vec<u8> = Vec::with_capacity(stop - start);
while let Some(next) = byte_stream.next().await {
let next = next?;
buf.extend(&next);
progress.update(next.len()).await;
}
let mut file = tokio::fs::OpenOptions::new()
.write(true)
.open(filename)
.await?;
file.seek(SeekFrom::Start(start as u64)).await?;
file.write_all(&buf).await?;
file.flush().await?;
Ok((start, stop))
}
pub async fn get(&self, filename: &str) -> Result<PathBuf, ApiError> {
if let Some(path) = self.api.cache.repo(self.repo.clone()).get(filename) {
Ok(path)
} else {
self.download(filename).await
}
}
pub async fn download(&self, filename: &str) -> Result<PathBuf, ApiError> {
if self.api.progress {
self.download_with_progress(filename, ProgressBar::new(0))
.await
} else {
self.download_with_progress(filename, ()).await
}
}
pub async fn download_with_progress<P: Progress + Clone + Send + Sync + 'static>(
&self,
filename: &str,
mut progress: P,
) -> Result<PathBuf, ApiError> {
let url = self.url(filename);
let metadata = self.api.metadata(&url).await?;
let cache = self.api.cache.repo(self.repo.clone());
let blob_path = cache.blob_path(&metadata.etag);
std::fs::create_dir_all(blob_path.parent().unwrap())?;
let lock = lock_file(blob_path.clone()).await?;
progress.init(metadata.size, filename).await;
let mut tmp_path = blob_path.clone();
tmp_path.set_extension(EXTENSION);
let tmp_filename = self
.download_tempfile(&url, metadata.size, tmp_path, progress)
.await?;
tokio::fs::rename(&tmp_filename, &blob_path).await?;
drop(lock);
let mut pointer_path = cache.pointer_path(&metadata.commit_hash);
pointer_path.push(filename);
std::fs::create_dir_all(pointer_path.parent().unwrap()).ok();
fs::symlink_or_rename(&blob_path, &pointer_path)?;
cache.create_ref(&metadata.commit_hash)?;
Ok(pointer_path)
}
pub async fn info(&self) -> Result<RepoInfo, ApiError> {
Ok(self
.info_request()
.send()
.await?
.error_for_status()?
.json()
.await?)
}
pub fn info_request(&self) -> RequestBuilder {
let url = format!("{}/api/{}", self.api.endpoint, self.repo.api_url());
self.api.client.get(url)
}
}
#[derive(Debug)]
pub struct ApiBucket {
api: Api,
bucket_id: String,
}
impl ApiBucket {
pub fn url(&self, path: &str) -> String {
proto::bucket_resolve_url(&self.api.endpoint, &self.bucket_id, path)
}
pub fn list(
&self,
prefix: Option<&str>,
recursive: bool,
) -> impl Stream<Item = Result<BucketEntry, ApiError>> {
struct State {
client: Client,
next_url: Option<String>,
page: std::vec::IntoIter<BucketEntry>,
}
async fn fetch(
client: &Client,
url: &str,
) -> Result<(Vec<BucketEntry>, Option<String>), ApiError> {
let response = client.get(url).send().await?.error_for_status()?;
let next = proto::parse_next_link(response.headers())?;
let page = response.json().await?;
Ok((page, next))
}
let state = State {
client: self.api.client.clone(),
next_url: Some(proto::bucket_tree_url(
&self.api.endpoint,
&self.bucket_id,
prefix,
recursive,
)),
page: Vec::new().into_iter(),
};
futures::stream::unfold(state, |mut state| async move {
loop {
if let Some(entry) = state.page.next() {
return Some((Ok(entry), state));
}
let url = state.next_url.take()?;
match fetch(&state.client, &url).await {
Ok((page, next)) => {
state.page = page.into_iter();
state.next_url = next;
}
Err(err) => return Some((Err(err), state)),
}
}
})
}
pub async fn get(&self, path: &str) -> Result<Vec<u8>, ApiError> {
let url = self.url(path);
let mut buf = Vec::new();
let mut res = self.fetch_into(&url, &mut buf).await;
if self.api.max_retries > 0 {
let mut i = 0;
while let Err(dlerr) = res {
let wait_time = proto::exponential_backoff(300, i, 10_000);
tokio::time::sleep(tokio::time::Duration::from_millis(wait_time as u64)).await;
res = self.fetch_into(&url, &mut buf).await;
i += 1;
if i > self.api.max_retries {
return Err(ApiError::TooManyRetries(dlerr.into()));
}
}
}
res?;
Ok(buf)
}
async fn fetch_into(&self, url: &str, buf: &mut Vec<u8>) -> Result<(), ApiError> {
let range = format!("bytes={}-", buf.len());
let response = self
.api
.client
.get(url)
.header(RANGE, range)
.send()
.await?
.error_for_status()?;
let mut byte_stream = response.bytes_stream();
while let Some(next) = byte_stream.next().await {
buf.extend(&next?);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::api::test_server::{BucketFixture, Fixture, TestServer};
use crate::api::Siblings;
use crate::assert_no_diff;
use core::panic;
use futures::TryStreamExt;
use rand::{distr::Alphanumeric, RngExt};
use serde_json::{json, Value};
use std::io::{Seek, Write};
use std::sync::atomic::AtomicBool;
use std::time::Duration;
fn fixture_content(len: usize) -> Vec<u8> {
(1..=255u8).cycle().take(len).collect()
}
struct TempDir {
path: PathBuf,
}
impl TempDir {
pub fn new() -> Self {
let s: String = rand::rng()
.sample_iter(&Alphanumeric)
.take(7)
.map(char::from)
.collect();
let mut path = std::env::temp_dir();
path.push(s);
std::fs::create_dir(&path).unwrap();
Self { path }
}
}
impl Drop for TempDir {
fn drop(&mut self) {
std::fs::remove_dir_all(&self.path).unwrap();
}
}
#[tokio::test]
async fn simple() {
let content: Vec<u8> = fixture_content(1025);
let server = TestServer::spawn(Fixture {
repo: "local/test",
filename: "model.bin",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: true,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(Some("hub_token".to_string()))
.with_cache_dir(tmp.path.clone())
.with_endpoint(server.endpoint())
.with_chunk_size(Some(128))
.with_max_files(4)
.build()
.unwrap();
let repo = api.model("local/test".to_string());
let downloaded_path = repo.download("model.bin").await.unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
assert!(downloaded_path.to_str().unwrap().contains("commitsha456"));
let blob = std::fs::canonicalize(&downloaded_path).unwrap();
assert!(blob.to_str().unwrap().ends_with("etag123abc"));
let log = server.requests();
assert_eq!(log[0].range.as_deref(), Some("bytes=0-0"));
assert_eq!(log[1].path, "/cdn/blob");
let chunks: Vec<_> = log[2..]
.iter()
.filter(|request| request.path == "/cdn/blob")
.collect();
assert_eq!(chunks.len(), 9);
assert!(chunks
.iter()
.any(|request| request.range.as_deref() == Some("bytes=1024-1025")));
drop(server);
let cached_path = api
.model("local/test".to_string())
.get("model.bin")
.await
.unwrap();
assert_eq!(cached_path, downloaded_path);
}
#[tokio::test]
async fn locking() {
use std::sync::Arc;
use tokio::sync::Mutex;
use tokio::task::JoinSet;
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "local/test",
filename: "model.bin",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: false,
});
let endpoint = server.endpoint();
let tmp = Arc::new(Mutex::new(TempDir::new()));
let mut handles = JoinSet::new();
for _ in 0..5 {
let tmp2 = tmp.clone();
let endpoint = endpoint.clone();
handles.spawn(async move {
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(endpoint)
.with_cache_dir(tmp2.lock().await.path.clone())
.build()
.unwrap();
let millis: u64 = rand::random::<u8>().into();
tokio::time::sleep(Duration::from_millis(millis)).await;
api.model("local/test".to_string())
.download("model.bin")
.await
.unwrap()
});
}
while let Some(handle) = handles.join_next().await {
let downloaded_path = handle.unwrap();
assert!(downloaded_path.exists());
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
}
drop(server);
}
#[tokio::test]
async fn resume() {
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "local/test",
filename: "model.bin",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: false,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.with_chunk_size(Some(512))
.build()
.unwrap();
let model_id = "local/test".to_string();
let downloaded_path = api
.model(model_id.clone())
.download("model.bin")
.await
.unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
let blob = std::fs::canonicalize(&downloaded_path).unwrap();
let file = std::fs::OpenOptions::new().write(true).open(&blob).unwrap();
let size = file.metadata().unwrap().len();
let truncate: f32 = rand::random();
let new_size = (size as f32 * truncate) as u64;
file.set_len(new_size).unwrap();
let mut blob_part = blob.clone();
blob_part.set_extension("sync.part");
std::fs::rename(blob, &blob_part).unwrap();
std::fs::remove_file(&downloaded_path).unwrap();
let new_downloaded_path = api
.model(model_id.clone())
.download("model.bin")
.await
.unwrap();
assert_eq!(downloaded_path, new_downloaded_path);
assert_eq!(std::fs::read(&new_downloaded_path).unwrap(), content);
let blob = std::fs::canonicalize(&downloaded_path).unwrap();
let mut file = std::fs::OpenOptions::new().write(true).open(&blob).unwrap();
let size = file.metadata().unwrap().len();
let truncate: f32 = rand::random();
let new_size = (size as f32 * truncate) as u64;
file.set_len(new_size).unwrap();
let total_size = size + size_of::<u64>() as u64;
file.set_len(total_size).unwrap();
file.seek(SeekFrom::Start(size)).unwrap();
file.write_all(&new_size.to_le_bytes()).unwrap();
let mut blob_part = blob.clone();
blob_part.set_extension("sync.part");
std::fs::rename(blob, &blob_part).unwrap();
std::fs::remove_file(&downloaded_path).unwrap();
let new_downloaded_path = api
.model(model_id.clone())
.download("model.bin")
.await
.unwrap();
assert_eq!(downloaded_path, new_downloaded_path);
assert_eq!(std::fs::read(&new_downloaded_path).unwrap(), content);
let blob = std::fs::canonicalize(&downloaded_path).unwrap();
let mut file = std::fs::OpenOptions::new().write(true).open(&blob).unwrap();
let size = file.metadata().unwrap().len();
let truncate: f32 = 0.5;
let new_size = (size as f32 * truncate) as u64;
file.set_len(new_size).unwrap();
let total_size = size + size_of::<u64>() as u64;
file.set_len(total_size).unwrap();
file.seek(SeekFrom::Start(size)).unwrap();
file.write_all(&new_size.to_le_bytes()).unwrap();
file.seek(SeekFrom::Start(new_size - 1)).unwrap();
file.write_all(&[0]).unwrap();
let mut blob_part = blob.clone();
blob_part.set_extension("sync.part");
std::fs::rename(blob, &blob_part).unwrap();
std::fs::remove_file(&downloaded_path).unwrap();
let new_downloaded_path = api
.model(model_id.clone())
.download("model.bin")
.await
.unwrap();
assert_eq!(downloaded_path, new_downloaded_path);
let mut expected = content.clone();
expected[new_size as usize - 1] = 0;
assert_eq!(std::fs::read(&new_downloaded_path).unwrap(), expected);
}
#[tokio::test]
async fn dataset_with_revision() {
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "datasets/wikitext",
filename: "wikitext-103-v1/test/0000.parquet",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: true,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.build()
.unwrap();
let repo = Repo::with_revision(
"wikitext".to_string(),
RepoType::Dataset,
"refs/convert/parquet".to_string(),
);
let downloaded_path = api
.repo(repo)
.download("wikitext-103-v1/test/0000.parquet")
.await
.unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
let log = server.requests();
assert_eq!(
log[0].path,
"/datasets/wikitext/resolve/refs%2Fconvert%2Fparquet/wikitext-103-v1/test/0000.parquet"
);
}
#[tokio::test]
async fn model_with_revision() {
let content = fixture_content(4096);
let server = TestServer::spawn(Fixture {
repo: "BAAI/bGe-reRanker-Base",
filename: "tokenizer.json",
content: content.clone(),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: true,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.build()
.unwrap();
let repo = Repo::with_revision(
"BAAI/bGe-reRanker-Base".to_string(),
RepoType::Model,
"refs/pr/5".to_string(),
);
let downloaded_path = api
.repo(repo.clone())
.download("tokenizer.json")
.await
.unwrap();
assert_eq!(std::fs::read(&downloaded_path).unwrap(), content);
let log = server.requests();
assert_eq!(
log[0].path,
"/BAAI/bGe-reRanker-Base/resolve/refs%2Fpr%2F5/tokenizer.json"
);
let cache_path = api.cache.repo(repo).get("tokenizer.json").unwrap();
assert_eq!(cache_path, downloaded_path);
}
#[tokio::test]
async fn info() {
let server = TestServer::spawn(Fixture {
repo: "datasets/wikitext",
filename: "wikitext-103-v1/test/0000.parquet",
content: fixture_content(16),
etag: "etag123abc",
commit: "infosha",
relative_redirects: 0,
cdn_redirect: false,
});
let tmp = TempDir::new();
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_cache_dir(tmp.path.clone())
.build()
.unwrap();
let repo = Repo::with_revision(
"wikitext".to_string(),
RepoType::Dataset,
"refs/convert/parquet".to_string(),
);
let model_info = api.repo(repo.clone()).info().await.unwrap();
assert_eq!(
model_info,
RepoInfo {
siblings: vec![Siblings {
rfilename: "wikitext-103-v1/test/0000.parquet".to_string()
}],
sha: "infosha".to_string()
}
);
let blobs_info: Value = api
.repo(repo)
.info_request()
.query(&[("blobs", "true")])
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_no_diff!(
blobs_info,
json!({
"siblings": [{"rfilename": "wikitext-103-v1/test/0000.parquet"}],
"sha": "infosha"
})
);
}
#[test]
fn headers_default() {
let headers = ApiBuilder::new().build_headers().unwrap();
assert_eq!(
headers.get(USER_AGENT),
Some(
&format!(
"unknown/None; {}/{}; rust/unknown",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
)
.try_into()
.unwrap()
)
);
}
#[test]
fn headers_custom() {
let headers = ApiBuilder::new()
.with_user_agent("origin", "custom")
.build_headers()
.unwrap();
assert_eq!(
headers.get(USER_AGENT),
Some(
&format!(
"unknown/None; {}/{}; rust/unknown; origin/custom",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
)
.try_into()
.unwrap()
)
);
}
#[tokio::test]
async fn repo_info_failed() {
let server = TestServer::spawn(Fixture {
repo: "local/other",
filename: "model.bin",
content: fixture_content(16),
etag: "etag123abc",
commit: "commitsha456",
relative_redirects: 0,
cdn_redirect: false,
});
let api = ApiBuilder::new()
.with_progress(true)
.with_endpoint(server.endpoint())
.with_token(Some("".to_string()))
.build()
.expect("failed to build API");
let repo = api.model("mcpotato/this-repo-does-not-exist".to_string());
let error = repo
.info()
.await
.expect_err("failed to raise authentication error");
match error {
ApiError::RequestError(error) if error.is_status() => {}
_ => panic!("unexpected request error"),
}
}
#[tokio::test]
async fn bucket_list_paginated() {
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![
r#"[{"type":"file","path":"data/a.bin","size":4,"xetHash":"hashaaa","mtime":"2024-01-15T10:30:00.123Z","uploadedAt":"2024-01-16T00:00:00Z"}]"#,
r#"[{"type":"directory","path":"data/sub","uploadedAt":"2024-01-16T00:00:00Z"},{"type":"file","path":"data/b.bin","size":7,"xetHash":"hashbbb","uploadedAt":"2024-01-17T00:00:00Z"}]"#,
],
files: vec![],
truncate_first_cdn_response: AtomicBool::new(false),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(Some("hub_token".to_string()))
.with_endpoint(server.endpoint())
.build()
.unwrap();
let bucket = api.bucket("myorg/mybucket".to_string());
let mut entries = std::pin::pin!(bucket.list(None, true));
assert_eq!(
entries.next().await.unwrap().unwrap(),
BucketEntry::File {
path: "data/a.bin".to_string(),
size: 4,
xet_hash: "hashaaa".to_string(),
uploaded_at: "2024-01-16T00:00:00Z".to_string(),
}
);
let log = server.requests();
assert_eq!(
log[0].path,
"/api/buckets/myorg/mybucket/tree?recursive=true"
);
assert_eq!(log[0].authorization.as_deref(), Some("Bearer hub_token"));
assert_eq!(log.len(), 1);
let rest: Vec<BucketEntry> = entries.try_collect().await.unwrap();
assert_eq!(
rest,
vec![
BucketEntry::Directory {
path: "data/sub".to_string(),
uploaded_at: "2024-01-16T00:00:00Z".to_string(),
},
BucketEntry::File {
path: "data/b.bin".to_string(),
size: 7,
xet_hash: "hashbbb".to_string(),
uploaded_at: "2024-01-17T00:00:00Z".to_string(),
},
]
);
let log = server.requests();
assert_eq!(log[0].path, "/api/buckets/myorg/mybucket/tree?cursor=1");
assert_eq!(log.len(), 1);
let _: Vec<BucketEntry> = bucket
.list(Some("data/sub"), false)
.try_collect()
.await
.unwrap();
let log = server.requests();
assert_eq!(
log[0].path,
"/api/buckets/myorg/mybucket/tree/data%2Fsub?recursive=false"
);
}
#[tokio::test]
async fn bucket_get() {
let content = fixture_content(1030);
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![],
files: vec![("data%2Ffile.bin", content.clone())],
truncate_first_cdn_response: AtomicBool::new(false),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(Some("hub_token".to_string()))
.with_endpoint(server.endpoint())
.build()
.unwrap();
let bytes = api
.bucket("myorg/mybucket".to_string())
.get("data/file.bin")
.await
.unwrap();
assert_eq!(bytes, content);
let log = server.requests();
assert_eq!(
log[0].path,
"/buckets/myorg/mybucket/resolve/data%2Ffile.bin"
);
assert_eq!(log[0].range.as_deref(), Some("bytes=0-"));
assert_eq!(log[0].authorization.as_deref(), Some("Bearer hub_token"));
assert_eq!(log[1].path, "/cdn/bucket/data%2Ffile.bin");
}
#[tokio::test]
async fn bucket_get_resume() {
let content = fixture_content(4096);
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![],
files: vec![("model.bin", content.clone())],
truncate_first_cdn_response: AtomicBool::new(true),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.with_retries(3)
.build()
.unwrap();
let bytes = api
.bucket("myorg/mybucket".to_string())
.get("model.bin")
.await
.unwrap();
assert_eq!(bytes, content);
let log = server.requests();
assert_eq!(log.last().unwrap().range.as_deref(), Some("bytes=2048-"));
}
#[tokio::test]
async fn bucket_unicode_path() {
let server = TestServer::spawn_bucket(BucketFixture {
bucket_id: "myorg/mybucket",
tree_pages: vec![],
files: vec![("sub%20dir%2Fh%C3%A9llo.txt", b"unicode!".to_vec())],
truncate_first_cdn_response: AtomicBool::new(false),
});
let api = ApiBuilder::new()
.with_progress(false)
.with_token(None)
.with_endpoint(server.endpoint())
.build()
.unwrap();
let bytes = api
.bucket("myorg/mybucket".to_string())
.get("sub dir/héllo.txt")
.await
.unwrap();
assert_eq!(bytes, b"unicode!");
let log = server.requests();
assert_eq!(
log[0].path,
"/buckets/myorg/mybucket/resolve/sub%20dir%2Fh%C3%A9llo.txt"
);
}
}