use alloc::{
collections::BTreeSet,
format,
string::{
String,
ToString,
},
vec::Vec,
};
#[cfg(feature = "cache")]
use crate::data::resources::ResourceCache;
use crate::{
data::{
builds::BuildProvider,
catalog::{
CatalogName,
ResolvedBuild,
document::{
ResourceEntry,
ResourceShard,
ResourceSource,
},
},
resources::{
Resource,
SAFETENSORS,
SAFETENSORS_INDEX,
Source,
hub::{
HUB_MAIN,
HUB_ORIGIN,
HubEntry,
HubRepo,
},
},
},
errors::{
BunsenError,
BunsenErrorKind,
BunsenResult,
ParseError,
},
};
pub const HF: &str = "hf";
const HF_SINGLE_FILE: &str = "model.safetensors";
const HF_INDEX_FILE: &str = "model.safetensors.index.json";
const HF_CONFIG_FILE: &str = "config.json";
pub const CONFIG: &str = "config";
const DEFAULT_CHECKPOINT_KEY: &str = "checkpoint";
fn shard_numbers(file: &str) -> Option<(usize, usize)> {
let rest = file.strip_prefix("model-")?.strip_suffix(".safetensors")?;
let (n, of) = rest.split_once("-of-")?;
Some((n.parse().ok()?, of.parse().ok()?))
}
fn missing_shards(
of: usize,
shards: &[(usize, usize, &HubEntry)],
) -> String {
let present: BTreeSet<usize> = shards
.iter()
.map(|(n, _, _)| *n)
.filter(|n| (1..=of).contains(n))
.collect();
let count = of - present.len();
if count == 0 {
return String::new();
}
let named: Vec<String> = (1..=of)
.filter(|i| !present.contains(i))
.take(8)
.map(|i| format!("model-{i:05}-of-{of:05}.safetensors"))
.collect();
let more = if count > named.len() {
format!(", and {} more", count - named.len())
} else {
String::new()
};
format!("; missing {}{more}", named.join(", "))
}
#[derive(Clone, Debug)]
pub struct HfProvider {
pub name: String,
pub origins: Vec<String>,
pub revision: String,
pub checkpoint_key: String,
pub extra_files: Vec<HfExtraFile>,
pub archive_checkpoints: Vec<HfArchiveCheckpoint>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct HfArchiveCheckpoint {
pub extension: String,
pub kind: String,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct HfExtraFile {
pub key: String,
pub path: String,
pub kind: String,
}
impl Default for HfProvider {
fn default() -> Self {
Self::new()
}
}
impl HfProvider {
pub fn new() -> Self {
let origins = crate::data::catalog::BuiltinCatalog::get()
.map(|catalog| catalog.hub_origins().to_vec())
.unwrap_or_default();
Self {
name: HF.to_string(),
origins: if origins.is_empty() {
vec![HUB_ORIGIN.to_string()]
} else {
origins
},
revision: HUB_MAIN.to_string(),
checkpoint_key: DEFAULT_CHECKPOINT_KEY.to_string(),
extra_files: Vec::new(),
archive_checkpoints: Vec::new(),
}
}
pub fn with_origin(
self,
origin: impl Into<String>,
) -> Self {
self.with_origins([origin])
}
pub fn with_origins(
mut self,
origins: impl IntoIterator<Item = impl Into<String>>,
) -> Self {
self.origins = origins.into_iter().map(Into::into).collect();
assert!(!self.origins.is_empty(), "a Hub provider needs an origin");
self
}
pub fn with_name(
mut self,
name: impl Into<String>,
) -> Self {
self.name = name.into();
self
}
pub fn with_revision(
mut self,
revision: impl Into<String>,
) -> Self {
self.revision = revision.into();
self
}
pub fn with_checkpoint_key(
mut self,
key: impl Into<String>,
) -> Self {
self.checkpoint_key = key.into();
self
}
pub fn with_extra_file(
mut self,
key: impl Into<String>,
path: impl Into<String>,
kind: impl Into<String>,
) -> Self {
self.extra_files.push(HfExtraFile {
key: key.into(),
path: path.into(),
kind: kind.into(),
});
self
}
pub fn with_archive_checkpoint(
mut self,
extension: impl Into<String>,
kind: impl Into<String>,
) -> Self {
self.archive_checkpoints.push(HfArchiveCheckpoint {
extension: extension.into(),
kind: kind.into(),
});
self
}
pub fn urls(
&self,
org: &str,
repo: &str,
file: &str,
) -> Vec<String> {
let hub = HubRepo::new(format!("{org}/{repo}"), &self.revision);
self.origins
.iter()
.map(|origin| hub.file_url_at(origin, file))
.collect()
}
pub fn tree_urls(
&self,
org: &str,
repo: &str,
) -> Vec<String> {
let hub = HubRepo::new(format!("{org}/{repo}"), &self.revision);
self.origins.iter().map(|o| hub.tree_url_at(o)).collect()
}
pub fn split<'a>(
&self,
name: &'a str,
) -> BunsenResult<(&'a str, &'a str)> {
match name.split_once('/') {
Some((org, repo)) if !org.is_empty() && !repo.is_empty() && !repo.contains('/') => {
Ok((org, repo))
}
_ => Err(BunsenError::from_cause(
BunsenErrorKind::Illegal,
ParseError::new("Hugging Face ref")
.input(format!("{}:{name}", self.name))
.because("a ref is org/repo"),
)),
}
}
pub fn listing_resource(
&self,
org: &str,
repo: &str,
) -> Resource {
Resource {
size: None,
dtype: None,
slot: "listing".to_string(),
file: format!("{org}--{repo}--{}.json", self.revision.replace('/', "-")),
sha256: None,
format: Some("json".to_string()),
name: self.resource_name(org, repo, "listing"),
sources: self
.tree_urls(org, repo)
.into_iter()
.map(Source::Url)
.collect(),
}
}
pub fn resource_name(
&self,
org: &str,
repo: &str,
slot: &str,
) -> String {
format!("{}/{org}/{repo}/{slot}", self.name)
}
fn entry(
&self,
org: &str,
repo: &str,
entry: &HubEntry,
format: &str,
) -> ResourceEntry {
let mut declared = ResourceEntry::new(
entry.path.clone(),
format,
self.urls(org, repo, &entry.path)
.into_iter()
.map(ResourceSource::Url)
.collect(),
);
declared.sha256 = entry.sha256();
declared.size = Some(entry.size);
declared
}
fn catalog_name(
&self,
org: &str,
repo: &str,
slot: Option<&str>,
) -> BunsenResult<CatalogName> {
CatalogName::parse(&match slot {
Some(slot) => self.resource_name(org, repo, slot),
None => format!("{}/{org}/{repo}", self.name),
})
}
pub fn build_from_listing(
&self,
name: &str,
listing: &[HubEntry],
) -> BunsenResult<ResolvedBuild> {
let (org, repo) = self.split(name)?;
let files: Vec<&HubEntry> = listing.iter().filter(|e| e.is_file()).collect();
let file = |path: &str| files.iter().copied().find(|e| e.path == path);
let origin = format!("{}/{org}/{repo}", self.origins[0]);
let key = self.checkpoint_key.as_str();
let mut resources: Vec<(&str, ResourceEntry)> = Vec::new();
let what;
if let Some(single) = file(HF_SINGLE_FILE) {
resources.push((key, self.entry(org, repo, single, SAFETENSORS)));
what = format!("{HF_SINGLE_FILE}, {}", megabytes(single.size));
} else if let Some(index) = file(HF_INDEX_FILE) {
let mut shards: Vec<(usize, usize, &HubEntry)> = files
.iter()
.filter_map(|e| shard_numbers(&e.path).map(|(n, of)| (n, of, *e)))
.collect();
shards.sort_by_key(|(n, _, _)| *n);
let of = shards.first().map(|(_, of, _)| *of).unwrap_or(0);
let one_set = shards.iter().all(|(_, o, _)| *o == of);
let complete = !shards.is_empty()
&& one_set
&& shards.len() == of
&& shards.iter().enumerate().all(|(i, (n, _, _))| *n == i + 1);
if !complete {
let has: Vec<&str> = shards.iter().map(|(_, _, e)| e.path.as_str()).collect();
let mut message = format!(
"{HF_INDEX_FILE} with an incomplete set of shards: {}",
if has.is_empty() {
"none".to_string()
} else {
has.join(", ")
}
);
if one_set {
message.push_str(&missing_shards(of, &shards));
}
return Err(
BunsenError::invalid_resource(message).context(format!("{}:{name}", self.name))
);
}
let mut checkpoint = self.entry(org, repo, index, SAFETENSORS_INDEX);
let mut total = 0;
for (_, _, entry) in &shards {
total += entry.size;
let sha256 = entry.sha256().ok_or_else(|| {
BunsenError::invalid_resource(format!(
"the shard {} is not in LFS, so nothing pins it",
entry.path
))
.context(format!("{}:{name}", self.name))
})?;
checkpoint.shards.push(ResourceShard {
file: entry.path.clone(),
sha256,
size: entry.size,
sources: self
.urls(org, repo, &entry.path)
.into_iter()
.map(ResourceSource::Url)
.collect(),
});
}
resources.push((key, checkpoint));
what = format!("{} shards, {}", shards.len(), megabytes(total));
} else if let Some((archive, kind)) = self.archive_checkpoint(name, &files)? {
resources.push((key, self.entry(org, repo, archive, kind)));
what = format!("{}, {}", archive.path, megabytes(archive.size));
} else {
let weights: Vec<&str> = files
.iter()
.map(|e| e.path.as_str())
.filter(|p| {
p.ends_with(".safetensors")
|| p.ends_with(".bin")
|| p.ends_with(".h5")
|| p.ends_with(".msgpack")
|| p.ends_with(".pt")
|| p.ends_with(".gguf")
|| p.ends_with(".nemo")
})
.collect();
let archives: String = self
.archive_checkpoints
.iter()
.map(|a| format!(", or one *{}", a.extension))
.collect();
return Err(BunsenError::invalid_resource(format!(
"no {HF_SINGLE_FILE} or {HF_INDEX_FILE}{archives} at {}; the repo has: {}",
self.revision,
if weights.is_empty() {
"no weights".to_string()
} else {
weights.join(", ")
}
))
.context(format!("{}:{name}", self.name)));
}
let mut with: Vec<&str> = Vec::new();
if let Some(config) = file(HF_CONFIG_FILE) {
resources.push((CONFIG, self.entry(org, repo, config, "json")));
with.push(HF_CONFIG_FILE);
}
for extra in &self.extra_files {
if let Some(entry) = file(&extra.path) {
resources.push((&extra.key, self.entry(org, repo, entry, &extra.kind)));
with.push(&extra.path);
}
}
let with = if with.is_empty() {
String::new()
} else {
format!(", with {}", with.join(" and "))
};
let description = format!(
"{org}/{repo} on Hugging Face at {}: {what}{with}",
self.revision
);
let mut build = ResolvedBuild::new(self.catalog_name(org, repo, None)?, description);
build.origin = Some(origin);
for (slot, entry) in resources {
build = build.with_resource(slot, self.catalog_name(org, repo, Some(slot))?, entry);
}
Ok(build)
}
fn archive_checkpoint<'e>(
&'e self,
name: &str,
files: &[&'e HubEntry],
) -> BunsenResult<Option<(&'e HubEntry, &'e str)>> {
for archive in &self.archive_checkpoints {
let found: Vec<&HubEntry> = files
.iter()
.copied()
.filter(|e| e.path.ends_with(&archive.extension))
.collect();
match found.as_slice() {
[] => continue,
[one] => return Ok(Some((one, archive.kind.as_str()))),
many => {
let names: Vec<&str> = many.iter().map(|e| e.path.as_str()).collect();
return Err(BunsenError::invalid_resource(format!(
"{} {} files, and no way to tell the checkpoint: {}",
many.len(),
archive.extension,
names.join(", ")
))
.context(format!("{}:{name}", self.name)));
}
}
}
Ok(None)
}
#[cfg(feature = "cache")]
fn listing(
&self,
name: &str,
cache: &ResourceCache,
) -> BunsenResult<Vec<HubEntry>> {
use crate::errors::{
LookupError,
LookupProblem,
ResultContext,
sys_at,
};
let (org, repo) = self.split(name)?;
let resource = self.listing_resource(org, repo);
let resolved = cache.resolve(&resource);
let unauthorized = resolved
.as_ref()
.err()
.and_then(|e| e.find::<LookupError>())
.is_some_and(|l| l.problem == LookupProblem::Unauthorized);
let resolved = resolved.with_context(|| {
if unauthorized {
format!(
"{}:{name}: the repo does not exist, or is gated (bunsen fetches anonymously, so a gated repo is unreachable)",
self.name
)
} else {
format!("{}:{name}: file listing", self.name)
}
})?;
let path = &resolved.path;
let file = std::fs::File::open(path)
.map_err(sys_at("open", path))
.with_context(|| format!("{}:{name}: file listing", self.name))?;
serde_json::from_reader(std::io::BufReader::new(file)).map_err(|e| {
BunsenError::from_cause(
BunsenErrorKind::InvalidResource,
ParseError::new("Hugging Face file listing")
.at(path.display())
.with_source(e),
)
.with_details(format!(
"the listing is cached unpinned; remove {} to fetch it again",
path.display()
))
.context(format!("{}:{name}: file listing", self.name))
})
}
}
impl BuildProvider for HfProvider {
fn name(&self) -> &str {
&self.name
}
fn description(&self) -> &str {
"Hugging Face repos by ref: `hf:org/repo` is its safetensors checkpoint at a revision, pinned by the hub's listing; lists nothing"
}
fn origin(&self) -> Option<&str> {
Some(HUB_ORIGIN)
}
fn list(&self) -> Vec<ResolvedBuild> {
Vec::new()
}
fn lookup(
&self,
name: &str,
) -> BunsenResult<Option<ResolvedBuild>> {
self.split(name)?;
Err(BunsenError::policy(
"a Hugging Face ref is resolved through a cache, which asks the hub what the repo holds",
)
.context(format!("{}:{name}", self.name)))
}
#[cfg(feature = "cache")]
fn resolve(
&self,
name: &str,
cache: &ResourceCache,
) -> BunsenResult<Option<ResolvedBuild>> {
let listing = self.listing(name, cache)?;
self.build_from_listing(name, &listing).map(Some)
}
fn answers_bare_names(&self) -> bool {
false
}
}
fn megabytes(bytes: u64) -> String {
if bytes >= 1_000_000_000 {
format!("{:.2} GB", bytes as f64 / 1e9)
} else {
format!("{:.1} MB", bytes as f64 / 1e6)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
data::resources::hub::HubLfs,
errors::testing::{
ErrorMatcher,
text,
},
};
fn entry(
path: &str,
size: u64,
sha256: Option<&str>,
) -> HubEntry {
HubEntry {
kind: "file".to_string(),
path: path.to_string(),
size,
lfs: sha256.map(|oid| HubLfs {
oid: oid.to_string(),
size,
}),
}
}
const TINY_SHA: &str = "7ebd0e69e78190ff5b1b2e6c7c0a6a9b1a4e2f0f0b3d2c1a9f8e7d6c5b4a3f2e";
const SHARD_1: &str = "1111111111111111111111111111111111111111111111111111111111111111";
const SHARD_2: &str = "2222222222222222222222222222222222222222222222222222222222222222";
#[test]
fn test_a_single_file_repo_is_one_pinned_checkpoint() {
let hf = HfProvider::new();
let listing = vec![
entry(".gitattributes", 1438, None),
entry("config.json", 1983, None),
entry("model.safetensors", 151_061_672, Some(TINY_SHA)),
entry("pytorch_model.bin", 151_095_027, Some(SHARD_1)),
entry("tokenizer.json", 2_480_466, None),
];
let row = hf
.build_from_listing("openai/whisper-tiny", &listing)
.unwrap();
assert_eq!(row.name, "hf/openai/whisper-tiny");
assert_eq!(hf.id(&row), "hf:openai/whisper-tiny");
assert_eq!(
row.origin.as_deref(),
Some("https://huggingface.co/openai/whisper-tiny")
);
assert_eq!(
row.description,
"openai/whisper-tiny on Hugging Face at main: model.safetensors, 151.1 MB, with config.json"
);
row.resource_set().validate().unwrap();
assert_eq!(row.resource_set().keys(), ["checkpoint", "config"]);
let set_checkpoint = row.resource_set();
let checkpoint = set_checkpoint.get("checkpoint").unwrap();
assert_eq!(checkpoint.file, "model.safetensors");
assert_eq!(checkpoint.sha256.as_deref(), Some(TINY_SHA));
assert_eq!(checkpoint.format.as_deref(), Some("safetensors"));
assert_eq!(checkpoint.name, "hf/openai/whisper-tiny/checkpoint");
assert_eq!(
checkpoint.sources,
[Source::Url(
"https://huggingface.co/openai/whisper-tiny/resolve/main/model.safetensors"
.to_string()
)]
);
let set_config = row.resource_set();
let config = set_config.get("config").unwrap();
assert_eq!(config.file, "config.json");
assert_eq!(config.sha256, None);
assert_eq!(config.format.as_deref(), Some("json"));
}
#[test]
fn test_extra_files_ride_along() {
let hf = HfProvider::new()
.with_extra_file("vocabulary", "tokenizer.json", "tokenizer.json")
.with_extra_file("merges", "merges.txt", "text");
let listing = vec![
entry("config.json", 1983, None),
entry("model.safetensors", 151_061_672, Some(TINY_SHA)),
entry("tokenizer.json", 2_480_466, Some(SHARD_1)),
];
let row = hf.build_from_listing("org/asr", &listing).unwrap();
assert_eq!(
row.resource_set().keys(),
["checkpoint", "config", "vocabulary"]
);
let set_vocab = row.resource_set();
let vocab = set_vocab.get("vocabulary").unwrap();
assert_eq!(vocab.file, "tokenizer.json");
assert_eq!(vocab.format.as_deref(), Some("tokenizer.json"));
assert_eq!(vocab.sha256.as_deref(), Some(SHARD_1));
assert_eq!(
row.description,
"org/asr on Hugging Face at main: model.safetensors, 151.1 MB, with config.json and tokenizer.json"
);
}
#[test]
fn test_a_sharded_repo_is_the_index_and_its_shards() {
let hf = HfProvider::new().with_checkpoint_key("weights");
let listing = vec![
entry(
"model-00002-of-00002.safetensors",
1_170_000_000,
Some(SHARD_2),
),
entry(
"model-00001-of-00002.safetensors",
5_000_000_000,
Some(SHARD_1),
),
entry("model.safetensors.index.json", 71_000, None),
];
let row = hf
.build_from_listing("ivrit-ai/whisper-large-v3", &listing)
.unwrap();
assert_eq!(
row.resource_set().keys(),
["weights.00001", "weights.00002", "weights.index"]
);
assert_eq!(
row.description,
"ivrit-ai/whisper-large-v3 on Hugging Face at main: 2 shards, 6.17 GB"
);
let set_index = row.resource_set();
let index = set_index.get("weights.index").unwrap();
assert_eq!(index.file, "model.safetensors.index.json");
assert_eq!(index.format.as_deref(), Some("safetensors-index"));
assert_eq!(index.sha256, None);
let set_first = row.resource_set();
let first = set_first.get("weights.00001").unwrap();
assert_eq!(first.file, "model-00001-of-00002.safetensors");
assert_eq!(first.sha256.as_deref(), Some(SHARD_1));
assert_eq!(first.format.as_deref(), Some("safetensors"));
let incomplete = vec![
entry("model-00002-of-00002.safetensors", 1, Some(SHARD_2)),
entry("model.safetensors.index.json", 71_000, None),
];
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("incomplete set of shards")
.frame_contains("hf:ivrit-ai/whisper-large-v3")
.assert_err(&hf.build_from_listing("ivrit-ai/whisper-large-v3", &incomplete));
}
#[test]
fn test_a_sharded_repo_missing_its_last_shards_is_refused() {
let hf = HfProvider::new();
let listing = vec![
entry("model.safetensors.index.json", 71_000, None),
entry("model-00001-of-00003.safetensors", 1, Some(SHARD_1)),
entry("model-00002-of-00003.safetensors", 1, Some(SHARD_2)),
];
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("incomplete set of shards")
.message_contains("missing model-00003-of-00003.safetensors")
.assert_err(&hf.build_from_listing("org/repo", &listing));
let listing = vec![
entry("model.safetensors.index.json", 71_000, None),
entry("model-00001-of-00012.safetensors", 1, Some(SHARD_1)),
];
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("missing model-00002-of-00012.safetensors, ")
.message(text::ends_with(
"model-00009-of-00012.safetensors, and 3 more",
))
.assert_err(&hf.build_from_listing("org/repo", &listing));
}
#[test]
fn test_what_is_refused() {
let hf = HfProvider::new();
let listing = vec![
entry("pytorch_model.bin", 10, Some(SHARD_1)),
entry("tf_model.h5", 10, Some(SHARD_2)),
entry("README.md", 10, None),
];
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("no model.safetensors")
.message_contains("pytorch_model.bin, tf_model.h5")
.assert_err(&hf.build_from_listing("some/repo", &listing));
for bad in [
"whisper-tiny",
"openai/",
"/whisper-tiny",
"openai/whisper/tiny",
"",
] {
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("org/repo")
.has_cause::<ParseError>()
.assert_err(&hf.build_from_listing(bad, &[]));
}
ErrorMatcher::kind(BunsenErrorKind::Policy)
.message_contains("through a cache")
.assert_err(&hf.lookup("openai/whisper-tiny"));
assert!(hf.list().is_empty());
assert!(hf.ids().is_empty());
assert!(!hf.answers_bare_names());
assert_eq!(hf.origin(), Some("https://huggingface.co"));
}
#[test]
fn test_an_archive_checkpoint_stands_in() {
let hf = HfProvider::new().with_archive_checkpoint(".nemo", "nemo");
let listing = vec![
entry(".gitattributes", 1438, None),
entry("README.md", 10, None),
entry(
"diar_streaming_sortformer_4spk-v2.nemo",
471_367_680,
Some(TINY_SHA),
),
];
let row = hf
.build_from_listing("nvidia/diar_streaming_sortformer_4spk-v2", &listing)
.unwrap();
row.resource_set().validate().unwrap();
assert_eq!(row.resource_set().keys(), ["checkpoint"]);
let set_checkpoint = row.resource_set();
let checkpoint = set_checkpoint.get("checkpoint").unwrap();
assert_eq!(checkpoint.file, "diar_streaming_sortformer_4spk-v2.nemo");
assert_eq!(checkpoint.format.as_deref(), Some("nemo"));
assert_eq!(checkpoint.sha256.as_deref(), Some(TINY_SHA));
assert!(
row.description
.ends_with(": diar_streaming_sortformer_4spk-v2.nemo, 471.4 MB"),
"{}",
row.description
);
let mut both = listing.clone();
both.push(entry("model.safetensors", 10, Some(SHARD_1)));
let row = hf.build_from_listing("org/repo", &both).unwrap();
assert_eq!(
row.resource_set()
.get("checkpoint")
.unwrap()
.format
.as_deref(),
Some("safetensors")
);
let mut two = listing.clone();
two.push(entry("other.nemo", 10, Some(SHARD_1)));
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("2 .nemo files")
.message_contains("other.nemo")
.frame_contains("hf:org/repo")
.assert_err(&hf.build_from_listing("org/repo", &two));
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("the repo has: diar_streaming_sortformer_4spk-v2.nemo")
.assert_err(&HfProvider::new().build_from_listing("org/repo", &listing));
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("or one *.nemo at main; the repo has: no weights")
.assert_err(&hf.build_from_listing("org/repo", &listing[..2]));
}
#[test]
fn test_urls_carry_the_revision() {
let hf = HfProvider::new().with_revision("06f233fe06e710322aca913c1bc4249a0d71fce1");
assert_eq!(
hf.urls("openai", "whisper-large-v3", "model.safetensors"),
[
"https://huggingface.co/openai/whisper-large-v3/resolve/06f233fe06e710322aca913c1bc4249a0d71fce1/model.safetensors"
]
);
assert_eq!(
hf.tree_urls("openai", "whisper-large-v3"),
[
"https://huggingface.co/api/models/openai/whisper-large-v3/tree/06f233fe06e710322aca913c1bc4249a0d71fce1"
]
);
let mirrored = HfProvider::new()
.with_revision("abc")
.with_origins(["https://huggingface.co", "https://mirror.example"]);
assert_eq!(
mirrored.urls("o", "r", "config.json"),
[
"https://huggingface.co/o/r/resolve/abc/config.json",
"https://mirror.example/o/r/resolve/abc/config.json",
]
);
assert_eq!(
mirrored.listing_resource("o", "r").urls(),
[
"https://huggingface.co/api/models/o/r/tree/abc",
"https://mirror.example/api/models/o/r/tree/abc",
]
);
let listing = hf.listing_resource("openai", "whisper-large-v3");
assert_eq!(
listing.file,
"openai--whisper-large-v3--06f233fe06e710322aca913c1bc4249a0d71fce1.json"
);
assert_eq!(listing.name, "hf/openai/whisper-large-v3/listing");
listing.validate().unwrap();
let named = HfProvider::new().with_name("hf-dev").with_revision("dev");
assert_eq!(named.name(), "hf-dev");
assert_eq!(named.listing_resource("o", "r").name, "hf-dev/o/r/listing");
}
#[test]
fn test_the_listing_parses() {
let json = r#"[
{"type":"file","oid":"abc","size":1983,"path":"config.json"},
{"type":"file","oid":"def","size":151061672,"path":"model.safetensors",
"lfs":{"oid":"7ebd0e69e78190ff5b1b2e6c7c0a6a9b1a4e2f0f0b3d2c1a9f8e7d6c5b4a3f2e","size":151061672,"pointerSize":135}},
{"type":"directory","oid":"ghi","size":0,"path":"onnx"}
]"#;
let listing: Vec<HubEntry> = serde_json::from_str(json).unwrap();
assert_eq!(listing.len(), 3);
assert_eq!(listing[0].sha256(), None);
assert_eq!(listing[1].sha256().as_deref(), Some(TINY_SHA));
assert_eq!(listing[2].kind, "directory");
let row = HfProvider::new()
.build_from_listing("openai/whisper-tiny", &listing)
.unwrap();
assert_eq!(row.resource_set().keys(), ["checkpoint", "config"]);
}
#[test]
fn test_an_uppercase_digest_pins_in_lowercase() {
let json = format!(
r#"[{{"type":"file","size":3,"path":"model.safetensors","lfs":{{"oid":"{}","size":3}}}}]"#,
TINY_SHA.to_uppercase()
);
let listing: Vec<HubEntry> = serde_json::from_str(&json).unwrap();
assert_eq!(listing[0].sha256().as_deref(), Some(TINY_SHA));
let row = HfProvider::new()
.build_from_listing("org/repo", &listing)
.unwrap();
row.resource_set().validate().unwrap();
assert_eq!(
row.resource_set()
.get("checkpoint")
.unwrap()
.sha256
.as_deref(),
Some(TINY_SHA)
);
}
#[cfg(feature = "fetch")]
#[test]
fn test_resolve_fetches_the_listing_once_and_keeps_it() {
use crate::data::{
cache::{
BunsenDiskCacheOptions,
testing::{
serve_once,
serve_status,
},
},
resources::{
CacheStatus,
ResourceCacheOptions,
},
};
static LISTING: &[u8] = br#"[
{"type":"file","oid":"abc","size":1983,"path":"config.json"},
{"type":"file","oid":"def","size":3,"path":"model.safetensors",
"lfs":{"oid":"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad","size":3,"pointerSize":135}}
]"#;
let dir = tempfile::tempdir().unwrap();
let cache_at = |offline: bool| {
ResourceCache::new(
ResourceCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.path().join("cache")))
.without_transfer_observers(),
)
.with_offline(offline),
)
.unwrap()
};
let origin_of = |url: String| url.rsplit_once('/').unwrap().0.to_string();
let hf = HfProvider::new().with_origin(origin_of(serve_once("any", LISTING)));
let online = cache_at(false);
let row = hf.resolve("org/repo", &online).unwrap().unwrap();
assert_eq!(row.name, "hf/org/repo");
let set_checkpoint = row.resource_set();
let checkpoint = set_checkpoint.get("checkpoint").unwrap();
assert_eq!(
checkpoint.sha256.as_deref(),
Some("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
);
assert_eq!(
checkpoint.sources,
[Source::Url(format!(
"{}/org/repo/resolve/main/model.safetensors",
hf.origins[0]
))]
);
assert_eq!(
online.status(&hf.listing_resource("org", "repo")),
CacheStatus::Cached,
"the listing stays in the cache"
);
let offline = cache_at(true);
let again = hf.resolve("org/repo", &offline).unwrap().unwrap();
assert_eq!(again, row);
ErrorMatcher::kind(BunsenErrorKind::Policy)
.display(text::starts_with("hf:org/other: file listing: "))
.message_contains("the cache is offline")
.assert_err(&hf.resolve("org/other", &offline));
let gone = HfProvider::new().with_origin(origin_of(serve_status("any", 401)));
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.display(text::starts_with(
"hf:nobody/nothing: the repo does not exist, or is gated",
))
.cause(crate::errors::testing::predicate(
"an unauthorized lookup",
|c: &crate::errors::LookupError| {
c.problem == crate::errors::LookupProblem::Unauthorized
},
))
.assert_err(&gone.resolve("nobody/nothing", &online));
}
}