use alloc::{
format,
string::{
String,
ToString,
},
vec::Vec,
};
use crate::data::{
catalog::{
ResolvedBuild,
document::{
ResourceEntry,
ResourceSource,
},
},
resources::{
Resource,
ResourceSet,
Source,
hub::HubRepo,
resolve_dir_template,
},
};
impl ResolvedBuild {
pub fn resource_set(&self) -> ResourceSet {
let mut set = ResourceSet::new(self.name.to_string());
set.description = self.description.clone();
set.license = self.license.clone();
set.origin = self.origin.clone();
for (slot, bound) in &self.resources {
let name = bound.resource.to_string();
for resource in bound.entry.to_resources(slot, &name, &self.hub_origins) {
set = set.with_resource(resource);
}
}
set
}
}
impl ResourceEntry {
pub(crate) fn to_slot_set(
&self,
name: &str,
slot: &str,
origins: &[String],
) -> ResourceSet {
let mut set = ResourceSet::new(name);
if let Some(description) = &self.description {
set.description = description.clone();
}
for r in self.to_resources(slot, name, origins) {
set = set.with_resource(r);
}
set
}
}
impl ResourceEntry {
fn to_resources(
&self,
slot: &str,
name: &str,
origins: &[String],
) -> Vec<Resource> {
let resource =
|slot: String, file: &str, sha256: Option<&String>, size, format: &str, sources| {
Resource {
slot,
file: file.to_string(),
sha256: sha256.cloned(),
format: Some(format.to_string()),
size,
dtype: self.dtype.clone(),
name: name.to_string(),
sources,
}
};
let sources_of = |sources: &mut dyn Iterator<Item = &ResourceSource>, file: &str| {
sources
.flat_map(|s| to_sources(s, file, origins))
.collect::<Vec<_>>()
};
if let (Some(width), Some(count)) = (self.index_width, self.count) {
return (0..count)
.map(|index| {
let pinned = self.shards.get(index);
let file = self.templated_file(index);
let sources = match pinned {
Some(shard) => sources_of(&mut self.shard_sources(shard), &file),
None => {
sources_of(&mut self.sources.iter().filter(|s| s.is_place()), &file)
}
};
resource(
format!("{slot}.{index:0width$}"),
&file,
pinned.map(|s| &s.sha256),
pinned.map(|s| s.size),
&self.format,
sources,
)
})
.collect();
}
if self.shards.is_empty() {
return alloc::vec![resource(
slot.to_string(),
&self.file,
self.sha256.as_ref(),
self.size,
&self.format,
sources_of(&mut self.sources.iter(), &self.file),
)];
}
let shard_format = self.format.strip_suffix("-index").unwrap_or(&self.format);
let mut out = alloc::vec![resource(
format!("{slot}.index"),
&self.file,
self.sha256.as_ref(),
self.size,
&self.format,
sources_of(&mut self.sources.iter(), &self.file),
)];
for (n, shard) in self.shards.iter().enumerate() {
out.push(resource(
format!("{slot}.{:05}", n + 1),
&shard.file,
Some(&shard.sha256),
Some(shard.size),
shard_format,
sources_of(&mut self.shard_sources(shard), &shard.file),
));
}
out
}
}
fn to_sources(
source: &ResourceSource,
file: &str,
origins: &[String],
) -> Vec<Source> {
match source {
ResourceSource::Url(url) => alloc::vec![Source::Url(url.clone())],
ResourceSource::BaseUrl(base) => {
alloc::vec![Source::Url(format!(
"{}/{file}",
base.trim_end_matches('/')
))]
}
ResourceSource::Hub {
repo,
revision,
dataset,
} => {
let hub = HubRepo::new(repo.as_str(), revision.as_str());
let hub = if *dataset { hub.dataset() } else { hub };
if origins.is_empty() {
alloc::vec![Source::Url(hub.file_url(file))]
} else {
origins
.iter()
.map(|origin| Source::Url(hub.file_url_at(origin, file)))
.collect()
}
}
ResourceSource::LocalDir { name, default } => alloc::vec![Source::LocalDir {
name: name.clone(),
dir: default.iter().find_map(|t| resolve_dir_template(t)),
}],
}
}
#[cfg(test)]
mod tests {
use alloc::vec;
use crate::data::catalog::{
BunsenCatalogStack,
CatalogName,
};
#[test]
fn test_a_templated_resource_is_a_slot_per_file() {
let mut stack = BunsenCatalogStack::new();
stack
.push_yaml(
"prelude",
r#"
$contracts:
data-set:
$kind: contract
$description: "Numbered files."
$resources: { shards: { $format: [parquet] } }
template_class_root: { $kind: template_class, $description: "The root." }
"#,
)
.unwrap();
stack
.push_yaml(
"data",
&format!(
r#"
data:
$kind: namespace
$description: "Data."
set:
$kind: build
$description: "A set."
$contract: data-set
$resources: {{ shards: shards }}
shards:
$kind: resource
$file: "shard_{{index}}.parquet"
$format: parquet
$index_width: 3
$count: 2
$sources: [{{ $hub: {{ $repo: o/set, $revision: abc, $dataset: true }} }}]
$shards:
- {{ $file: shard_000.parquet, $sha256: "{a}", $size: 9 }}
- {{ $file: shard_001.parquet, $sha256: "{b}", $size: 8 }}
loose:
$kind: build
$description: "An unpinned set."
$contract: data-set
$resources: {{ shards: shards }}
shards:
$kind: resource
$file: "shard_{{index}}.parquet"
$format: parquet
$index_width: 3
$count: 2
$unpinned: true
$sources: [{{ $base_url: "https://data.example/set" }}]
"#,
a = "a".repeat(64),
b = "b".repeat(64)
),
)
.unwrap();
let n = |name: &str| CatalogName::parse(name).unwrap();
let set = stack.resolve_build(&n("data/set")).unwrap().resource_set();
assert_eq!(set.keys(), ["shards.000", "shards.001"]);
let first = set.get("shards.000").unwrap();
assert_eq!(first.file, "shard_000.parquet");
assert_eq!(first.sha256, Some("a".repeat(64)));
assert_eq!(first.size, Some(9));
assert_eq!(first.name, "data/set/shards");
assert_eq!(
first.urls(),
vec!["https://huggingface.co/datasets/o/set/resolve/abc/shard_000.parquet"]
);
let loose = stack
.resolve_build(&n("data/loose"))
.unwrap()
.resource_set();
let second = loose.get("shards.001").unwrap();
assert_eq!(second.sha256, None);
assert_eq!(
second.urls(),
vec!["https://data.example/set/shard_001.parquet"]
);
}
#[test]
fn test_a_hub_source_is_had_from_each_mirror() {
let mut stack = BunsenCatalogStack::new();
stack
.push_yaml(
"prelude",
&format!(
r#"
$contracts:
{}
one-file:
$kind: contract
$description: "One file."
$resources: {{ file: {{ $format: [json] }} }}
template_class_root: {{ $kind: template_class, $description: "The root." }}
"#,
crate::data::resources::hub::HF_MIRROR_CONTRACT
.lines()
.filter(|l| !l.starts_with('#'))
.map(|l| format!(" {l}\n"))
.collect::<String>()
),
)
.unwrap();
let mirror = |name: &str, origin: &str| {
format!(
"{name}:\n $kind: build\n $description: \"A mirror.\"\n $contract: hf-mirror\n \
$args: {{ origin: \"{origin}\" }}\n"
)
};
stack
.push_yaml(
"hub",
&format!(
"{}m:\n $kind: build\n $description: \"M.\"\n $contract: one-file\n \
$resources: {{ file: file }}\n file:\n $kind: resource\n $file: \
config.json\n $format: json\n $sources: [{{ $hub: {{ $repo: o/m, \
$revision: abc }} }}]\n",
mirror("hub", "https://huggingface.co")
),
)
.unwrap();
stack
.push_yaml("mirror", &mirror("other", "https://mirror.example"))
.unwrap();
assert_eq!(
stack.hub_origins(),
["https://huggingface.co", "https://mirror.example"]
);
let set = stack
.resolve_build(&CatalogName::parse("m").unwrap())
.unwrap()
.resource_set();
assert_eq!(
set.get("file").unwrap().urls(),
vec![
"https://huggingface.co/o/m/resolve/abc/config.json",
"https://mirror.example/o/m/resolve/abc/config.json",
]
);
}
#[test]
fn test_the_builtin_mirror_set_starts_at_the_hub() {
let catalog = crate::data::catalog::BuiltinCatalog::get().unwrap();
assert_eq!(
catalog.hub_origins().first().map(String::as_str),
Some(crate::data::resources::hub::HUB_ORIGIN)
);
}
}