use alloc::{
collections::BTreeSet,
fmt,
string::String,
vec::Vec,
};
use burn::tensor::Slice;
use serde::{
Deserialize,
Serialize,
};
use crate::{
data::{
catalog::{
BuiltinCatalog,
ResolvedBuild,
document::ResourceSource,
},
resources::{
Resource,
ResourceSet,
Source,
hub::{
HUB_ORIGIN,
HubRepo,
},
},
},
errors::{
BunsenError,
BunsenResult,
ConstraintError,
LookupError,
ResultContext,
Rule,
},
};
pub const INDEX_PLACEHOLDER: &str = "{index}";
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ShardId(pub usize);
impl From<usize> for ShardId {
fn from(index: usize) -> Self {
Self(index)
}
}
impl fmt::Display for ShardId {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
write!(f, "{}", self.0)
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub enum ShardDigests {
#[default]
Unpinned,
Table(Vec<String>),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ShardSetDescriptor {
pub name: String,
pub description: String,
pub license: Option<String>,
pub origin: Option<String>,
pub base_urls: Vec<String>,
pub template: String,
pub index_width: usize,
pub count: usize,
pub format: String,
pub digests: ShardDigests,
}
impl ShardSetDescriptor {
pub fn from_build(build: &ResolvedBuild) -> BunsenResult<Self> {
let name = build.name.to_string();
let bound = build.resources.get(super::SHARDS).ok_or_else(|| {
BunsenError::lookup(LookupError::missing("resource slot", super::SHARDS)).context(&name)
})?;
let entry = &bound.entry;
let (Some(index_width), Some(count)) = (entry.index_width, entry.count) else {
return Err(BunsenError::illegal(format!(
"its `{}` resource is no templated set of files",
super::SHARDS
))
.context(&name));
};
let origins: Vec<&str> = if build.hub_origins.is_empty() {
vec![HUB_ORIGIN]
} else {
build.hub_origins.iter().map(String::as_str).collect()
};
let base_urls = entry
.sources
.iter()
.flat_map(|source| match source {
ResourceSource::Hub {
repo,
revision,
dataset,
} => {
let hub = HubRepo::new(repo.as_str(), revision.as_str());
let hub = if *dataset { hub.dataset() } else { hub };
origins
.iter()
.map(|origin| {
hub.file_url_at(origin, "")
.trim_end_matches('/')
.to_string()
})
.collect()
}
ResourceSource::BaseUrl(base) => vec![base.clone()],
ResourceSource::Url(_) | ResourceSource::LocalDir { .. } => Vec::new(),
})
.collect();
let digests = if entry.shards.is_empty() {
ShardDigests::Unpinned
} else {
ShardDigests::Table(entry.shards.iter().map(|s| s.sha256.clone()).collect())
};
let descriptor = Self {
name,
description: build.description.clone(),
license: build.license.clone(),
origin: build.origin.clone(),
base_urls,
template: entry.file.clone(),
index_width,
count,
format: entry.format.clone(),
digests,
};
descriptor.validate()?;
Ok(descriptor)
}
pub fn from_builtin(name: &str) -> BunsenResult<Self> {
Self::from_build(BuiltinCatalog::get()?.build(name)?)
}
pub fn validate(&self) -> BunsenResult<()> {
self.validate_fields().context(&self.name)
}
fn validate_fields(&self) -> BunsenResult<()> {
const OWNER: &str = "ShardSetDescriptor";
let placeholders = self.template.matches(INDEX_PLACEHOLDER).count();
if placeholders != 1 {
return Err(BunsenError::illegal(format!(
"template {:?} must contain {INDEX_PLACEHOLDER} exactly once, not {placeholders} times",
self.template
)));
}
if self.base_urls.is_empty() {
return Err(ConstraintError::zero_or_empty(OWNER, "base_urls").into());
}
if self.count == 0 {
return Err(ConstraintError::zero_or_empty(OWNER, "count").into());
}
if let ShardDigests::Table(table) = &self.digests {
if table.len() != self.count {
return Err(ConstraintError::new(
OWNER,
"digests",
Rule::Relation {
lhs: ("digests.len()".to_string(), table.len().to_string()),
op: "==",
rhs: ("count".to_string(), self.count.to_string()),
},
)
.into());
}
let is_hex = |d: &str| {
d.len() == 64 && d.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
};
if let Some((i, bad)) = table.iter().enumerate().find(|(_, d)| !is_hex(d)) {
return Err(BunsenError::illegal(format!(
"shard {i}'s digest {bad:?} is not 64 lowercase hex digits"
)));
}
}
Ok(())
}
pub fn is_pinned(&self) -> bool {
!matches!(self.digests, ShardDigests::Unpinned)
}
pub fn ids(&self) -> impl Iterator<Item = ShardId> {
(0..self.count).map(ShardId)
}
pub fn contains(
&self,
id: ShardId,
) -> bool {
id.0 < self.count
}
pub fn file_name(
&self,
id: ShardId,
) -> String {
let index = format!("{:0width$}", id.0, width = self.index_width);
self.template.replace(INDEX_PLACEHOLDER, &index)
}
pub fn parse_file_name(
&self,
file_name: &str,
) -> Option<ShardId> {
let (prefix, suffix) = self.template.split_once(INDEX_PLACEHOLDER)?;
let digits = file_name.strip_prefix(prefix)?.strip_suffix(suffix)?;
if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
let index: usize = digits.parse().ok()?;
(index < self.count).then_some(ShardId(index))
}
pub fn urls(
&self,
id: ShardId,
) -> Vec<String> {
let file_name = self.file_name(id);
self.base_urls
.iter()
.map(|base| format!("{}/{file_name}", base.trim_end_matches('/')))
.collect()
}
pub fn digest(
&self,
id: ShardId,
) -> Option<&str> {
match &self.digests {
ShardDigests::Unpinned => None,
ShardDigests::Table(table) => table.get(id.0).map(String::as_str),
}
}
pub fn to_resource_set(
&self,
ids: &[ShardId],
) -> BunsenResult<ResourceSet> {
let mut map = ResourceSet::new(&self.name);
map.description = self.description.clone();
map.license = self.license.clone();
map.origin = self.origin.clone();
for &id in ids {
if !self.contains(id) {
return Err(BunsenError::lookup(LookupError::out_of_range(
"shard", id, self.count,
))
.context(&self.name));
}
let file = self.file_name(id);
map.insert(Resource {
size: None,
dtype: None,
slot: file.clone(),
file,
sha256: self.digest(id).map(str::to_string),
format: Some(self.format.clone()),
name: self.name.clone(),
sources: self.urls(id).into_iter().map(Source::Url).collect(),
});
}
Ok(map)
}
pub fn select(
&self,
slices: &[Slice],
) -> BunsenResult<Vec<ShardId>> {
let count = self.count;
let resolve = |bound: isize| -> BunsenResult<usize> {
let resolved = if bound < 0 {
count as isize + bound
} else {
bound
};
if resolved < 0 || resolved > count as isize {
return Err(BunsenError::from(ConstraintError::out_of_range(
"ShardSetDescriptor::select",
"slices",
format!("shard index {bound}"),
format!("-{count}..={count}"),
))
.context(&self.name));
}
Ok(resolved as usize)
};
let mut ids = BTreeSet::new();
for slice in slices {
if slice.is_reversed() {
return Err(BunsenError::illegal(format!(
"shard slice {slice} is reversed; shards are selected in order"
))
.context(&self.name));
}
let start = resolve(slice.start)?;
let end = match slice.end {
Some(end) => resolve(end)?,
None => count,
};
let step = slice.step().unsigned_abs().max(1);
ids.extend((start..end).step_by(step).map(ShardId));
}
Ok(ids.into_iter().collect())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::errors::{
BunsenErrorKind,
LookupProblem,
testing::{
ErrorMatcher,
predicate,
},
};
pub(crate) fn tiny() -> ShardSetDescriptor {
ShardSetDescriptor {
name: "tiny".into(),
description: "a tiny set".into(),
license: Some("CC0".into()),
origin: Some("https://example.invalid/tiny".into()),
base_urls: vec![
"https://a.example/tiny/".into(),
"https://b.example/mirror".into(),
],
template: "shard_{index}.bin".into(),
index_width: 3,
count: 12,
format: "bin".into(),
digests: ShardDigests::Unpinned,
}
}
static TINY_SHA256: [&str; 12] =
["ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; 12];
fn tiny_pinned() -> ShardSetDescriptor {
ShardSetDescriptor {
digests: ShardDigests::Table(TINY_SHA256.iter().map(|d| d.to_string()).collect()),
..tiny()
}
}
#[test]
fn test_table_digests() {
let d = tiny_pinned();
assert!(d.is_pinned());
assert!(!tiny().is_pinned());
assert_eq!(
d.digests,
ShardDigests::Table(vec![TINY_SHA256[0].to_string(); 12])
);
assert_eq!(d.digest(ShardId(0)), Some(TINY_SHA256[0]));
assert_eq!(d.digest(ShardId(11)), Some(TINY_SHA256[11]));
assert_eq!(d.digest(ShardId(12)), None);
d.validate().unwrap();
let json = serde_json::to_string(&d).unwrap();
assert_eq!(
serde_json::from_str::<ShardSetDescriptor>(&json).unwrap(),
d
);
let mut short = d.clone();
short.digests = ShardDigests::Table(vec![TINY_SHA256[0].to_string(); 11]);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.display_contains(
"tiny: ShardSetDescriptor.digests: digests.len() (11) must be == count (12)",
)
.cause(predicate("the digests field", |c: &ConstraintError| {
c.field == "digests"
}))
.assert_err(&short.validate());
let mut bad = d.clone();
let mut table = vec![TINY_SHA256[0].to_string(); 12];
table[3] = TINY_SHA256[0].to_uppercase();
bad.digests = ShardDigests::Table(table);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("shard 3's digest")
.frame_contains("tiny")
.assert_err(&bad.validate());
}
#[test]
fn test_serde_round_trip() {
let d = tiny();
d.validate().unwrap();
let json = serde_json::to_string(&d).unwrap();
assert_eq!(
serde_json::from_str::<ShardSetDescriptor>(&json).unwrap(),
d
);
}
#[test]
fn test_a_builtin_build_is_a_shard_set() {
let d = ShardSetDescriptor::from_builtin("fineweb-edu-100b-shuffle").unwrap();
assert_eq!(d.name, "karpathy/fineweb-edu-100b-shuffle");
assert_eq!(d.count, 1823);
assert_eq!(d.format, "parquet");
assert_eq!(d.file_name(ShardId(7)), "shard_00007.parquet");
assert_eq!(
d.base_urls,
[
"https://huggingface.co/datasets/karpathy/fineweb-edu-100b-shuffle/resolve/4c8f30d6756da75362432a4d5569e1b229263b71"
]
);
assert!(d.is_pinned());
assert_eq!(
d.digest(ShardId(0)),
Some("6a3450ea745ee6b6830c09819c0f2c72682eba443022de595bcb50ceb3092c22")
);
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.assert_err(&ShardSetDescriptor::from_builtin("openai/whisper/tiny"));
}
#[test]
fn test_validate_names_the_problem() {
let mut d = tiny();
d.template = "shard.bin".to_string();
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("exactly once")
.assert_err(&d.validate());
d.template = "{index}-{index}".to_string();
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("not 2 times")
.assert_err(&d.validate());
let zero_or_empty = |field: &'static str| {
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.frame_contains("tiny")
.cause(predicate(field, move |c: &ConstraintError| {
c.field == field && c.rule == Rule::ZeroOrEmpty
}))
};
let mut d = tiny();
d.base_urls.clear();
zero_or_empty("base_urls").assert_err(&d.validate());
let mut d = tiny();
d.count = 0;
zero_or_empty("count").assert_err(&d.validate());
}
#[test]
fn test_names_and_urls() {
let d = tiny();
assert_eq!(d.file_name(ShardId(0)), "shard_000.bin");
assert_eq!(d.file_name(ShardId(11)), "shard_011.bin");
assert_eq!(
d.urls(ShardId(7)),
vec![
"https://a.example/tiny/shard_007.bin",
"https://b.example/mirror/shard_007.bin",
]
);
assert_eq!(d.digest(ShardId(7)), None);
assert_eq!(ShardId(7).to_string(), "7");
assert_eq!(ShardId::from(3), ShardId(3));
assert_eq!(d.parse_file_name("shard_007.bin"), Some(ShardId(7)));
assert_eq!(d.parse_file_name("shard_7.bin"), Some(ShardId(7)));
assert_eq!(d.parse_file_name("shard_012.bin"), None, "past count");
assert_eq!(d.parse_file_name("shard_.bin"), None);
assert_eq!(d.parse_file_name("shard_0x7.bin"), None);
assert_eq!(d.parse_file_name("other_007.bin"), None);
assert_eq!(d.parse_file_name("shard_007.bin.partial"), None);
assert!(d.contains(ShardId(11)));
assert!(!d.contains(ShardId(12)));
assert_eq!(d.ids().count(), 12);
assert_eq!(d.ids().last(), Some(ShardId(11)));
}
#[test]
fn test_select_resolves_slices_in_order_without_repeats() {
let d = tiny();
let ids = |v: &[usize]| v.iter().copied().map(ShardId).collect::<Vec<_>>();
assert_eq!(d.select(&[]).unwrap(), ids(&[]));
assert_eq!(d.select(&[Slice::index(3)]).unwrap(), ids(&[3]));
assert_eq!(
d.select(&[Slice::from(..3), Slice::index(1)]).unwrap(),
ids(&[0, 1, 2])
);
assert_eq!(d.select(&[Slice::from(9..)]).unwrap(), ids(&[9, 10, 11]));
assert_eq!(d.select(&[Slice::from(-2..)]).unwrap(), ids(&[10, 11]));
assert_eq!(
d.select(&[Slice::from_range_stepped(0..12, 5)]).unwrap(),
ids(&[0, 5, 10])
);
assert_eq!(d.select(&[Slice::from(..)]).unwrap().len(), 12);
assert_eq!(
d.select(&[Slice::from(12..)]).unwrap(),
ids(&[]),
"empty at the end"
);
let out_of_range = ErrorMatcher::kind(BunsenErrorKind::Illegal)
.frame_contains("tiny")
.has_cause::<ConstraintError>();
out_of_range
.message_contains("shard index 13 is outside -12..=12")
.assert_err(&d.select(&[Slice::from(..13)]));
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("shard index -13 is outside -12..=12")
.assert_err(&d.select(&[Slice::from(-13..)]));
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("is reversed")
.assert_err(&d.select(&[Slice::with_step(5, Some(0), -1)]));
}
#[test]
fn test_to_resource_set() {
let d = tiny_pinned();
let ids = d.select(&[Slice::from(..2)]).unwrap();
let map = d.to_resource_set(&ids).unwrap();
assert_eq!(map.name, "tiny");
assert_eq!(map.description, "a tiny set");
assert_eq!(map.license.as_deref(), Some("CC0"));
assert_eq!(map.keys(), ["shard_000.bin", "shard_001.bin"]);
let r = map.get("shard_001.bin").unwrap();
assert_eq!(r.file, "shard_001.bin");
assert_eq!(r.sha256.as_deref(), Some(TINY_SHA256[1]));
assert_eq!(r.format.as_deref(), Some("bin"));
assert_eq!(r.name, "tiny");
assert_eq!(
r.urls(),
[
"https://a.example/tiny/shard_001.bin",
"https://b.example/mirror/shard_001.bin",
]
);
map.validate().unwrap();
let unpinned = tiny().to_resource_set(&[ShardId(3)]).unwrap();
assert!(unpinned.get("shard_003.bin").unwrap().sha256.is_none());
assert_eq!(unpinned.len(), 1);
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.display_contains("tiny: shard 12 is out of range; there are 12")
.cause(predicate("shard 12 of 12", |l: &LookupError| {
l.problem == LookupProblem::OutOfRange { len: 12 }
}))
.assert_err(&d.to_resource_set(&[ShardId(12)]));
}
#[test]
fn test_select_is_ordered_without_repeats_twice_over() {
let d = tiny();
let ids = |v: &[usize]| v.iter().copied().map(ShardId).collect::<Vec<_>>();
assert_eq!(
d.select(&[Slice::index(1), Slice::index(1)]).unwrap(),
ids(&[1])
);
}
}