use std::{
fs,
io,
path::{
Path,
PathBuf,
},
};
#[cfg(feature = "fetch")]
use crate::{
data::cache::{
FetchJob,
FetchPolicy,
FetchReport,
},
errors::ResultContext,
};
use crate::{
data::{
cache::BunsenDiskCache,
datasets::{
ShardId,
ShardSetDescriptor,
},
},
errors::{
BunsenError,
BunsenResult,
LookupError,
sys_at,
},
};
pub const SHARDS_DIR: &str = "shards";
pub struct ShardSet<'c> {
cache: &'c BunsenDiskCache,
desc: ShardSetDescriptor,
root: PathBuf,
}
impl<'c> ShardSet<'c> {
pub fn in_cache(
cache: &'c BunsenDiskCache,
desc: ShardSetDescriptor,
) -> Self {
let root = cache.data_path(&[SHARDS_DIR], &desc.name);
Self { cache, desc, root }
}
pub fn at_dir(
cache: &'c BunsenDiskCache,
desc: ShardSetDescriptor,
dir: impl Into<PathBuf>,
) -> Self {
Self {
cache,
desc,
root: dir.into(),
}
}
pub fn descriptor(&self) -> &ShardSetDescriptor {
&self.desc
}
pub fn cache(&self) -> &BunsenDiskCache {
self.cache
}
pub fn root(&self) -> &Path {
&self.root
}
pub fn path(
&self,
id: ShardId,
) -> PathBuf {
self.root.join(self.desc.file_name(id))
}
pub fn is_cached(
&self,
id: ShardId,
) -> bool {
self.path(id).is_file()
}
pub fn cached_ids(&self) -> BunsenResult<Vec<ShardId>> {
let entries = match fs::read_dir(&self.root) {
Ok(entries) => entries,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(sys_at("read directory", &self.root)(e)),
};
let mut ids = Vec::new();
for entry in entries {
let entry = entry.map_err(sys_at("read directory", &self.root))?;
let file_type = entry.file_type().map_err(sys_at("stat", entry.path()))?;
if !file_type.is_file() {
continue;
}
let name = entry.file_name();
if let Some(id) = name.to_str().and_then(|n| self.desc.parse_file_name(n)) {
ids.push(id);
}
}
ids.sort_unstable();
Ok(ids)
}
pub fn locate(
&self,
id: ShardId,
download: bool,
) -> BunsenResult<PathBuf> {
self.check(id)?;
let path = self.path(id);
if path.is_file() {
return Ok(path);
}
if !download {
return Err(BunsenError::policy(format!(
"shard {id} is not at {}, and downloads are off",
path.display()
))
.context(&self.desc.name));
}
self.fetch(id)
}
#[cfg(feature = "fetch")]
pub fn fetch(
&self,
id: ShardId,
) -> BunsenResult<PathBuf> {
self.check(id)?;
let path = self.path(id);
if path.is_file() {
return Ok(path);
}
let urls = self.desc.urls(id);
let urls: Vec<&str> = urls.iter().map(String::as_str).collect();
self.cache
.fetch_from_urls(&urls, &path, self.desc.digest(id))
.with_context(|| format!("{}: shard {id}", self.desc.name))?;
Ok(path)
}
#[cfg(not(feature = "fetch"))]
pub fn fetch(
&self,
id: ShardId,
) -> BunsenResult<PathBuf> {
self.check(id)?;
let path = self.path(id);
if path.is_file() {
return Ok(path);
}
Err(BunsenError::unsupported(format!(
"shard {id} is not at {}, and fetching needs the `fetch` feature",
path.display()
))
.context(&self.desc.name))
}
#[cfg(feature = "fetch")]
pub fn jobs(
&self,
ids: &[ShardId],
) -> BunsenResult<Vec<FetchJob>> {
ids.iter()
.map(|&id| {
self.check(id)?;
let mut job = FetchJob::new(self.desc.urls(id), self.path(id));
job.sha256 = self.desc.digest(id).map(str::to_string);
Ok(job)
})
.collect()
}
#[cfg(feature = "fetch")]
pub fn fetch_many(
&self,
ids: &[ShardId],
policy: &FetchPolicy,
) -> BunsenResult<FetchReport> {
let jobs = self.jobs(ids)?;
Ok(self.cache.fetch_many(&jobs, policy))
}
fn check(
&self,
id: ShardId,
) -> BunsenResult<()> {
if self.desc.contains(id) {
Ok(())
} else {
Err(
BunsenError::lookup(LookupError::out_of_range("shard", id, self.desc.count))
.context(&self.desc.name),
)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "fetch")]
use crate::data::cache::{
FetchFailure,
FetchOutcome,
OnFailure,
testing::{
ABC_SHA256,
refused_url,
serve_n,
},
};
use crate::{
data::{
cache::BunsenDiskCacheOptions,
datasets::ShardDigests,
},
errors::{
BunsenErrorKind,
LookupProblem,
testing::{
ErrorMatcher,
predicate,
},
},
};
fn cache_in(dir: &Path) -> BunsenDiskCache {
BunsenDiskCache::new(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.join("cache")))
.with_data_dir(Some(dir.join("data")))
.without_transfer_observers(),
)
.unwrap()
}
fn served(
base_urls: &[&str],
count: usize,
digests: ShardDigests,
) -> ShardSetDescriptor {
ShardSetDescriptor {
name: "served".into(),
description: "served off a loopback port".into(),
license: None,
origin: None,
base_urls: base_urls.iter().map(|b| b.to_string()).collect(),
template: "shard_{index}.bin".into(),
index_width: 2,
count,
format: "bin".into(),
digests,
}
}
#[test]
fn test_roots_and_paths() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path());
let desc = served(&["https://a.example"], 3, ShardDigests::Unpinned);
let set = ShardSet::in_cache(&cache, desc.clone());
assert_eq!(
set.root(),
dir.path().join("data").join("shards").join("served")
);
assert_eq!(set.path(ShardId(2)), set.root().join("shard_02.bin"));
assert_eq!(set.descriptor(), &desc);
assert_eq!(set.cached_ids().unwrap(), vec![], "no root yet");
let set = ShardSet::at_dir(&cache, desc, dir.path().join("elsewhere"));
assert_eq!(set.root(), dir.path().join("elsewhere"));
assert!(!set.is_cached(ShardId(0)));
ErrorMatcher::kind(BunsenErrorKind::Policy)
.display_contains("served: shard 0 is not at ")
.assert_err(&set.locate(ShardId(0), false));
let out_of_range = ErrorMatcher::kind(BunsenErrorKind::Lookup)
.display_contains("served: shard 3 is out of range; there are 3")
.cause(predicate("shard 3 of 3", |l: &LookupError| {
l.key == "3" && l.problem == LookupProblem::OutOfRange { len: 3 }
}));
out_of_range.assert_err(&set.locate(ShardId(3), true));
out_of_range.assert_err(&set.fetch(ShardId(3)));
}
#[test]
fn test_cached_ids_reads_the_root() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path());
let set = ShardSet::at_dir(
&cache,
served(&["https://a.example"], 10, ShardDigests::Unpinned),
dir.path(),
);
for name in [
"shard_07.bin",
"shard_02.bin",
"shard_02.bin.partial",
"notes.txt",
] {
fs::write(dir.path().join(name), b"x").unwrap();
}
fs::create_dir(dir.path().join("shard_05.bin")).unwrap();
assert_eq!(set.cached_ids().unwrap(), vec![ShardId(2), ShardId(7)]);
assert!(set.is_cached(ShardId(7)));
assert!(!set.is_cached(ShardId(5)), "a directory is not a shard");
assert_eq!(
set.locate(ShardId(7), false).unwrap(),
dir.path().join("shard_07.bin")
);
}
#[cfg(feature = "fetch")]
#[test]
fn test_fetch_and_fetch_many() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path());
let dead = refused_url("");
let live = serve_n("", b"abc", 3);
let bases: &'static [&'static str] = Box::leak(
vec![
Box::leak(dead.into_boxed_str()) as &str,
Box::leak(live.into_boxed_str()) as &str,
]
.into_boxed_slice(),
);
let set = ShardSet::in_cache(&cache, served(bases, 3, ShardDigests::Unpinned));
let path = set.fetch(ShardId(1)).unwrap();
assert_eq!(path, set.path(ShardId(1)));
assert_eq!(fs::read(&path).unwrap(), b"abc");
assert_eq!(set.locate(ShardId(1), false).unwrap(), path);
let ids = set
.descriptor()
.select(&[burn::tensor::Slice::from(..)])
.unwrap();
let jobs = set.jobs(&ids).unwrap();
assert_eq!(jobs.len(), 3);
assert_eq!(jobs[0].urls.len(), 2);
assert_eq!(jobs[0].sha256, None);
let policy = FetchPolicy::default()
.with_parallel(2)
.with_on_failure(OnFailure::Continue);
let report = set.fetch_many(&ids, &policy).unwrap();
assert!(
matches!(report.outcomes[1], FetchOutcome::Cached(_)),
"{report}"
);
assert!(report.is_complete(), "{report}");
assert_eq!(
report.paths().unwrap(),
ids.iter().map(|&id| set.path(id)).collect::<Vec<_>>()
);
assert_eq!(set.cached_ids().unwrap(), ids);
}
#[cfg(feature = "fetch")]
#[test]
fn test_pinned_set_verifies_each_shard() {
static PINNED: [&str; 2] = [
ABC_SHA256,
"0000000000000000000000000000000000000000000000000000000000000000",
];
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path());
let live = serve_n("", b"abc", 3);
let bases: &'static [&'static str] =
Box::leak(vec![Box::leak(live.into_boxed_str()) as &str].into_boxed_slice());
let set = ShardSet::in_cache(
&cache,
served(
bases,
2,
ShardDigests::Table(PINNED.iter().map(|d| d.to_string()).collect()),
),
);
assert!(set.descriptor().is_pinned());
let good = set.fetch(ShardId(0)).unwrap();
assert_eq!(fs::read(&good).unwrap(), b"abc");
let bad = set.fetch(ShardId(1));
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.frame_contains("served: shard 1")
.cause(predicate("a digest failure", |f: &FetchFailure| {
matches!(f, FetchFailure::Digest(_))
}))
.assert_err(&bad);
assert!(!set.path(ShardId(1)).exists());
let jobs = set.jobs(&[ShardId(0), ShardId(1)]).unwrap();
assert_eq!(jobs[0].sha256.as_deref(), Some(ABC_SHA256));
assert_eq!(jobs[1].sha256.as_deref(), Some(PINNED[1]));
let policy = FetchPolicy::default().with_on_failure(OnFailure::Continue);
let report = set.fetch_many(&[ShardId(0), ShardId(1)], &policy).unwrap();
assert!(
matches!(report.outcomes[0], FetchOutcome::Cached(_)),
"{report}"
);
assert!(
matches!(report.outcomes[1], FetchOutcome::Failed { .. }),
"{report}"
);
assert_eq!(set.cached_ids().unwrap(), vec![ShardId(0)]);
}
#[cfg(feature = "fetch")]
#[test]
fn test_a_set_loads_as_a_map_through_the_pretrained_cache() {
use crate::data::resources::{
Provenance,
ResourceCache,
ResourceCacheOptions,
};
let dir = tempfile::tempdir().unwrap();
let live = serve_n("", b"abc", 2);
let bases: &'static [&'static str] =
Box::leak(vec![Box::leak(live.into_boxed_str()) as &str].into_boxed_slice());
let desc = served(bases, 2, ShardDigests::Unpinned);
let map = desc.to_resource_set(&[ShardId(0), ShardId(1)]).unwrap();
assert_eq!(map.keys(), ["shard_00.bin", "shard_01.bin"]);
let cache = ResourceCache::new(
ResourceCacheOptions::default().with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.path().join("cache")))
.without_transfer_observers(),
),
)
.unwrap();
let loaded = cache.load(&map).unwrap();
for (key, part) in loaded.iter() {
assert_eq!(part.provenance, Provenance::Downloaded, "{key}");
assert_eq!(fs::read(&part.path).unwrap(), b"abc", "{key}");
assert!(
part.path
.starts_with(dir.path().join("cache").join("resources"))
);
}
let again = cache.load(&map).unwrap();
for (_, part) in again.iter() {
assert_eq!(part.provenance, Provenance::Cached);
}
}
}