use alloc::collections::BTreeMap;
use std::{
fs,
path::{
Path,
PathBuf,
},
};
use crate::{
data::{
cache::link_or_copy,
resources::{
ResolvedResource,
ResourceSet,
},
support::not_found,
},
errors::{
BunsenError,
BunsenResult,
sys_at,
},
};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct LoadedResourceSet {
pub map: ResourceSet,
pub parts: BTreeMap<String, ResolvedResource>,
}
impl LoadedResourceSet {
pub fn get(
&self,
key: &str,
) -> Option<&ResolvedResource> {
self.parts.get(key)
}
pub fn expect(
&self,
key: &str,
) -> BunsenResult<&Path> {
self.get(key)
.map(|r| r.path.as_path())
.ok_or_else(|| not_found(Some(&self.map.name), "resource", key, &self.keys()))
}
pub fn format(
&self,
slot: &str,
) -> Option<&str> {
self.map.get(slot).and_then(|r| r.format.as_deref())
}
pub fn family(
&self,
key: &str,
) -> Vec<(&str, &ResolvedResource)> {
let prefix = format!("{key}.");
self.parts
.iter()
.filter(|(k, _)| *k == key || k.starts_with(&prefix))
.map(|(k, r)| (k.as_str(), r))
.collect()
}
pub fn keys(&self) -> Vec<&str> {
self.parts.keys().map(String::as_str).collect()
}
pub fn iter(&self) -> impl Iterator<Item = (&str, &ResolvedResource)> {
self.parts.iter().map(|(k, r)| (k.as_str(), r))
}
pub fn materialize(
&self,
dir: impl Into<PathBuf>,
) -> BunsenResult<PathBuf> {
let dir = dir.into();
fs::create_dir_all(&dir).map_err(sys_at("create", &dir))?;
let mut placed: BTreeMap<&str, &str> = BTreeMap::new();
for (key, part) in self.iter() {
let file = self
.map
.get(key)
.map(|r| r.file.as_str())
.unwrap_or_else(|| key);
if let Some(other) = placed.insert(file, key) {
return Err(BunsenError::illegal(format!(
"{key} and {other} would both land as {file:?}"
))
.context(&self.map.name));
}
let dest = dir.join(file);
if dest.symlink_metadata().is_ok() {
fs::remove_file(&dest).map_err(sys_at("remove", &dest))?;
}
link_or_copy(&part.path, &dest)?;
}
Ok(dir)
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use super::*;
use crate::{
data::resources::{
Provenance,
Resource,
},
errors::{
BunsenErrorKind,
LookupError,
testing::{
ErrorMatcher,
predicate,
},
},
};
fn loaded() -> LoadedResourceSet {
let mut checkpoint = Resource::given("checkpoint", "/models/tiny.pt");
checkpoint.format = Some("pytorch fp16".to_string());
let map = ResourceSet::new("m")
.with_resource(checkpoint)
.with_resource(Resource::given("vocabulary", "/models/gpt2.tiktoken"));
let parts = [
("checkpoint", "/models/tiny.pt", Provenance::LocalDir),
("vocabulary", "/models/gpt2.tiktoken", Provenance::LocalDir),
]
.into_iter()
.map(|(key, path, provenance)| {
(
key.to_string(),
ResolvedResource {
path: PathBuf::from(path),
provenance,
},
)
})
.collect();
LoadedResourceSet { map, parts }
}
#[test]
fn test_lookups() {
let loaded = loaded();
assert_eq!(loaded.keys(), ["checkpoint", "vocabulary"]);
assert_eq!(
loaded.get("checkpoint").map(|r| r.provenance),
Some(Provenance::LocalDir)
);
assert_eq!(
loaded.expect("vocabulary").unwrap(),
Path::new("/models/gpt2.tiktoken")
);
assert_eq!(loaded.format("checkpoint"), Some("pytorch fp16"));
assert_eq!(loaded.format("vocabulary"), None);
assert_eq!(loaded.format("config"), None);
assert_eq!(
loaded.iter().map(|(k, _)| k).collect::<Vec<_>>(),
["checkpoint", "vocabulary"]
);
}
#[test]
fn test_expect_names_the_keys_there_are() {
let loaded = loaded();
let err = loaded.expect("config").unwrap_err();
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.cause(predicate("a missing \"config\"", |c: &LookupError| {
c.key == "config" && c.candidates == ["checkpoint", "vocabulary"]
}))
.assert(&err);
assert_eq!(
err.to_string(),
"m: no resource \"config\"; there are: checkpoint, vocabulary"
);
let empty = LoadedResourceSet {
map: ResourceSet::new("empty"),
parts: BTreeMap::new(),
};
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.message_eq("no resource \"x\"")
.frame_contains("empty")
.assert_err(&empty.expect("x"));
}
#[test]
fn test_materialize_lays_the_parts_out() {
let dir = tempfile::tempdir().unwrap();
let checkpoint = dir.path().join("src").join("tiny.pt");
let vocabulary = dir.path().join("src").join("gpt2.tiktoken");
fs::create_dir_all(checkpoint.parent().unwrap()).unwrap();
fs::write(&checkpoint, b"weights").unwrap();
fs::write(&vocabulary, b"ranks").unwrap();
let map = ResourceSet::new("m")
.with_resource(Resource::given("checkpoint", &checkpoint))
.with_resource(Resource::given("vocabulary", &vocabulary));
let parts = [("checkpoint", &checkpoint), ("vocabulary", &vocabulary)]
.into_iter()
.map(|(key, path)| {
(
key.to_string(),
ResolvedResource {
path: path.clone(),
provenance: Provenance::LocalDir,
},
)
})
.collect();
let loaded = LoadedResourceSet { map, parts };
let view = loaded.materialize(dir.path().join("view")).unwrap();
assert_eq!(view, dir.path().join("view"));
assert_eq!(fs::read(view.join("tiny.pt")).unwrap(), b"weights");
assert_eq!(fs::read(view.join("gpt2.tiktoken")).unwrap(), b"ranks");
assert!(checkpoint.is_file(), "the part stays where it is");
let again = loaded.materialize(&view).unwrap();
assert_eq!(fs::read(again.join("tiny.pt")).unwrap(), b"weights");
let clash = LoadedResourceSet {
map: ResourceSet::new("clash")
.with_resource(Resource::given(
"a",
dir.path().join("one").join("same.bin"),
))
.with_resource(Resource::given(
"b",
dir.path().join("two").join("same.bin"),
)),
parts: [("a", &checkpoint), ("b", &vocabulary)]
.into_iter()
.map(|(key, path)| {
(
key.to_string(),
ResolvedResource {
path: path.clone(),
provenance: Provenance::LocalDir,
},
)
})
.collect(),
};
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("same.bin")
.frame_contains("clash")
.assert_err(&clash.materialize(dir.path().join("clash")));
}
}