use alloc::{
collections::BTreeMap,
format,
string::{
String,
ToString,
},
vec,
vec::Vec,
};
use std::path::{
Path,
PathBuf,
};
use serde::{
Deserialize,
Serialize,
};
use super::{
Resource,
Source,
StaticBase,
StaticResource,
not_found,
};
use crate::errors::{
BunsenError,
BunsenErrorKind,
BunsenResult,
LookupError,
ResultContext,
};
#[derive(Debug)]
pub struct StaticResourceMap<'a> {
pub name: &'a str,
pub description: &'a str,
pub license: Option<&'a str>,
pub origin: Option<&'a str>,
pub namespace: &'a str,
pub bases: &'a [StaticBase<'a>],
pub resources: &'a [StaticResource<'a>],
}
impl StaticResourceMap<'_> {
pub fn to_map(&self) -> ResourceMap {
ResourceMap {
name: self.name.to_string(),
description: self.description.to_string(),
license: self.license.map(str::to_string),
origin: self.origin.map(str::to_string),
resources: self
.resources
.iter()
.map(|r| (r.key.to_string(), r.to_resource(self.namespace, self.bases)))
.collect(),
}
}
pub fn keys(&self) -> Vec<&str> {
self.resources.iter().map(|r| r.key).collect()
}
pub fn get(
&self,
key: &str,
) -> Option<&StaticResource<'_>> {
self.resources.iter().find(|r| r.key == key)
}
}
impl From<&StaticResourceMap<'_>> for ResourceMap {
fn from(map: &StaticResourceMap<'_>) -> Self {
map.to_map()
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Fuse {
Strict,
Overlay,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResourceMap {
pub name: String,
pub description: String,
pub license: Option<String>,
pub origin: Option<String>,
pub resources: BTreeMap<String, Resource>,
}
impl ResourceMap {
pub fn new(name: impl Into<String>) -> Self {
Self {
name: name.into(),
description: String::new(),
license: None,
origin: None,
resources: BTreeMap::new(),
}
}
pub fn given(
name: impl Into<String>,
key: impl Into<String>,
path: impl Into<PathBuf>,
) -> Self {
Self::new(name).with_resource(Resource::given(key, path))
}
pub fn with_resource(
mut self,
resource: Resource,
) -> Self {
self.insert(resource);
self
}
pub fn insert(
&mut self,
resource: Resource,
) -> Option<Resource> {
self.resources.insert(resource.key.clone(), resource)
}
pub fn keys(&self) -> Vec<&str> {
self.resources.keys().map(String::as_str).collect()
}
pub fn family(
&self,
key: &str,
) -> ResourceMap {
let prefix = format!("{key}.");
let mut family = Self::new(self.name.clone());
family.description.clone_from(&self.description);
family.license.clone_from(&self.license);
family.origin.clone_from(&self.origin);
for (k, resource) in &self.resources {
if k == key || k.starts_with(&prefix) {
family.insert(resource.clone());
}
}
family
}
pub fn len(&self) -> usize {
self.resources.len()
}
pub fn is_empty(&self) -> bool {
self.resources.is_empty()
}
pub fn get(
&self,
key: &str,
) -> Option<&Resource> {
self.resources.get(key)
}
pub fn try_get(
&self,
key: &str,
) -> BunsenResult<&Resource> {
self.get(key)
.ok_or_else(|| not_found(Some(&self.name), "resource", key, &self.keys()))
}
pub fn validate(&self) -> BunsenResult<()> {
for (key, resource) in &self.resources {
if key != &resource.key {
return Err(BunsenError::illegal(format!(
"resource {:?} is filed under {key:?}",
resource.key
))
.context(&self.name));
}
resource.validate().context(&self.name)?;
}
Ok(())
}
pub fn fuse(
mut self,
other: Self,
how: Fuse,
) -> BunsenResult<Self> {
for (key, resource) in other.resources {
if how == Fuse::Strict && self.resources.contains_key(&key) {
return Err(BunsenError::from_cause(
BunsenErrorKind::Illegal,
LookupError::duplicate("resource", key),
)
.context(format!("fusing {} into {}", other.name, self.name)));
}
self.resources.insert(key, resource);
}
Ok(self)
}
pub fn with_local_files(
mut self,
name: &str,
files: &[(&str, &Path)],
) -> BunsenResult<Self> {
for (key, path) in files {
let resource = self.try_get(key)?;
let file_name = path.file_name().map(|n| n.to_string_lossy().into_owned());
if file_name.as_deref() != Some(resource.file.as_str()) {
return Err(BunsenError::illegal(format!(
"{key} is {}, and {} is not it",
resource.file,
path.display()
))
.context(&self.name));
}
let dir = path.parent().map(Path::to_path_buf);
let resource = self.resources.get_mut(*key).expect("try_get found it");
resource.sources = vec![Source::LocalDir {
name: name.to_string(),
dir,
}];
}
Ok(self)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::errors::{
ConstraintError,
LookupProblem,
testing::{
ErrorMatcher,
predicate,
text,
},
};
#[test]
fn test_with_local_files_replaces_sources_and_checks_names() {
let map = CHECKPOINT
.to_map()
.fuse(VOCABULARY.to_map(), Fuse::Strict)
.unwrap();
let bundled = map
.clone()
.with_local_files(
"bundled",
&[("checkpoint", Path::new("/build/out/tiny.en.pt"))],
)
.unwrap();
let r = bundled.get("checkpoint").unwrap();
assert_eq!(
r.sources,
vec![Source::LocalDir {
name: "bundled".to_string(),
dir: Some(PathBuf::from("/build/out")),
}]
);
assert_eq!(r.sha256, map.get("checkpoint").unwrap().sha256);
assert_eq!(r.namespace, "a");
assert_eq!(
bundled.get("vocabulary").unwrap().sources,
map.get("vocabulary").unwrap().sources,
"an unnamed resource is untouched"
);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("tiny.en.pt")
.message_contains("other.pt")
.assert_err(
&map.clone()
.with_local_files("bundled", &[("checkpoint", Path::new("/x/other.pt"))]),
);
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.has_cause::<LookupError>()
.assert_err(
&map.with_local_files("bundled", &[("config", Path::new("/x/config.json"))]),
);
}
use crate::data::pretrained::{
GIVEN_NAMESPACE,
Source,
StaticSource,
};
const ABC_SHA256: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
fn upstream_dir() -> Option<PathBuf> {
Some(PathBuf::from("/home/someone/.cache/whisper"))
}
static CHECKPOINT: StaticResourceMap<'static> = StaticResourceMap {
name: "a/tiny.en.pt",
description: "a checkpoint",
license: Some("MIT"),
origin: Some("https://a.example"),
namespace: "a",
bases: &[
StaticBase::LocalDir {
name: "upstream",
default: upstream_dir,
},
StaticBase::Url("https://a.example/models"),
],
resources: &[StaticResource {
key: "checkpoint",
file: "tiny.en.pt",
sha256: Some(ABC_SHA256),
kind: Some("pytorch fp16"),
sources: &[],
}],
};
static VOCABULARY: StaticResourceMap<'static> = StaticResourceMap {
name: "a/gpt2.tiktoken",
description: "a vocabulary",
license: None,
origin: None,
namespace: "a",
bases: &[StaticBase::Url("https://raw.example/assets/")],
resources: &[StaticResource {
key: "vocabulary",
file: "gpt2.tiktoken",
sha256: None,
kind: Some("tiktoken"),
sources: &[StaticSource::Url("https://mirror.example/gpt2.tiktoken")],
}],
};
#[test]
fn test_static_to_map() {
assert_eq!(CHECKPOINT.keys(), ["checkpoint"]);
assert_eq!(
CHECKPOINT.get("checkpoint").map(|r| r.file),
Some("tiny.en.pt")
);
assert!(CHECKPOINT.get("vocabulary").is_none());
let map = CHECKPOINT.to_map();
assert_eq!(map, ResourceMap::from(&CHECKPOINT));
assert_eq!(map.name, "a/tiny.en.pt");
assert_eq!(map.license.as_deref(), Some("MIT"));
assert_eq!(map.len(), 1);
assert!(!map.is_empty());
assert_eq!(map.keys(), ["checkpoint"]);
let r = map.get("checkpoint").unwrap();
assert_eq!(r.namespace, "a");
assert_eq!(
r.sources,
vec![
Source::LocalDir {
name: "upstream".to_string(),
dir: upstream_dir(),
},
Source::Url("https://a.example/models/tiny.en.pt".to_string()),
]
);
map.validate().unwrap();
let json = serde_json::to_string(&map).unwrap();
assert_eq!(serde_json::from_str::<ResourceMap>(&json).unwrap(), map);
}
#[test]
fn test_fuse_strict() {
let fused = CHECKPOINT
.to_map()
.fuse(VOCABULARY.to_map(), Fuse::Strict)
.unwrap();
assert_eq!(fused.name, "a/tiny.en.pt");
assert_eq!(fused.keys(), ["checkpoint", "vocabulary"]);
assert_eq!(
fused.get("vocabulary").unwrap().sources,
vec![
Source::Url("https://mirror.example/gpt2.tiktoken".to_string()),
Source::Url("https://raw.example/assets/gpt2.tiktoken".to_string()),
]
);
fused.validate().unwrap();
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_eq("duplicate resource \"checkpoint\"")
.frame_contains("fusing a/tiny.en.pt into a/tiny.en.pt")
.cause(predicate(
"a duplicate \"checkpoint\"",
|c: &LookupError| c.problem == LookupProblem::Duplicate && c.key == "checkpoint",
))
.assert_err(&CHECKPOINT.to_map().fuse(CHECKPOINT.to_map(), Fuse::Strict));
}
#[test]
fn test_fuse_overlay() {
let row = CHECKPOINT
.to_map()
.fuse(VOCABULARY.to_map(), Fuse::Strict)
.unwrap();
let overlaid = row
.fuse(
ResourceMap::given("flags", "vocabulary", "/mine/vocab.tiktoken"),
Fuse::Overlay,
)
.unwrap();
assert_eq!(overlaid.name, "a/tiny.en.pt");
assert_eq!(overlaid.keys(), ["checkpoint", "vocabulary"]);
let vocab = overlaid.get("vocabulary").unwrap();
assert_eq!(vocab.namespace, GIVEN_NAMESPACE);
assert_eq!(vocab.file, "vocab.tiktoken");
assert_eq!(vocab.kind, None);
assert_eq!(
vocab.sources,
vec![Source::LocalDir {
name: GIVEN_NAMESPACE.to_string(),
dir: Some(PathBuf::from("/mine")),
}]
);
overlaid.validate().unwrap();
}
#[test]
fn test_given_insert_and_lookup() {
let mut map = ResourceMap::given("flags", "checkpoint", "/models/my.pt");
assert_eq!(map.keys(), ["checkpoint"]);
assert_eq!(map.try_get("checkpoint").unwrap().file, "my.pt");
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.display(text::eq(
"flags: no resource \"vocabulary\"; there are: checkpoint",
))
.cause(predicate("candidates: checkpoint", |c: &LookupError| {
c.candidates == ["checkpoint"]
}))
.assert_err(&map.try_get("vocabulary"));
let replaced = map.insert(Resource::given("checkpoint", "/models/other.pt"));
assert_eq!(replaced.map(|r| r.file), Some("my.pt".to_string()));
assert_eq!(map.get("checkpoint").unwrap().file, "other.pt");
let empty = ResourceMap::new("empty");
assert!(empty.is_empty());
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.display(text::eq("empty: no resource \"x\""))
.assert_err(&empty.try_get("x"));
empty.validate().unwrap();
}
#[test]
fn test_validate_names_the_problem() {
let mut misfiled = ResourceMap::new("m");
misfiled.resources.insert(
"vocabulary".to_string(),
Resource::given("checkpoint", "/x.pt"),
);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.display(text::eq(
"m: resource \"checkpoint\" is filed under \"vocabulary\"",
))
.assert_err(&misfiled.validate());
let mut no_source = ResourceMap::given("m", "checkpoint", "/x.pt");
no_source
.resources
.get_mut("checkpoint")
.unwrap()
.sources
.clear();
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.display(text::eq(
"m: checkpoint: Resource.sources: must not be zero or empty",
))
.has_cause::<ConstraintError>()
.assert_err(&no_source.validate());
}
}