use alloc::{
format,
string::{
String,
ToString,
},
vec,
vec::Vec,
};
use core::fmt;
use std::path::{
Path,
PathBuf,
};
use serde::{
Deserialize,
Serialize,
};
use crate::errors::{
BunsenError,
BunsenResult,
};
pub const GIVEN_NAMESPACE: &str = "given";
const X25: crc::Crc<u16> = crc::Crc::<u16>::new(&crc::CRC_16_IBM_SDLC);
pub fn url_to_cache_key(
name: Option<&str>,
url: &str,
) -> String {
let hash = X25.checksum(url.as_bytes()).to_string();
let base_name = url.rsplit_once('/').map_or(url, |(_, name)| name);
match name {
Some(n) => format!("{}-{}-{}", n, hash, base_name),
None => format!("{}-{}", hash, base_name),
}
}
pub const SAFETENSORS: &str = "safetensors";
pub const SAFETENSORS_INDEX: &str = "safetensors-index";
#[derive(Clone, Copy)]
pub enum StaticSource<'a> {
Url(&'a str),
LocalDir {
name: &'a str,
default: fn() -> Option<PathBuf>,
},
Bundled(&'static [u8]),
}
impl StaticSource<'_> {
pub fn to_source(&self) -> Source {
match self {
Self::Url(url) => Source::Url(url.to_string()),
Self::LocalDir { name, default } => Source::LocalDir {
name: name.to_string(),
dir: default(),
},
Self::Bundled(bytes) => Source::Bundled(BundledBytes(bytes)),
}
}
}
impl fmt::Debug for StaticSource<'_> {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
match self {
Self::Url(url) => f.debug_tuple("Url").field(url).finish(),
Self::LocalDir { name, .. } => f.debug_struct("LocalDir").field("name", name).finish(),
Self::Bundled(bytes) => write!(f, "Bundled({} bytes)", bytes.len()),
}
}
}
impl fmt::Display for StaticSource<'_> {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
match self {
Self::Url(url) => write!(f, "url {url}"),
Self::LocalDir { name, .. } => write!(f, "local dir {name}"),
Self::Bundled(bytes) => write!(f, "bundled ({} bytes)", bytes.len()),
}
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub struct BundledBytes(pub &'static [u8]);
impl BundledBytes {
pub fn len(&self) -> usize {
self.0.len()
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
impl fmt::Debug for BundledBytes {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
write!(f, "BundledBytes({} bytes)", self.len())
}
}
impl Serialize for BundledBytes {
fn serialize<S: serde::Serializer>(
&self,
_serializer: S,
) -> Result<S::Ok, S::Error> {
Err(serde::ser::Error::custom(
"bundled bytes have no manifest form",
))
}
}
impl<'de> Deserialize<'de> for BundledBytes {
fn deserialize<D: serde::Deserializer<'de>>(_deserializer: D) -> Result<Self, D::Error> {
Err(serde::de::Error::custom(
"bundled bytes have no manifest form",
))
}
}
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Source {
Url(String),
LocalDir {
name: String,
dir: Option<PathBuf>,
},
Bundled(BundledBytes),
}
impl fmt::Debug for Source {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
match self {
Self::Url(url) => f.debug_tuple("Url").field(url).finish(),
Self::LocalDir { name, dir } => f
.debug_struct("LocalDir")
.field("name", name)
.field("dir", dir)
.finish(),
Self::Bundled(bytes) => write!(f, "Bundled({} bytes)", bytes.len()),
}
}
}
impl fmt::Display for Source {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
match self {
Self::Url(url) => write!(f, "url {url}"),
Self::LocalDir {
name,
dir: Some(dir),
} => write!(f, "local dir {name} ({})", dir.display()),
Self::LocalDir { name, dir: None } => write!(f, "local dir {name} (unresolved)"),
Self::Bundled(bytes) => write!(f, "bundled ({} bytes)", bytes.len()),
}
}
}
#[derive(Clone, Copy, Debug)]
pub enum StaticBase<'a> {
Url(&'a str),
LocalDir {
name: &'a str,
default: fn() -> Option<PathBuf>,
},
}
impl StaticBase<'_> {
pub fn source_for(
&self,
file: &str,
) -> Source {
match self {
Self::Url(url) => Source::Url(format!("{}/{file}", url.trim_end_matches('/'))),
Self::LocalDir { name, default } => Source::LocalDir {
name: name.to_string(),
dir: default(),
},
}
}
}
impl fmt::Display for StaticBase<'_> {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
match self {
Self::Url(url) => write!(f, "url {url}"),
Self::LocalDir { name, .. } => write!(f, "local dir {name}"),
}
}
}
#[derive(Debug)]
pub struct StaticResource<'a> {
pub key: &'a str,
pub file: &'a str,
pub sha256: Option<&'a str>,
pub kind: Option<&'a str>,
pub sources: &'a [StaticSource<'a>],
}
impl StaticResource<'_> {
pub fn to_resource(
&self,
namespace: &str,
bases: &[StaticBase<'_>],
) -> Resource {
Resource {
key: self.key.to_string(),
file: self.file.to_string(),
sha256: self.sha256.map(str::to_string),
kind: self.kind.map(str::to_string),
namespace: namespace.to_string(),
sources: self
.sources
.iter()
.map(StaticSource::to_source)
.chain(bases.iter().map(|b| b.source_for(self.file)))
.collect(),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Resource {
pub key: String,
pub file: String,
pub sha256: Option<String>,
pub kind: Option<String>,
pub namespace: String,
pub sources: Vec<Source>,
}
impl Resource {
pub fn given(
key: impl Into<String>,
path: impl Into<PathBuf>,
) -> Self {
let path = path.into();
let file = path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| path.display().to_string());
let dir = path.parent().map(Path::to_path_buf);
Self {
key: key.into(),
file,
sha256: None,
kind: None,
namespace: GIVEN_NAMESPACE.to_string(),
sources: vec![Source::LocalDir {
name: GIVEN_NAMESPACE.to_string(),
dir,
}],
}
}
pub fn is_pinned(&self) -> bool {
self.sha256.is_some()
}
pub fn urls(&self) -> Vec<&str> {
self.sources
.iter()
.filter_map(|s| match s {
Source::Url(url) => Some(url.as_str()),
_ => None,
})
.collect()
}
pub fn bundled(&self) -> Option<&'static [u8]> {
self.sources.iter().find_map(|s| match s {
Source::Bundled(bytes) => Some(bytes.0),
_ => None,
})
}
pub fn validate(&self) -> BunsenResult<()> {
if self.key.is_empty() {
return Err(BunsenError::Invalid(format!(
"{}: a resource has no key",
self.file
)));
}
if self.file.is_empty() {
return Err(BunsenError::Invalid(format!("{}: no file name", self.key)));
}
if self.namespace.is_empty() {
return Err(BunsenError::Invalid(format!("{}: no namespace", self.key)));
}
if self.sources.is_empty() {
return Err(BunsenError::Invalid(format!("{}: no source", self.key)));
}
if let Some(sha256) = &self.sha256
&& !is_sha256_hex(sha256)
{
return Err(BunsenError::Invalid(format!(
"{}: sha256 {sha256:?} is not 64 lowercase hex digits",
self.key
)));
}
if self.bundled().is_some() && self.sha256.is_none() {
return Err(BunsenError::Invalid(format!(
"{}: a bundled source needs a digest to be cached under",
self.key
)));
}
Ok(())
}
pub fn cache_key(&self) -> String {
match self.urls().first() {
Some(url) => url_to_cache_key(None, url),
None => self.file.clone(),
}
}
}
impl fmt::Display for Resource {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
write!(f, "{}: {}", self.key, self.file)?;
let pin = if self.is_pinned() {
"pinned"
} else {
"unpinned"
};
match &self.kind {
Some(kind) => write!(f, " ({kind}, {pin})"),
None => write!(f, " ({pin})"),
}
}
}
pub(crate) fn is_sha256_hex(s: &str) -> bool {
s.len() == 64 && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
}
#[cfg(test)]
mod tests {
use super::*;
fn upstream_dir_for_display() -> Option<PathBuf> {
Some(PathBuf::from("/home/someone/.cache/upstream"))
}
#[test]
fn test_source_display() {
let sources = [
StaticSource::LocalDir {
name: "upstream",
default: upstream_dir_for_display,
},
StaticSource::Url("https://a.example/my_model.pt"),
];
let shown: Vec<String> = sources.iter().map(|s| s.to_string()).collect();
assert_eq!(
shown,
["local dir upstream", "url https://a.example/my_model.pt"]
);
assert_eq!(
sources[0].to_source().to_string(),
"local dir upstream (/home/someone/.cache/upstream)"
);
assert_eq!(
sources[1].to_source(),
Source::Url("https://a.example/my_model.pt".to_string())
);
assert_eq!(
Source::LocalDir {
name: "x".to_string(),
dir: None
}
.to_string(),
"local dir x (unresolved)"
);
let bundled = StaticSource::Bundled(b"abc");
assert_eq!(bundled.to_string(), "bundled (3 bytes)");
assert_eq!(format!("{bundled:?}"), "Bundled(3 bytes)");
assert_eq!(bundled.to_source(), Source::Bundled(BundledBytes(b"abc")));
assert_eq!(format!("{:?}", bundled.to_source()), "Bundled(3 bytes)");
assert!(
serde_json::to_string(&Source::Bundled(BundledBytes(b"abc"))).is_err(),
"in-binary bytes have no manifest form"
);
let bundled = StaticSource::Bundled(b"abc");
assert_eq!(bundled.to_string(), "bundled (3 bytes)");
assert_eq!(format!("{bundled:?}"), "Bundled(3 bytes)");
assert_eq!(bundled.to_source(), Source::Bundled(BundledBytes(b"abc")));
assert_eq!(format!("{:?}", bundled.to_source()), "Bundled(3 bytes)");
assert!(
serde_json::to_string(&Source::Bundled(BundledBytes(b"abc"))).is_err(),
"in-binary bytes have no manifest form"
);
assert_eq!(
url_to_cache_key(Some("m"), "https://a.example/m.pt"),
format!("m-{}-m.pt", X25.checksum(b"https://a.example/m.pt"))
);
}
#[test]
fn test_a_bundled_source_needs_a_digest() {
let mut r = Resource {
key: "burnpack".to_string(),
file: "model.bpk".to_string(),
sha256: None,
kind: None,
namespace: "bunsen".to_string(),
sources: vec![Source::Bundled(BundledBytes(b"abc"))],
};
assert_eq!(r.bundled(), Some(&b"abc"[..]));
assert!(r.urls().is_empty());
let err = r.validate().unwrap_err();
assert!(
matches!(&err, BunsenError::Invalid(m) if m.contains("bundled source needs a digest")),
"{err}"
);
r.sha256 =
Some("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad".to_string());
r.validate().unwrap();
assert!(r.is_pinned());
}
const ABC_SHA256: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
fn upstream_dir() -> Option<PathBuf> {
Some(PathBuf::from("/home/someone/.cache/whisper"))
}
fn nowhere() -> Option<PathBuf> {
None
}
static BASES: [StaticBase<'static>; 2] = [
StaticBase::LocalDir {
name: "upstream",
default: upstream_dir,
},
StaticBase::Url("https://a.example/models/"),
];
static CHECKPOINT: StaticResource<'static> = StaticResource {
key: "checkpoint",
file: "tiny.en.pt",
sha256: Some(ABC_SHA256),
kind: Some("pytorch fp16"),
sources: &[StaticSource::Url("https://mirror.example/tiny.en.pt")],
};
#[test]
fn test_base_source_for() {
assert_eq!(
BASES[1].source_for("tiny.en.pt"),
Source::Url("https://a.example/models/tiny.en.pt".to_string())
);
assert_eq!(
StaticBase::Url("https://a.example/models").source_for("tiny.en.pt"),
Source::Url("https://a.example/models/tiny.en.pt".to_string())
);
assert_eq!(
BASES[0].source_for("tiny.en.pt"),
Source::LocalDir {
name: "upstream".to_string(),
dir: upstream_dir(),
}
);
assert_eq!(
StaticBase::LocalDir {
name: "gone",
default: nowhere,
}
.source_for("x"),
Source::LocalDir {
name: "gone".to_string(),
dir: None,
}
);
assert_eq!(BASES[0].to_string(), "local dir upstream");
assert_eq!(BASES[1].to_string(), "url https://a.example/models/");
}
#[test]
fn test_to_resource_orders_sources() {
let r = CHECKPOINT.to_resource("openai", &BASES);
assert_eq!(r.key, "checkpoint");
assert_eq!(r.file, "tiny.en.pt");
assert_eq!(r.sha256.as_deref(), Some(ABC_SHA256));
assert_eq!(r.kind.as_deref(), Some("pytorch fp16"));
assert_eq!(r.namespace, "openai");
assert_eq!(
r.sources,
vec![
Source::Url("https://mirror.example/tiny.en.pt".to_string()),
Source::LocalDir {
name: "upstream".to_string(),
dir: upstream_dir(),
},
Source::Url("https://a.example/models/tiny.en.pt".to_string()),
]
);
assert!(r.is_pinned());
assert_eq!(
r.urls(),
vec![
"https://mirror.example/tiny.en.pt",
"https://a.example/models/tiny.en.pt"
]
);
assert_eq!(
r.cache_key(),
url_to_cache_key(None, "https://mirror.example/tiny.en.pt")
);
assert_eq!(
r.to_string(),
"checkpoint: tiny.en.pt (pytorch fp16, pinned)"
);
r.validate().unwrap();
let json = serde_json::to_string(&r).unwrap();
assert_eq!(serde_json::from_str::<Resource>(&json).unwrap(), r);
}
#[test]
fn test_given() {
let r = Resource::given("checkpoint", "/models/my.pt");
assert_eq!(r.key, "checkpoint");
assert_eq!(r.file, "my.pt");
assert_eq!(r.namespace, GIVEN_NAMESPACE);
assert!(!r.is_pinned());
assert_eq!(r.kind, None);
assert_eq!(
r.sources,
vec![Source::LocalDir {
name: GIVEN_NAMESPACE.to_string(),
dir: Some(PathBuf::from("/models")),
}]
);
assert!(r.urls().is_empty());
assert_eq!(r.cache_key(), "my.pt");
assert_eq!(r.to_string(), "checkpoint: my.pt (unpinned)");
r.validate().unwrap();
let bare = Resource::given("checkpoint", "my.pt");
assert_eq!(bare.file, "my.pt");
assert_eq!(
bare.sources,
vec![Source::LocalDir {
name: GIVEN_NAMESPACE.to_string(),
dir: Some(PathBuf::from("")),
}]
);
}
#[test]
fn test_validate_names_the_problem() {
let r = CHECKPOINT.to_resource("openai", &BASES);
let mut no_key = r.clone();
no_key.key.clear();
assert!(matches!(no_key.validate(), Err(BunsenError::Invalid(m)) if m.contains("no key")));
let mut no_file = r.clone();
no_file.file.clear();
assert!(
matches!(no_file.validate(), Err(BunsenError::Invalid(m)) if m == "checkpoint: no file name")
);
let mut no_namespace = r.clone();
no_namespace.namespace.clear();
assert!(
matches!(no_namespace.validate(), Err(BunsenError::Invalid(m)) if m.contains("no namespace"))
);
let mut no_source = r.clone();
no_source.sources.clear();
assert!(
matches!(no_source.validate(), Err(BunsenError::Invalid(m)) if m == "checkpoint: no source")
);
let mut bad_digest = r.clone();
bad_digest.sha256 = Some("ABC".to_string());
assert!(
matches!(bad_digest.validate(), Err(BunsenError::Invalid(m)) if m.contains("sha256"))
);
let mut unpinned = r;
unpinned.sha256 = None;
assert!(!unpinned.is_pinned());
unpinned.validate().unwrap();
assert_eq!(
unpinned.to_string(),
"checkpoint: tiny.en.pt (pytorch fp16, unpinned)"
);
}
#[test]
fn test_is_sha256_hex() {
assert!(is_sha256_hex(ABC_SHA256));
assert!(!is_sha256_hex(&ABC_SHA256.to_uppercase()));
assert!(!is_sha256_hex(&ABC_SHA256[1..]));
assert!(!is_sha256_hex(""));
}
}