use std::{
collections::BTreeMap,
fmt,
fs,
path::{
Path,
PathBuf,
},
};
use super::{
PretrainedWeightsDescriptor,
WeightsSource,
pretrained_weights_resource_key,
};
use crate::{
data::cache::{
BunsenDiskCache,
BunsenDiskCacheOptions,
link_or_copy,
verify_sha256,
},
errors::{
BunsenError,
BunsenResult,
},
};
pub const WEIGHTS_DIR: &str = "weights";
#[derive(Clone, Debug, Default)]
pub struct WeightsCacheOptions {
pub disk: BunsenDiskCacheOptions,
pub offline: bool,
pub local_dirs: BTreeMap<String, PathBuf>,
}
impl WeightsCacheOptions {
pub fn with_disk(
mut self,
disk: BunsenDiskCacheOptions,
) -> Self {
self.disk = disk;
self
}
pub fn with_offline(
mut self,
offline: bool,
) -> Self {
self.offline = offline;
self
}
pub fn with_local_dir(
mut self,
name: impl Into<String>,
dir: impl Into<PathBuf>,
) -> Self {
self.local_dirs.insert(name.into(), dir.into());
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Provenance {
Cached,
File,
LocalDir,
Downloaded,
Given,
}
impl fmt::Display for Provenance {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
f.write_str(match self {
Self::Cached => "cached",
Self::File => "bundled file",
Self::LocalDir => "local dir",
Self::Downloaded => "downloaded",
Self::Given => "given",
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedWeights {
pub path: PathBuf,
pub provenance: Provenance,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CacheStatus {
Cached,
File,
LocalDir,
Remote,
}
impl fmt::Display for CacheStatus {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
f.write_str(match self {
Self::Cached => "cached",
Self::File => "bundled file",
Self::LocalDir => "local dir",
Self::Remote => "remote",
})
}
}
pub struct WeightsCache {
disk: BunsenDiskCache,
offline: bool,
local_dirs: BTreeMap<String, PathBuf>,
}
impl WeightsCache {
pub fn new(options: WeightsCacheOptions) -> BunsenResult<Self> {
Ok(Self {
disk: BunsenDiskCache::new(options.disk)?,
offline: options.offline,
local_dirs: options.local_dirs,
})
}
pub fn disk(&self) -> &BunsenDiskCache {
&self.disk
}
pub fn cache_dir(&self) -> &Path {
self.disk.cache_dir()
}
pub fn offline(&self) -> bool {
self.offline
}
pub fn local_dirs(&self) -> &BTreeMap<String, PathBuf> {
&self.local_dirs
}
pub fn local_dir(
&self,
source: &WeightsSource,
) -> Option<PathBuf> {
match source {
WeightsSource::LocalDir { name, dir } => {
self.local_dirs.get(name).cloned().or_else(|| dir.clone())
}
_ => None,
}
}
pub fn cached_path(
&self,
kit: &str,
provider: &str,
desc: &PretrainedWeightsDescriptor,
) -> PathBuf {
match &desc.sha256 {
Some(sha256) => self
.disk
.cache_path(&[WEIGHTS_DIR, kit, provider, sha256], &desc.file),
None => {
let key = pretrained_weights_resource_key(&desc.cache_key());
self.disk.cache_path(&key, &desc.file)
}
}
}
pub fn status(
&self,
kit: &str,
provider: &str,
desc: &PretrainedWeightsDescriptor,
) -> CacheStatus {
if self.cached_path(kit, provider, desc).is_file() {
return CacheStatus::Cached;
}
for source in &desc.sources {
match source {
WeightsSource::File(path) if path.is_file() => return CacheStatus::File,
WeightsSource::LocalDir { .. } => {
if let Some(dir) = self.local_dir(source)
&& dir.join(&desc.file).is_file()
{
return CacheStatus::LocalDir;
}
}
_ => {}
}
}
CacheStatus::Remote
}
pub fn resolve(
&self,
kit: &str,
provider: &str,
desc: &PretrainedWeightsDescriptor,
) -> BunsenResult<ResolvedWeights> {
let id = format!("{provider}/{}", desc.name);
let dest = self.cached_path(kit, provider, desc);
if dest.is_file() {
return Ok(ResolvedWeights {
path: dest,
provenance: Provenance::Cached,
});
}
if dest.symlink_metadata().is_ok() {
fs::remove_file(&dest).map_err(BunsenError::external)?;
}
let mut urls = Vec::new();
for source in &desc.sources {
match source {
WeightsSource::File(path) => {
if path.is_file() {
return Ok(ResolvedWeights {
path: path.clone(),
provenance: Provenance::File,
});
}
}
WeightsSource::LocalDir { .. } => {
let Some(dir) = self.local_dir(source) else {
continue;
};
let path = dir.join(&desc.file);
if !path.is_file() {
continue;
}
let checks_out = match &desc.sha256 {
Some(sha256) => verify_sha256(&path, sha256).is_ok(),
None => true,
};
if checks_out {
link_or_copy(&path, &dest)?;
return Ok(ResolvedWeights {
path: dest,
provenance: Provenance::LocalDir,
});
}
}
WeightsSource::Url(url) => urls.push(url.as_str()),
}
}
if urls.is_empty() {
return Err(BunsenError::ResourceNotFound(format!(
"{id}: not local, and no URL to fetch it from"
)));
}
if self.offline {
return Err(BunsenError::ResourceNotFound(format!(
"{id}: not in the cache ({}) and the cache is offline",
dest.display()
)));
}
self.download(dest, &urls, desc.sha256.as_deref())
}
#[cfg(feature = "fetch")]
fn download(
&self,
dest: PathBuf,
urls: &[&str],
sha256: Option<&str>,
) -> BunsenResult<ResolvedWeights> {
self.disk.fetch_from_urls(urls, &dest, sha256)?;
Ok(ResolvedWeights {
path: dest,
provenance: Provenance::Downloaded,
})
}
#[cfg(not(feature = "fetch"))]
fn download(
&self,
dest: PathBuf,
_urls: &[&str],
_sha256: Option<&str>,
) -> BunsenResult<ResolvedWeights> {
Err(BunsenError::ResourceNotFound(format!(
"{}: not local, and fetching needs the `fetch` feature",
dest.display()
)))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "fetch")]
use crate::data::cache::testing::serve_once;
use crate::data::{
cache::testing::ABC_SHA256,
pretrained::WeightsFormat,
};
fn cache_in(
dir: &Path,
offline: bool,
) -> WeightsCache {
WeightsCache::new(
WeightsCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.join("cache")))
.without_transfer_observers(),
)
.with_offline(offline),
)
.unwrap()
}
fn abc(
sha256: Option<&str>,
sources: Vec<WeightsSource>,
) -> PretrainedWeightsDescriptor {
PretrainedWeightsDescriptor {
name: "abc".to_string(),
description: "three bytes".to_string(),
license: None,
origin: None,
prefab: "abc".to_string(),
aliases: vec![],
file: "abc.pt".to_string(),
sha256: sha256.map(str::to_string),
format: WeightsFormat::PYTORCH_F16,
sources,
}
}
fn url(u: &str) -> WeightsSource {
WeightsSource::Url(u.to_string())
}
#[test]
fn test_cached_paths_by_pin() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let root = dir.path().join("cache").join("weights");
let pinned = abc(Some(ABC_SHA256), vec![url("https://a.example/abc.pt")]);
assert_eq!(
cache.cached_path("kit", "prov", &pinned),
root.join("kit")
.join("prov")
.join(ABC_SHA256)
.join("abc.pt")
);
let unpinned = abc(None, vec![url("https://a.example/abc.pt")]);
assert_eq!(
cache.cached_path("kit", "prov", &unpinned),
root.join(unpinned.cache_key()).join("abc.pt"),
"unpinned keeps the URL-keyed layout"
);
assert_eq!(cache.cache_dir(), dir.path().join("cache"));
assert!(cache.offline());
}
#[test]
fn test_offline_without_a_local_source_is_not_found() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let desc = abc(Some(ABC_SHA256), vec![url("https://a.example/abc.pt")]);
assert_eq!(cache.status("kit", "prov", &desc), CacheStatus::Remote);
assert!(matches!(
cache.resolve("kit", "prov", &desc),
Err(BunsenError::ResourceNotFound(_))
));
let no_url = abc(Some(ABC_SHA256), vec![]);
assert!(matches!(
cache_in(dir.path(), false).resolve("kit", "prov", &no_url),
Err(BunsenError::ResourceNotFound(_))
));
}
#[test]
fn test_a_cached_file_is_trusted_in_place() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let desc = abc(Some(ABC_SHA256), vec![url("https://a.example/abc.pt")]);
let dest = cache.cached_path("kit", "prov", &desc);
fs::create_dir_all(dest.parent().unwrap()).unwrap();
fs::write(&dest, b"not really abc").unwrap();
assert_eq!(cache.status("kit", "prov", &desc), CacheStatus::Cached);
assert_eq!(
cache.resolve("kit", "prov", &desc).unwrap(),
ResolvedWeights {
path: dest,
provenance: Provenance::Cached,
}
);
}
#[test]
fn test_local_dir_is_checked_before_adoption() {
let dir = tempfile::tempdir().unwrap();
let upstream = dir.path().join("upstream");
fs::create_dir_all(&upstream).unwrap();
let desc = abc(
Some(ABC_SHA256),
vec![
WeightsSource::LocalDir {
name: "up".to_string(),
dir: Some(upstream.clone()),
},
url("https://a.example/abc.pt"),
],
);
let cache = cache_in(dir.path(), true);
fs::write(upstream.join("abc.pt"), b"a partial download").unwrap();
assert_eq!(cache.status("kit", "prov", &desc), CacheStatus::LocalDir);
assert!(matches!(
cache.resolve("kit", "prov", &desc),
Err(BunsenError::ResourceNotFound(_))
));
assert!(!cache.cached_path("kit", "prov", &desc).exists());
fs::write(upstream.join("abc.pt"), b"abc").unwrap();
let resolved = cache.resolve("kit", "prov", &desc).unwrap();
assert_eq!(resolved.provenance, Provenance::LocalDir);
assert_eq!(resolved.path, cache.cached_path("kit", "prov", &desc));
assert_eq!(fs::read(&resolved.path).unwrap(), b"abc");
assert_eq!(cache.status("kit", "prov", &desc), CacheStatus::Cached);
let elsewhere = dir.path().join("elsewhere");
fs::create_dir_all(&elsewhere).unwrap();
fs::write(elsewhere.join("abc.pt"), b"abc").unwrap();
let cache = WeightsCache::new(
WeightsCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.path().join("cache2")))
.without_transfer_observers(),
)
.with_offline(true)
.with_local_dir("up", elsewhere.clone()),
)
.unwrap();
assert_eq!(cache.local_dir(&desc.sources[0]), Some(elsewhere));
assert_eq!(cache.local_dirs().len(), 1);
assert_eq!(
cache.resolve("kit", "prov", &desc).unwrap().provenance,
Provenance::LocalDir
);
}
#[test]
fn test_a_bundled_file_is_used_in_place() {
let dir = tempfile::tempdir().unwrap();
let bundled = dir.path().join("bundled").join("abc.pt");
fs::create_dir_all(bundled.parent().unwrap()).unwrap();
fs::write(&bundled, b"abc").unwrap();
let desc = abc(
Some(ABC_SHA256),
vec![
WeightsSource::File(bundled.clone()),
url("https://a.example/abc.pt"),
],
);
let cache = cache_in(dir.path(), true);
assert_eq!(cache.status("kit", "prov", &desc), CacheStatus::File);
assert_eq!(
cache.resolve("kit", "prov", &desc).unwrap(),
ResolvedWeights {
path: bundled,
provenance: Provenance::File,
}
);
assert!(!dir.path().join("cache").join("weights").exists());
}
#[cfg(feature = "fetch")]
#[test]
fn test_a_url_is_fetched_then_cached() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), false);
let live = serve_once("abc.pt", b"abc");
let desc = abc(Some(ABC_SHA256), vec![url(&live)]);
let resolved = cache.resolve("kit", "prov", &desc).unwrap();
assert_eq!(resolved.provenance, Provenance::Downloaded);
assert_eq!(resolved.path, cache.cached_path("kit", "prov", &desc));
assert_eq!(fs::read(&resolved.path).unwrap(), b"abc");
assert_eq!(
cache.resolve("kit", "prov", &desc).unwrap().provenance,
Provenance::Cached
);
let wrong = serve_once("abc.pt", b"abd");
let desc = abc(Some(ABC_SHA256), vec![url(&wrong)]);
assert!(matches!(
cache.resolve("kit", "other", &desc),
Err(BunsenError::Invalid(_))
));
assert!(!cache.cached_path("kit", "other", &desc).exists());
}
#[test]
fn test_display() {
assert_eq!(Provenance::LocalDir.to_string(), "local dir");
assert_eq!(Provenance::Given.to_string(), "given");
assert_eq!(CacheStatus::File.to_string(), "bundled file");
assert_eq!(CacheStatus::Remote.to_string(), "remote");
}
}