use alloc::{
collections::BTreeMap,
fmt,
};
use std::{
fs,
io::Write,
path::{
Path,
PathBuf,
},
};
pub use bunsen_fetch::RESOURCES_DIR;
use super::cached_files::is_sha256;
pub(super) const FETCHING_DIR: &str = ".fetching";
use sha2::{
Digest,
Sha256,
};
#[cfg(feature = "fetch")]
use crate::{
data::cache::{
FetchJob,
FetchOutcome,
FetchPolicy,
},
errors::Multiple,
};
use crate::{
data::{
cache::{
BunsenDiskCache,
BunsenDiskCacheOptions,
partial_path,
verify_sha256,
},
resources::{
LoadedResourceSet,
Resource,
ResourceSet,
Source,
},
},
errors::{
BunsenError,
BunsenResult,
DigestMismatch,
DigestOrigin,
LookupError,
LookupProblem,
sys_at,
},
};
#[derive(Clone, Debug, Default)]
pub struct ResourceCacheOptions {
pub disk: BunsenDiskCacheOptions,
pub offline: bool,
pub local_dirs: BTreeMap<String, PathBuf>,
pub verify_local_dirs: bool,
#[cfg(feature = "fetch")]
pub fetch: FetchPolicy,
}
impl ResourceCacheOptions {
pub fn with_disk(
mut self,
disk: BunsenDiskCacheOptions,
) -> Self {
self.disk = disk;
self
}
pub fn with_offline(
mut self,
offline: bool,
) -> Self {
self.offline = offline;
self
}
pub fn with_local_dir(
mut self,
name: impl Into<String>,
dir: impl Into<PathBuf>,
) -> Self {
self.local_dirs.insert(name.into(), dir.into());
self
}
pub fn with_verify_local_dirs(
mut self,
verify: bool,
) -> Self {
self.verify_local_dirs = verify;
self
}
#[cfg(feature = "fetch")]
pub fn with_fetch_policy(
mut self,
fetch: FetchPolicy,
) -> Self {
self.fetch = fetch;
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Provenance {
Cached,
LocalDir,
Downloaded,
Bundled,
}
impl fmt::Display for Provenance {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
f.write_str(match self {
Self::Cached => "cached",
Self::LocalDir => "local dir",
Self::Downloaded => "downloaded",
Self::Bundled => "bundled",
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedResource {
pub path: PathBuf,
pub provenance: Provenance,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CacheStatus {
Cached,
Conflicting,
LocalDir,
Bundled,
Remote,
}
impl fmt::Display for CacheStatus {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
f.write_str(match self {
Self::Cached => "cached",
Self::Conflicting => "conflicting copies",
Self::LocalDir => "local dir",
Self::Bundled => "bundled",
Self::Remote => "remote",
})
}
}
enum Local {
Found(ResolvedResource),
Remote { dest: PathBuf, urls: Vec<String> },
}
#[cfg_attr(not(feature = "fetch"), allow(dead_code))]
struct RemotePart {
key: String,
dest: PathBuf,
urls: Vec<String>,
res: Resource,
}
fn remote_keys(remote: &[RemotePart]) -> String {
remote
.iter()
.map(|r| r.key.as_str())
.collect::<Vec<_>>()
.join(", ")
}
fn offline(
dest: &Path,
res: &Resource,
) -> BunsenError {
BunsenError::policy(format!(
"not in the cache ({}), and the cache is offline",
dest.display()
))
.context(&res.slot)
}
#[cfg(feature = "fetch")]
fn fetch_prefix(
_dest: &Path,
urls: &[String],
len: u64,
) -> BunsenResult<Vec<u8>> {
let mut last = None;
for url in urls {
match crate::data::cache::fetch_prefix(url, len) {
Ok(prefix) => return Ok(prefix),
Err(e) => last = Some(e),
}
}
Err(last.unwrap_or_else(|| BunsenError::illegal("a remote resource with no URL")))
}
#[cfg(not(feature = "fetch"))]
fn fetch_prefix(
dest: &Path,
_urls: &[String],
_len: u64,
) -> BunsenResult<Vec<u8>> {
Err(BunsenError::unsupported(format!(
"{}: not local, and fetching needs the `fetch` feature",
dest.display()
)))
}
pub struct ResourceCache {
disk: BunsenDiskCache,
offline: bool,
local_dirs: BTreeMap<String, PathBuf>,
verify_local_dirs: bool,
#[cfg(feature = "fetch")]
fetch: FetchPolicy,
}
impl ResourceCache {
pub fn new(options: ResourceCacheOptions) -> BunsenResult<Self> {
Ok(Self {
disk: BunsenDiskCache::new(options.disk)?,
offline: options.offline,
local_dirs: options.local_dirs,
verify_local_dirs: options.verify_local_dirs,
#[cfg(feature = "fetch")]
fetch: options.fetch,
})
}
pub fn disk(&self) -> &BunsenDiskCache {
&self.disk
}
pub fn cache_dir(&self) -> &Path {
self.disk.cache_dir()
}
pub fn offline(&self) -> bool {
self.offline
}
pub fn verify_local_dirs(&self) -> bool {
self.verify_local_dirs
}
pub fn local_dirs(&self) -> &BTreeMap<String, PathBuf> {
&self.local_dirs
}
pub fn local_dir(
&self,
source: &Source,
) -> Option<PathBuf> {
match source {
Source::LocalDir { name, dir } => {
self.local_dirs.get(name).cloned().or_else(|| dir.clone())
}
Source::Url(_) | Source::Bundled(_) => None,
}
}
pub fn resource_path(
&self,
res: &Resource,
) -> Option<PathBuf> {
res.sha256
.as_deref()
.map(|sha256| self.path_under(res, sha256))
}
pub fn cached_path(
&self,
res: &Resource,
) -> BunsenResult<Option<PathBuf>> {
if let Some(path) = self.resource_path(res) {
return Ok(path.is_file().then_some(path));
}
let mut copies = self.unpinned_copies(res)?;
match copies.len() {
0 => Ok(None),
1 => Ok(copies.pop()),
_ => {
let listed = copies
.iter()
.map(|p| format!("\n {}", p.display()))
.collect::<String>();
Err(BunsenError::lookup(LookupError::path(
copies.swap_remove(0),
LookupProblem::Duplicate,
))
.with_details(format!(
"an unpinned resource has {} copies in the cache; remove all but \
one to choose it, or all to fetch it afresh:{listed}",
copies.len() + 1
))
.context(&res.slot))
}
}
}
fn path_under(
&self,
res: &Resource,
pin: &str,
) -> PathBuf {
self.disk.cache_path(
&core::iter::once(RESOURCES_DIR)
.chain(res.name.split('/'))
.chain(core::iter::once(pin))
.collect::<Vec<_>>(),
&res.file,
)
}
fn name_dir(
&self,
res: &Resource,
) -> PathBuf {
let mut dir = self.disk.cache_path(&[RESOURCES_DIR], "");
dir.extend(res.name.split('/'));
dir
}
fn staging_path(
&self,
res: &Resource,
) -> PathBuf {
self.path_under(res, FETCHING_DIR)
}
fn unpinned_copies(
&self,
res: &Resource,
) -> BunsenResult<Vec<PathBuf>> {
let dir = self.name_dir(res);
let entries = match fs::read_dir(&dir) {
Ok(entries) => entries,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(sys_at("read the directory", &dir)(e)),
};
let mut copies = Vec::new();
for entry in entries {
let entry = entry.map_err(sys_at("read the directory", &dir))?;
if entry.file_name().to_str().is_some_and(is_sha256) {
let path = entry.path().join(&res.file);
if path.is_file() {
copies.push(path);
}
}
}
copies.sort();
Ok(copies)
}
#[cfg(feature = "fetch")]
fn file_under_digest(
&self,
res: &Resource,
staged: &Path,
) -> BunsenResult<PathBuf> {
let sha256 = crate::data::cache::sha256_of(staged)?;
let dest = self.path_under(res, &sha256);
if dest.is_file() {
fs::remove_file(staged).map_err(sys_at("remove", staged))?;
} else {
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).map_err(sys_at("create", parent))?;
}
fs::rename(staged, &dest).map_err(sys_at("move", staged))?;
}
let top = self.name_dir(res);
let mut dir = staged.parent();
while let Some(d) = dir
&& d != top
&& fs::remove_dir(d).is_ok()
{
dir = d.parent();
}
Ok(dest)
}
pub fn status(
&self,
res: &Resource,
) -> CacheStatus {
match self.cached_path(res) {
Ok(Some(_)) => return CacheStatus::Cached,
Err(_) => return CacheStatus::Conflicting,
Ok(None) => {}
}
for source in &res.sources {
match source {
Source::LocalDir { .. } => {
if let Some(dir) = self.local_dir(source)
&& dir.join(&res.file).is_file()
{
return CacheStatus::LocalDir;
}
}
Source::Bundled(_) => return CacheStatus::Bundled,
Source::Url(_) => {}
}
}
CacheStatus::Remote
}
pub fn map_status(
&self,
map: &ResourceSet,
) -> BTreeMap<String, CacheStatus> {
map.resources
.iter()
.map(|(key, res)| (key.clone(), self.status(res)))
.collect()
}
pub fn resolve(
&self,
res: &Resource,
) -> BunsenResult<ResolvedResource> {
match self.resolve_local(res)? {
Local::Found(found) => {
found.mark_used();
Ok(found)
}
Local::Remote { dest, urls } => {
if self.offline {
return Err(offline(&dest, res));
}
let urls: Vec<&str> = urls.iter().map(String::as_str).collect();
self.download(res, dest, &urls)
}
}
}
pub fn read_prefix(
&self,
res: &Resource,
len: u64,
) -> BunsenResult<Vec<u8>> {
match self.resolve_local(res)? {
Local::Found(found) => {
use std::io::Read;
let path = &found.path;
let mut prefix = Vec::new();
fs::File::open(path)
.map_err(sys_at("open", path))?
.take(len)
.read_to_end(&mut prefix)
.map_err(sys_at("read", path))?;
Ok(prefix)
}
Local::Remote { dest, urls } => {
if self.offline {
return Err(offline(&dest, res));
}
fetch_prefix(&dest, &urls, len).map_err(|e| e.context(&res.slot))
}
}
}
pub fn load(
&self,
map: &ResourceSet,
) -> BunsenResult<LoadedResourceSet> {
let mut parts = BTreeMap::new();
let mut remote = Vec::new();
for (key, res) in &map.resources {
match self.resolve_local(res)? {
Local::Found(found) => {
found.mark_used();
parts.insert(key.clone(), found);
}
Local::Remote { dest, urls } => remote.push(RemotePart {
key: key.clone(),
dest,
urls,
res: res.clone(),
}),
}
}
if !remote.is_empty() {
if self.offline {
return Err(BunsenError::policy(format!(
"not in the cache, and the cache is offline: {}",
remote_keys(&remote)
))
.context(&map.name));
}
self.fetch_remote(&map.name, remote, &mut parts)?;
}
Ok(LoadedResourceSet {
map: map.clone(),
parts,
})
}
fn resolve_local(
&self,
res: &Resource,
) -> BunsenResult<Local> {
if let Some(path) = self.cached_path(res)? {
return Ok(Local::Found(ResolvedResource {
path,
provenance: Provenance::Cached,
}));
}
let dest = match self.resource_path(res) {
Some(path) => path,
None => self.staging_path(res),
};
let mut urls = Vec::new();
let mut tried = Vec::new();
for source in &res.sources {
match source {
Source::LocalDir { .. } => {
let Some(dir) = self.local_dir(source) else {
continue;
};
let path = dir.join(&res.file);
if !path.is_file() {
tried.push(path);
continue;
}
if self.verify_local_dirs
&& let Some(sha256) = &res.sha256
{
verify_sha256(&path, sha256)?;
}
return Ok(Local::Found(ResolvedResource {
path,
provenance: Provenance::LocalDir,
}));
}
Source::Bundled(bytes) => {
return Ok(Local::Found(self.write_bundled(res, &dest, bytes.0)?));
}
Source::Url(url) => urls.push(url.clone()),
}
}
if urls.is_empty() {
let missing = tried
.first()
.cloned()
.unwrap_or_else(|| PathBuf::from(&res.file));
let looked_in = tried
.iter()
.map(|p| format!("\n {}", p.display()))
.collect::<String>();
return Err(
BunsenError::lookup(LookupError::path(missing, LookupProblem::Missing))
.with_details(format!("no URL to fetch it from; looked in:{looked_in}"))
.context(&res.slot),
);
}
Ok(Local::Remote { dest, urls })
}
fn write_bundled(
&self,
res: &Resource,
dest: &Path,
bytes: &[u8],
) -> BunsenResult<ResolvedResource> {
let Some(sha256) = res.sha256.as_deref() else {
return Err(BunsenError::from(Resource::bundled_without_digest()).context(&res.slot));
};
let found = Sha256::digest(bytes)
.iter()
.map(|b| format!("{b:02x}"))
.collect::<String>();
if found != sha256 {
return Err(
DigestMismatch::new(&res.slot, DigestOrigin::Bundled, sha256, found).into(),
);
}
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).map_err(sys_at("create", parent))?;
}
let partial = partial_path(dest);
let written = (|| -> std::io::Result<()> {
let mut file = fs::File::create(&partial)?;
file.write_all(bytes)?;
file.sync_all()?;
fs::rename(&partial, dest)
})();
if let Err(e) = written {
let _ = fs::remove_file(&partial);
return Err(sys_at("write", dest)(e));
}
Ok(ResolvedResource {
path: dest.to_path_buf(),
provenance: Provenance::Bundled,
})
}
#[cfg(feature = "fetch")]
fn download(
&self,
res: &Resource,
dest: PathBuf,
urls: &[&str],
) -> BunsenResult<ResolvedResource> {
self.disk.fetch_from_urls_retrying(
urls,
&dest,
res.sha256.as_deref(),
self.fetch.retries,
)?;
let path = if res.is_pinned() {
dest
} else {
self.file_under_digest(res, &dest)?
};
Ok(ResolvedResource {
path,
provenance: Provenance::Downloaded,
})
}
#[cfg(not(feature = "fetch"))]
fn download(
&self,
_res: &Resource,
dest: PathBuf,
_urls: &[&str],
) -> BunsenResult<ResolvedResource> {
Err(BunsenError::unsupported(format!(
"{}: not local, and fetching needs the `fetch` feature",
dest.display()
)))
}
#[cfg(feature = "fetch")]
fn fetch_remote(
&self,
name: &str,
remote: Vec<RemotePart>,
parts: &mut BTreeMap<String, ResolvedResource>,
) -> BunsenResult<()> {
let jobs: Vec<FetchJob> = remote
.iter()
.map(|r| {
let mut job = FetchJob::new(r.urls.clone(), r.dest.clone());
job.sha256 = r.res.sha256.clone();
job
})
.collect();
let report = self.disk.fetch_many(&jobs, &self.fetch);
let mut failed = Vec::new();
let mut skipped = Vec::new();
for (outcome, part) in report.outcomes.into_iter().zip(remote) {
let provenance = match outcome {
FetchOutcome::Cached(_) => Provenance::Cached,
FetchOutcome::Fetched(_) => Provenance::Downloaded,
FetchOutcome::Failed { error, .. } => {
failed.push((part.key, error));
continue;
}
FetchOutcome::Skipped { .. } => {
skipped.push(part.key);
continue;
}
};
let path = if part.res.is_pinned() {
part.dest
} else {
match self.file_under_digest(&part.res, &part.dest) {
Ok(path) => path,
Err(error) => {
failed.push((part.key, error));
continue;
}
}
};
parts.insert(part.key, ResolvedResource { path, provenance });
}
if failed.is_empty() && skipped.is_empty() {
return Ok(());
}
let mut summary = format!(
"{} of {} resources did not land",
failed.len() + skipped.len(),
jobs.len()
);
if !skipped.is_empty() {
summary.push_str(&format!(
"; skipped after a failure: {}",
skipped.join(", ")
));
}
Err(BunsenError::from(Multiple::new(summary, failed)).context(name))
}
#[cfg(not(feature = "fetch"))]
fn fetch_remote(
&self,
name: &str,
remote: Vec<RemotePart>,
_parts: &mut BTreeMap<String, ResolvedResource>,
) -> BunsenResult<()> {
Err(BunsenError::unsupported(format!(
"not local, and fetching needs the `fetch` feature: {}",
remote_keys(&remote)
))
.context(name))
}
}
impl ResolvedResource {
fn mark_used(&self) {
if self.provenance != Provenance::Cached {
return;
}
let now = std::time::SystemTime::now();
if let Ok(file) = fs::OpenOptions::new().write(true).open(&self.path) {
let _ = file.set_times(fs::FileTimes::new().set_accessed(now).set_modified(now));
}
}
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
use crate::{
data::cache::testing::ABC_SHA256,
errors::{
BunsenErrorKind,
ConstraintError,
testing::{
ErrorMatcher,
predicate,
text,
},
},
};
#[cfg(feature = "fetch")]
use crate::{
data::cache::{
OnFailure,
testing::{
refused_url,
serve_once,
},
},
errors::testing::not,
};
fn cache_in(
dir: &Path,
offline: bool,
) -> ResourceCache {
ResourceCache::new(
ResourceCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.join("cache")))
.without_transfer_observers(),
)
.with_offline(offline),
)
.unwrap()
}
use crate::data::resources::BundledBytes;
fn url(u: &str) -> Source {
Source::Url(u.to_string())
}
fn res(
key: &str,
sha256: Option<&str>,
sources: Vec<Source>,
) -> Resource {
Resource {
size: None,
dtype: None,
slot: key.to_string(),
file: format!("{key}.bin"),
sha256: sha256.map(str::to_string),
format: None,
name: "prov".to_string(),
sources,
}
}
fn local_dir(dir: &Path) -> Source {
Source::LocalDir {
name: "up".to_string(),
dir: Some(dir.to_path_buf()),
}
}
#[test]
fn test_read_prefix() {
let dir = tempfile::tempdir().unwrap();
let up = dir.path().join("up");
fs::create_dir_all(&up).unwrap();
fs::write(up.join("local.bin"), b"abcdef").unwrap();
let local = res("local", None, vec![local_dir(&up)]);
let cache = cache_in(dir.path(), true);
assert_eq!(cache.read_prefix(&local, 4).unwrap(), b"abcd");
assert_eq!(cache.read_prefix(&local, 64).unwrap(), b"abcdef");
let remote = res("remote", None, vec![url("http://h.invalid/remote.bin")]);
ErrorMatcher::kind(BunsenErrorKind::Policy).assert_err(&cache.read_prefix(&remote, 4));
#[cfg(feature = "fetch")]
{
use crate::data::cache::testing::serve_ranges;
let cache = cache_in(dir.path(), false);
let remote = res(
"remote",
Some(ABC_SHA256),
vec![
url(&refused_url("remote.bin")),
url(&serve_ranges("remote.bin", b"abc", 1)),
],
);
assert_eq!(cache.read_prefix(&remote, 2).unwrap(), b"ab");
assert!(!cache.resource_path(&remote).unwrap().exists());
assert_eq!(cache.status(&remote), CacheStatus::Remote);
}
}
#[test]
fn test_a_bundled_source_lands_under_its_digest() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let bundled = res(
"abc",
Some(ABC_SHA256),
vec![Source::Bundled(BundledBytes(b"abc"))],
);
bundled.validate().unwrap();
assert_eq!(cache.status(&bundled), CacheStatus::Bundled);
let resolved = cache.resolve(&bundled).unwrap();
assert_eq!(resolved.provenance, Provenance::Bundled);
assert_eq!(resolved.path, cache.resource_path(&bundled).unwrap());
assert_eq!(std::fs::read(&resolved.path).unwrap(), b"abc");
assert_eq!(cache.status(&bundled), CacheStatus::Cached);
assert_eq!(
cache.resolve(&bundled).unwrap().provenance,
Provenance::Cached
);
let wrong = res(
"abd",
Some(ABC_SHA256),
vec![Source::Bundled(BundledBytes(b"abd"))],
);
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("not the pinned")
.cause(predicate("a bundled mismatch", |c: &DigestMismatch| {
c.origin == DigestOrigin::Bundled && c.subject == "abd" && c.expected == ABC_SHA256
}))
.assert_err(&cache.resolve(&wrong));
assert!(!cache.resource_path(&wrong).unwrap().exists());
let unpinned = res("abe", None, vec![Source::Bundled(BundledBytes(b"abe"))]);
let unpinned_err = |e: &BunsenError| {
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.display(text::eq(
"abe: Resource.sha256: a bundled source needs a digest to be cached under",
))
.has_cause::<ConstraintError>()
.assert(e);
};
unpinned_err(&unpinned.validate().unwrap_err());
unpinned_err(&cache.resolve(&unpinned).unwrap_err());
let up = dir.path().join("up");
std::fs::create_dir_all(&up).unwrap();
std::fs::write(up.join("abf.bin"), b"abc").unwrap();
let ordered = res(
"abf",
Some(ABC_SHA256),
vec![
local_dir(&up),
Source::Bundled(BundledBytes(b"abc")),
url("https://a.example/abf.bin"),
],
);
assert_eq!(cache.status(&ordered), CacheStatus::LocalDir);
assert_eq!(
cache.resolve(&ordered).unwrap().provenance,
Provenance::LocalDir
);
let map = ResourceSet::new("m").with_resource(bundled.clone());
let loaded = cache.load(&map).unwrap();
assert_eq!(loaded.get("abc").unwrap().provenance, Provenance::Cached);
}
#[cfg(unix)]
#[test]
fn test_a_bundled_write_is_renamed_into_place() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let bundled = res(
"abc",
Some(ABC_SHA256),
vec![Source::Bundled(BundledBytes(b"abc"))],
);
let dest = cache.resource_path(&bundled).unwrap();
fs::create_dir_all(dest.parent().unwrap()).unwrap();
fs::write(&dest, b"ab").unwrap();
assert_eq!(
cache.resolve(&bundled).unwrap().provenance,
Provenance::Cached
);
fs::remove_file(&dest).unwrap();
let outside = dir.path().join("outside.bin");
std::os::unix::fs::symlink(&outside, &dest).unwrap();
assert_eq!(
cache.resolve(&bundled).unwrap().provenance,
Provenance::Bundled
);
assert!(
!outside.exists(),
"the bytes were written through the cache path"
);
assert!(fs::symlink_metadata(&dest).unwrap().file_type().is_file());
assert_eq!(fs::read(&dest).unwrap(), b"abc");
assert!(!partial_path(&dest).exists());
}
#[test]
fn test_resource_paths_by_pin() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let root = dir.path().join("cache").join("resources");
let mut pinned = res(
"abc",
Some(ABC_SHA256),
vec![url("https://a.example/abc.bin")],
);
pinned.name = "pub/fam/build/checkpoint".to_string();
assert_eq!(
cache.resource_path(&pinned).unwrap(),
root.join("pub")
.join("fam")
.join("build")
.join("checkpoint")
.join(ABC_SHA256)
.join("abc.bin")
);
assert_eq!(
cache.resource_path(&pinned).unwrap(),
bunsen_fetch::resource_cache_path(
cache.cache_dir(),
"pub/fam/build/checkpoint",
ABC_SHA256,
"abc.bin"
)
);
let unpinned = res("abc", None, vec![url("https://a.example/abc.bin")]);
assert_eq!(
cache.resource_path(&unpinned),
None,
"an unpinned file's directory is the digest of its copy"
);
assert_eq!(cache.cache_dir(), dir.path().join("cache"));
assert!(cache.offline());
assert!(!cache.verify_local_dirs());
assert!(cache.local_dirs().is_empty());
}
#[test]
fn test_a_cached_resource_is_trusted_in_place() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let r = res(
"abc",
Some(ABC_SHA256),
vec![url("https://a.example/abc.bin")],
);
assert_eq!(cache.status(&r), CacheStatus::Remote);
let dest = cache.resource_path(&r).unwrap();
fs::create_dir_all(dest.parent().unwrap()).unwrap();
fs::write(&dest, b"not really abc").unwrap();
assert_eq!(cache.status(&r), CacheStatus::Cached);
assert_eq!(
cache.resolve(&r).unwrap(),
ResolvedResource {
path: dest,
provenance: Provenance::Cached,
}
);
}
#[test]
fn test_a_load_marks_a_cached_file_used() {
use core::time::Duration;
use std::time::SystemTime;
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let r = res(
"abc",
Some(ABC_SHA256),
vec![url("https://a.example/abc.bin")],
);
let dest = cache.resource_path(&r).unwrap();
fs::create_dir_all(dest.parent().unwrap()).unwrap();
fs::write(&dest, b"abc").unwrap();
let old = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000_000);
let age = || {
fs::File::options()
.write(true)
.open(&dest)
.unwrap()
.set_modified(old)
.unwrap()
};
let modified = || fs::metadata(&dest).unwrap().modified().unwrap();
let map = ResourceSet::new("m").with_resource(r.clone());
age();
assert_eq!(cache.status(&r), CacheStatus::Cached);
cache.map_status(&map);
cache.read_prefix(&r, 2).unwrap();
let listed = cache.cached_files(&Default::default()).unwrap();
assert_eq!(listed[0].last_used, Some(old));
assert_eq!(modified(), old, "a probe marks nothing");
cache.resolve(&r).unwrap();
assert!(modified() > old, "resolve marks it used");
age();
cache.load(&map).unwrap();
assert!(modified() > old, "load marks it used");
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
age();
fs::set_permissions(&dest, fs::Permissions::from_mode(0o444)).unwrap();
assert_eq!(
cache.load(&map).unwrap().get("abc").unwrap().provenance,
Provenance::Cached
);
if fs::File::options().write(true).open(&dest).is_err() {
assert_eq!(modified(), old, "a read-only file keeps its times");
}
fs::set_permissions(&dest, fs::Permissions::from_mode(0o644)).unwrap();
}
}
#[test]
fn test_local_dir_is_used_in_place_and_trusted_unless_asked() {
let dir = tempfile::tempdir().unwrap();
let upstream = dir.path().join("upstream");
fs::create_dir_all(&upstream).unwrap();
let r = res(
"abc",
Some(ABC_SHA256),
vec![local_dir(&upstream), url("https://a.example/abc.bin")],
);
fs::write(upstream.join("abc.bin"), b"a partial download").unwrap();
let trusting = cache_in(dir.path(), true);
assert_eq!(trusting.status(&r), CacheStatus::LocalDir);
let resolved = trusting.resolve(&r).unwrap();
assert_eq!(resolved.provenance, Provenance::LocalDir);
assert_eq!(resolved.path, upstream.join("abc.bin"));
assert!(!dir.path().join("cache").join("resources").exists());
let verifying = ResourceCache::new(
ResourceCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.path().join("cache")))
.without_transfer_observers(),
)
.with_offline(true)
.with_verify_local_dirs(true),
)
.unwrap();
assert!(verifying.verify_local_dirs());
ErrorMatcher::kind(BunsenErrorKind::InvalidResource).assert_err(&verifying.resolve(&r));
fs::write(upstream.join("abc.bin"), b"abc").unwrap();
assert_eq!(
verifying.resolve(&r).unwrap().provenance,
Provenance::LocalDir
);
assert!(!dir.path().join("cache").join("resources").exists());
let elsewhere = dir.path().join("elsewhere");
fs::create_dir_all(&elsewhere).unwrap();
fs::write(elsewhere.join("abc.bin"), b"abc").unwrap();
let overridden = ResourceCache::new(
ResourceCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.path().join("cache")))
.without_transfer_observers(),
)
.with_offline(true)
.with_local_dir("up", elsewhere.clone()),
)
.unwrap();
assert_eq!(overridden.local_dir(&r.sources[0]), Some(elsewhere.clone()));
assert_eq!(overridden.local_dir(&r.sources[1]), None);
assert_eq!(
overridden.resolve(&r).unwrap().path,
elsewhere.join("abc.bin")
);
}
#[test]
fn test_a_given_resource_resolves_in_place() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("mine.pt");
fs::write(&file, b"x").unwrap();
let cache = cache_in(dir.path(), true);
let r = Resource::given("checkpoint", &file);
assert_eq!(cache.status(&r), CacheStatus::LocalDir);
assert_eq!(
cache.resolve(&r).unwrap(),
ResolvedResource {
path: file,
provenance: Provenance::LocalDir,
}
);
}
#[test]
fn test_resource_offline_or_without_a_url() {
let dir = tempfile::tempdir().unwrap();
let offline = cache_in(dir.path(), true);
let remote = res(
"abc",
Some(ABC_SHA256),
vec![url("https://a.example/abc.bin")],
);
ErrorMatcher::kind(BunsenErrorKind::Policy)
.message_contains("the cache is offline")
.frame_contains("abc")
.assert_err(&offline.resolve(&remote));
let no_url = res("abc", Some(ABC_SHA256), vec![]);
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.details_contains("no URL")
.assert_err(&cache_in(dir.path(), false).resolve(&no_url));
let absent = dir.path().join("absent.pt");
let given = Resource::given("checkpoint", &absent);
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.frame_contains("checkpoint")
.cause(predicate("a missing path", move |c: &LookupError| {
c.problem == LookupProblem::Missing && c.key == absent.display().to_string()
}))
.assert_err(&offline.resolve(&given));
}
#[cfg(feature = "fetch")]
#[test]
fn test_a_resource_url_is_fetched_then_cached() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), false);
let live = serve_once("abc.bin", b"abc");
let r = res("abc", Some(ABC_SHA256), vec![url(&live)]);
let resolved = cache.resolve(&r).unwrap();
assert_eq!(resolved.provenance, Provenance::Downloaded);
assert_eq!(resolved.path, cache.resource_path(&r).unwrap());
assert_eq!(fs::read(&resolved.path).unwrap(), b"abc");
assert_eq!(cache.resolve(&r).unwrap().provenance, Provenance::Cached);
let wrong = serve_once("abd.bin", b"abd");
let r = res("abd", Some(ABC_SHA256), vec![url(&wrong)]);
ErrorMatcher::kind(BunsenErrorKind::InvalidResource).assert_err(&cache.resolve(&r));
assert!(!cache.resource_path(&r).unwrap().exists());
}
#[cfg(feature = "fetch")]
#[test]
fn test_an_unpinned_download_is_filed_under_its_digest() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), false);
let prov = cache.resources_dir().join("prov");
let live = serve_once("abc.bin", b"abc");
let r = res("abc", None, vec![url(&live)]);
assert_eq!(cache.status(&r), CacheStatus::Remote);
let resolved = cache.resolve(&r).unwrap();
let filed = prov.join(ABC_SHA256).join("abc.bin");
assert_eq!(resolved.provenance, Provenance::Downloaded);
assert_eq!(resolved.path, filed);
assert_eq!(fs::read(&filed).unwrap(), b"abc");
assert!(!prov.join(FETCHING_DIR).exists());
assert_eq!(cache.cached_path(&r).unwrap(), Some(filed.clone()));
assert_eq!(cache.status(&r), CacheStatus::Cached);
let again = cache.resolve(&r).unwrap();
assert_eq!((again.path, again.provenance), (filed, Provenance::Cached));
let live = serve_once("abd.bin", b"abd");
let abd = res("abd", None, vec![url(&live)]);
let loaded = cache
.load(&ResourceSet::new("m").with_resource(abd.clone()))
.unwrap();
let path = loaded.expect("abd").unwrap();
assert_eq!(
path,
prov.join(crate::data::cache::sha256_of(&path).unwrap())
.join("abd.bin")
);
assert_eq!(cache.cached_path(&abd).unwrap(), Some(path.to_path_buf()));
assert!(!prov.join(FETCHING_DIR).exists());
}
#[test]
fn test_unpinned_copies() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let prov = cache.resources_dir().join("prov");
let put = |pin: &str, bytes: &[u8]| {
let path = prov.join(pin).join("abc.bin");
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(&path, bytes).unwrap();
path
};
let r = res("abc", None, vec![url("https://a.example/abc.bin")]);
assert_eq!(cache.cached_path(&r).unwrap(), None);
put(FETCHING_DIR, b"ab");
put("12345-abc.bin", b"abc");
assert_eq!(cache.cached_path(&r).unwrap(), None);
assert_eq!(cache.status(&r), CacheStatus::Remote);
let one = put(ABC_SHA256, b"abc");
assert_eq!(cache.cached_path(&r).unwrap(), Some(one.clone()));
let found = cache.resolve(&r).unwrap();
assert_eq!((found.path, found.provenance), (one, Provenance::Cached));
put(&"0".repeat(64), b"abd");
let listed = || {
let mut files = Vec::new();
for (path, _) in super::super::cached_files::files_under(&prov).unwrap() {
files.push((path.clone(), fs::read(path).unwrap()));
}
files
};
let before = listed();
assert_eq!(cache.status(&r), CacheStatus::Conflicting);
let duplicate = || {
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.cause(predicate("a duplicate copy", |c: &LookupError| {
c.problem == LookupProblem::Duplicate
}))
};
duplicate().assert_err(&cache.cached_path(&r));
duplicate().assert_err(&cache.resolve(&r));
duplicate().assert_err(&cache.load(&ResourceSet::new("m").with_resource(r.clone())));
assert_eq!(listed(), before);
}
#[cfg(feature = "fetch")]
#[test]
fn test_load_mixes_provenances() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), false);
let upstream = dir.path().join("upstream");
fs::create_dir_all(&upstream).unwrap();
fs::write(upstream.join("local.bin"), b"abc").unwrap();
let live = serve_once("remote.bin", b"abc");
let cached = res(
"cached",
Some(ABC_SHA256),
vec![url("https://a.example/cached.bin")],
);
let dest = cache.resource_path(&cached).unwrap();
fs::create_dir_all(dest.parent().unwrap()).unwrap();
fs::write(&dest, b"abc").unwrap();
let local = res(
"local",
Some(ABC_SHA256),
vec![local_dir(&upstream), url("https://a.example/local.bin")],
);
let remote = res("remote", Some(ABC_SHA256), vec![url(&live)]);
let map = ResourceSet::new("m")
.with_resource(cached)
.with_resource(local.clone())
.with_resource(remote.clone());
let status = cache.map_status(&map);
assert_eq!(status["cached"], CacheStatus::Cached);
assert_eq!(status["local"], CacheStatus::LocalDir);
assert_eq!(status["remote"], CacheStatus::Remote);
let loaded = cache.load(&map).unwrap();
assert_eq!(loaded.map, map);
assert_eq!(loaded.keys(), ["cached", "local", "remote"]);
assert_eq!(loaded.get("cached").unwrap().provenance, Provenance::Cached);
assert_eq!(
loaded.get("local").unwrap().provenance,
Provenance::LocalDir
);
assert_eq!(
loaded.get("local").unwrap().path,
upstream.join("local.bin")
);
assert_eq!(
loaded.get("remote").unwrap().provenance,
Provenance::Downloaded
);
assert_eq!(
loaded.expect("remote").unwrap(),
cache.resource_path(&remote).unwrap()
);
assert!(!cache.resource_path(&local).unwrap().exists());
assert_eq!(cache.map_status(&map)["remote"], CacheStatus::Cached);
}
#[cfg(feature = "fetch")]
#[test]
fn test_load_names_the_key_that_did_not_land() {
let dir = tempfile::tempdir().unwrap();
let cache = ResourceCache::new(
ResourceCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.path().join("cache")))
.without_transfer_observers(),
)
.with_fetch_policy(FetchPolicy::default().with_on_failure(OnFailure::Continue)),
)
.unwrap();
let live = res(
"live",
Some(ABC_SHA256),
vec![url(&serve_once("live.bin", b"abc"))],
);
let dead = res(
"dead",
Some(ABC_SHA256),
vec![url(&refused_url("dead.bin"))],
);
let map = ResourceSet::new("m")
.with_resource(live.clone())
.with_resource(dead);
let err = cache.load(&map).unwrap_err();
ErrorMatcher::new()
.display(text::eq("m: 1 of 2 resources did not land"))
.details_contains("dead: ")
.details(not(text::contains("live:")))
.cause(predicate("one member, dead", |c: &Multiple| {
c.members.len() == 1 && c.members[0].0 == "dead"
}))
.assert(&err);
assert!(cache.resource_path(&live).unwrap().is_file());
}
#[test]
fn test_load_offline_names_the_remote_keys() {
let dir = tempfile::tempdir().unwrap();
let cache = cache_in(dir.path(), true);
let cached = res(
"cached",
Some(ABC_SHA256),
vec![url("https://a.example/cached.bin")],
);
let dest = cache.resource_path(&cached).unwrap();
fs::create_dir_all(dest.parent().unwrap()).unwrap();
fs::write(&dest, b"abc").unwrap();
let remote = res(
"remote",
Some(ABC_SHA256),
vec![url("https://a.example/remote.bin")],
);
let map = ResourceSet::new("m")
.with_resource(cached)
.with_resource(remote);
ErrorMatcher::kind(BunsenErrorKind::Policy)
.message(text::ends_with("offline: remote"))
.frame_contains("m")
.assert_err(&cache.load(&map));
let local_only = ResourceSet::new("m").with_resource(res(
"cached",
Some(ABC_SHA256),
vec![url("https://a.example/cached.bin")],
));
let loaded = cache.load(&local_only).unwrap();
assert_eq!(loaded.get("cached").unwrap().provenance, Provenance::Cached);
}
#[test]
fn test_display() {
assert_eq!(Provenance::Cached.to_string(), "cached");
assert_eq!(Provenance::LocalDir.to_string(), "local dir");
assert_eq!(Provenance::Downloaded.to_string(), "downloaded");
assert_eq!(CacheStatus::Cached.to_string(), "cached");
assert_eq!(CacheStatus::LocalDir.to_string(), "local dir");
assert_eq!(CacheStatus::Remote.to_string(), "remote");
}
}