use crate::asset::{BuildCtx, CacheInputs, SourceFiles};
use crate::file_stamp::FileStamp;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::path::Path;
use std::sync::{Mutex, OnceLock};
fn file_content_hash(path: &str) -> Option<[u8; 32]> {
type HashMemo = Mutex<HashMap<String, (FileStamp, [u8; 32])>>;
static MEMO: OnceLock<HashMemo> = OnceLock::new();
let stamp = FileStamp::read(path)?;
let memoizable = stamp.settled();
let memo = MEMO.get_or_init(|| Mutex::new(HashMap::new()));
if let Some(&(s, h)) = memo
.lock()
.expect("file-stamp memo lock is not poisoned")
.get(path)
&& s == stamp
{
return Some(h);
}
let bytes = std::fs::read(path).ok()?;
let mut hasher = Sha256::new();
hasher.update(&bytes);
let hash: [u8; 32] = hasher.finalize().into();
if memoizable {
memo.lock()
.expect("file-stamp memo lock is not poisoned")
.insert(path.to_string(), (stamp, hash));
}
Some(hash)
}
include!(concat!(env!("OUT_DIR"), "/compile_source_hash.rs"));
pub(crate) fn payload_key(
discriminant: u8,
args: &serde_json::Value,
ctx: &BuildCtx<'_>,
inputs: &CacheInputs,
) -> String {
let files = match &inputs.sources {
SourceFiles::Extra(extra) => {
let mut files = referenced_files(args, ctx);
for path in extra {
if let Some(h) = file_content_hash(path) {
files.push((path.clone(), h));
}
}
files
}
SourceFiles::Only(paths) => paths
.iter()
.filter_map(|p| file_content_hash(p).map(|h| (p.clone(), h)))
.collect(),
};
let target = inputs
.target_dependent
.then(|| concinnity_core::platform::Platform::current().key());
key_from_parts(discriminant, args, &files, target)
}
const EXPAND_FORMAT_VERSION: u32 = 4;
pub(crate) fn expand_key(
source: &str,
args: &serde_json::Value,
assets_dir: Option<&Path>,
) -> String {
let mut hasher = Sha256::new();
hasher.update(EXPAND_FORMAT_VERSION.to_le_bytes());
if let Some(h) = file_content_hash(source) {
hasher.update(h);
}
if source.to_lowercase().ends_with(".gltf") {
for path in crate::gltf_source::referenced_files(source, assets_dir) {
if let Some(h) = file_content_hash(&path) {
hasher.update(h);
}
}
}
let args_bytes = serde_json::to_vec(args).unwrap_or_default();
hasher.update((args_bytes.len() as u64).to_le_bytes());
hasher.update(&args_bytes);
format!("{:x}", hasher.finalize())
}
pub fn load(key: &str) -> Option<Vec<u8>> {
if cfg!(test) {
return None;
}
std::fs::read(crate::paths::cache_dir()?.join(key)).ok()
}
pub fn store(key: &str, bytes: &[u8]) {
if cfg!(test) {
return;
}
let Some(dir) = crate::paths::cache_dir() else {
return;
};
store_in(&dir, key, bytes);
}
fn key_from_parts(
discriminant: u8,
args: &serde_json::Value,
files: &[(String, [u8; 32])],
target: Option<&str>,
) -> String {
let mut hasher = Sha256::new();
hasher.update(COMPILE_SOURCE_HASH.to_le_bytes());
hasher.update(concinnity_core::SCHEMA_VERSION.to_le_bytes());
hasher.update([discriminant]);
let args_bytes = serde_json::to_vec(args).unwrap_or_default();
hasher.update((args_bytes.len() as u64).to_le_bytes());
hasher.update(&args_bytes);
match target {
Some(t) => {
hasher.update([1u8]);
hasher.update((t.len() as u64).to_le_bytes());
hasher.update(t.as_bytes());
}
None => hasher.update([0u8]),
}
let mut files = files.to_vec();
files.sort();
for (path, content_hash) in &files {
hasher.update((path.len() as u64).to_le_bytes());
hasher.update(path.as_bytes());
hasher.update(content_hash);
}
format!("{:x}", hasher.finalize())
}
fn referenced_files(args: &serde_json::Value, ctx: &BuildCtx<'_>) -> Vec<(String, [u8; 32])> {
let mut strings = Vec::new();
collect_strings(args, &mut strings);
let mut out = Vec::new();
for s in strings {
let Some(path) = resolve_source(&s, ctx) else {
continue;
};
if let Some(h) = file_content_hash(&path) {
out.push((path, h));
}
}
out
}
fn collect_strings(v: &serde_json::Value, out: &mut Vec<String>) {
match v {
serde_json::Value::String(s) => out.push(s.clone()),
serde_json::Value::Array(a) => a.iter().for_each(|e| collect_strings(e, out)),
serde_json::Value::Object(m) => m.values().for_each(|e| collect_strings(e, out)),
_ => {}
}
}
fn resolve_source(s: &str, ctx: &BuildCtx<'_>) -> Option<String> {
let looks_like_file = s.contains('/') || s.contains('\\') || Path::new(s).extension().is_some();
if !looks_like_file {
return None;
}
if Path::new(s).is_file() {
return Some(s.to_string());
}
if let Some(p) = ctx
.assets_dir
.and_then(|dir| crate::source::find_in(dir, s))
{
return Some(p);
}
if let Some(dir) = ctx.artifacts_dir {
let p = format!("{dir}/{s}");
if Path::new(&p).is_file() {
return Some(p);
}
}
None
}
fn store_in(dir: &Path, key: &str, bytes: &[u8]) {
if std::fs::create_dir_all(dir).is_err() {
return;
}
let tmp = dir.join(format!("{key}.{}.tmp", std::process::id()));
if std::fs::write(&tmp, bytes).is_ok() {
let _ = std::fs::rename(&tmp, dir.join(key));
}
}
#[cfg(test)]
fn load_in(dir: &Path, key: &str) -> Option<Vec<u8>> {
std::fs::read(dir.join(key)).ok()
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn ctx() -> BuildCtx<'static> {
BuildCtx {
name: "test",
assets_dir: None,
artifacts_dir: None,
all_assets: &[],
}
}
#[test]
fn key_is_stable_for_same_inputs() {
let a = json!({"generator": "box", "half_extents": [1, 2, 3]});
assert_eq!(
key_from_parts(7, &a, &[], None),
key_from_parts(7, &a, &[], None)
);
}
#[test]
fn key_changes_with_args_discriminant_and_files() {
let a = json!({"generator": "box"});
let b = json!({"generator": "sphere"});
let base = key_from_parts(1, &a, &[], None);
assert_ne!(
base,
key_from_parts(1, &b, &[], None),
"args must affect the key"
);
assert_ne!(
base,
key_from_parts(2, &a, &[], None),
"discriminant must affect the key"
);
assert_ne!(
base,
key_from_parts(1, &a, &[("x.hdr".into(), [9u8; 32])], None),
"a referenced file must affect the key"
);
}
#[test]
fn key_ignores_referenced_file_order() {
let a = json!({});
let f1 = ("a.hdr".to_string(), [1u8; 32]);
let f2 = ("b.hdr".to_string(), [2u8; 32]);
assert_eq!(
key_from_parts(0, &a, &[f1.clone(), f2.clone()], None),
key_from_parts(0, &a, &[f2, f1], None),
);
}
#[test]
fn key_changes_with_the_compile_target() {
let a = json!({"sources": {"hlsl": "shared.inc", "glsl": "shared.inc"}});
let hlsl = key_from_parts(1, &a, &[], Some("hlsl"));
let glsl = key_from_parts(1, &a, &[], Some("glsl"));
assert_ne!(hlsl, glsl, "the compile target must affect the key");
assert_ne!(
hlsl,
key_from_parts(1, &a, &[], None),
"a target-dependent key must differ from a target-independent one"
);
assert_ne!(
key_from_parts(1, &a, &[], Some("")),
key_from_parts(1, &a, &[], None),
"an empty target must not hash as no target"
);
}
#[test]
fn key_tracks_referenced_file_contents() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("env.hdr");
std::fs::write(&file, b"first").unwrap();
let args = json!({ "source": file.to_str().unwrap() });
let before = payload_key(3, &args, &ctx(), &CacheInputs::extra(vec![]));
std::fs::write(&file, b"second").unwrap();
let after = payload_key(3, &args, &ctx(), &CacheInputs::extra(vec![]));
assert_ne!(
before, after,
"key must change when a referenced file changes"
);
}
#[test]
fn key_tracks_an_equal_length_edit() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("env.hdr");
std::fs::write(&file, b"aaaaa").unwrap();
let args = json!({ "source": file.to_str().unwrap() });
let before = payload_key(3, &args, &ctx(), &CacheInputs::extra(vec![]));
std::fs::write(&file, b"bbbbb").unwrap();
let after = payload_key(3, &args, &ctx(), &CacheInputs::extra(vec![]));
assert_ne!(
before, after,
"an equal-length edit in the same mtime tick must still bust the key"
);
}
#[test]
fn key_tracks_extra_source_file_contents() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("shader.metal");
std::fs::write(&file, b"void shade() {}").unwrap();
let path = file.to_str().unwrap().to_string();
let args = json!({ "fragment_shader": "chrome" });
let before = payload_key(11, &args, &ctx(), &CacheInputs::extra(vec![path.clone()]));
std::fs::write(&file, b"void shade(float) {}").unwrap();
let after = payload_key(11, &args, &ctx(), &CacheInputs::extra(vec![path.clone()]));
assert_ne!(
before, after,
"an extra source file's contents must affect the key"
);
}
#[test]
fn key_ignores_unreadable_extra_source_file() {
let args = json!({ "fragment_shader": "chrome" });
let missing = "/definitely/not/a/real/path.metal".to_string();
assert_eq!(
payload_key(11, &args, &ctx(), &CacheInputs::extra(vec![])),
payload_key(11, &args, &ctx(), &CacheInputs::extra(vec![missing])),
);
}
#[test]
fn non_file_strings_are_not_resolved() {
assert!(referenced_files(&json!({"generator": "box"}), &ctx()).is_empty());
}
#[test]
fn only_inputs_replace_the_generic_args_walk() {
let dir = tempfile::tempdir().unwrap();
let used = dir.path().join("used.hlsl");
let unused = dir.path().join("unused.glsl");
std::fs::write(&used, b"used").unwrap();
std::fs::write(&unused, b"unused").unwrap();
let args = json!({"sources": {
"hlsl": used.to_str().unwrap(),
"glsl": unused.to_str().unwrap(),
}});
let only = |inputs: &CacheInputs| payload_key(9, &args, &ctx(), inputs);
let reported = CacheInputs {
sources: SourceFiles::Only(vec![used.to_str().unwrap().to_string()]),
target_dependent: false,
};
let before = only(&reported);
std::fs::write(&unused, b"edited").unwrap();
assert_eq!(
before,
only(&reported),
"editing an unreported file must not affect the key"
);
std::fs::write(&used, b"edited").unwrap();
assert_ne!(
before,
only(&reported),
"editing a reported file must affect the key"
);
assert_eq!(referenced_files(&args, &ctx()).len(), 2);
}
#[test]
fn expand_key_tracks_source_contents_and_args() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("scene.fbx");
std::fs::write(&file, b"first").unwrap();
let src = file.to_str().unwrap();
let args = json!({ "prefix": "scn", "texture_max_size": 512 });
let base = expand_key(src, &args, None);
assert_eq!(base, expand_key(src, &args, None));
assert_ne!(
base,
expand_key(
src,
&json!({ "prefix": "scn", "texture_max_size": 256 }),
None
)
);
std::fs::write(&file, b"second").unwrap();
assert_ne!(base, expand_key(src, &args, None));
}
#[test]
fn expand_key_tracks_gltf_sibling_file_contents() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("geo.bin"), b"first").unwrap();
let gltf = dir.path().join("scene.gltf");
std::fs::write(
&gltf,
serde_json::to_vec(&json!({
"asset": {"version": "2.0"},
"buffers": [{"byteLength": 5, "uri": "geo.bin"}]
}))
.unwrap(),
)
.unwrap();
let src = gltf.to_str().unwrap();
let args = json!({ "prefix": "scn" });
let before = expand_key(src, &args, None);
assert_eq!(before, expand_key(src, &args, None));
std::fs::write(dir.path().join("geo.bin"), b"second").unwrap();
assert_ne!(
before,
expand_key(src, &args, None),
"editing a referenced .bin must bust the expansion key"
);
}
#[test]
fn expansion_and_payload_key_spaces_stay_distinct() {
let args = json!({ "prefix": "scn" });
assert_ne!(
expand_key("/no/such/scene.glb", &args, None),
payload_key(0, &args, &ctx(), &CacheInputs::extra(vec![])),
);
}
#[test]
fn store_then_load_round_trips() {
let dir = tempfile::tempdir().unwrap();
store_in(dir.path(), "abc123", b"payload bytes");
assert_eq!(
load_in(dir.path(), "abc123").as_deref(),
Some(&b"payload bytes"[..])
);
assert_eq!(load_in(dir.path(), "missing"), None);
}
#[test]
fn store_in_creates_the_directory_and_overwrites() {
let dir = tempfile::tempdir().unwrap();
let nested = dir.path().join("cache").join("deep");
store_in(&nested, "k", b"one");
assert_eq!(load_in(&nested, "k").as_deref(), Some(&b"one"[..]));
store_in(&nested, "k", b"two");
assert_eq!(load_in(&nested, "k").as_deref(), Some(&b"two"[..]));
}
#[test]
fn payload_keys_are_bare_hashes_with_no_prefix() {
let key = payload_key(1, &json!({}), &ctx(), &CacheInputs::extra(vec![]));
assert_eq!(
key.len(),
64,
"key '{key}' must be a bare sha256 hex digest"
);
assert!(
key.chars().all(|c| c.is_ascii_hexdigit()),
"key '{key}' must contain no namespacing prefix"
);
assert_eq!(
key,
key_from_parts(1, &json!({}), &[], None),
"a target-independent asset must not fold the platform into its key"
);
}
#[test]
fn target_dependent_payload_keys_fold_in_the_platform() {
let args = json!({});
let dependent = CacheInputs {
sources: SourceFiles::Only(Vec::new()),
target_dependent: true,
};
let platform = concinnity_core::platform::Platform::current().key();
assert_eq!(
payload_key(1, &args, &ctx(), &dependent),
key_from_parts(1, &args, &[], Some(platform)),
);
assert_ne!(
payload_key(1, &args, &ctx(), &dependent),
payload_key(
1,
&args,
&ctx(),
&CacheInputs {
sources: SourceFiles::Only(Vec::new()),
target_dependent: false,
}
),
);
}
#[test]
fn collect_strings_walks_nested_arrays_and_objects() {
let mut out = Vec::new();
collect_strings(
&json!({"a": ["x", {"b": "y"}], "n": 5, "f": true, "z": null}),
&mut out,
);
out.sort();
assert_eq!(out, vec!["x".to_string(), "y".to_string()]);
}
#[test]
fn referenced_files_resolve_through_the_artifacts_dir() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("fx.hdr"), b"pixels").unwrap();
let artifacts = dir.path().to_str().unwrap().to_string();
let artifact_ctx = BuildCtx {
name: "test",
assets_dir: None,
artifacts_dir: Some(&artifacts),
all_assets: &[],
};
let args = json!({"maps": [{"source": "fx.hdr"}]});
let files = referenced_files(&args, &artifact_ctx);
assert_eq!(files.len(), 1);
assert!(files[0].0.ends_with("fx.hdr"));
assert!(referenced_files(&args, &ctx()).is_empty());
}
#[test]
fn resolve_source_finds_a_bare_filename_under_the_assets_tree() {
let dir = tempfile::tempdir().unwrap();
let nested = dir.path().join("hdri");
std::fs::create_dir_all(&nested).expect("assets tree");
std::fs::write(nested.join("sky.hdr"), b"radiance").expect("write source");
let assets_ctx = BuildCtx {
name: "test",
assets_dir: Some(dir.path()),
artifacts_dir: None,
all_assets: &[],
};
let found = resolve_source("sky.hdr", &assets_ctx).expect("bare filename resolves");
assert!(found.ends_with("sky.hdr"), "got: {found}");
assert_eq!(resolve_source("missing.hdr", &assets_ctx), None);
assert_eq!(resolve_source("sky.hdr", &ctx()), None);
}
#[test]
fn resolve_source_falls_through_an_artifacts_dir_without_the_file() {
let dir = tempfile::tempdir().unwrap();
let artifacts = dir.path().to_str().unwrap().to_string();
let artifact_ctx = BuildCtx {
name: "test",
assets_dir: None,
artifacts_dir: Some(&artifacts),
all_assets: &[],
};
assert_eq!(resolve_source("absent.hdr", &artifact_ctx), None);
}
#[test]
fn store_in_gives_up_when_the_directory_cannot_be_created() {
let dir = tempfile::tempdir().unwrap();
let blocker = dir.path().join("blocker");
std::fs::write(&blocker, b"a file, not a directory").unwrap();
let unusable = blocker.join("cache");
store_in(&unusable, "k", b"bytes");
assert_eq!(load_in(&unusable, "k"), None);
assert!(!unusable.exists());
}
#[test]
fn resolve_source_skips_non_file_looking_strings() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("chrome"), b"x").unwrap();
let artifacts = dir.path().to_str().unwrap().to_string();
let artifact_ctx = BuildCtx {
name: "test",
assets_dir: None,
artifacts_dir: Some(&artifacts),
all_assets: &[],
};
assert_eq!(resolve_source("chrome", &artifact_ctx), None);
}
#[test]
fn expand_key_is_stable_when_the_source_is_missing() {
let args = json!({ "prefix": "scn" });
let a = expand_key("/no/such/scene.glb", &args, None);
let b = expand_key("/no/such/scene.glb", &args, None);
assert_eq!(a, b);
assert_ne!(
a,
expand_key("/no/such/scene.glb", &json!({ "prefix": "x" }), None)
);
}
}