use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use xlog_core::{Result, XlogError};
use crate::xgcf::XgcfNodeType;
const MAGIC: u32 = 0x584C4743; const FORMAT_VERSION: u32 = 1;
const HEADER_SIZE: usize = 60;
#[derive(Debug, Clone)]
pub(crate) struct CircuitCacheKey {
pub cnf_hash: u64,
pub config_hash: u64,
pub random_vars_hash: u64,
pub sm: u32,
}
#[derive(Debug)]
pub(crate) struct CircuitArtifact {
pub num_nodes: u32,
pub num_edges: u32,
pub num_levels: u32,
pub root: u32,
pub max_var: u32,
pub has_free_var_mask: bool,
pub node_type: Vec<u8>,
pub child_offsets: Vec<u32>,
pub child_indices: Vec<u32>,
pub lit: Vec<i32>,
pub decision_var: Vec<u32>,
pub decision_child_false: Vec<u32>,
pub decision_child_true: Vec<u32>,
pub level_nodes: Vec<u32>,
pub level_offsets: Vec<u32>,
pub free_var_mask: Vec<u8>,
}
pub(crate) fn cache_dir() -> PathBuf {
if let Ok(dir) = std::env::var("XLOG_CIRCUIT_CACHE_DIR") {
PathBuf::from(dir)
} else if let Ok(xdg) = std::env::var("XDG_CACHE_HOME") {
PathBuf::from(xdg).join("xlog").join("circuits")
} else {
PathBuf::from(std::env::var("HOME").unwrap_or_default())
.join(".cache")
.join("xlog")
.join("circuits")
}
}
fn artifact_filename(key: &CircuitCacheKey) -> String {
format!(
"{:016x}_{:016x}_{:016x}_{:08x}_{:08x}.bin",
key.cnf_hash, key.config_hash, key.random_vars_hash, key.sm, FORMAT_VERSION,
)
}
pub(crate) fn write_artifact(key: &CircuitCacheKey, artifact: &CircuitArtifact) -> Result<()> {
write_artifact_to(&cache_dir(), key, artifact)
}
pub(crate) fn read_artifact(key: &CircuitCacheKey) -> Result<Option<CircuitArtifact>> {
read_artifact_from(&cache_dir(), key)
}
fn io_err(e: std::io::Error) -> XlogError {
XlogError::Compilation(format!("circuit cache IO error: {}", e))
}
fn read_u32_vec(data: &[u8], offset: usize, count: usize) -> Vec<u32> {
(0..count)
.map(|i| {
let s = offset + i * 4;
u32::from_le_bytes([data[s], data[s + 1], data[s + 2], data[s + 3]])
})
.collect()
}
fn read_i32_vec(data: &[u8], offset: usize, count: usize) -> Vec<i32> {
(0..count)
.map(|i| {
let s = offset + i * 4;
i32::from_le_bytes([data[s], data[s + 1], data[s + 2], data[s + 3]])
})
.collect()
}
fn checked_bytes(elems: usize, bytes_per_elem: usize) -> Option<usize> {
elems.checked_mul(bytes_per_elem)
}
fn checked_sum(parts: &[usize]) -> Option<usize> {
parts
.iter()
.try_fold(0usize, |acc, part| acc.checked_add(*part))
}
fn valid_node_type(ty: u8) -> bool {
ty <= XgcfNodeType::Decision as u8
}
fn artifact_topology_is_valid(artifact: &CircuitArtifact) -> bool {
if artifact.num_nodes == 0 || artifact.num_levels == 0 || artifact.root >= artifact.num_nodes {
return false;
}
let num_nodes = artifact.num_nodes as usize;
let num_edges = artifact.num_edges as usize;
let num_levels = artifact.num_levels as usize;
let Ok(max_var) = usize::try_from(artifact.max_var) else {
return false;
};
let Some(free_var_mask_len) = max_var.checked_add(1) else {
return false;
};
if artifact.node_type.len() != num_nodes
|| artifact.child_offsets.len() != num_nodes + 1
|| artifact.child_indices.len() != num_edges
|| artifact.lit.len() != num_nodes
|| artifact.decision_var.len() != num_nodes
|| artifact.decision_child_false.len() != num_nodes
|| artifact.decision_child_true.len() != num_nodes
|| artifact.level_nodes.len() != num_nodes
|| artifact.level_offsets.len() != num_levels + 1
|| artifact.free_var_mask.len() != free_var_mask_len
{
return false;
}
if artifact.child_offsets.first().copied() != Some(0)
|| artifact.child_offsets.last().copied() != Some(artifact.num_edges)
{
return false;
}
let mut prev = 0u32;
for &offset in &artifact.child_offsets {
if offset < prev || offset > artifact.num_edges {
return false;
}
prev = offset;
}
if artifact
.child_indices
.iter()
.any(|&child| child >= artifact.num_nodes)
{
return false;
}
if artifact.level_offsets.first().copied() != Some(0)
|| artifact.level_offsets.last().copied() != Some(artifact.num_nodes)
{
return false;
}
let mut prev = 0u32;
for &offset in &artifact.level_offsets {
if offset < prev || offset > artifact.num_nodes {
return false;
}
prev = offset;
}
if artifact
.level_nodes
.iter()
.any(|&node| node >= artifact.num_nodes)
{
return false;
}
for idx in 0..num_nodes {
let ty = artifact.node_type[idx];
if !valid_node_type(ty) {
return false;
}
match ty {
t if t == XgcfNodeType::Lit as u8
&& artifact.lit[idx].unsigned_abs() > artifact.max_var =>
{
return false;
}
t if t == XgcfNodeType::Decision as u8 => {
if artifact.decision_var[idx] > artifact.max_var {
return false;
}
if artifact.decision_child_false[idx] >= artifact.num_nodes
|| artifact.decision_child_true[idx] >= artifact.num_nodes
{
return false;
}
}
_ => {}
}
}
true
}
fn write_artifact_to(dir: &Path, key: &CircuitCacheKey, artifact: &CircuitArtifact) -> Result<()> {
fs::create_dir_all(dir).map_err(io_err)?;
let path = dir.join(artifact_filename(key));
let tmp = path.with_extension("tmp");
let mut f = fs::File::create(&tmp).map_err(io_err)?;
f.write_all(&MAGIC.to_le_bytes()).map_err(io_err)?;
f.write_all(&FORMAT_VERSION.to_le_bytes()).map_err(io_err)?;
f.write_all(&key.cnf_hash.to_le_bytes()).map_err(io_err)?;
f.write_all(&key.config_hash.to_le_bytes())
.map_err(io_err)?;
f.write_all(&key.random_vars_hash.to_le_bytes())
.map_err(io_err)?;
f.write_all(&key.sm.to_le_bytes()).map_err(io_err)?;
f.write_all(&artifact.num_nodes.to_le_bytes())
.map_err(io_err)?;
f.write_all(&artifact.num_edges.to_le_bytes())
.map_err(io_err)?;
f.write_all(&artifact.num_levels.to_le_bytes())
.map_err(io_err)?;
f.write_all(&artifact.root.to_le_bytes()).map_err(io_err)?;
f.write_all(&artifact.max_var.to_le_bytes())
.map_err(io_err)?;
f.write_all(&[artifact.has_free_var_mask as u8])
.map_err(io_err)?;
f.write_all(&[0u8; 3]).map_err(io_err)?;
f.write_all(&artifact.node_type).map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.child_offsets))
.map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.child_indices))
.map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.lit))
.map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.decision_var))
.map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.decision_child_false))
.map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.decision_child_true))
.map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.level_nodes))
.map_err(io_err)?;
f.write_all(bytemuck::cast_slice(&artifact.level_offsets))
.map_err(io_err)?;
f.write_all(&artifact.free_var_mask).map_err(io_err)?;
drop(f);
fs::rename(&tmp, &path).map_err(io_err)?;
evict_if_needed_in(dir)?;
Ok(())
}
fn read_artifact_from(dir: &Path, key: &CircuitCacheKey) -> Result<Option<CircuitArtifact>> {
let path = dir.join(artifact_filename(key));
let data = match fs::read(&path) {
Ok(d) => d,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => {
return Err(XlogError::Compilation(format!(
"Failed to read cache file: {}",
e
)))
}
};
parse_artifact(&data, key)
}
fn parse_artifact(data: &[u8], key: &CircuitCacheKey) -> Result<Option<CircuitArtifact>> {
if data.len() < HEADER_SIZE {
return Ok(None);
}
let mut cursor = 0usize;
macro_rules! read_u32 {
() => {{
let val = u32::from_le_bytes([
data[cursor],
data[cursor + 1],
data[cursor + 2],
data[cursor + 3],
]);
cursor += 4;
val
}};
}
macro_rules! read_u64 {
() => {{
let val = u64::from_le_bytes([
data[cursor],
data[cursor + 1],
data[cursor + 2],
data[cursor + 3],
data[cursor + 4],
data[cursor + 5],
data[cursor + 6],
data[cursor + 7],
]);
cursor += 8;
val
}};
}
let magic = read_u32!();
if magic != MAGIC {
return Ok(None);
}
let version = read_u32!();
if version != FORMAT_VERSION {
return Ok(None);
}
let cnf_hash = read_u64!();
let config_hash = read_u64!();
let random_vars_hash = read_u64!();
let sm = read_u32!();
if cnf_hash != key.cnf_hash
|| config_hash != key.config_hash
|| random_vars_hash != key.random_vars_hash
|| sm != key.sm
{
return Ok(None);
}
let num_nodes = read_u32!();
let num_edges = read_u32!();
let num_levels = read_u32!();
let root = read_u32!();
let max_var = read_u32!();
let has_free_var_mask = data[cursor] != 0;
cursor += 1;
cursor += 3;
debug_assert_eq!(cursor, HEADER_SIZE);
if num_nodes == 0 || num_levels == 0 || root >= num_nodes {
return Ok(None);
}
let Ok(num_nodes_usize) = usize::try_from(num_nodes) else {
return Ok(None);
};
let Ok(num_edges_usize) = usize::try_from(num_edges) else {
return Ok(None);
};
let Ok(num_levels_usize) = usize::try_from(num_levels) else {
return Ok(None);
};
let Ok(max_var_usize) = usize::try_from(max_var) else {
return Ok(None);
};
let Some(child_offsets_elems) = num_nodes_usize.checked_add(1) else {
return Ok(None);
};
let Some(level_offsets_elems) = num_levels_usize.checked_add(1) else {
return Ok(None);
};
let Some(free_var_mask_bytes) = max_var_usize.checked_add(1) else {
return Ok(None);
};
let node_type_bytes = num_nodes_usize;
let Some(child_offsets_bytes) = checked_bytes(child_offsets_elems, 4) else {
return Ok(None);
};
let Some(child_indices_bytes) = checked_bytes(num_edges_usize, 4) else {
return Ok(None);
};
let Some(lit_bytes) = checked_bytes(num_nodes_usize, 4) else {
return Ok(None);
};
let Some(decision_var_bytes) = checked_bytes(num_nodes_usize, 4) else {
return Ok(None);
};
let Some(decision_child_false_bytes) = checked_bytes(num_nodes_usize, 4) else {
return Ok(None);
};
let Some(decision_child_true_bytes) = checked_bytes(num_nodes_usize, 4) else {
return Ok(None);
};
let Some(level_nodes_bytes) = checked_bytes(num_nodes_usize, 4) else {
return Ok(None);
};
let Some(level_offsets_bytes) = checked_bytes(level_offsets_elems, 4) else {
return Ok(None);
};
let Some(expected_total) = checked_sum(&[
HEADER_SIZE,
node_type_bytes,
child_offsets_bytes,
child_indices_bytes,
lit_bytes,
decision_var_bytes,
decision_child_false_bytes,
decision_child_true_bytes,
level_nodes_bytes,
level_offsets_bytes,
free_var_mask_bytes,
]) else {
return Ok(None);
};
if data.len() < expected_total {
return Ok(None);
}
let node_type = data[cursor..cursor + node_type_bytes].to_vec();
cursor += node_type_bytes;
let child_offsets = read_u32_vec(data, cursor, child_offsets_elems);
cursor += child_offsets_bytes;
let child_indices = read_u32_vec(data, cursor, num_edges_usize);
cursor += child_indices_bytes;
let lit = read_i32_vec(data, cursor, num_nodes_usize);
cursor += lit_bytes;
let decision_var = read_u32_vec(data, cursor, num_nodes_usize);
cursor += decision_var_bytes;
let decision_child_false = read_u32_vec(data, cursor, num_nodes_usize);
cursor += decision_child_false_bytes;
let decision_child_true = read_u32_vec(data, cursor, num_nodes_usize);
cursor += decision_child_true_bytes;
let level_nodes = read_u32_vec(data, cursor, num_nodes_usize);
cursor += level_nodes_bytes;
let level_offsets = read_u32_vec(data, cursor, level_offsets_elems);
cursor += level_offsets_bytes;
let free_var_mask = data[cursor..cursor + free_var_mask_bytes].to_vec();
let artifact = CircuitArtifact {
num_nodes,
num_edges,
num_levels,
root,
max_var,
has_free_var_mask,
node_type,
child_offsets,
child_indices,
lit,
decision_var,
decision_child_false,
decision_child_true,
level_nodes,
level_offsets,
free_var_mask,
};
if !artifact_topology_is_valid(&artifact) {
return Ok(None);
}
Ok(Some(artifact))
}
fn evict_if_needed_in(dir: &Path) -> Result<()> {
let max_mb: u64 = std::env::var("XLOG_CIRCUIT_CACHE_MAX_MB")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(512);
evict_if_needed_in_with_limit(dir, max_mb)
}
fn evict_if_needed_in_with_limit(dir: &Path, max_mb: u64) -> Result<()> {
let max_bytes = max_mb * 1024 * 1024;
let entries = match fs::read_dir(dir) {
Ok(e) => e,
Err(_) => return Ok(()), };
let mut files: Vec<(PathBuf, u64, std::time::SystemTime)> = Vec::new();
let mut total_size: u64 = 0;
for entry in entries {
let entry = match entry {
Ok(e) => e,
Err(_) => continue,
};
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("bin") {
continue;
}
let meta = match entry.metadata() {
Ok(m) => m,
Err(_) => continue,
};
if !meta.is_file() {
continue;
}
let size = meta.len();
let mtime = meta.modified().unwrap_or(std::time::UNIX_EPOCH);
total_size += size;
files.push((path, size, mtime));
}
if total_size <= max_bytes {
return Ok(());
}
files.sort_by_key(|&(_, _, mtime)| mtime);
for (path, size, _) in &files {
if total_size <= max_bytes {
break;
}
if fs::remove_file(path).is_ok() {
total_size = total_size.saturating_sub(*size);
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU64, Ordering};
static TEST_COUNTER: AtomicU64 = AtomicU64::new(0);
fn test_cache_dir() -> PathBuf {
let id = TEST_COUNTER.fetch_add(1, Ordering::SeqCst);
let pid = std::process::id();
let dir = std::env::temp_dir()
.join("xlog_disk_cache_test")
.join(format!("{}_{}", pid, id));
fs::create_dir_all(&dir).expect("create test cache dir");
dir
}
fn make_key(cnf_hash: u64) -> CircuitCacheKey {
CircuitCacheKey {
cnf_hash,
config_hash: 0xDEADBEEF,
random_vars_hash: 0xCAFEBABE,
sm: 89,
}
}
fn make_artifact() -> CircuitArtifact {
CircuitArtifact {
num_nodes: 4,
num_edges: 3,
num_levels: 2,
root: 0,
max_var: 2,
has_free_var_mask: true,
node_type: vec![1, 2, 3, 4],
child_offsets: vec![0, 1, 2, 3, 3], child_indices: vec![1, 2, 3], lit: vec![0, 1, -1, 2], decision_var: vec![0, 1, 2, 0],
decision_child_false: vec![0, 2, 3, 0],
decision_child_true: vec![0, 1, 3, 0],
level_nodes: vec![0, 1, 2, 3], level_offsets: vec![0, 1, 4], free_var_mask: vec![0, 1, 0], }
}
#[test]
fn test_roundtrip() {
let dir = test_cache_dir();
let key = make_key(0x1234);
let artifact = make_artifact();
write_artifact_to(&dir, &key, &artifact).expect("write should succeed");
let loaded = read_artifact_from(&dir, &key)
.expect("read should not error")
.expect("read should return Some");
assert_eq!(loaded.num_nodes, artifact.num_nodes);
assert_eq!(loaded.num_edges, artifact.num_edges);
assert_eq!(loaded.num_levels, artifact.num_levels);
assert_eq!(loaded.root, artifact.root);
assert_eq!(loaded.max_var, artifact.max_var);
assert_eq!(loaded.has_free_var_mask, artifact.has_free_var_mask);
assert_eq!(loaded.node_type, artifact.node_type);
assert_eq!(loaded.child_offsets, artifact.child_offsets);
assert_eq!(loaded.child_indices, artifact.child_indices);
assert_eq!(loaded.lit, artifact.lit);
assert_eq!(loaded.decision_var, artifact.decision_var);
assert_eq!(loaded.decision_child_false, artifact.decision_child_false);
assert_eq!(loaded.decision_child_true, artifact.decision_child_true);
assert_eq!(loaded.level_nodes, artifact.level_nodes);
assert_eq!(loaded.level_offsets, artifact.level_offsets);
assert_eq!(loaded.free_var_mask, artifact.free_var_mask);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn test_roundtrip_unaligned_num_nodes() {
let dir = test_cache_dir();
let key = make_key(0x5555);
let artifact = CircuitArtifact {
num_nodes: 5,
num_edges: 4,
num_levels: 3,
root: 0,
max_var: 3,
has_free_var_mask: false,
node_type: vec![1, 2, 3, 4, 5],
child_offsets: vec![0, 1, 2, 3, 4, 4], child_indices: vec![1, 2, 3, 4], lit: vec![0, 1, -1, 2, -2], decision_var: vec![0, 1, 2, 3, 0],
decision_child_false: vec![0, 2, 3, 4, 0],
decision_child_true: vec![0, 1, 3, 4, 0],
level_nodes: vec![0, 1, 2, 3, 4], level_offsets: vec![0, 1, 3, 5], free_var_mask: vec![0, 1, 0, 1], };
write_artifact_to(&dir, &key, &artifact).expect("write should succeed");
let loaded = read_artifact_from(&dir, &key)
.expect("read should not error")
.expect("read should return Some");
assert_eq!(loaded.num_nodes, artifact.num_nodes);
assert_eq!(loaded.num_edges, artifact.num_edges);
assert_eq!(loaded.num_levels, artifact.num_levels);
assert_eq!(loaded.root, artifact.root);
assert_eq!(loaded.max_var, artifact.max_var);
assert_eq!(loaded.has_free_var_mask, artifact.has_free_var_mask);
assert_eq!(loaded.node_type, artifact.node_type);
assert_eq!(loaded.child_offsets, artifact.child_offsets);
assert_eq!(loaded.child_indices, artifact.child_indices);
assert_eq!(loaded.lit, artifact.lit);
assert_eq!(loaded.decision_var, artifact.decision_var);
assert_eq!(loaded.decision_child_false, artifact.decision_child_false);
assert_eq!(loaded.decision_child_true, artifact.decision_child_true);
assert_eq!(loaded.level_nodes, artifact.level_nodes);
assert_eq!(loaded.level_offsets, artifact.level_offsets);
assert_eq!(loaded.free_var_mask, artifact.free_var_mask);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn test_mismatched_key_returns_none() {
let dir = test_cache_dir();
let key = make_key(0xAAAA);
let artifact = make_artifact();
write_artifact_to(&dir, &key, &artifact).expect("write should succeed");
let different_key = make_key(0xBBBB);
let result = read_artifact_from(&dir, &different_key).expect("read should not error");
assert!(result.is_none(), "mismatched key should return None");
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn test_missing_file_returns_none() {
let dir = test_cache_dir();
let key = make_key(0x9999);
let result = read_artifact_from(&dir, &key).expect("read should not error");
assert!(result.is_none(), "missing file should return None");
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn test_truncated_file_returns_none() {
let dir = test_cache_dir();
let key = make_key(0x7777);
let artifact = make_artifact();
write_artifact_to(&dir, &key, &artifact).expect("write should succeed");
let path = dir.join(artifact_filename(&key));
let data = fs::read(&path).unwrap();
fs::write(&path, &data[..HEADER_SIZE / 2]).unwrap();
let result = read_artifact_from(&dir, &key).expect("read should not error");
assert!(result.is_none(), "truncated file should return None");
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn test_corrupted_magic_returns_none() {
let dir = test_cache_dir();
let key = make_key(0x6666);
let artifact = make_artifact();
write_artifact_to(&dir, &key, &artifact).expect("write should succeed");
let path = dir.join(artifact_filename(&key));
let mut data = fs::read(&path).unwrap();
data[0] = 0xFF;
data[1] = 0xFF;
fs::write(&path, &data).unwrap();
let result = read_artifact_from(&dir, &key).expect("read should not error");
assert!(result.is_none(), "corrupted magic should return None");
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn test_eviction() {
let dir = test_cache_dir();
let key1 = make_key(0x1111);
let key2 = make_key(0x2222);
let artifact = make_artifact();
write_artifact_to(&dir, &key1, &artifact).expect("write 1 should succeed");
write_artifact_to(&dir, &key2, &artifact).expect("write 2 should succeed");
assert!(read_artifact_from(&dir, &key1).unwrap().is_some());
assert!(read_artifact_from(&dir, &key2).unwrap().is_some());
evict_if_needed_in_with_limit(&dir, 0).expect("eviction should succeed");
let r1 = read_artifact_from(&dir, &key1).unwrap();
let r2 = read_artifact_from(&dir, &key2).unwrap();
assert!(r1.is_none(), "key1 should have been evicted");
assert!(r2.is_none(), "key2 should have been evicted");
let _ = fs::remove_dir_all(&dir);
}
}