use anyhow::{Context, Result};
use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use crate::build_context::BuildContext;
use crate::checksum::{ChecksumPath, checksum_fast};
const RSBUILD_DIR: &str = ".rsconstruct";
const DEPS_DB_FILE: &str = "deps.redb";
const DEPS_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("deps");
#[derive(Debug, Serialize, Deserialize)]
struct DepsEntry {
source_checksum: String,
dependencies: Vec<String>,
#[serde(default)]
dependency_checksums: Vec<String>,
#[serde(default)]
absent: Vec<String>,
#[serde(default)]
config_fingerprint: String,
#[serde(default)]
analyzer: String,
}
#[derive(Debug, Clone, Copy)]
pub struct AnalyzerId<'a> {
pub iname: &'a str,
pub fingerprint: &'a str,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClassifyResult {
MtimeHit,
ContentHit,
Miss,
}
#[derive(Debug, Default, Clone)]
pub struct DepsCacheStats {
pub hits: usize,
pub mtime_hits: usize,
pub content_hits: usize,
pub misses: usize,
}
pub struct DepsCache {
db: Database,
stats: DepsCacheStats,
pending: HashMap<String, DepsEntry>,
}
impl DepsCache {
pub fn open() -> Result<Self> {
Self::open_in(Path::new(""))
}
pub fn open_in(base: &Path) -> Result<Self> {
let rsconstruct_dir = base.join(RSBUILD_DIR);
let db_path = rsconstruct_dir.join(DEPS_DB_FILE);
fs::create_dir_all(&rsconstruct_dir).context("Failed to create .rsconstruct directory")?;
let db = crate::db::open_or_recreate(&db_path, "Dependency cache")?;
Ok(Self {
db,
stats: DepsCacheStats::default(),
pending: HashMap::new(),
})
}
pub fn get(
&mut self,
ctx: &BuildContext,
analyzer: AnalyzerId<'_>,
source: &Path,
) -> Option<Vec<PathBuf>> {
let Some((deps, checksum_path)) = self.lookup(ctx, analyzer, source) else {
self.stats.misses += 1;
return None;
};
self.stats.hits += 1;
match checksum_path {
ChecksumPath::MtimeShortcut => self.stats.mtime_hits += 1,
ChecksumPath::FullRead => self.stats.content_hits += 1,
}
Some(deps)
}
pub fn classify(
&self,
ctx: &BuildContext,
analyzer: AnalyzerId<'_>,
source: &Path,
) -> ClassifyResult {
match self.lookup(ctx, analyzer, source) {
None => ClassifyResult::Miss,
Some((_, ChecksumPath::MtimeShortcut)) => ClassifyResult::MtimeHit,
Some((_, ChecksumPath::FullRead)) => ClassifyResult::ContentHit,
}
}
fn lookup(
&self,
ctx: &BuildContext,
analyzer: AnalyzerId<'_>,
source: &Path,
) -> Option<(Vec<PathBuf>, ChecksumPath)> {
let key = key_for(analyzer.iname, source);
if let Some(entry) = self.pending.get(&key) {
return Self::validate(ctx, analyzer, source, entry);
}
let read_txn = self.db.begin_read().ok()?;
let table = read_txn.open_table(DEPS_TABLE).ok()?;
let data = table.get(key.as_str()).ok()??;
let entry = serde_json::from_slice::<DepsEntry>(data.value()).ok()?;
Self::validate(ctx, analyzer, source, &entry)
}
fn validate(
ctx: &BuildContext,
analyzer: AnalyzerId<'_>,
source: &Path,
entry: &DepsEntry,
) -> Option<(Vec<PathBuf>, ChecksumPath)> {
if entry.config_fingerprint != analyzer.fingerprint {
return None;
}
let (current_checksum, checksum_path) = checksum_fast(ctx, source).ok()?;
if entry.source_checksum != current_checksum {
return None;
}
if !dependencies_unchanged(ctx, entry) || !absent_still_absent(entry) {
return None;
}
let deps = entry.dependencies.iter().map(PathBuf::from).collect();
Some((deps, checksum_path))
}
pub fn source_checksum(ctx: &BuildContext, source: &Path) -> Result<String> {
let (checksum, _) = checksum_fast(ctx, source)?;
Ok(checksum)
}
pub fn set(
&mut self,
ctx: &BuildContext,
analyzer: AnalyzerId<'_>,
source: &Path,
source_checksum: String,
dependencies: &[PathBuf],
absent: &[PathBuf],
) -> Result<()> {
let key = key_for(analyzer.iname, source);
let dependency_checksums = dependencies
.iter()
.map(|dep| {
checksum_fast(ctx, dep)
.map(|(checksum, _)| checksum)
.with_context(|| format!("Failed to checksum dependency {}", dep.display()))
})
.collect::<Result<Vec<String>>>()?;
let entry = DepsEntry {
source_checksum,
dependencies: dependencies
.iter()
.map(|p| p.display().to_string())
.collect(),
dependency_checksums,
absent: absent.iter().map(|p| p.display().to_string()).collect(),
config_fingerprint: analyzer.fingerprint.to_string(),
analyzer: analyzer.iname.to_string(),
};
self.pending.insert(key, entry);
Ok(())
}
pub fn flush(&mut self) -> Result<()> {
if self.pending.is_empty() {
return Ok(());
}
let pending = std::mem::take(&mut self.pending);
let write_txn = self
.db
.begin_write()
.context("Failed to begin write transaction")?;
{
let mut table = write_txn
.open_table(DEPS_TABLE)
.context("Failed to open deps table")?;
for (key, entry) in &pending {
let data =
serde_json::to_vec(entry).context("Failed to serialize dependency entry")?;
table
.insert(key.as_str(), data.as_slice())
.context("Failed to write to dependency cache")?;
}
}
write_txn
.commit()
.context("Failed to commit dependency cache write")?;
Ok(())
}
pub const fn count_uncached_scan(&mut self) {
self.stats.misses += 1;
}
pub const fn stats(&self) -> &DepsCacheStats {
&self.stats
}
fn collect_entries(&self) -> Vec<(String, PathBuf, DepsEntry)> {
let Ok(read_txn) = self.db.begin_read() else {
return Vec::new();
};
let Ok(table) = read_txn.open_table(DEPS_TABLE) else {
return Vec::new();
};
let Ok(iter) = table.iter() else {
return Vec::new();
};
iter.filter_map(|item| {
let (key, value) = item.ok()?;
let (analyzer, source) = parse_key(key.value())?;
let entry: DepsEntry = serde_json::from_slice(value.value()).ok()?;
Some((analyzer, source, entry))
})
.collect()
}
pub fn get_raw_for_path(&self, source: &Path) -> Vec<(Vec<PathBuf>, String)> {
self.collect_entries()
.into_iter()
.filter(|(_a, s, _e)| s == source)
.map(|(analyzer, _s, entry)| {
let deps = entry.dependencies.iter().map(PathBuf::from).collect();
(deps, analyzer)
})
.collect()
}
pub fn list_all(&self) -> Vec<(PathBuf, Vec<PathBuf>, String)> {
self.collect_entries()
.into_iter()
.map(|(analyzer, source, entry)| {
let deps: Vec<PathBuf> = entry.dependencies.iter().map(PathBuf::from).collect();
(source, deps, analyzer)
})
.collect()
}
pub fn stats_by_analyzer(&self) -> std::collections::HashMap<String, (usize, usize)> {
let mut stats: std::collections::HashMap<String, (usize, usize)> =
std::collections::HashMap::new();
for (analyzer, _source, entry) in self.collect_entries() {
let name = if analyzer.is_empty() {
"unknown".to_string()
} else {
analyzer
};
let (files, deps) = stats.entry(name).or_insert((0, 0));
*files += 1;
*deps += entry.dependencies.len();
}
stats
}
pub fn list_by_analyzers(&self, analyzers: &[String]) -> Vec<(PathBuf, Vec<PathBuf>, String)> {
self.collect_entries()
.into_iter()
.filter_map(|(analyzer, source, entry)| {
if !analyzers.contains(&analyzer) {
return None;
}
let deps: Vec<PathBuf> = entry.dependencies.iter().map(PathBuf::from).collect();
Some((source, deps, analyzer))
})
.collect()
}
pub fn remove_by_analyzer(&self, analyzer: &str) -> Result<usize> {
let keys_to_remove: Vec<String> = self
.collect_entries()
.into_iter()
.filter_map(|(a, source, _entry)| {
if a == analyzer {
Some(key_for(&a, &source))
} else {
None
}
})
.collect();
let mut removed = 0;
if !keys_to_remove.is_empty() {
let write_txn = self
.db
.begin_write()
.context("Failed to begin write transaction")?;
{
let mut table = write_txn
.open_table(DEPS_TABLE)
.context("Failed to open deps table")?;
for key in &keys_to_remove {
if table.remove(key.as_str()).is_ok() {
removed += 1;
}
}
}
write_txn
.commit()
.context("Failed to commit dependency cache removal")?;
}
Ok(removed)
}
}
fn dependencies_unchanged(ctx: &BuildContext, entry: &DepsEntry) -> bool {
entry.dependencies.len() == entry.dependency_checksums.len()
&& entry
.dependencies
.iter()
.zip(&entry.dependency_checksums)
.all(|(dep, stored)| {
checksum_fast(ctx, Path::new(dep)).is_ok_and(|(current, _)| ¤t == stored)
})
}
fn absent_still_absent(entry: &DepsEntry) -> bool {
entry
.absent
.iter()
.all(|path| fs::symlink_metadata(path).is_err())
}
fn key_for(analyzer: &str, path: &Path) -> String {
let mut s = String::with_capacity(analyzer.len() + 1 + path.as_os_str().len());
s.push_str(analyzer);
s.push('\0');
s.push_str(&path.display().to_string());
s
}
fn parse_key(key: &str) -> Option<(String, PathBuf)> {
let (analyzer, path) = key.split_once('\0')?;
Some((analyzer.to_string(), PathBuf::from(path)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn key_and_parse_roundtrip() {
let key = key_for("python", Path::new("src/foo/bar.py"));
assert_eq!(key, "python\0src/foo/bar.py");
let (analyzer, path) = parse_key(&key).expect("must parse");
assert_eq!(analyzer, "python");
assert_eq!(path, PathBuf::from("src/foo/bar.py"));
}
#[test]
fn keys_differ_by_analyzer() {
let k1 = key_for("python", Path::new("foo.py"));
let k2 = key_for("mypy", Path::new("foo.py"));
assert_ne!(k1, k2);
}
#[test]
fn keys_differ_by_instance_name() {
let k1 = key_for("cpp.kernel", Path::new("foo.c"));
let k2 = key_for("cpp.userspace", Path::new("foo.c"));
assert_ne!(k1, k2);
}
#[test]
fn parse_rejects_key_without_separator() {
assert!(parse_key("just/a/path.py").is_none());
}
#[test]
fn parse_handles_path_with_colons() {
let key = key_for("cpp", Path::new("src/a:b.c"));
let (analyzer, path) = parse_key(&key).unwrap();
assert_eq!(analyzer, "cpp");
assert_eq!(path, PathBuf::from("src/a:b.c"));
}
#[test]
fn get_misses_when_a_dependency_changes() {
let fresh_ctx = || {
let ctx = crate::build_context::BuildContext::new();
ctx.set_mtime_check(false);
ctx
};
let tmp = tempfile::TempDir::new().unwrap();
let ctx = fresh_ctx();
let mut cache = DepsCache::open_in(tmp.path()).expect("open fresh cache");
let source = tmp.path().join("main.c");
let header = tmp.path().join("a.h");
fs::write(&source, "#include \"a.h\"\n").unwrap();
fs::write(&header, "#define V 1\n").unwrap();
let checksum = DepsCache::source_checksum(&ctx, &source).unwrap();
cache
.set(
&ctx,
ICPP,
&source,
checksum,
std::slice::from_ref(&header),
&[],
)
.unwrap();
assert_eq!(cache.get(&ctx, ICPP, &source), Some(vec![header.clone()]));
fs::write(&header, "#include \"b.h\"\n").unwrap();
let ctx = fresh_ctx();
assert_eq!(cache.get(&ctx, ICPP, &source), None);
}
const ICPP: AnalyzerId<'static> = AnalyzerId {
iname: "icpp",
fingerprint: "config-a",
};
fn content_only_ctx() -> crate::build_context::BuildContext {
let ctx = crate::build_context::BuildContext::new();
ctx.set_mtime_check(false);
ctx
}
#[test]
fn set_is_visible_at_once_and_persisted_by_flush() {
let tmp = tempfile::TempDir::new().unwrap();
let ctx = content_only_ctx();
let a = tmp.path().join("a.c");
let b = tmp.path().join("b.c");
fs::write(&a, "a").unwrap();
fs::write(&b, "b").unwrap();
{
let mut cache = DepsCache::open_in(tmp.path()).expect("open fresh cache");
let checksum = DepsCache::source_checksum(&ctx, &a).unwrap();
cache.set(&ctx, ICPP, &a, checksum, &[], &[]).unwrap();
assert_eq!(cache.get(&ctx, ICPP, &a), Some(Vec::new()), "staged entry");
cache.flush().unwrap();
let checksum = DepsCache::source_checksum(&ctx, &b).unwrap();
cache.set(&ctx, ICPP, &b, checksum, &[], &[]).unwrap();
}
let mut reopened = DepsCache::open_in(tmp.path()).expect("reopen cache");
assert_eq!(reopened.get(&ctx, ICPP, &a), Some(Vec::new()), "flushed");
assert_eq!(reopened.get(&ctx, ICPP, &b), None, "never flushed");
}
#[test]
fn get_misses_when_the_config_fingerprint_changes() {
let tmp = tempfile::TempDir::new().unwrap();
let ctx = content_only_ctx();
let mut cache = DepsCache::open_in(tmp.path()).expect("open fresh cache");
let source = tmp.path().join("main.c");
fs::write(&source, "int main;\n").unwrap();
let checksum = DepsCache::source_checksum(&ctx, &source).unwrap();
cache.set(&ctx, ICPP, &source, checksum, &[], &[]).unwrap();
assert_eq!(cache.get(&ctx, ICPP, &source), Some(Vec::new()));
let other = AnalyzerId {
iname: "icpp",
fingerprint: "config-b",
};
assert_eq!(cache.get(&ctx, other, &source), None);
assert_eq!(cache.classify(&ctx, other, &source), ClassifyResult::Miss);
}
#[test]
fn get_misses_when_an_absent_path_appears() {
let tmp = tempfile::TempDir::new().unwrap();
let ctx = content_only_ctx();
let mut cache = DepsCache::open_in(tmp.path()).expect("open fresh cache");
let source = tmp.path().join("main.c");
let found = tmp.path().join("include_x.h");
let shadow = tmp.path().join("x.h");
fs::write(&source, "#include \"x.h\"\n").unwrap();
fs::write(&found, "\n").unwrap();
let checksum = DepsCache::source_checksum(&ctx, &source).unwrap();
cache
.set(
&ctx,
ICPP,
&source,
checksum,
std::slice::from_ref(&found),
std::slice::from_ref(&shadow),
)
.unwrap();
assert_eq!(cache.get(&ctx, ICPP, &source), Some(vec![found]));
fs::write(&shadow, "\n").unwrap();
assert_eq!(cache.get(&ctx, ICPP, &source), None);
assert_eq!(cache.classify(&ctx, ICPP, &source), ClassifyResult::Miss);
}
#[test]
fn get_counts_miss_even_when_db_is_fresh() {
let tmp = tempfile::TempDir::new().unwrap();
let ctx = crate::build_context::BuildContext::new();
let mut cache = DepsCache::open_in(tmp.path()).expect("open fresh cache");
let nonexistent = tmp.path().join("does_not_exist.py");
let python = AnalyzerId {
iname: "python",
fingerprint: "",
};
let result = cache.get(&ctx, python, &nonexistent);
assert!(result.is_none(), "missing entry must return None");
let stats = cache.stats();
assert_eq!(
stats.hits + stats.misses,
1,
"exactly one of hits/misses must advance per get call (hits={}, misses={})",
stats.hits,
stats.misses
);
assert_eq!(stats.misses, 1, "missing entry counts as a miss");
}
}