pub mod blobs;
mod config_diff;
mod descriptors;
mod management;
mod operations;
mod owners;
mod restore;
use anyhow::{Context, Result};
use redb::{Database, TableDefinition};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::{Path, PathBuf};
use crate::config::RestoreMethod;
use crate::remote_cache::RemoteCache;
pub const CHECKSUM_PREFIX_LEN: usize = 2;
pub fn walk_files(dir: &Path) -> Result<Vec<PathBuf>> {
let mut result = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(current) = stack.pop() {
let entries = match fs::read_dir(¤t) {
Ok(entries) => entries,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => {
return Err(e)
.with_context(|| format!("Failed to read directory {}", current.display()));
}
};
for entry in entries {
let entry = entry.with_context(|| {
format!("Failed to read an entry of directory {}", current.display())
})?;
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else if path.is_file() {
result.push(path);
}
}
}
Ok(result)
}
fn safe_entry_path(raw: &str) -> Result<&Path> {
let path = Path::new(raw);
if path
.components()
.any(|c| matches!(c, std::path::Component::ParentDir))
{
anyhow::bail!("cache descriptor entry path '{raw}' escapes the project root");
}
Ok(path)
}
fn shard(key: &str) -> (&str, &str) {
key.split_at_checked(CHECKSUM_PREFIX_LEN.min(key.len()))
.unwrap_or((key, ""))
}
fn remote_object_key(checksum: &str) -> String {
let (prefix, rest) = shard(checksum);
format!("objects/{prefix}/{rest}")
}
fn is_object_checksum(text: &str) -> bool {
text.len() == 64
&& text
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
const RSBUILD_DIR: &str = ".rsconstruct";
const OBJECTS_DIR: &str = "objects";
const DESCRIPTORS_DIR: &str = "descriptors";
const DB_FILE: &str = "db.redb";
const CONFIGS_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("processor_configs");
const LAST_TREE_TABLE: TableDefinition<&str, &str> = TableDefinition::new("product_last_tree");
#[derive(Debug)]
pub enum RebuildReason {
NoCacheEntry,
OutputMissing(String),
Force,
DependencyChanged,
InputUnreadable,
}
impl std::fmt::Display for RebuildReason {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::NoCacheEntry => write!(f, "no cache entry"),
Self::OutputMissing(path) => write!(f, "output missing: {path}"),
Self::Force => write!(f, "forced"),
Self::DependencyChanged => write!(f, "a dependency will be rebuilt or restored"),
Self::InputUnreadable => write!(f, "an input could not be read"),
}
}
}
#[derive(Debug)]
pub enum ExplainAction {
Skip,
Restore(RebuildReason),
Rebuild(RebuildReason),
}
impl std::fmt::Display for ExplainAction {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Skip => write!(f, "SKIP (inputs unchanged)"),
Self::Restore(reason) => write!(f, "RESTORE ({reason})"),
Self::Rebuild(reason) => write!(f, "BUILD ({reason})"),
}
}
}
pub struct ObjectStore {
pub(super) objects_dir: PathBuf,
pub(super) descriptors_dir: PathBuf,
pub(super) db: Database,
pub(super) restore_method: RestoreMethod,
pub(super) compression: bool,
pub(super) remote: Option<Box<dyn RemoteCache>>,
pub(super) remote_push: bool,
pub(super) remote_pull: bool,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(tag = "type")]
pub enum CacheDescriptor {
#[serde(rename = "marker")]
Marker,
#[serde(rename = "blob")]
Blob {
checksum: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
mode: Option<u32>,
},
#[serde(rename = "tree")]
Tree { entries: Vec<TreeEntry> },
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct TreeEntry {
pub(super) path: String,
pub(super) checksum: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(super) mode: Option<u32>,
}
#[derive(Debug, Default, Serialize)]
pub struct ProcessorCacheStats {
pub entry_count: usize,
pub output_count: usize,
pub output_bytes: u64,
}
#[derive(Debug, Serialize)]
pub struct CacheListEntry {
pub cache_key: String,
pub outputs: Vec<CacheListOutput>,
}
#[derive(Debug, Serialize)]
pub struct CacheListOutput {
pub path: String,
pub exists: bool,
}
pub struct ObjectStoreOptions {
pub restore_method: RestoreMethod,
pub compression: bool,
pub remote: Option<Box<dyn RemoteCache>>,
pub remote_push: bool,
pub remote_pull: bool,
}
impl ObjectStore {
pub fn new(opts: ObjectStoreOptions) -> Result<Self> {
let rsconstruct_dir = PathBuf::from(RSBUILD_DIR);
let objects_dir = rsconstruct_dir.join(OBJECTS_DIR);
let db_path = rsconstruct_dir.join(DB_FILE);
fs::create_dir_all(&rsconstruct_dir).context("Failed to create .rsconstruct directory")?;
let db = crate::db::open(&db_path, "Cache database")?;
let descriptors_dir = rsconstruct_dir.join(DESCRIPTORS_DIR);
Ok(Self {
objects_dir,
descriptors_dir,
db,
restore_method: opts.restore_method,
compression: opts.compression,
remote: opts.remote,
remote_push: opts.remote_push,
remote_pull: opts.remote_pull,
})
}
#[cfg(test)]
pub(crate) fn new_at(dir: &Path, db_name: &str) -> Self {
Self {
objects_dir: dir.join(OBJECTS_DIR),
descriptors_dir: dir.join(DESCRIPTORS_DIR),
db: crate::db::open(&dir.join(db_name), "test cache database").unwrap(),
restore_method: RestoreMethod::Hardlink,
compression: false,
remote: None,
remote_push: false,
remote_pull: false,
}
}
#[cfg(test)]
pub(crate) fn new_in(dir: &Path) -> Self {
Self::new_at(dir, "db.redb")
}
#[cfg(test)]
pub(crate) fn new_with_remote(dir: &Path, db_name: &str, remote_dir: &Path) -> Self {
fs::create_dir_all(dir).unwrap();
let mut store = Self::new_at(dir, db_name);
store.remote = Some(Box::new(
crate::remote_cache::FileBackend::new(&format!("file://{}", remote_dir.display()))
.unwrap(),
));
store.remote_push = true;
store.remote_pull = true;
store
}
}