use anyhow::{Context, Result};
use std::cell::RefCell;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::atomic::{AtomicU64, Ordering};
use crate::config::{ExecutionModel, StandardConfig};
pub const SANDBOX_TOOL: &str = "bwrap";
const SANDBOX_DIR: &str = ".rsconstruct/sandbox";
const OPAQUE_XATTR: &str = "user.overlay.opaque";
#[derive(Clone, Debug)]
pub struct Spec {
pub label: String,
pub outputs: Vec<PathBuf>,
pub output_dirs: Vec<PathBuf>,
pub network: bool,
pub writable: Vec<String>,
pub stale: Vec<PathBuf>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Writable {
Path(PathBuf),
Named(String),
}
impl Writable {
pub fn parse(entry: &str) -> std::result::Result<Self, String> {
let bad = |why: &str| Err(format!("sandbox_writable entry '{entry}': {why}"));
if let Some(name) = entry.strip_prefix("**/") {
if name.is_empty() || name.contains('/') || name.contains('*') {
return bad("after '**/' must come one directory name, without '/' or '*'");
}
return Ok(Self::Named(name.to_string()));
}
if entry.is_empty() {
return bad("empty");
}
if entry.contains('*') {
return bad("wildcards are only allowed as a leading '**/'");
}
let path = Path::new(entry);
if path.is_absolute() {
return bad("must be relative to the project root");
}
if path
.components()
.any(|c| matches!(c, std::path::Component::ParentDir))
{
return bad("must not contain '..'");
}
let clean: PathBuf = path
.components()
.filter(|c| !matches!(c, std::path::Component::CurDir))
.collect();
if clean.as_os_str().is_empty() {
return bad("names the project root itself");
}
Ok(Self::Path(clean))
}
fn covers(&self, path: &Path) -> bool {
match self {
Self::Path(p) => path.starts_with(p),
Self::Named(name) => path.components().any(|c| c.as_os_str() == name.as_str()),
}
}
}
thread_local! {
static SPEC: RefCell<Option<Spec>> = const { RefCell::new(None) };
}
pub fn set_spec(spec: Option<Spec>) {
SPEC.with(|s| *s.borrow_mut() = spec);
}
pub fn declare_output_dir(dir: &Path) {
SPEC.with(|s| {
if let Some(spec) = s.borrow_mut().as_mut() {
spec.output_dirs.push(dir.to_path_buf());
}
});
}
pub fn declare_replaced_outputs(paths: &[&Path]) -> bool {
SPEC.with(|s| {
let mut spec = s.borrow_mut();
let Some(spec) = spec.as_mut() else {
return false;
};
for path in paths {
spec.outputs.push(path.to_path_buf());
spec.stale.push(path.to_path_buf());
}
true
})
}
const INSIDE_ENV: &str = "RSCONSTRUCT_IN_SANDBOX";
pub fn active() -> bool {
SPEC.with(|s| s.borrow().is_some()) || std::env::var_os(INSIDE_ENV).is_some()
}
fn current_spec() -> Option<Spec> {
SPEC.with(|s| s.borrow().clone())
}
pub fn allow_output_dirs(dirs: &[PathBuf]) {
SPEC.with(|s| {
if let Some(spec) = s.borrow_mut().as_mut() {
for dir in dirs {
if !spec.output_dirs.contains(dir) {
spec.output_dirs.push(dir.clone());
}
}
}
});
}
fn clear_stale() {
SPEC.with(|s| {
if let Some(spec) = s.borrow_mut().as_mut() {
spec.stale.clear();
}
});
}
const PRELUDE: &str = "while [ \"$1\" != -- ]; do rm -f -- \"$1\" || exit; shift; done; shift; \
while [ \"$1\" != -- ]; do mkdir -p -- \"$1\" || exit; shift; done; shift; \
exec \"$@\"";
fn plugin_is_sandboxable(pname: &str) -> bool {
crate::registries::find_plugin(pname).is_none_or(|p| p.sandboxable)
}
fn unsandboxable_instances(
instances: &[crate::config::ProcessorInstance],
build_default: ExecutionModel,
sandboxable: &dyn Fn(&str) -> bool,
) -> Vec<String> {
let mut out: Vec<String> = instances
.iter()
.filter(|inst| {
let table = &inst.config_toml;
let enabled = table
.get("enabled")
.and_then(toml::Value::as_bool)
.unwrap_or(true);
let model = table
.get(crate::config::EXECUTION_MODEL)
.and_then(|v| v.clone().try_into::<ExecutionModel>().ok())
.unwrap_or(build_default);
enabled && model == ExecutionModel::Sandbox && !sandboxable(&inst.pname)
})
.map(|inst| inst.instance_name.clone())
.collect();
out.sort();
out
}
pub fn check_sandboxable(
instances: &[crate::config::ProcessorInstance],
build_default: ExecutionModel,
) -> Result<()> {
check_sandboxable_by(instances, build_default, &plugin_is_sandboxable)
}
fn check_sandboxable_by(
instances: &[crate::config::ProcessorInstance],
build_default: ExecutionModel,
sandboxable: &dyn Fn(&str) -> bool,
) -> Result<()> {
let bad = unsandboxable_instances(instances, build_default, sandboxable);
if bad.is_empty() {
return Ok(());
}
let msg = format!(
"These processors would run under execution_model = \"sandbox\" but cannot run in a sandbox:\n {}\nSet execution_model = \"native\" on each of them, or disable them.",
bad.join("\n ")
);
Err(crate::exit_code::RsconstructError::new(
crate::exit_code::RsconstructExitCode::ConfigError,
msg,
)
.into())
}
pub fn is_sandboxed(standard: &StandardConfig, build_default: ExecutionModel) -> bool {
std::env::var_os(INSIDE_ENV).is_none() && resolves_to_sandbox(standard, build_default)
}
fn resolves_to_sandbox(standard: &StandardConfig, build_default: ExecutionModel) -> bool {
standard.execution_model.unwrap_or(build_default) == ExecutionModel::Sandbox
}
pub fn sandboxed_processors(
processors: &crate::processor::ProcessorMap,
is_enabled: &dyn Fn(&str) -> bool,
build_default: ExecutionModel,
) -> Vec<String> {
let mut names: Vec<String> = processors
.iter()
.filter(|(name, p)| is_enabled(name) && is_sandboxed(p.scan_config(), build_default))
.map(|(name, _)| name.clone())
.collect();
names.sort();
names
}
pub fn writable_for(standard: &StandardConfig, tools: &[String]) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
let declared = tools
.iter()
.filter_map(|t| crate::tools::tool_info(t))
.flat_map(|info| info.sandbox_writable.iter().map(|s| (*s).to_string()));
for entry in declared.chain(standard.sandbox_writable.iter().cloned()) {
if !out.contains(&entry) {
out.push(entry);
}
}
out
}
pub fn network_for(standard: &StandardConfig, tools: &[String]) -> bool {
standard.sandbox_network
|| tools
.iter()
.filter_map(|t| crate::tools::tool_info(t))
.any(|info| info.needs_network)
}
pub fn spec_for(
standard: &StandardConfig,
build_default: ExecutionModel,
products: &[&crate::graph::Product],
tools: &[String],
) -> Option<Spec> {
if !is_sandboxed(standard, build_default) {
return None;
}
let label = products
.first()
.map(|p| p.processor.clone())
.unwrap_or_default();
let outputs = products
.iter()
.flat_map(|p| p.outputs.iter().cloned())
.collect();
let output_dirs = products
.iter()
.flat_map(|p| p.output_dirs.iter().map(|d| d.as_ref().clone()))
.collect();
Some(Spec {
label,
outputs,
output_dirs,
network: network_for(standard, tools),
writable: writable_for(standard, tools),
stale: Vec::new(),
})
}
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
pub struct BuiltinInstance {
pub instance: String,
pub pname: String,
pub config: toml::Value,
}
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct BuiltinSetup {
pub instances: Vec<BuiltinInstance>,
pub force_dirs: Vec<String>,
pub exclude_roots: Vec<String>,
pub warn_symlinks: bool,
pub python: String,
pub max_arg_len: usize,
pub command_timeout_secs: u64,
pub webcache_ttl_secs: u64,
pub worker_stack_mib: usize,
}
impl BuiltinSetup {
pub fn from_config(config: &crate::config::Config) -> Self {
let (exclude_roots, force_dirs) = config.file_index_walk_dirs();
Self {
instances: config
.processor
.instances
.iter()
.filter(|inst| crate::registries::processor::is_builtin(&inst.pname))
.map(|inst| BuiltinInstance {
instance: inst.instance_name.clone(),
pname: inst.pname.clone(),
config: inst.config_toml.clone(),
})
.collect(),
force_dirs,
exclude_roots,
warn_symlinks: config.build.warn_symlinks,
python: config.build.python.clone(),
max_arg_len: config.build.max_arg_len,
command_timeout_secs: config.build.command_timeout_secs,
webcache_ttl_secs: config.cache.webcache_ttl_secs,
worker_stack_mib: config.build.worker_stack_mib,
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct JobResults {
results: Vec<Option<String>>,
fetched: Vec<crate::webcache::Download>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct JobProduct {
id: usize,
inputs: Vec<PathBuf>,
outputs: Vec<PathBuf>,
output_dirs: Vec<PathBuf>,
variant: Option<String>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct BuiltinJob {
setup: BuiltinSetup,
instance: BuiltinInstance,
products: Vec<JobProduct>,
batch: bool,
}
const RESULTS_MARKER: &str = "\u{1e}rsconstruct-sandbox-results\u{1e}";
pub fn execute_builtin(
ctx: &crate::build_context::BuildContext,
setup: &BuiltinSetup,
instance: &str,
products: &[&crate::graph::Product],
batch: bool,
) -> Vec<Result<()>> {
let all = |e: &anyhow::Error| {
products
.iter()
.map(|_| Err(anyhow::anyhow!("{e:#}")))
.collect()
};
match run_builtin_job(ctx, setup, instance, products, batch) {
Ok(results) => results
.into_iter()
.map(|r| r.map_or(Ok(()), |e| Err(anyhow::anyhow!(e))))
.collect(),
Err(e) => all(&e),
}
}
fn run_builtin_job(
ctx: &crate::build_context::BuildContext,
setup: &BuiltinSetup,
instance: &str,
products: &[&crate::graph::Product],
batch: bool,
) -> Result<Vec<Option<String>>> {
let instance = setup
.instances
.iter()
.find(|i| i.instance == instance)
.cloned()
.with_context(|| format!("internal error: no config for builtin processor '{instance}'"))?;
let label = instance.instance.clone();
let job = BuiltinJob {
setup: BuiltinSetup {
instances: Vec::new(),
..setup.clone()
},
instance,
products: products
.iter()
.map(|p| JobProduct {
id: p.id,
inputs: p.inputs.clone(),
outputs: p.outputs.clone(),
output_dirs: p.output_dirs.iter().map(|d| d.as_ref().clone()).collect(),
variant: p.variant.clone(),
})
.collect(),
batch,
};
let payload = serde_json::to_vec(&job).context("Failed to encode the sandboxed job")?;
let exe = std::env::current_exe().context("Failed to locate the rsconstruct executable")?;
let mut cmd = Command::new(&exe);
cmd.arg("--color")
.arg(if crate::runtime_flags::color_enabled() {
"always"
} else {
"never"
});
for (on, flag) in [
(crate::runtime_flags::json_mode(), "--json"),
(crate::runtime_flags::quiet(), "--quiet"),
(crate::runtime_flags::show_output(), "--show-output"),
(
crate::runtime_flags::show_child_processes(),
"--show-child-processes",
),
] {
if on {
cmd.arg(flag);
}
}
cmd.arg("sandbox-run");
let output = crate::processor::run_command_with_stdin(ctx, &cmd, &payload)?;
let stdout = String::from_utf8_lossy(&output.stdout);
let (printed, results) = match stdout.rfind(RESULTS_MARKER) {
Some(at) => (&stdout[..at], Some(&stdout[at + RESULTS_MARKER.len()..])),
None => (&*stdout, None),
};
relay(printed, &String::from_utf8_lossy(&output.stderr));
if !output.status.success() {
anyhow::bail!(
"[{label}] the sandboxed run of rsconstruct failed ({})",
output.status
);
}
let results = results.with_context(|| {
format!("[{label}] the sandboxed run of rsconstruct reported no results")
})?;
let JobResults { results, fetched } = serde_json::from_str(results.trim())
.with_context(|| format!("[{label}] Failed to read the sandboxed run's results"))?;
crate::webcache::defer_store(fetched);
anyhow::ensure!(
results.len() == products.len(),
"[{label}] the sandboxed run reported {} results for {} products",
results.len(),
products.len()
);
Ok(results)
}
fn relay(stdout: &str, stderr: &str) {
use std::io::Write;
if !stdout.is_empty() {
let mut out = std::io::stdout().lock();
let _ = out.write_all(stdout.as_bytes());
let _ = out.flush();
}
if !stderr.is_empty() {
let _ = std::io::stderr().lock().write_all(stderr.as_bytes());
}
}
pub fn sandbox_run(ctx: &crate::build_context::BuildContext) -> Result<()> {
let input = std::io::read_to_string(std::io::stdin())
.context("Failed to read the sandboxed job from standard input")?;
let job: BuiltinJob =
serde_json::from_str(&input).context("Failed to decode the sandboxed job")?;
let setup = &job.setup;
ctx.set_mtime_check(false);
ctx.set_max_arg_len(setup.max_arg_len);
ctx.set_command_timeout_secs(setup.command_timeout_secs);
ctx.set_webcache_ttl_secs(setup.webcache_ttl_secs);
crate::webcache::set_child_mode();
crate::python_env::set_interpreter(&setup.python);
let processor =
crate::builder::create_processor_for_instance(&job.instance.pname, &job.instance.config)?
.with_context(|| format!("Unknown processor type: '{}'", job.instance.pname))?;
let force_refs: Vec<&str> = setup.force_dirs.iter().map(String::as_str).collect();
let file_index = crate::file_index::FileIndex::build_with_force_dirs(
&force_refs,
&setup.exclude_roots,
setup.warn_symlinks,
)?;
processor.discover(
&mut crate::graph::BuildGraph::new(),
&file_index,
&job.instance.instance,
)?;
let products: Vec<crate::graph::Product> = job
.products
.into_iter()
.map(|p| {
let mut product =
crate::graph::Product::new(p.inputs, p.outputs, &job.instance.instance, p.id, None);
product.output_dirs = p.output_dirs.into_iter().map(std::sync::Arc::new).collect();
product.variant = p.variant;
product
})
.collect();
let refs: Vec<&crate::graph::Product> = products.iter().collect();
let results: Vec<Result<()>> = std::thread::scope(|scope| {
std::thread::Builder::new()
.name("sandbox-run".to_string())
.stack_size(setup.worker_stack_mib * 1024 * 1024)
.spawn_scoped(scope, || {
crate::processor::set_declared_tools(Some(processor.required_tools()));
let results = if job.batch {
processor.execute_batch(ctx, &refs)
} else {
refs.iter().map(|p| processor.execute(ctx, p)).collect()
};
crate::processor::set_declared_tools(None);
results
})
.context("Failed to start the sandboxed run's worker thread")?
.join()
.map_err(|_| anyhow::anyhow!("the sandboxed run's worker thread panicked"))
})?;
let results = JobResults {
results: results
.into_iter()
.map(|r| r.err().map(|e| format!("{e:#}")))
.collect(),
fetched: crate::webcache::take_fetched(),
};
let encoded = serde_json::to_string(&results).context("Failed to encode the results")?;
use std::io::Write;
let mut out = std::io::stdout().lock();
writeln!(out, "{RESULTS_MARKER}{encoded}")
.and_then(|()| out.flush())
.context("Failed to write the sandboxed run's results")?;
Ok(())
}
pub struct Run {
pub command: Command,
root: PathBuf,
dir: PathBuf,
upper: PathBuf,
spec: Spec,
writable: Vec<Writable>,
}
pub fn prepare(cmd: &Command) -> Result<Option<Run>> {
let Some(mut spec) = current_spec() else {
return Ok(None);
};
let root =
std::env::current_dir().context("Failed to determine the project root for the sandbox")?;
spec.outputs = spec.outputs.iter().map(|p| relative_to(&root, p)).collect();
spec.output_dirs = spec
.output_dirs
.iter()
.map(|p| relative_to(&root, p))
.collect();
spec.stale = spec.stale.iter().map(|p| relative_to(&root, p)).collect();
let writable = spec
.writable
.iter()
.map(|entry| Writable::parse(entry).map_err(|e| anyhow::anyhow!("[{}] {e}", spec.label)))
.collect::<Result<Vec<_>>>()?;
static NEXT_RUN: AtomicU64 = AtomicU64::new(0);
let dir = root.join(SANDBOX_DIR).join(format!(
"{}-{}",
std::process::id(),
NEXT_RUN.fetch_add(1, Ordering::Relaxed)
));
let upper = dir.join("upper");
let work = dir.join("work");
for d in [&upper, &work] {
fs::create_dir_all(d)
.with_context(|| format!("Failed to create sandbox directory {}", d.display()))?;
}
let mut command = Command::new(SANDBOX_TOOL);
command.args(["--die-with-parent", "--ro-bind", "/", "/"]);
if let Some(home) = std::env::var_os("HOME").filter(|h| !h.is_empty()) {
command.arg("--bind").arg(&home).arg(&home);
}
command.args([
"--bind",
"/tmp",
"/tmp",
"--dev-bind",
"/dev",
"/dev",
"--proc",
"/proc",
]);
if !spec.network {
command.arg("--unshare-net");
}
command
.arg("--overlay-src")
.arg(&root)
.arg("--overlay")
.arg(&upper)
.arg(&work)
.arg(&root);
for place in &writable {
if let Writable::Path(rel) = place {
let path = root.join(rel);
if fs::symlink_metadata(&path).is_ok() {
command.arg("--bind").arg(&path).arg(&path);
}
}
}
let cwd = cmd
.get_current_dir()
.map_or_else(|| root.clone(), |d| root.join(d));
command.arg("--chdir").arg(&cwd);
command.arg("--").args(inner_command(&root, &spec, cmd));
command.env(INSIDE_ENV, "1");
for (key, value) in cmd.get_envs() {
if let Some(value) = value {
command.env(key, value);
}
}
Ok(Some(Run {
command,
root,
dir,
upper,
spec,
writable,
}))
}
fn inner_command(root: &Path, spec: &Spec, cmd: &Command) -> Vec<std::ffi::OsString> {
let mut args: Vec<std::ffi::OsString> = Vec::new();
if !spec.stale.is_empty() || !spec.output_dirs.is_empty() {
args.extend(["sh", "-c", PRELUDE, "sh"].map(Into::into));
args.extend(spec.stale.iter().map(|p| root.join(p).into_os_string()));
args.push("--".into());
args.extend(
spec.output_dirs
.iter()
.map(|d| root.join(d).into_os_string()),
);
args.push("--".into());
}
args.push(cmd.get_program().to_os_string());
args.extend(cmd.get_args().map(std::ffi::OsStr::to_os_string));
args
}
fn relative_to(root: &Path, path: &Path) -> PathBuf {
let path = path.strip_prefix(root).unwrap_or(path);
path.strip_prefix(".").unwrap_or(path).to_path_buf()
}
#[derive(Debug, PartialEq, Eq)]
enum Change {
Written(PathBuf),
Removed(PathBuf),
Replaced(PathBuf),
}
impl Change {
fn path(&self) -> &Path {
match self {
Self::Written(p) | Self::Removed(p) | Self::Replaced(p) => p,
}
}
fn describe(&self) -> String {
match self {
Self::Written(p) => format!("wrote {}", p.display()),
Self::Removed(p) => format!("removed {}", p.display()),
Self::Replaced(p) => format!("replaced {}", p.display()),
}
}
}
impl Run {
pub fn finish(self, tool_succeeded: bool) -> Result<()> {
let applied = if tool_succeeded { self.apply() } else { Ok(()) };
if tool_succeeded && applied.is_ok() {
clear_stale();
}
let cleaned = remove_run_dir(&self.dir);
applied?;
cleaned
}
fn declares(&self, path: &Path) -> bool {
self.spec.stale.iter().any(|s| s == path)
|| self.spec.outputs.iter().any(|o| o == path)
|| self.spec.output_dirs.iter().any(|d| path.starts_with(d))
|| self.writable.iter().any(|w| w.covers(path))
}
fn undeclared_in(&self, change: &Change) -> Result<Vec<String>> {
let path = change.path();
if self.declares(path) {
return Ok(Vec::new());
}
let (Change::Removed(_) | Change::Replaced(_)) = change else {
return Ok(vec![change.describe()]);
};
let real = self.root.join(path);
if !fs::symlink_metadata(&real).is_ok_and(|m| m.is_dir()) {
return Ok(vec![change.describe()]);
}
let mut files = Vec::new();
real_files_under(&self.root, path, &mut files)?;
Ok(files
.into_iter()
.filter(|f| !self.declares(f))
.map(|f| Change::Removed(f).describe())
.collect())
}
fn apply(&self) -> Result<()> {
let mut changes = Vec::new();
collect_changes(&self.upper, Path::new(""), &self.root, &mut changes)?;
let mut undeclared: Vec<String> = Vec::new();
for change in &changes {
undeclared.extend(self.undeclared_in(change)?);
}
if !undeclared.is_empty() {
anyhow::bail!(
"[{}] the tool changed paths its product does not declare (execution_model = \"sandbox\"):\n {}\n\
Nothing it wrote was kept. Declare these paths as outputs, stop the tool from writing them, \
or set execution_model = \"native\" on this processor.",
self.spec.label,
undeclared.join("\n ")
);
}
for change in &changes {
if let Change::Removed(p) | Change::Replaced(p) = change {
remove_real(&self.root.join(p))?;
}
}
for change in &changes {
if let Change::Written(p) = change {
let from = self.upper.join(p);
let to = self.root.join(p);
if let Some(parent) = to.parent() {
fs::create_dir_all(parent).with_context(|| {
format!("Failed to create directory {}", parent.display())
})?;
}
fs::rename(&from, &to).with_context(|| {
format!(
"Failed to move sandbox output {} to {}",
from.display(),
to.display()
)
})?;
}
}
Ok(())
}
}
fn collect_changes(upper: &Path, rel: &Path, root: &Path, out: &mut Vec<Change>) -> Result<()> {
collect_changes_in(upper, rel, root, false, out)
}
fn collect_changes_in(
upper: &Path,
rel: &Path,
root: &Path,
replacing: bool,
out: &mut Vec<Change>,
) -> Result<()> {
let dir = upper.join(rel);
let mut entries: Vec<_> = fs::read_dir(&dir)
.with_context(|| format!("Failed to read sandbox directory {}", dir.display()))?
.collect::<std::io::Result<_>>()
.with_context(|| format!("Failed to read sandbox directory {}", dir.display()))?;
entries.sort_by_key(fs::DirEntry::file_name);
for entry in entries {
let child = rel.join(entry.file_name());
let path = entry.path();
let meta = fs::symlink_metadata(&path)
.with_context(|| format!("Failed to stat {}", path.display()))?;
if crate::platform::is_overlay_whiteout(&meta) {
out.push(Change::Removed(child));
} else if meta.is_dir() {
let opaque = crate::platform::get_xattr(&path, OPAQUE_XATTR)
.with_context(|| format!("Failed to read xattrs of {}", path.display()))?
.is_some_and(|v| v == b"y");
let replaces = opaque && fs::symlink_metadata(root.join(&child)).is_ok();
if replaces {
out.push(Change::Replaced(child.clone()));
}
collect_changes_in(upper, &child, root, replacing || replaces, out)?;
} else if replacing || !same_as_real(&path, &meta, &root.join(&child))? {
out.push(Change::Written(child));
}
}
Ok(())
}
fn same_as_real(upper: &Path, upper_meta: &fs::Metadata, real: &Path) -> Result<bool> {
let Ok(real_meta) = fs::symlink_metadata(real) else {
return Ok(false);
};
if !upper_meta.is_file()
|| !real_meta.is_file()
|| upper_meta.len() != real_meta.len()
|| crate::platform::get_mode(upper_meta) != crate::platform::get_mode(&real_meta)
{
return Ok(false);
}
same_content(upper, real)
}
fn same_content(a: &Path, b: &Path) -> Result<bool> {
use std::io::Read;
let open =
|p: &Path| fs::File::open(p).with_context(|| format!("Failed to open {}", p.display()));
let (mut fa, mut fb) = (open(a)?, open(b)?);
let mut ba = vec![0u8; 64 * 1024];
let mut bb = vec![0u8; 64 * 1024];
loop {
let n = fa
.read(&mut ba)
.with_context(|| format!("Failed to read {}", a.display()))?;
if n == 0 {
return Ok(true);
}
fb.read_exact(&mut bb[..n])
.with_context(|| format!("Failed to read {}", b.display()))?;
if ba[..n] != bb[..n] {
return Ok(false);
}
}
}
fn real_files_under(root: &Path, rel: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
let dir = root.join(rel);
let mut entries: Vec<_> = fs::read_dir(&dir)
.with_context(|| format!("Failed to read directory {}", dir.display()))?
.collect::<std::io::Result<_>>()
.with_context(|| format!("Failed to read directory {}", dir.display()))?;
entries.sort_by_key(fs::DirEntry::file_name);
for entry in entries {
let child = rel.join(entry.file_name());
let meta = fs::symlink_metadata(entry.path())
.with_context(|| format!("Failed to stat {}", entry.path().display()))?;
if meta.is_dir() {
real_files_under(root, &child, out)?;
} else {
out.push(child);
}
}
Ok(())
}
fn remove_real(path: &Path) -> Result<()> {
let Ok(meta) = fs::symlink_metadata(path) else {
return Ok(());
};
if meta.is_dir() {
fs::remove_dir_all(path)
} else {
fs::remove_file(path)
}
.with_context(|| format!("Failed to remove {}", path.display()))
}
pub fn remove_leftover_runs(root: &Path) -> Result<usize> {
let sandbox = root.join(SANDBOX_DIR);
let entries = match fs::read_dir(&sandbox) {
Ok(entries) => entries,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0),
Err(e) => {
return Err(e).with_context(|| {
format!("Failed to read sandbox directory {}", sandbox.display())
});
}
};
let mut removed = 0;
for entry in entries {
let entry = entry
.with_context(|| format!("Failed to read sandbox directory {}", sandbox.display()))?;
let path = entry.path();
if fs::symlink_metadata(&path)
.with_context(|| format!("Failed to stat {}", path.display()))?
.is_dir()
{
remove_run_dir(&path)?;
} else {
fs::remove_file(&path)
.with_context(|| format!("Failed to remove {}", path.display()))?;
}
removed += 1;
}
fs::remove_dir(&sandbox)
.with_context(|| format!("Failed to remove sandbox directory {}", sandbox.display()))?;
Ok(removed)
}
fn remove_run_dir(dir: &Path) -> Result<()> {
let inner = dir.join("work").join("work");
if inner.exists() {
crate::platform::set_permissions_mode(&inner, 0o700)
.with_context(|| format!("Failed to open up {}", inner.display()))?;
}
fs::remove_dir_all(dir)
.with_context(|| format!("Failed to remove sandbox directory {}", dir.display()))
}
#[cfg(test)]
mod tests {
use super::*;
fn instance(
instance_name: &str,
pname: &str,
toml_src: &str,
) -> crate::config::ProcessorInstance {
crate::config::ProcessorInstance {
instance_name: instance_name.into(),
pname: pname.into(),
config_toml: toml::from_str(toml_src).expect("instance toml"),
provenance: crate::config::ProvenanceMap::new(),
}
}
fn marp_is_unsandboxable(pname: &str) -> bool {
pname != "processor.generator.marp"
}
#[test]
fn unsandboxable_processors_are_refused_only_when_sandboxed() {
let marp = instance("processor.generator.marp", "processor.generator.marp", "");
let ruff = instance("processor.checker.ruff", "processor.checker.ruff", "");
let native_marp = instance(
"processor.generator.marp.a",
"processor.generator.marp",
"execution_model = \"native\"",
);
let disabled_marp = instance(
"processor.generator.marp.b",
"processor.generator.marp",
"enabled = false",
);
let sandboxed_marp = instance(
"processor.generator.marp.c",
"processor.generator.marp",
"execution_model = \"sandbox\"",
);
let all = [marp, ruff, native_marp, disabled_marp, sandboxed_marp];
assert_eq!(
unsandboxable_instances(&all, ExecutionModel::Sandbox, &marp_is_unsandboxable),
["processor.generator.marp", "processor.generator.marp.c"]
);
assert_eq!(
unsandboxable_instances(&all, ExecutionModel::Native, &marp_is_unsandboxable),
["processor.generator.marp.c"]
);
assert!(
check_sandboxable_by(&all[1..3], ExecutionModel::Sandbox, &marp_is_unsandboxable)
.is_ok()
);
let err = check_sandboxable_by(&all, ExecutionModel::Sandbox, &marp_is_unsandboxable)
.unwrap_err()
.to_string();
assert!(
err.contains("processor.generator.marp")
&& err.contains("execution_model = \"native\""),
"{err}"
);
}
#[test]
fn every_processor_is_sandboxable() {
let unsandboxable: Vec<String> = crate::registries::processor::all_plugins()
.filter(|p| !p.sandboxable)
.map(crate::registries::ProcessorPlugin::pname)
.collect();
assert_eq!(unsandboxable, Vec::<String>::new());
let marp = instance("processor.generator.marp", "processor.generator.marp", "");
assert!(check_sandboxable(&[marp], ExecutionModel::Sandbox).is_ok());
}
fn run_with(outputs: &[&str], dirs: &[&str]) -> Run {
Run {
command: Command::new("true"),
root: PathBuf::from("/p"),
dir: PathBuf::from("/p/.rsconstruct/sandbox/x"),
upper: PathBuf::from("/p/.rsconstruct/sandbox/x/upper"),
spec: Spec {
label: "processor.checker.x".into(),
outputs: outputs.iter().map(PathBuf::from).collect(),
output_dirs: dirs.iter().map(PathBuf::from).collect(),
network: false,
writable: Vec::new(),
stale: Vec::new(),
},
writable: Vec::new(),
}
}
#[test]
fn writable_entries_parse_as_paths_or_named_directories() {
assert_eq!(
Writable::parse("target"),
Ok(Writable::Path(PathBuf::from("target")))
);
assert_eq!(
Writable::parse("./sub/.cache"),
Ok(Writable::Path(PathBuf::from("sub/.cache")))
);
assert_eq!(
Writable::parse("**/__pycache__"),
Ok(Writable::Named("__pycache__".into()))
);
for bad in [
"", ".", "/abs", "../up", "a/../b", "*.pyc", "a/**/b", "**/", "**/a/b", "**/*",
] {
assert!(Writable::parse(bad).is_err(), "{bad:?} must be rejected");
}
}
#[test]
fn writable_places_are_declared() {
let mut run = run_with(&["out/a"], &[]);
run.writable = vec![
Writable::Path(PathBuf::from("target")),
Writable::Named("__pycache__".into()),
];
assert!(run.declares(Path::new("target/debug/x")));
assert!(run.declares(Path::new(
".venv/lib/python3/site-packages/m/__pycache__/m.pyc"
)));
assert!(run.declares(Path::new("__pycache__/a.pyc")));
assert!(!run.declares(Path::new("targets/x")));
assert!(!run.declares(Path::new("src/m.py")));
}
#[test]
fn every_tool_declares_valid_writable_places() {
for tool in crate::tools::all_tools() {
for entry in tool.sandbox_writable {
assert!(
Writable::parse(entry).is_ok(),
"tool {}: {:?}",
tool.name,
Writable::parse(entry)
);
}
}
}
#[test]
fn writable_for_unions_tools_and_config() {
let standard = StandardConfig {
sandbox_writable: vec!["extra".into(), "target".into()],
..Default::default()
};
let tools = [
"cargo".to_string(),
"python".to_string(),
"no-such-tool".to_string(),
];
assert_eq!(
writable_for(&standard, &tools),
[
"target",
"**/target",
"**/Cargo.lock",
"**/__pycache__",
"extra"
]
);
}
#[test]
fn network_comes_from_the_tools_or_the_config() {
let standard = StandardConfig::default();
let tools = |names: &[&str]| names.iter().map(|s| (*s).to_string()).collect::<Vec<_>>();
assert!(!network_for(&standard, &tools(&["cargo", "ruff"])));
assert!(network_for(&standard, &tools(&["cargo", "cargo-deny"])));
assert!(network_for(&standard, &tools(&["taplo"])));
let configured = StandardConfig {
sandbox_network: true,
..Default::default()
};
assert!(network_for(&configured, &tools(&[])));
}
#[test]
fn network_tools_are_the_installers_and_fetchers() {
let mut names: Vec<&str> = crate::tools::all_tools()
.filter(|t| t.needs_network)
.map(|t| t.name)
.collect();
names.sort_unstable();
assert_eq!(names, ["bundle", "cargo-deny", "npm", "pip", "taplo", "uv"]);
}
#[test]
fn declared_output_dirs_join_the_running_spec() {
declare_output_dir(Path::new(".rsconstruct/mass_generator/x.staging"));
assert!(
current_spec().is_none(),
"no product running: nothing to join"
);
set_spec(Some(run_with(&[], &[]).spec));
declare_output_dir(Path::new(".rsconstruct/mass_generator/x.staging"));
let spec = current_spec().unwrap();
set_spec(None);
assert_eq!(
spec.output_dirs,
[PathBuf::from(".rsconstruct/mass_generator/x.staging")]
);
}
#[test]
fn the_prelude_runs_only_when_needed() {
let mut cmd = Command::new("tool");
cmd.args(["-x", "--", "in"]);
let root = Path::new("/p");
let mut spec = run_with(&[], &[]).spec;
assert_eq!(inner_command(root, &spec, &cmd), ["tool", "-x", "--", "in"]);
spec.stale = vec![PathBuf::from("out/a"), PathBuf::from("out/b")];
spec.output_dirs = vec![PathBuf::from("out")];
assert_eq!(
inner_command(root, &spec, &cmd),
[
"sh", "-c", PRELUDE, "sh", "/p/out/a", "/p/out/b", "--", "/p/out", "--", "tool",
"-x", "--", "in"
]
);
}
#[test]
fn the_prelude_clears_stale_outputs_then_runs_the_tool() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
fs::create_dir(root.join("out")).unwrap();
fs::write(root.join("out/old"), "old").unwrap();
fs::write(root.join("out/restored"), "restored").unwrap();
crate::platform::set_permissions_mode(&root.join("out/restored"), 0o444).unwrap();
let mut spec = run_with(&[], &["out", "made/here"]).spec;
spec.stale = vec![
PathBuf::from("out/old"),
PathBuf::from("out/restored"),
PathBuf::from("out/gone"),
];
let mut tool = Command::new("sh");
tool.args([
"-c",
"printf '%s|' \"$@\" > out/ran",
"sh",
"a b",
"--",
"c",
]);
let args = inner_command(root, &spec, &tool);
let status = Command::new(&args[0])
.args(&args[1..])
.current_dir(root)
.status()
.unwrap();
assert!(status.success());
assert!(!root.join("out/old").exists());
assert!(!root.join("out/restored").exists());
assert!(root.join("made/here").is_dir());
assert_eq!(
fs::read_to_string(root.join("out/ran")).unwrap(),
"a b|--|c|"
);
}
#[test]
fn leftover_runs_are_removed() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
assert_eq!(remove_leftover_runs(root).unwrap(), 0);
let run = root.join(SANDBOX_DIR).join("4242-0");
fs::create_dir_all(run.join("upper/out")).unwrap();
fs::write(run.join("upper/out/half.html"), "half").unwrap();
fs::create_dir_all(run.join("work/work")).unwrap();
crate::platform::set_permissions_mode(&run.join("work/work"), 0o000).unwrap();
fs::create_dir_all(root.join(SANDBOX_DIR).join("4242-1")).unwrap();
assert_eq!(remove_leftover_runs(root).unwrap(), 2);
assert!(!root.join(SANDBOX_DIR).exists());
assert!(root.join(".rsconstruct").is_dir(), "only the sandbox goes");
}
#[test]
fn stale_outputs_may_be_removed() {
let mut run = run_with(&[], &[]);
assert!(!run.declares(Path::new("old/out.html")));
run.spec.stale = vec![PathBuf::from("old/out.html")];
assert!(run.declares(Path::new("old/out.html")));
}
#[test]
fn stale_outputs_are_forgotten_once_a_run_is_applied() {
let settle = |succeeded: bool| {
let tmp = tempfile::tempdir().unwrap();
let mut run = run_with(&[], &[]);
run.root = tmp.path().to_path_buf();
run.dir = tmp.path().join("run");
run.upper = run.dir.join("upper");
fs::create_dir_all(&run.upper).unwrap();
let mut spec = run.spec.clone();
spec.stale = vec![PathBuf::from("out/old")];
set_spec(Some(spec));
run.finish(succeeded).unwrap();
let left = current_spec().unwrap().stale;
set_spec(None);
left
};
assert_eq!(settle(false), [PathBuf::from("out/old")]);
let none: Vec<PathBuf> = Vec::new();
assert_eq!(settle(true), none);
}
#[test]
fn replaced_outputs_are_declared_and_removed_in_the_overlay() {
set_spec(None);
assert!(!declare_replaced_outputs(&[Path::new("_site/a.html")]));
set_spec(Some(run_with(&["_site/a.html"], &[]).spec));
assert!(declare_replaced_outputs(&[Path::new("_site/b.html")]));
let spec = current_spec().unwrap();
set_spec(None);
assert!(spec.outputs.contains(&PathBuf::from("_site/b.html")));
assert_eq!(spec.stale, [PathBuf::from("_site/b.html")]);
}
#[test]
fn replaced_directory_is_checked_file_by_file() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path().to_path_buf();
fs::create_dir_all(root.join("docx/sub")).unwrap();
fs::write(root.join("docx/a.docx"), "a").unwrap();
fs::write(root.join("docx/sub/b.docx"), "b").unwrap();
let mut run = run_with(&["docx/a.docx", "docx/sub/b.docx"], &[]);
run.root = root.clone();
let replaced = Change::Replaced(PathBuf::from("docx"));
let none: Vec<String> = Vec::new();
assert_eq!(run.undeclared_in(&replaced).unwrap(), none);
assert_eq!(
run.undeclared_in(&Change::Removed(PathBuf::from("docx")))
.unwrap(),
none
);
fs::write(root.join("docx/notes.txt"), "n").unwrap();
assert_eq!(
run.undeclared_in(&replaced).unwrap(),
["removed docx/notes.txt"]
);
assert_eq!(
run.undeclared_in(&Change::Removed(PathBuf::from("other")))
.unwrap(),
["removed other"]
);
}
#[test]
fn declares_outputs_and_everything_under_output_dirs() {
let run = run_with(&["out/a.html"], &["docs"]);
assert!(run.declares(Path::new("out/a.html")));
assert!(run.declares(Path::new("docs")));
assert!(run.declares(Path::new("docs/x/y.html")));
assert!(!run.declares(Path::new("out/b.html")));
assert!(!run.declares(Path::new("out")));
assert!(!run.declares(Path::new("docsx/y.html")));
}
#[test]
fn relative_to_strips_the_root_and_a_leading_dot() {
let root = Path::new("/p");
assert_eq!(
relative_to(root, Path::new("/p/out/a")),
PathBuf::from("out/a")
);
assert_eq!(
relative_to(root, Path::new("./out/a")),
PathBuf::from("out/a")
);
assert_eq!(
relative_to(root, Path::new("out/a")),
PathBuf::from("out/a")
);
}
#[test]
fn native_unless_the_processor_or_the_build_says_sandbox() {
let mut standard = StandardConfig::default();
assert!(!resolves_to_sandbox(&standard, ExecutionModel::Native));
assert!(resolves_to_sandbox(&standard, ExecutionModel::Sandbox));
standard.execution_model = Some(ExecutionModel::Native);
assert!(!resolves_to_sandbox(&standard, ExecutionModel::Sandbox));
standard.execution_model = Some(ExecutionModel::Sandbox);
assert!(resolves_to_sandbox(&standard, ExecutionModel::Native));
}
#[test]
fn collect_changes_reports_files_and_ignores_plain_directories() {
let tmp = tempfile::tempdir().unwrap();
let upper = tmp.path().join("upper");
let root = tmp.path().join("root");
fs::create_dir_all(upper.join("out/sub")).unwrap();
fs::create_dir_all(upper.join("empty")).unwrap();
fs::create_dir_all(&root).unwrap();
fs::write(upper.join("out/sub/a.txt"), "a").unwrap();
fs::write(upper.join("stray"), "s").unwrap();
let mut changes = Vec::new();
collect_changes(&upper, Path::new(""), &root, &mut changes).unwrap();
assert_eq!(
changes,
vec![
Change::Written(PathBuf::from("out/sub/a.txt")),
Change::Written(PathBuf::from("stray")),
]
);
}
#[test]
fn files_copied_up_unchanged_are_not_changes() {
let tmp = tempfile::tempdir().unwrap();
let upper = tmp.path().join("upper");
let root = tmp.path().join("root");
fs::create_dir_all(&upper).unwrap();
fs::create_dir_all(&root).unwrap();
for (name, real, written) in [("same", "abc", "abc"), ("edited", "abc", "abd")] {
fs::write(root.join(name), real).unwrap();
fs::write(upper.join(name), written).unwrap();
}
let mut changes = Vec::new();
collect_changes(&upper, Path::new(""), &root, &mut changes).unwrap();
assert_eq!(changes, vec![Change::Written(PathBuf::from("edited"))]);
}
#[test]
fn identical_files_in_a_replaced_directory_are_written() {
let tmp = tempfile::tempdir().unwrap();
let upper = tmp.path().join("upper");
let root = tmp.path().join("root");
fs::create_dir_all(upper.join("marp")).unwrap();
fs::create_dir_all(root.join("marp")).unwrap();
fs::write(root.join("marp/a.css"), "a").unwrap();
fs::write(upper.join("marp/a.css"), "a").unwrap();
xattr::set(upper.join("marp"), OPAQUE_XATTR, b"y").unwrap();
let mut changes = Vec::new();
collect_changes(&upper, Path::new(""), &root, &mut changes).unwrap();
assert_eq!(
changes,
vec![
Change::Replaced(PathBuf::from("marp")),
Change::Written(PathBuf::from("marp/a.css")),
]
);
}
}