bob 1.0.0

/*
 * Copyright (c) 2026 Jonathan Perkin <jonathan@perkin.org.uk>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*!
 * Sandbox creation and management.
 *
 * This module provides the [`Sandbox`] struct for creating isolated build
 * environments using chroot. The implementation varies by platform but
 * presents a uniform interface.
 *
 * # Platform Support
 *
 * | Platform | Implementation |
 * |----------|---------------|
 * | Linux | Mount namespaces + chroot |
 * | macOS | bindfs/devfs + chroot |
 * | NetBSD | Native mounts + chroot |
 * | illumos/Solaris | Platform mounts + chroot |
 *
 * # Sandbox Lifecycle
 *
 * 1. **Create**: Set up the sandbox directory and perform configured actions
 * 2. **Execute**: Run build scripts inside the sandbox via chroot
 * 3. **Destroy**: Reverse actions and clean up the sandbox directory
 *
 * # Configuration
 *
 * Sandboxes are configured in the `sandboxes` section of the Lua config file.
 * See the [`action`](crate::action) module for available actions.
 *
 * ```lua
 * sandboxes = {
 *     basedir = "/data/chroot",
 *     setup = {
 *         { action = "mount", fs = "proc", dir = "/proc" },
 *         { action = "mount", fs = "dev", dir = "/dev" },
 *         { action = "mount", fs = "bind", dir = "/usr/bin", opts = "ro" },
 *         { action = "copy", dir = "/etc" },
 *     },
 * }
 * ```
 *
 * # Multiple Sandboxes
 *
 * Multiple sandboxes can be created for parallel builds. Each sandbox is
 * identified by an integer ID (0, 1, 2, ...) and created as a subdirectory
 * of `basedir`.
 *
 * With `build_threads = 4`, sandboxes are created at:
 * - `/data/chroot/0`
 * - `/data/chroot/1`
 * - `/data/chroot/2`
 * - `/data/chroot/3`
 */
#[cfg(target_os = "linux")]
mod sandbox_linux;
#[cfg(target_os = "macos")]
mod sandbox_macos;
#[cfg(target_os = "netbsd")]
mod sandbox_netbsd;
#[cfg(any(target_os = "illumos", target_os = "solaris"))]
mod sandbox_sunos;

use crate::action::{Action, ActionContext, ActionType, FSType};
use crate::config::{Config, Pkgsrc, PkgsrcEnv};
use crate::try_println;
use crate::{Interrupted, RunState};
use anyhow::{Context, Result, bail};
use rayon::prelude::*;
use std::fs;
use std::io::{BufReader, Read, Write};
use std::os::unix::process::CommandExt;
use std::os::unix::process::ExitStatusExt;
use std::path::{Component, Path, PathBuf};
use std::process::{Child, ChildStdout, Command, ExitStatus, Output, Stdio};
use std::sync::mpsc::RecvTimeoutError;
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
use tracing::{debug, info, info_span, warn};

/**
 * Extension trait to place a child process in its own session.
 *
 * Calling `.setsid()` detaches the child from the controlling terminal,
 * preventing build tools (e.g. pax) from writing to `/dev/tty`.  This
 * registers a `pre_exec` hook, which forces Rust to use `fork+exec`
 * instead of `posix_spawn`.  On illumos `fork()` suspends all threads,
 * so only use this when terminal isolation is required (builds, not
 * scans).
 */
pub(crate) trait CommandSetsid {
    fn new_session(&mut self) -> &mut Self;
}

impl CommandSetsid for Command {
    fn new_session(&mut self) -> &mut Self {
        unsafe {
            self.pre_exec(|| {
                libc::setsid();
                Ok(())
            });
        }
        self
    }
}

/*
 * The libc crate does not expose wait4() on illumos, but it is available
 * in libc as a wrapper around waitid().  Declare it here.
 */
#[cfg(target_os = "illumos")]
unsafe extern "C" {
    fn wait4(
        pid: libc::pid_t,
        status: *mut libc::c_int,
        options: libc::c_int,
        rusage: *mut libc::rusage,
    ) -> libc::pid_t;
}

/// How often to check the shutdown flag while waiting for something else.
/// This determines the maximum latency between Ctrl+C and response.
/// 100ms provides responsive feel without excessive polling overhead.
pub(crate) const SHUTDOWN_POLL_INTERVAL: Duration = Duration::from_millis(100);

/// Maximum number of retries when killing processes in a sandbox.
/// Uses exponential backoff: 64ms, 128ms, 256ms, 512ms, 1024ms = ~2s total.
pub(crate) const KILL_PROCESSES_MAX_RETRIES: u32 = 5;
pub(crate) const KILL_PROCESSES_INITIAL_DELAY_MS: u64 = 64;

/**
 * Retry backoff when unable to unmount a busy mount point.
 *
 * Short, minimal delay up to around 5 seconds on most operating systems.  On
 * macOS a clean unmount of /System takes over a minute so we retry a lot
 * longer there.
 */
#[cfg(not(target_os = "macos"))]
pub(crate) const UNMOUNT_MAX_RETRIES: u32 = 8;
#[cfg(target_os = "macos")]
pub(crate) const UNMOUNT_MAX_RETRIES: u32 = 180;
#[cfg(not(target_os = "macos"))]
pub(crate) const UNMOUNT_INITIAL_DELAY_MS: u64 = 64;
#[cfg(target_os = "macos")]
pub(crate) const UNMOUNT_INITIAL_DELAY_MS: u64 = 1024;
pub(crate) const UNMOUNT_MAX_DELAY_MS: u64 = 1024;

/**
 * Poll for child process exit while checking a run state flag.  If
 * shutdown is requested, kill the child and return an error.  During
 * stop the child is allowed to continue running.
 *
 * Uses wait4() instead of try_wait() to collect resource usage from the
 * child process.  Returns the exit status and CPU time (user + system).
 */
pub(crate) fn wait_with_shutdown(
    child: &mut Child,
    state: &RunState,
) -> Result<(ExitStatus, Duration)> {
    let pid = child.id() as libc::pid_t;
    loop {
        if state.is_shutdown() {
            let _ = child.kill();
            let _ = child.wait();
            bail!("Interrupted by shutdown");
        }
        let mut status: libc::c_int = 0;
        let mut rusage: libc::rusage = unsafe { std::mem::zeroed() };
        #[cfg(target_os = "illumos")]
        let ret = unsafe { wait4(pid, &mut status, libc::WNOHANG, &mut rusage) };
        #[cfg(not(target_os = "illumos"))]
        let ret = unsafe { libc::wait4(pid, &mut status, libc::WNOHANG, &mut rusage) };
        if ret < 0 {
            let err = std::io::Error::last_os_error();
            bail!("wait4 failed for pid {}: {}", pid, err);
        }
        if ret == 0 {
            std::thread::sleep(SHUTDOWN_POLL_INTERVAL);
            continue;
        }
        let utime = Duration::new(
            rusage.ru_utime.tv_sec as u64,
            rusage.ru_utime.tv_usec as u32 * 1000,
        );
        let stime = Duration::new(
            rusage.ru_stime.tv_sec as u64,
            rusage.ru_stime.tv_usec as u32 * 1000,
        );
        return Ok((ExitStatus::from_raw(status), utime + stime));
    }
}

/**
 * Poll a helper thread's result channel while checking a shutdown flag.
 * If shutdown is requested, kill the child's process group and return an
 * error.  This avoids the polling latency of try_wait() while still
 * allowing shutdown interruption.
 */
fn wait_channel_with_shutdown<T>(
    pid: u32,
    rx: std::sync::mpsc::Receiver<Result<T>>,
    state: &RunState,
) -> Result<T> {
    loop {
        if state.is_shutdown() {
            unsafe {
                libc::kill(-(pid as libc::pid_t), libc::SIGKILL);
            }
            let _ = rx.recv();
            bail!("Interrupted by shutdown");
        }
        match rx.recv_timeout(SHUTDOWN_POLL_INTERVAL) {
            Ok(result) => return result,
            Err(RecvTimeoutError::Timeout) => continue,
            Err(RecvTimeoutError::Disconnected) => {
                bail!("wait thread disconnected unexpectedly");
            }
        }
    }
}

/**
 * Wait for child process exit while checking a shutdown flag, returning
 * the full output (stdout/stderr).  If shutdown is requested, kill the
 * child and return an error.
 *
 * Uses a single helper thread that calls wait_with_output() (which handles
 * pipe draining correctly via internal threads).
 */
pub(crate) fn wait_output_with_shutdown(child: Child, state: &RunState) -> Result<Output> {
    let pid = child.id();
    let (tx, rx) = std::sync::mpsc::channel();

    crate::spawn_named("wait-output", move || {
        let _ = tx.send(child.wait_with_output().map_err(Into::into));
    });

    wait_channel_with_shutdown(pid, rx, state)
}

/**
 * Wait for child process exit while checking a shutdown flag, streaming
 * the child's stdout through `parse` as it is produced and collecting
 * stderr.  If shutdown is requested, kill the child and return an error.
 *
 * The parse result is returned alongside the exit status and stderr,
 * leaving the caller to decide which takes precedence on failure.
 */
pub(crate) fn wait_parse_with_shutdown<T, F>(
    mut child: Child,
    state: &RunState,
    parse: F,
) -> Result<(ExitStatus, T, String)>
where
    T: Send + 'static,
    F: FnOnce(&mut BufReader<ChildStdout>) -> T + Send + 'static,
{
    let pid = child.id();
    let (tx, rx) = std::sync::mpsc::channel();

    crate::spawn_named("wait-parse", move || {
        let run = || {
            let stdout = child.stdout.take().context("child stdout not piped")?;
            let stderr = child.stderr.take().context("child stderr not piped")?;
            let drain = crate::spawn_named("stderr-drain", move || {
                let mut out = String::new();
                let mut stderr = stderr;
                let _ = stderr.read_to_string(&mut out);
                out
            });
            let mut reader = BufReader::new(stdout);
            let parsed = parse(&mut reader);
            /* Consume any unread stdout so the child can exit. */
            let _ = std::io::copy(&mut reader, &mut std::io::sink());
            let status = child.wait()?;
            let stderr = drain.join().unwrap_or_default();
            Ok((status, parsed, stderr))
        };
        let _ = tx.send(run());
    });

    wait_channel_with_shutdown(pid, rx, state)
}

/**
 * Build sandbox manager.
 *
 * Provides a uniform interface for running commands regardless of
 * whether sandboxes are enabled. When enabled, commands execute inside
 * a chroot; when disabled, they run directly on the host.
 *
 * Use [`SandboxScope`] for RAII lifecycle management in build/scan
 * operations.
 */
#[derive(Clone, Debug, Default)]
pub struct Sandbox {
    config: Config,
    pkgsrc: Option<Pkgsrc>,
    context: ActionContext,
    pkgsrc_env: Arc<OnceLock<PkgsrcEnv>>,
}

impl Sandbox {
    /**
     * Create a new [`Sandbox`] instance for build operations.  This is
     * used even if sandboxes have not been enabled, as it provides a
     * consistent interface to run commands through.  If sandboxes are
     * enabled then commands are executed via `chroot(8)`, otherwise
     * they are executed directly.
     */
    pub fn new(config: &Config, pkgsrc: Option<&Pkgsrc>) -> Sandbox {
        Self::with_context(config, pkgsrc, ActionContext::Build)
    }

    /**
     * Create a new [`Sandbox`] instance for `bob dev`
     * interactive sessions.  Actions whose `only.context` is set to
     * `"dev"` will run; actions restricted to `"build"` will be skipped.
     */
    pub fn new_dev(config: &Config, pkgsrc: Option<&Pkgsrc>) -> Sandbox {
        Self::with_context(config, pkgsrc, ActionContext::Dev)
    }

    fn with_context(config: &Config, pkgsrc: Option<&Pkgsrc>, context: ActionContext) -> Sandbox {
        Sandbox {
            config: config.clone(),
            pkgsrc: pkgsrc.cloned(),
            context,
            pkgsrc_env: Arc::new(OnceLock::new()),
        }
    }

    /**
     * Store the pkgsrc environment for this sandbox.
     *
     * Once stored, subsequent calls are silently ignored: the value is
     * fixed for the lifetime of the sandbox.  All consumers
     * (script_env(), pkgsrc_env(), run_post_build()) read from this
     * single cell.
     */
    pub fn set_pkgsrc_env(&self, env: PkgsrcEnv) {
        let _ = self.pkgsrc_env.set(env);
    }

    /**
     * Return the stored pkgsrc environment, if [`set_pkgsrc_env`] has
     * been called.  Returns `None` before that point (e.g. during
     * initial sandbox setup, before bootstrap unpack).
     *
     * [`set_pkgsrc_env`]: Sandbox::set_pkgsrc_env
     */
    pub fn pkgsrc_env(&self) -> Option<&PkgsrcEnv> {
        self.pkgsrc_env.get()
    }

    /**
     * Return script environment variables for use by hook commands and
     * shell exports.
     *
     * Always includes host-side variables derived from config.  Once
     * [`set_pkgsrc_env`] has been called, also includes the
     * pkgsrc-derived variables and the configured cachevars.
     *
     * [`set_pkgsrc_env`]: Sandbox::set_pkgsrc_env
     */
    pub fn script_env(&self) -> Vec<(String, String)> {
        let mut envs = vec![(
            "bob_logdir".to_string(),
            format!("{}", self.config.logdir().display()),
        )];
        if let Some(pkgsrc) = &self.pkgsrc {
            envs.push(("bob_make".to_string(), format!("{}", pkgsrc.make.display())));
            envs.push((
                "bob_pkgsrc".to_string(),
                format!("{}", pkgsrc.basedir.display()),
            ));
            if let Some(build_user) = &pkgsrc.build_user {
                envs.push(("bob_build_user".to_string(), build_user.clone()));
            }
            if let Some(home) = &pkgsrc.build_user_home {
                envs.push((
                    "bob_build_user_home".to_string(),
                    home.display().to_string(),
                ));
            }
            if let Some(bootstrap) = &pkgsrc.bootstrap {
                envs.push((
                    "bob_bootstrap".to_string(),
                    format!("{}", bootstrap.display()),
                ));
            }
        }
        if let Some(env) = self.pkgsrc_env.get() {
            envs.push((
                "bob_packages".to_string(),
                env.packages.display().to_string(),
            ));
            envs.push((
                "bob_pkgtools".to_string(),
                env.pkgtools.display().to_string(),
            ));
            envs.push(("bob_prefix".to_string(), env.prefix.display().to_string()));
            envs.push((
                "bob_pkg_dbdir".to_string(),
                env.pkg_dbdir.display().to_string(),
            ));
            envs.push((
                "bob_pkg_refcount_dbdir".to_string(),
                env.pkg_refcount_dbdir.display().to_string(),
            ));
            if let Some(varbase) = env.metadata.get("VARBASE") {
                envs.push(("bob_varbase".to_string(), varbase.clone()));
            }
            for (key, value) in &env.cachevars {
                envs.push((key.clone(), value.clone()));
            }
        }
        envs
    }

    /// Return whether sandboxes have been enabled.
    ///
    /// This is based on whether a valid `sandboxes` section has been
    /// specified in the config file.
    pub fn enabled(&self) -> bool {
        self.config.sandboxes().is_some()
    }

    fn basedir(&self) -> Option<&PathBuf> {
        self.config.sandboxes().as_ref().map(|s| &s.basedir)
    }

    /**
     * Return full path to a sandbox by id.
     */
    pub fn path(&self, id: usize) -> PathBuf {
        let sandbox = &self.config.sandboxes().as_ref().unwrap();
        let mut p = PathBuf::from(&sandbox.basedir);
        p.push(id.to_string());
        p
    }

    /**
     * Create a Command that runs in the sandbox (via chroot) if enabled,
     * or directly if sandboxes are disabled.
     *
     * The returned command uses `posix_spawn` where available.  Call
     * [`.new_session()`](CommandSetsid::new_session) on the result to place the
     * child in a new session (prevents `/dev/tty` access by build tools,
     * but forces `fork+exec` which is expensive on illumos).
     */
    pub(crate) fn command(&self, id: Option<usize>, cmd: &Path) -> Command {
        let build_vars = matches!(self.context, ActionContext::Build)
            .then(|| self.config.environment().and_then(|e| e.build.as_ref()))
            .flatten()
            .map(|ctx| &ctx.vars);

        let mut c = match id {
            Some(id) => {
                let mut c = Command::new("/usr/sbin/chroot");
                c.arg(self.path(id));
                if let Some(vars) = build_vars
                    && !vars.is_empty()
                {
                    c.arg("/usr/bin/env");
                    for (name, value) in vars {
                        c.arg(format!("{name}={value}"));
                    }
                }
                c.arg(cmd);
                c
            }
            None => {
                let mut c = Command::new(cmd);
                if let Some(vars) = build_vars {
                    for (name, value) in vars {
                        c.env(name, value);
                    }
                }
                c
            }
        };
        match self.context {
            ActionContext::Build => self.apply_build_environment(&mut c),
            ActionContext::Dev => self.apply_dev_environment(&mut c),
        }
        c
    }

    /**
     * Apply the build-time `clear` / `inherit` policy to a Command.
     */
    pub(crate) fn apply_build_environment(&self, cmd: &mut Command) {
        let Some(ctx) = self.config.environment().and_then(|e| e.build.as_ref()) else {
            return;
        };
        Self::apply_env_context(cmd, ctx);
    }

    /**
     * Apply only the dev-context parent-environment policy
     * (`clear`/`inherit`) to a Command, without setting any variables.
     *
     * Used by interactive `bob dev` sessions, where the user's
     * `environment.dev.vars` are written to a wrapper init script and
     * exported by the shell itself rather than via `cmd.env()`.
     */
    pub fn apply_dev_environment(&self, cmd: &mut Command) {
        let Some(ctx) = self.config.environment().and_then(|e| e.dev.as_ref()) else {
            return;
        };
        Self::apply_env_context(cmd, ctx);
    }

    fn apply_env_context(cmd: &mut Command, ctx: &crate::config::EnvContext) {
        if ctx.clear {
            cmd.env_clear();
            for name in &ctx.inherit {
                if let Ok(value) = std::env::var(name) {
                    cmd.env(name, value);
                }
            }
        }
    }

    /**
     * Kill all processes in a sandbox by id.
     * This is used for graceful shutdown on Ctrl+C.
     */
    pub(crate) fn kill_processes_by_id(&self, id: Option<usize>) {
        let Some(id) = id else { return };
        let sandbox = self.path(id);
        if sandbox.exists() {
            let span = info_span!("kill_processes", sandbox_id = id);
            let _guard = span.enter();
            self.kill_processes_by_path(&sandbox);
        }
    }

    /**
     * Kill all processes with open references under a sandbox path.
     *
     * Uses platform-specific `find_pids()` to locate processes, then
     * sends SIGKILL via `kill -9`.  Retries with exponential backoff.
     */
    fn kill_processes_by_path(&self, sandbox: &Path) {
        for iteration in 0..KILL_PROCESSES_MAX_RETRIES {
            let pids = self.find_pids(sandbox);

            if pids.is_empty() {
                debug!(retries = iteration, "No processes found in sandbox");
                return;
            }

            info!(pids = %pids.join(" "), "Killed processes using sandbox");

            let _ = Command::new("kill")
                .arg("-9")
                .args(&pids)
                .stderr(Stdio::null())
                .process_group(0)
                .status();

            let delay_ms = KILL_PROCESSES_INITIAL_DELAY_MS << iteration;
            std::thread::sleep(Duration::from_millis(delay_ms));
        }
        if let Some(proc_info) = self.get_process_info(sandbox) {
            warn!(
                max_retries = KILL_PROCESSES_MAX_RETRIES,
                remaining = %proc_info,
                "Gave up killing processes after max retries"
            );
        } else {
            warn!(
                max_retries = KILL_PROCESSES_MAX_RETRIES,
                "Gave up killing processes after max retries"
            );
        }
    }

    /**
     * Find PIDs of processes using files under the sandbox path.
     *
     * Uses `fuser` which matches by path, not filesystem.
     * macOS and NetBSD override this in their platform files.
     */
    #[cfg(not(any(target_os = "macos", target_os = "netbsd")))]
    fn find_pids(&self, sandbox: &Path) -> Vec<String> {
        let output = Command::new("fuser")
            .arg(sandbox)
            .stdout(Stdio::piped())
            .stderr(Stdio::null())
            .process_group(0)
            .output();
        let Ok(out) = output else { return vec![] };
        if !out.status.success() {
            return vec![];
        }
        String::from_utf8_lossy(&out.stdout)
            .split_whitespace()
            .map(|s| s.to_string())
            .collect()
    }

    /**
     * Get human-readable info about processes still using the sandbox.
     */
    fn get_process_info(&self, sandbox: &Path) -> Option<String> {
        let pids = self.find_pids(sandbox);
        self.format_process_info(&pids)
    }

    /**
     * Format process info for a list of PIDs using ps.
     *
     * illumos overrides this with pargs (ps truncates on illumos).
     */
    #[cfg(not(any(target_os = "illumos", target_os = "solaris")))]
    fn format_process_info(&self, pids: &[String]) -> Option<String> {
        if pids.is_empty() {
            return None;
        }
        let out = Command::new("ps")
            .arg("-ww")
            .arg("-o")
            .arg("pid,args")
            .arg("-p")
            .arg(pids.join(","))
            .process_group(0)
            .output()
            .ok()?;

        let info: Vec<String> = String::from_utf8_lossy(&out.stdout)
            .lines()
            .skip(1)
            .filter_map(|line| {
                let mut parts = line.split_whitespace();
                let pid = parts.next()?;
                let cmd: String = parts.collect::<Vec<_>>().join(" ");
                Some(format!("pid={} cmd='{}'", pid, cmd))
            })
            .collect();

        if info.is_empty() {
            None
        } else {
            Some(info.join(", "))
        }
    }

    /**
     * Return full path to a specified mount point in a sandbox.
     * The returned path is guaranteed to be within the sandbox directory.
     */
    fn mountpath(&self, id: usize, mnt: &PathBuf) -> PathBuf {
        /*
         * Note that .push() on a PathBuf will replace the path if
         * it is absolute, so we need to trim any leading "/".
         */
        let mut p = self.path(id);
        match mnt.strip_prefix("/") {
            Ok(s) => p.push(s),
            Err(_) => p.push(mnt),
        };
        p
    }

    /**
     * Verify that a path is safely contained within the sandbox.
     * This prevents path traversal attacks via ".." or symlinks.
     * Returns error if the path escapes the sandbox boundary.
     */
    fn verify_path_in_sandbox(&self, id: usize, path: &Path) -> Result<()> {
        let sandbox_root = self.path(id);
        // Canonicalize both paths to resolve any ".." or symlinks
        // Note: canonicalize requires the path to exist, so we check
        // the parent directory for paths that don't exist yet
        let canonical_sandbox = sandbox_root.canonicalize().unwrap_or(sandbox_root.clone());

        // For the target path, try to canonicalize what exists
        let canonical_path = if path.exists() {
            path.canonicalize()?
        } else {
            // Path doesn't exist yet, check its parent
            if let Some(parent) = path.parent()
                && parent.exists()
            {
                let canonical_parent = parent.canonicalize()?;
                if !canonical_parent.starts_with(&canonical_sandbox) {
                    bail!(
                        "Path escapes sandbox: {} is not within {}",
                        path.display(),
                        sandbox_root.display()
                    );
                }
            }
            return Ok(());
        };

        if !canonical_path.starts_with(&canonical_sandbox) {
            bail!(
                "Path escapes sandbox: {} resolves to {} which is not within {}",
                path.display(),
                canonical_path.display(),
                canonical_sandbox.display()
            );
        }
        Ok(())
    }

    /*
     * Marker directory functions for sandbox lifecycle management.
     *
     * Each sandbox has a .bob directory that marks it as bob-managed.
     * Inside .bob, a "completed" directory indicates the sandbox was set up
     * successfully.  This two-stage marker allows detecting incomplete sandboxes.
     */
    fn bobmarker(&self, id: usize) -> PathBuf {
        self.path(id).join(".bob")
    }

    fn completedpath(&self, id: usize) -> PathBuf {
        self.bobmarker(id).join("completed")
    }

    fn mark_complete(&self, id: usize) -> Result<()> {
        let path = self.completedpath(id);
        fs::create_dir(&path).with_context(|| format!("Failed to create {}", path.display()))?;
        Ok(())
    }

    fn is_bob_sandbox(&self, id: usize) -> bool {
        self.bobmarker(id).exists()
    }

    fn is_sandbox_complete(&self, id: usize) -> bool {
        self.completedpath(id).exists()
    }

    /**
     * Discover all bob-managed sandboxes by scanning basedir.
     * Returns sorted list of sandbox IDs.
     */
    fn discover_sandboxes(&self) -> Result<Vec<usize>> {
        let Some(basedir) = self.basedir() else {
            return Ok(vec![]);
        };
        if !basedir.exists() {
            return Ok(vec![]);
        }
        let mut ids = Vec::new();
        let entries = fs::read_dir(basedir)
            .with_context(|| format!("Failed to read {}", basedir.display()))?;
        for entry in entries {
            let entry = entry?;
            let path = entry.path();
            if !path.is_dir() {
                continue;
            }
            let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
                continue;
            };
            let Ok(id) = name.parse::<usize>() else {
                continue;
            };
            if self.is_bob_sandbox(id) {
                ids.push(id);
            }
        }
        ids.sort_unstable();
        Ok(ids)
    }

    /**
     * Claim the next available sandbox ID.
     */
    pub fn claim_id(&self) -> Result<usize> {
        let mut candidate = 0;
        loop {
            if self.create(candidate)? {
                return Ok(candidate);
            }
            candidate += 1;
        }
    }

    /**
     * Claim and set up `n` sandboxes in parallel.
     *
     * Each parallel task independently claims the next available ID
     * (the atomic mkdir on `.bob` prevents races) and sets it up.
     * On any failure, all successfully created sandboxes are destroyed.
     */
    pub(crate) fn claim_ids(&self, n: usize) -> Result<Vec<usize>> {
        let results: Vec<Result<usize>> = (0..n).into_par_iter().map(|_| self.claim_id()).collect();

        let mut claimed = Vec::with_capacity(n);
        let mut first_error: Option<anyhow::Error> = None;
        for result in results {
            match result {
                Ok(id) => claimed.push(id),
                Err(e) => {
                    if first_error.is_none() {
                        first_error = Some(e);
                    }
                }
            }
        }

        if let Some(e) = first_error {
            for &id in &claimed {
                let _ = self.destroy(id);
            }
            return Err(e);
        }

        claimed.sort_unstable();
        Ok(claimed)
    }

    /**
     * Atomically claims the ID via the `.bob` marker directory, then
     * runs configured actions and marks complete.  Returns `Ok(false)`
     * if the ID is already taken by another process.
     */
    pub(crate) fn create(&self, id: usize) -> Result<bool> {
        let sandbox = self.path(id);
        fs::create_dir_all(&sandbox)
            .with_context(|| format!("Failed to create {}", sandbox.display()))?;
        let marker = self.bobmarker(id);
        match fs::create_dir(&marker) {
            Ok(()) => {}
            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(false),
            Err(e) => {
                return Err(e).with_context(|| format!("Failed to create {}", marker.display()));
            }
        }
        let Some(sandbox) = &self.config.sandboxes() else {
            bail!("Internal error: trying to create sandbox when sandboxes disabled.");
        };
        let envs = self.script_env();
        self.perform_actions(id, &sandbox.setup, &envs)?;
        self.mark_complete(id)?;
        Ok(true)
    }

    /**
     * Execute inline script content via /bin/sh.
     */
    pub(crate) fn execute_script(
        &self,
        id: Option<usize>,
        content: &str,
        envs: Vec<(String, String)>,
    ) -> Result<Child> {
        let mut cmd = self.command(id, Path::new("/bin/sh"));
        cmd.new_session();
        cmd.current_dir("/").arg("-s");

        for (key, val) in envs {
            cmd.env(key, val);
        }

        let mut child = cmd
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()?;

        if let Some(mut stdin) = child.stdin.take() {
            stdin.write_all(content.as_bytes())?;
        }

        Ok(child)
    }

    /**
     * Execute a command directly without shell interpretation.
     */
    pub(crate) fn execute_command<I, S>(
        &self,
        id: Option<usize>,
        cmd: &Path,
        args: I,
        envs: Vec<(String, String)>,
    ) -> Result<Child>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<std::ffi::OsStr>,
    {
        let mut command = self.command(id, cmd);
        command.args(args);
        for (key, val) in envs {
            command.env(key, val);
        }
        command
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .process_group(0)
            .spawn()
            .map_err(Into::into)
    }

    /**
     * Run pre-build operations:
     *  1. Unpack bootstrap kit if configured
     *  2. Execute hook action create commands in order
     *
     * Returns Err if any step fails; the caller decides whether the
     * failure is fatal (sandbox-level) or per-package.
     */
    pub fn run_pre_build(&self, sandbox_id: Option<usize>) -> Result<()> {
        if let Some(bootstrap) = self.pkgsrc.as_ref().and_then(|p| p.bootstrap.as_deref()) {
            let Some(sandbox_id) = sandbox_id else {
                bail!("bootstrap requires sandboxes to be enabled");
            };
            let dest = self.path(sandbox_id);
            info!(bootstrap = %bootstrap.display(), dest = %dest.display(), "Unpacking bootstrap kit");
            let file = fs::File::open(bootstrap)
                .with_context(|| format!("Failed to open bootstrap {}", bootstrap.display()))?;
            let gz = flate2::read::GzDecoder::new(file);
            let mut archive = tar::Archive::new(gz);
            archive.set_preserve_permissions(true);
            archive.set_preserve_ownerships(true);
            self.unpack_archive(&mut archive, &dest).with_context(|| {
                format!(
                    "Failed to unpack bootstrap {} to {}",
                    bootstrap.display(),
                    dest.display()
                )
            })?;
        }

        let hooks = self.config.hooks();
        if !hooks.is_empty() {
            let Some(sandbox_id) = sandbox_id else {
                bail!("hooks require sandboxes to be enabled");
            };
            let envs = self.script_env();
            self.perform_actions(sandbox_id, hooks, &envs)?;
        }

        Ok(())
    }

    /*
     * Unpack a tar archive into dest, matching tar(1) hard link
     * behaviour: an entry whose target is its own destination and
     * already exists is skipped, and an existing file at the
     * destination of any other hard link is replaced.
     */
    fn unpack_archive<R: Read>(&self, archive: &mut tar::Archive<R>, dest: &Path) -> Result<()> {
        for entry in archive.entries()? {
            let mut entry = entry?;
            if entry.header().entry_type().is_hard_link()
                && let Some(dst) = Self::entry_dest(dest, &entry.path()?)
                && let Ok(meta) = fs::symlink_metadata(&dst)
                && !meta.is_dir()
            {
                let src = entry.link_name()?.and_then(|p| Self::entry_dest(dest, &p));
                if src.as_deref() == Some(dst.as_path()) {
                    continue;
                }
                fs::remove_file(&dst)
                    .with_context(|| format!("Failed to remove {}", dst.display()))?;
            }
            entry.unpack_in(dest)?;
        }
        Ok(())
    }

    /*
     * Resolve an archive entry path within dest using the same rules
     * as Entry::unpack_in: root components are dropped, and paths
     * containing parent references return None.
     */
    fn entry_dest(dest: &Path, path: &Path) -> Option<PathBuf> {
        let mut out = dest.to_path_buf();
        for comp in path.components() {
            match comp {
                Component::Prefix(_) | Component::RootDir | Component::CurDir => {}
                Component::ParentDir => return None,
                Component::Normal(c) => out.push(c),
            }
        }
        Some(out)
    }

    /**
     * Run post-build operations:
     *  1. Execute hook destroy commands in reverse order
     *  2. Remove prefix, pkg_dbdir, and pkg_refcount_dbdir
     *
     * Returns Err if a hook destroy command fails; the caller decides
     * whether to log and continue or treat as fatal.  Prefix and
     * pkg_dbdir cleanup is best-effort and errors are ignored: the
     * prefix is commonly a mount point, so removing the directory
     * itself is expected to fail.
     *
     * The pkgsrc paths are removed only when [`set_pkgsrc_env`] has
     * been called, since they are unknown otherwise.  If they were
     * populated without the cell ever being set, destroy()'s
     * bail-on-non-empty watchdog surfaces the resulting state.
     *
     * [`set_pkgsrc_env`]: Sandbox::set_pkgsrc_env
     */
    pub fn run_post_build(&self, sandbox_id: Option<usize>) -> Result<()> {
        // Capture any hook destroy error but still run the
        // best-effort prefix/pkgdb cleanup below so the sandbox is
        // left as clean as possible for reuse.  The captured error is
        // returned at the end.
        let hook_result: Result<()> = if let Some(sandbox_id) = sandbox_id {
            let hooks = self.config.hooks();
            if hooks.is_empty() {
                Ok(())
            } else {
                let envs = self.script_env();
                self.reverse_actions(sandbox_id, hooks, &envs)
            }
        } else if !self.config.hooks().is_empty() {
            bail!("hooks require sandboxes to be enabled");
        } else {
            Ok(())
        };

        if let Some(pkgsrc) = self.pkgsrc_env.get() {
            let mut targets = Vec::new();
            for path in [
                &pkgsrc.prefix,
                &pkgsrc.pkg_dbdir,
                &pkgsrc.pkg_refcount_dbdir,
            ] {
                let target = match sandbox_id {
                    Some(id) => self.resolve_path(id, path),
                    None => path.clone(),
                };
                if target.exists() {
                    debug!(path = %target.display(), "Removing");
                    targets.push(target);
                }
            }
            self.remove_dirs(&targets);
        }

        hook_result
    }

    /**
     * Resolve a path relative to the sandbox root.
     */
    fn resolve_path(&self, sandbox_id: usize, path: &Path) -> PathBuf {
        self.mountpath(sandbox_id, &path.to_path_buf())
    }

    /**
     * Remove directories recursively by running rm(1), keeping the
     * unlink load in a child process.  Removal is best-effort and
     * errors are ignored.  If rm cannot be spawned the directories
     * are removed in-process instead.
     */
    fn remove_dirs(&self, paths: &[PathBuf]) {
        if paths.is_empty() {
            return;
        }
        let mut cmd = Command::new("/bin/rm");
        cmd.arg("-rf")
            .arg("--")
            .args(paths)
            .stdin(Stdio::null())
            .stdout(Stdio::null())
            .stderr(Stdio::null())
            .process_group(0);
        if cmd.status().is_err() {
            for path in paths {
                let _ = fs::remove_dir_all(path);
            }
        }
    }

    /**
     * Run a command as part of an action.
     *
     * When `chroot` is true, the command runs inside the sandbox via
     * chroot.  Otherwise it runs on the host with the sandbox root as
     * the working directory.
     */
    fn run_action_cmd(
        &self,
        sandbox_id: usize,
        cmd: &str,
        chroot: bool,
        envs: &[(String, String)],
    ) -> Result<Option<Output>> {
        let sandbox_path = self.path(sandbox_id);
        let output = if chroot {
            let mut c = Command::new("/usr/sbin/chroot");
            for (key, val) in envs {
                c.env(key, val);
            }
            c.arg(&sandbox_path)
                .arg("/bin/sh")
                .arg("-ceu")
                .arg(cmd)
                .stdin(Stdio::null())
                .stdout(Stdio::piped())
                .stderr(Stdio::piped())
                .process_group(0);
            c.output()?
        } else {
            let mut c = Command::new("/bin/sh");
            for (key, val) in envs {
                c.env(key, val);
            }
            c.arg("-ceu")
                .arg(cmd)
                .env("bob_sandbox_path", &sandbox_path)
                .current_dir(&sandbox_path)
                .stdin(Stdio::null())
                .stdout(Stdio::piped())
                .stderr(Stdio::piped())
                .process_group(0);
            c.output()?
        };
        Ok(Some(output))
    }

    /**
     * Destroy a single sandbox by id.
     */
    pub fn destroy(&self, id: usize) -> anyhow::Result<()> {
        let sandbox = self.path(id);
        if !sandbox.exists() {
            return Ok(());
        }
        /*
         * First, remove the "completed" marker to indicate this sandbox is no
         * longer available for builds.  If cleanup fails after this point,
         * the sandbox remains discoverable (has .bob) but is marked incomplete.
         */
        let completeddir = self.completedpath(id);
        if completeddir.exists() {
            self.remove_empty_hierarchy(&completeddir)?;
        }
        let Some(sandbox_config) = &self.config.sandboxes() else {
            bail!("Internal error: trying to destroy sandbox when sandboxes disabled.");
        };
        let envs = self.script_env();
        self.kill_processes_by_path(&self.path(id));
        self.reverse_actions(id, &sandbox_config.setup, &envs)?;
        /*
         * Remove all sandbox contents EXCEPT .bob.  If removal fails due to
         * unexpected files, the .bob marker remains and the sandbox can still
         * be discovered for retry.
         */
        if sandbox.exists() {
            let bobmarker = self.bobmarker(id);
            let entries = fs::read_dir(&sandbox)
                .with_context(|| format!("Failed to read {}", sandbox.display()))?;
            for entry in entries {
                let entry = entry?;
                let path = entry.path();
                if path == bobmarker {
                    continue;
                }
                self.remove_empty_hierarchy(&path)?;
            }
            /*
             * All other contents removed successfully.  Remove .bob as the
             * absolute last step - this is what makes the sandbox no longer
             * discoverable.
             */
            if bobmarker.exists() {
                self.remove_empty_hierarchy(&bobmarker)?;
            }
            self.remove_empty_hierarchy(&sandbox)?;
        }
        Ok(())
    }

    /**
     * Create sandboxes, claiming available IDs.  Returns the IDs in
     * ascending order so callers can pair them with paths via
     * [`Sandbox::path`].
     */
    pub fn create_all(&self, count: usize) -> Result<Vec<usize>> {
        let msg = if count == 1 {
            "Creating sandbox".to_string()
        } else {
            format!("Creating {} sandboxes", count)
        };
        crate::print_status(&msg);
        let start = Instant::now();
        let ids = self.claim_ids(count)?;
        crate::print_elapsed(&msg, start.elapsed());
        Ok(ids)
    }

    /**
     * Destroy all discovered sandboxes in parallel.  Runs post-build cleanup
     * on each sandbox first, then destroys them.  Continue on errors to ensure
     * all sandboxes are attempted, printing each error as it occurs.
     *
     * Removal of pkgsrc paths uses the pkgsrc environment stored on
     * this sandbox via [`set_pkgsrc_env`].  Callers that have a pkgsrc
     * environment available (loaded from the database, fetched fresh)
     * should call `set_pkgsrc_env` before invoking this method.
     *
     * [`set_pkgsrc_env`]: Sandbox::set_pkgsrc_env
     */
    pub fn destroy_all(&self) -> Result<()> {
        let ids = self.discover_sandboxes()?;
        self.destroy_set(ids)
    }

    /**
     * Destroy the listed sandbox IDs.  IDs that are not currently
     * allocated are silently skipped (rm -f semantics).  Both complete
     * and incomplete sandboxes are valid targets.
     */
    pub fn destroy_ids(&self, ids: &[usize]) -> Result<()> {
        let discovered = self.discover_sandboxes()?;
        let mut targets: Vec<usize> = discovered
            .into_iter()
            .filter(|id| ids.contains(id))
            .collect();
        targets.sort_unstable();
        self.destroy_set(targets)
    }

    /**
     * Run an unmount command, retrying a busy mountpoint before giving up.
     *
     * Failures are retried with exponential backoff, defined per-platform.
     */
    fn run_umount(&self, cmd: &mut Command, dest: &Path) -> Result<()> {
        let mut out = cmd.output().context("Unable to execute unmount")?;
        for retry in 0..UNMOUNT_MAX_RETRIES {
            if out.status.success() {
                if retry > 0 {
                    debug!(
                        dest = %dest.display(),
                        retries = retry,
                        "Unmount succeeded after retries"
                    );
                }
                return Ok(());
            }
            /* Clamp the shift so the delay cannot overflow over many retries. */
            let delay = (UNMOUNT_INITIAL_DELAY_MS << retry.min(10)).min(UNMOUNT_MAX_DELAY_MS);
            std::thread::sleep(Duration::from_millis(delay));
            out = cmd.output().context("Unable to execute unmount")?;
        }
        if out.status.success() {
            return Ok(());
        }
        let reason = String::from_utf8_lossy(&out.stderr);
        let reason = reason.trim();
        warn!(
            dest = %dest.display(),
            retries = UNMOUNT_MAX_RETRIES,
            reason,
            "Failed to unmount"
        );
        if reason.is_empty() {
            bail!("Failed to unmount {}", dest.display());
        }
        bail!("{reason}");
    }

    fn destroy_set(&self, sandboxes: Vec<usize>) -> Result<()> {
        if sandboxes.is_empty() {
            return Ok(());
        }
        for &id in &sandboxes {
            if self.path(id).exists()
                && let Err(e) = self.run_post_build(Some(id))
            {
                warn!(error = format!("{e:#}"), sandbox = id, "post-build error");
            }
        }
        let msg = if sandboxes.len() == 1 {
            "Destroying sandbox".to_string()
        } else {
            format!("Destroying {} sandboxes", sandboxes.len())
        };
        crate::print_status(&msg);
        let start = Instant::now();
        let results: Vec<(usize, Result<()>)> = sandboxes
            .into_par_iter()
            .map(|i| (i, self.destroy(i)))
            .collect();
        let mut failed = 0;
        for (i, result) in results {
            if let Err(e) = result {
                if failed == 0 {
                    println!();
                }
                eprintln!("sandbox {}: {:#}", i, e);
                warn!(
                    error = format!("{e:#}"),
                    sandbox = i,
                    "Failed to destroy sandbox"
                );
                failed += 1;
            }
        }
        if failed == 0 {
            crate::print_elapsed(&msg, start.elapsed());
            Ok(())
        } else {
            Err(anyhow::anyhow!(
                "Failed to destroy {} sandbox{}.\n\
                 Remove unexpected files, then try again.",
                failed,
                if failed == 1 { "" } else { "es" }
            ))
        }
    }

    /**
     * List all discovered sandboxes.
     */
    pub fn list_all(&self) -> Result<()> {
        for id in self.discover_sandboxes()? {
            let sandbox = self.path(id);
            let line = if self.is_sandbox_complete(id) {
                format!("{}", sandbox.display())
            } else {
                format!("{} (incomplete)", sandbox.display())
            };
            if !try_println(&line) {
                break;
            }
        }
        Ok(())
    }

    /*
     * Remove any empty directories from a mount point up to the root of the
     * sandbox.
     */
    fn remove_empty_dirs(&self, id: usize, mountpoint: &Path) {
        for p in mountpoint.ancestors() {
            /*
             * Sanity check we are within the chroot.
             */
            if !p.starts_with(self.path(id)) {
                break;
            }
            /*
             * Go up to next parent if this path does not exist.
             */
            if !p.exists() {
                continue;
            }
            /*
             * Otherwise attempt to remove.  If this fails then skip any
             * parent directories.
             */
            if fs::remove_dir(p).is_err() {
                break;
            }
        }
    }

    /// Remove a directory hierarchy only if it contains nothing but empty
    /// directories and symlinks. Walks depth-first. Removes symlinks and
    /// empty directories. Fails if any regular files, device nodes, pipes,
    /// sockets, or other non-removable entries are encountered.
    #[allow(clippy::only_used_in_recursion)]
    fn remove_empty_hierarchy(&self, path: &Path) -> Result<()> {
        // Use symlink_metadata to not follow symlinks
        let meta = fs::symlink_metadata(path)?;

        if meta.is_symlink() {
            // Symlinks can be removed
            fs::remove_file(path).map_err(|e| {
                anyhow::anyhow!("Failed to remove symlink {}: {}", path.display(), e)
            })?;
            return Ok(());
        }

        if !meta.is_dir() {
            // Regular file, device node, pipe, socket, etc. - fail
            bail!(
                "Cannot remove sandbox: non-directory exists at {}",
                path.display()
            );
        }

        // It's a directory - process contents first (depth-first)
        let entries =
            fs::read_dir(path).with_context(|| format!("Failed to read {}", path.display()))?;
        for entry in entries {
            let entry = entry?;
            self.remove_empty_hierarchy(&entry.path())?;
        }

        // Directory should now be empty, remove it
        fs::remove_dir(path).map_err(|e| {
            anyhow::anyhow!(
                "Failed to remove directory {}: {}. Directory may not be empty.",
                path.display(),
                e
            )
        })
    }

    /**
     * Execute action create commands in order.
     */
    fn perform_actions(
        &self,
        sandbox_id: usize,
        actions: &[Action],
        envs: &[(String, String)],
    ) -> Result<()> {
        for action in actions {
            if !action.only().matches(self.context) {
                debug!(
                    sandbox = sandbox_id,
                    action = action
                        .action_type()
                        .map(|t| format!("{:?}", t))
                        .unwrap_or_default(),
                    "Skipped (only predicate not satisfied)"
                );
                continue;
            }
            action.validate()?;
            let action_type = action.action_type()?;

            let src = action.src().or(action.dest());
            let dest = action
                .dest()
                .or(action.src())
                .map(|d| self.resolve_path(sandbox_id, d));
            if let Some(dest_path) = &dest {
                self.verify_path_in_sandbox(sandbox_id, dest_path)?;
            }

            let mut opts = vec![];
            if let Some(o) = action.opts() {
                for opt in o.split(' ').collect::<Vec<&str>>() {
                    opts.push(opt);
                }
            }

            let status = match action_type {
                ActionType::Mount => {
                    let fs_type = action.fs_type()?;
                    let src =
                        src.ok_or_else(|| anyhow::anyhow!("mount action requires src or dest"))?;
                    let dest = dest.ok_or_else(|| anyhow::anyhow!("mount action requires dest"))?;
                    debug!(
                        sandbox = sandbox_id,
                        action = "mount",
                        fs = ?fs_type,
                        src = %src.display(),
                        dest = %dest.display(),
                        "Mounting"
                    );
                    /*
                     * Create the mount point only if it doesn't already
                     * exist.  This allows mounts to overlay paths within
                     * an earlier read-only mount (e.g. an fdfs mount on
                     * /dev/fd inside a read-only lofs /dev).
                     */
                    if !dest.exists() {
                        fs::create_dir_all(&dest)
                            .with_context(|| format!("Failed to create {}", dest.display()))?;
                    }
                    match fs_type {
                        FSType::Bind => self.mount_bindfs(src, &dest, &opts)?,
                        FSType::Dev => self.mount_devfs(src, &dest, &opts)?,
                        FSType::Fd => self.mount_fdfs(src, &dest, &opts)?,
                        FSType::Nfs => self.mount_nfs(src, &dest, &opts)?,
                        FSType::Proc => self.mount_procfs(src, &dest, &opts)?,
                        FSType::Tmp => self.mount_tmpfs(src, &dest, &opts)?,
                    }
                }
                ActionType::Copy => {
                    let src =
                        src.ok_or_else(|| anyhow::anyhow!("copy action requires src or dest"))?;
                    let dest = dest.ok_or_else(|| anyhow::anyhow!("copy action requires dest"))?;
                    debug!(
                        sandbox = sandbox_id,
                        action = "copy",
                        src = %src.display(),
                        dest = %dest.display(),
                        "Copying"
                    );
                    if let Some(parent) = dest.parent() {
                        fs::create_dir_all(parent)
                            .with_context(|| format!("Failed to create {}", parent.display()))?;
                    }
                    copy_dir::copy_dir(src, &dest).with_context(|| {
                        format!("Failed to copy {} to {}", src.display(), dest.display())
                    })?;
                    fs::set_permissions(&dest, fs::metadata(src)?.permissions())?;
                    None
                }
                ActionType::Cmd => {
                    if let Some(create_cmd) = action.create_cmd() {
                        debug!(
                            sandbox = sandbox_id,
                            action = "cmd",
                            cmd = %create_cmd.run,
                            chroot = action.chroot(),
                            "Running create command"
                        );
                        let mut merged_envs: Vec<(String, String)> = envs.to_vec();
                        merged_envs.extend(create_cmd.env.iter().cloned());
                        if let Some(out) = self.run_action_cmd(
                            sandbox_id,
                            &create_cmd.run,
                            action.chroot(),
                            &merged_envs,
                        )? && !out.status.success()
                        {
                            let stderr = String::from_utf8_lossy(&out.stderr);
                            let stderr = stderr.trim();
                            if stderr.is_empty() {
                                bail!(
                                    "create command failed (exit code {}): {}",
                                    out.status
                                        .code()
                                        .map_or("signal".to_string(), |c| c.to_string()),
                                    create_cmd.run,
                                );
                            } else {
                                bail!(
                                    "create command failed (exit code {}): {}\n{}",
                                    out.status
                                        .code()
                                        .map_or("signal".to_string(), |c| c.to_string()),
                                    create_cmd.run,
                                    stderr,
                                );
                            }
                        }
                    }
                    None
                }
                ActionType::Symlink => {
                    let src = action
                        .src()
                        .ok_or_else(|| anyhow::anyhow!("symlink action requires src"))?;
                    let dest_path = self.resolve_path(
                        sandbox_id,
                        action
                            .dest()
                            .ok_or_else(|| anyhow::anyhow!("symlink action requires dest"))?,
                    );
                    debug!(
                        sandbox = sandbox_id,
                        action = "symlink",
                        src = %src.display(),
                        dest = %dest_path.display(),
                        "Creating symlink"
                    );
                    if let Some(parent) = dest_path.parent()
                        && !parent.exists()
                    {
                        fs::create_dir_all(parent)
                            .with_context(|| format!("Failed to create {}", parent.display()))?;
                    }
                    std::os::unix::fs::symlink(src, &dest_path).with_context(|| {
                        format!(
                            "Failed to create symlink {} -> {}",
                            dest_path.display(),
                            src.display()
                        )
                    })?;
                    None
                }
                ActionType::MacosMdnsListener => {
                    #[cfg(not(target_os = "macos"))]
                    bail!("'macos-mdns-listener' action is only supported on macOS");
                    #[cfg(target_os = "macos")]
                    {
                        debug!(
                            sandbox = sandbox_id,
                            action = "macos-mdns-listener",
                            "Creating mDNS listener"
                        );
                        self.create_mdns_listener(sandbox_id)?;
                        None
                    }
                }
            };
            if let Some(s) = status
                && !s.success()
            {
                bail!(
                    "Action failed (exit code {:?})",
                    s.code().map_or("signal".to_string(), |c| c.to_string()),
                );
            }
        }
        Ok(())
    }

    /**
     * Execute action destroy commands in reverse order.
     */
    fn reverse_actions(
        &self,
        sandbox_id: usize,
        actions: &[Action],
        envs: &[(String, String)],
    ) -> Result<()> {
        for action in actions.iter().rev() {
            if !action.only().matches(self.context) {
                continue;
            }
            let action_type = action.action_type()?;
            let dest = action
                .dest()
                .or(action.src())
                .map(|d| self.resolve_path(sandbox_id, d));

            match action_type {
                ActionType::Cmd => {
                    if let Some(destroy_cmd) = action.destroy_cmd() {
                        debug!(
                            action = "cmd",
                            cmd = %destroy_cmd.run,
                            chroot = action.chroot(),
                            "Running destroy command"
                        );
                        let mut merged_envs: Vec<(String, String)> = envs.to_vec();
                        merged_envs.extend(destroy_cmd.env.iter().cloned());
                        if let Some(out) = self.run_action_cmd(
                            sandbox_id,
                            &destroy_cmd.run,
                            action.chroot(),
                            &merged_envs,
                        )? && !out.status.success()
                        {
                            let stderr = String::from_utf8_lossy(&out.stderr);
                            let stderr = stderr.trim();
                            if stderr.is_empty() {
                                bail!(
                                    "destroy command failed (exit code {}): {}",
                                    out.status
                                        .code()
                                        .map_or("signal".to_string(), |c| c.to_string()),
                                    destroy_cmd.run,
                                );
                            } else {
                                bail!(
                                    "destroy command failed (exit code {}): {}\n{}",
                                    out.status
                                        .code()
                                        .map_or("signal".to_string(), |c| c.to_string()),
                                    destroy_cmd.run,
                                    stderr,
                                );
                            }
                        }
                    }
                }
                ActionType::Copy => {
                    let Some(dest) = dest else { continue };
                    if !dest.exists() {
                        self.remove_empty_dirs(sandbox_id, &dest);
                        continue;
                    }
                    if fs::remove_dir(&dest).is_ok() {
                        continue;
                    }
                    /*
                     * Use remove_dir_recursive which fails if non-empty
                     * directories remain, rather than blindly deleting.
                     * First verify the path is within the sandbox.
                     */
                    debug!(
                        sandbox = sandbox_id,
                        action = "copy",
                        dest = %dest.display(),
                        "Removing copied path"
                    );
                    self.verify_path_in_sandbox(sandbox_id, &dest)?;
                    self.remove_dir_recursive(&dest)?;
                    self.remove_empty_dirs(sandbox_id, &dest);
                }
                ActionType::Symlink => {
                    let Some(dest) = dest else { continue };
                    if dest.is_symlink() {
                        debug!(
                            sandbox = sandbox_id,
                            action = "symlink",
                            dest = %dest.display(),
                            "Removing symlink"
                        );
                        fs::remove_file(&dest)?;
                    }
                    self.remove_empty_dirs(sandbox_id, &dest);
                }
                ActionType::Mount => {
                    let Some(dest) = dest else { continue };
                    let fs_type = action.fs_type()?;

                    /*
                     * If the mount point does not exist then do not try
                     * to unmount it, but do try to clean up any empty
                     * parent directories up to the sandbox root.
                     */
                    if !dest.exists() {
                        self.remove_empty_dirs(sandbox_id, &dest);
                        continue;
                    }

                    /*
                     * Try removing the directory first, in case it was
                     * never mounted.  Avoids errors trying to unmount a
                     * filesystem that is not mounted.
                     */
                    if fs::remove_dir(&dest).is_ok() {
                        continue;
                    }

                    /*
                     * Unmount the filesystem.  It is critical that all
                     * mounts are successfully unmounted before we attempt
                     * to remove the sandbox directory.
                     */
                    debug!(
                        sandbox = sandbox_id,
                        action = "mount",
                        fs = ?fs_type,
                        dest = %dest.display(),
                        "Unmounting"
                    );
                    match fs_type {
                        FSType::Bind => self.unmount_bindfs(&dest)?,
                        FSType::Dev => self.unmount_devfs(&dest)?,
                        FSType::Fd => self.unmount_fdfs(&dest)?,
                        FSType::Nfs => self.unmount_nfs(&dest)?,
                        FSType::Proc => self.unmount_procfs(&dest)?,
                        FSType::Tmp => self.unmount_tmpfs(&dest)?,
                    }
                    self.remove_empty_dirs(sandbox_id, &dest);
                }
                ActionType::MacosMdnsListener => {
                    #[cfg(not(target_os = "macos"))]
                    bail!("'macos-mdns-listener' action is only supported on macOS");
                    #[cfg(target_os = "macos")]
                    {
                        debug!(
                            sandbox = sandbox_id,
                            action = "macos-mdns-listener",
                            "Destroying mDNS listener"
                        );
                        self.destroy_mdns_listener(sandbox_id)?;
                    }
                }
            }
        }
        Ok(())
    }

    /**
     * Recursively remove a directory by walking it depth-first and removing
     * files and empty directories.  Unlike remove_dir_all, this will fail
     * if it encounters a non-empty directory that cannot be removed, which
     * would indicate an active mount point.
     *
     * IMPORTANT: This function explicitly does NOT follow symlinks to avoid
     * deleting files outside the sandbox via symlink attacks.
     */
    #[allow(clippy::only_used_in_recursion)]
    fn remove_dir_recursive(&self, path: &Path) -> Result<()> {
        // Use symlink_metadata to check type WITHOUT following symlinks
        let meta = fs::symlink_metadata(path)
            .with_context(|| format!("Failed to stat {}", path.display()))?;
        if meta.is_symlink() {
            // Remove the symlink itself, don't follow it
            fs::remove_file(path)
                .with_context(|| format!("Failed to remove {}", path.display()))?;
            return Ok(());
        }
        if !meta.is_dir() {
            fs::remove_file(path)
                .with_context(|| format!("Failed to remove {}", path.display()))?;
            return Ok(());
        }
        let entries =
            fs::read_dir(path).with_context(|| format!("Failed to read {}", path.display()))?;
        for entry in entries {
            let entry = entry?;
            let entry_path = entry.path();
            // Use file_type() from DirEntry which doesn't follow symlinks
            let file_type = entry.file_type()?;
            if file_type.is_symlink() {
                // Remove symlink itself, don't follow
                fs::remove_file(&entry_path)
                    .with_context(|| format!("Failed to remove {}", entry_path.display()))?;
            } else if file_type.is_dir() {
                self.remove_dir_recursive(&entry_path)?;
            } else {
                fs::remove_file(&entry_path)
                    .with_context(|| format!("Failed to remove {}", entry_path.display()))?;
            }
        }
        fs::remove_dir(path).with_context(|| format!("Failed to remove {}", path.display()))?;
        Ok(())
    }
}

/**
 * RAII scope for sandbox lifecycle management.
 *
 * Creates sandboxes on demand via `ensure()`, destroys them on drop.
 * This ensures sandboxes are always cleaned up, even on error paths.
 * If sandboxes are disabled, all operations are no-ops.
 */
#[derive(Debug)]
pub struct SandboxScope {
    sandbox: Sandbox,
    owned: Vec<usize>,
    state: RunState,
}

impl SandboxScope {
    /**
     * Create a new scope with no sandboxes.
     *
     * Use `ensure()` to create sandboxes when needed.
     */
    pub fn new(sandbox: Sandbox, state: RunState) -> Self {
        Self {
            sandbox,
            owned: Vec::new(),
            state,
        }
    }

    /**
     * Ensure at least `n` sandboxes are owned by this scope.
     *
     * Claims available IDs atomically, skipping any already held by other
     * processes.  Sets up new sandboxes in parallel.
     *
     * On error or interrupt, newly created sandboxes are rolled back but
     * previously owned sandboxes remain (they'll be cleaned up on drop).
     */
    pub(crate) fn ensure(&mut self, n: usize) -> Result<&[usize]> {
        if n <= self.owned.len() {
            return Ok(&self.owned);
        }
        let needed = n - self.owned.len();
        let new_ids = self.sandbox.claim_ids(needed)?;
        if self.state.interrupted() {
            for &id in &new_ids {
                let _ = self.sandbox.destroy(id);
            }
            return Err(Interrupted.into());
        }
        self.owned.extend(new_ids);
        Ok(&self.owned)
    }

    /**
     * Return the sandbox IDs owned by this scope.
     *
     * Index 0 is the first sandbox allocated, etc.  Use this to map
     * worker indices to sandbox IDs.
     */
    pub(crate) fn ids(&self) -> Option<&[usize]> {
        if self.owned.is_empty() {
            None
        } else {
            Some(&self.owned)
        }
    }

    /**
     * Access the underlying sandbox for operations.
     */
    pub(crate) fn sandbox(&self) -> &Sandbox {
        &self.sandbox
    }

    /**
     * Return whether sandboxes are enabled.
     */
    pub(crate) fn enabled(&self) -> bool {
        self.sandbox.enabled()
    }

    /**
     * Return the number of sandboxes currently managed.
     */
    pub(crate) fn count(&self) -> usize {
        self.owned.len()
    }

    /**
     * Access the run state flag.
     */
    pub(crate) fn state(&self) -> &RunState {
        &self.state
    }
}

impl Drop for SandboxScope {
    fn drop(&mut self) {
        if !self.sandbox.enabled() || self.owned.is_empty() {
            return;
        }
        for &id in &self.owned {
            if self.sandbox.path(id).exists()
                && let Err(e) = self.sandbox.run_post_build(Some(id))
            {
                warn!(error = format!("{e:#}"), sandbox = id, "post-build error");
            }
        }
        let msg = if self.owned.len() == 1 {
            "Destroying sandbox".to_string()
        } else {
            format!("Destroying {} sandboxes", self.owned.len())
        };
        crate::print_status(&msg);
        let start = Instant::now();
        let ids = self.owned.clone();
        let results: Vec<(usize, Result<()>)> = ids
            .into_par_iter()
            .map(|id| (id, self.sandbox.destroy(id)))
            .collect();
        let mut failed = 0;
        for (id, result) in results {
            if let Err(e) = result {
                if failed == 0 {
                    println!();
                }
                eprintln!("sandbox {}: {:#}", id, e);
                warn!(
                    error = format!("{e:#}"),
                    sandbox = id,
                    "Failed to destroy sandbox"
                );
                failed += 1;
            }
        }
        if failed == 0 {
            crate::print_elapsed(&msg, start.elapsed());
        } else {
            eprintln!(
                "Warning: failed to destroy {} sandbox{}",
                failed,
                if failed == 1 { "" } else { "es" }
            );
        }
    }
}