rch 2.1.3

//! File transfer and remote execution pipeline.
//!
//! Handles synchronizing project files to remote workers, executing compilation
//! commands, and retrieving build artifacts.

use crate::error::TransferError;
use anyhow::{Context, Result};
use glob::Pattern;
use rch_common::mock::{self, MockConfig, MockRsync, MockRsyncConfig, MockSshClient};
use rch_common::rsync_flavor::{ResolvedRsync, RsyncCapabilities, RsyncFlavor, RsyncSource};
use rch_common::ssh_utils::{
    EnvPrefix, is_retryable_transport_error, is_retryable_transport_error_text, is_valid_env_key,
    shell_escape_value,
};
use rch_common::{
    ColorMode, CommandResult, CompilationKind, PathTopologyPolicy, RemoteBuildJobs, RetryConfig,
    ToolchainInfo, TransferConfig, WorkerConfig, normalize_project_path_with_policy,
    wrap_command_with_color, wrap_command_with_toolchain,
};
#[cfg(unix)]
use rch_common::{SshClient, SshOptions};
use shell_escape::escape;
use std::borrow::Cow;
use std::collections::{BTreeSet, HashMap};
use std::future::Future;
use std::path::{Component, Path, PathBuf};
use std::process::Stdio;
use std::time::Duration;
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::process::Command;
use tokio::time::Instant as TokioInstant;
use tokio::time::sleep;
use tracing::{debug, info, warn};

/// The remote worker's OS family, which selects the transport and path
/// conventions the pipeline uses. Derived from the worker's declared `os`
/// (`os = "windows"` in workers.toml). Defaults to `Posix`, so every existing
/// worker — and every code path that does not thread a platform — behaves
/// exactly as before.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum WorkerPlatform {
    /// Linux/macOS: rsync transport, POSIX paths, `/data/tmp/rch` base. The
    /// only behaviour that has ever existed.
    #[default]
    Posix,
    /// Windows worker reached through Git's POSIX `sh`: tar-over-ssh transport,
    /// `C:/rch` base, no `timeout(1)` wrapper (Windows `timeout.exe` is unrelated).
    Windows,
}

impl WorkerPlatform {
    /// Derive the platform from a worker's declared OS. Only `windows` selects
    /// the Windows path; everything else — including an undeclared OS — is
    /// `Posix`, so linux/darwin workers are untouched.
    #[must_use]
    pub fn from_worker(worker: &WorkerConfig) -> Self {
        match rch_common::declared_os(&worker.tags).as_deref() {
            Some("windows") => Self::Windows,
            _ => Self::Posix,
        }
    }

    #[must_use]
    pub(crate) fn is_windows(self) -> bool {
        matches!(self, Self::Windows)
    }
}

/// Remote build root for Windows workers. Drive letter + forward slashes: Git's
/// POSIX `sh` (`mkdir`/`cd`) and `cargo.exe` (`CARGO_TARGET_DIR`) both accept
/// this form, and it sidesteps rsync's `C:`-as-host parsing (Windows uses the
/// tar transport instead). Empirically validated on a real Surface build host.
pub(crate) const WINDOWS_DEFAULT_REMOTE_BASE: &str = "C:/rch";

/// Whether `path` is a Windows drive-letter absolute path like `C:/rch` or
/// `C:\rch`. Windows workers use these as remote build roots, so a remote-path
/// override must accept them as absolute even though a leading-`/` check (Unix
/// `Path::is_absolute` semantics) would reject them. A relative path — the case
/// the override guard exists to reject — matches neither form.
fn is_windows_drive_abs_path(path: &str) -> bool {
    let bytes = path.as_bytes();
    bytes.len() >= 3
        && bytes[0].is_ascii_alphabetic()
        && bytes[1] == b':'
        && (bytes[2] == b'/' || bytes[2] == b'\\')
}

pub(crate) async fn read_bounded_output_stream<R>(
    mut reader: R,
    max_bytes: usize,
) -> std::io::Result<Vec<u8>>
where
    R: AsyncRead + Unpin,
{
    let mut output = Vec::new();
    let mut chunk = [0_u8; 16 * 1024];
    loop {
        let read = reader.read(&mut chunk).await?;
        if read == 0 {
            return Ok(output);
        }
        let next_len = output.len().checked_add(read).ok_or_else(|| {
            std::io::Error::new(
                std::io::ErrorKind::InvalidData,
                "command output size overflow",
            )
        })?;
        if next_len > max_bytes {
            return Err(std::io::Error::new(
                std::io::ErrorKind::InvalidData,
                format!("command output exceeded {max_bytes} bytes"),
            ));
        }
        output.extend_from_slice(&chunk[..read]);
    }
}

async fn run_source_content_rsync_capture(
    mut cmd: Command,
    operation: &str,
    operation_timeout: std::time::Duration,
) -> Result<std::process::Output> {
    cmd.kill_on_drop(true);
    let mut child = cmd.spawn().with_context(|| format!("spawn {operation}"))?;
    let stdout = child
        .stdout
        .take()
        .context("source-content rsync stdout was not piped")?;
    let stderr = child
        .stderr
        .take()
        .context("source-content rsync stderr was not piped")?;

    let collect = async {
        let (stdout, stderr) = tokio::try_join!(
            read_bounded_output_stream(stdout, MAX_SOURCE_CONTENT_RSYNC_OUTPUT_BYTES),
            read_bounded_output_stream(stderr, MAX_SOURCE_CONTENT_RSYNC_OUTPUT_BYTES),
        )?;
        let status = child.wait().await?;
        Ok::<_, std::io::Error>(std::process::Output {
            status,
            stdout,
            stderr,
        })
    };

    match tokio::time::timeout(operation_timeout, collect).await {
        Ok(result) => result.with_context(|| format!("collect {operation}")),
        Err(_) => anyhow::bail!(
            "{operation} timed out after {}ms",
            operation_timeout.as_millis()
        ),
    }
}

/// Windows uses the same rooted include and directory-exclusion policy as
/// rsync. Compile it locally; shell expansion must never choose archive members.
struct WindowsArtifactFilter {
    pattern: Pattern,
    rooted: bool,
    directory_only: bool,
}

impl WindowsArtifactFilter {
    fn new(raw: &str, include: bool) -> Result<Self> {
        let rooted = include || raw.starts_with('/');
        let directory_only = raw.ends_with('/');
        let raw = raw.trim_start_matches('/').trim_end_matches('/');
        anyhow::ensure!(!raw.is_empty(), "empty Windows artifact filter");
        // The source-integrity guard escapes literal glob characters with
        // backslashes. glob::Pattern uses bracket quoting instead.
        let mut converted = String::new();
        let mut chars = raw.chars();
        while let Some(ch) = chars.next() {
            if ch == '\\' {
                let literal = chars.next().context("incomplete artifact filter escape")?;
                converted.push_str(&Pattern::escape(&literal.to_string()));
            } else {
                converted.push(ch);
            }
        }
        Ok(Self {
            pattern: Pattern::new(&converted)
                .with_context(|| format!("invalid Windows artifact filter {raw:?}"))?,
            rooted,
            directory_only,
        })
    }

    fn matches(&self, path: &str, directory: bool) -> bool {
        if self.directory_only && !directory {
            return false;
        }
        let options = glob::MatchOptions {
            case_sensitive: true,
            require_literal_separator: true,
            require_literal_leading_dot: false,
        };
        if self.pattern.matches_with(path, options) {
            return true;
        }
        !self.rooted
            && path
                .match_indices('/')
                .any(|(index, _)| self.pattern.matches_with(&path[index + 1..], options))
    }

    fn excludes(&self, path: &str) -> bool {
        self.matches(path, false)
            || path
                .match_indices('/')
                .any(|(index, _)| self.matches(&path[..index], true))
    }
}

fn windows_artifact_relative_path(path: &str) -> Result<&str> {
    let path = path.strip_prefix("./").unwrap_or(path);
    anyhow::ensure!(
        !path.is_empty()
            && !path.contains(['\\', ':'])
            && !path.chars().any(char::is_control)
            && path.split('/').all(|part| !matches!(part, "" | "." | "..")),
        "invalid Windows artifact member {path:?}"
    );
    Ok(path)
}

fn windows_artifact_selection(
    inventory: &[u8],
    includes: &[WindowsArtifactFilter],
    excludes: &[WindowsArtifactFilter],
) -> Result<std::collections::BTreeMap<String, u64>> {
    anyhow::ensure!(
        inventory.is_empty() || inventory.ends_with(&[0]),
        "incomplete Windows artifact inventory"
    );
    let mut selected = std::collections::BTreeMap::new();
    for entry in inventory
        .split(|byte| *byte == 0)
        .filter(|entry| !entry.is_empty())
    {
        let split = entry
            .iter()
            .position(|byte| *byte == b' ')
            .context("artifact inventory lacks size")?;
        let size = std::str::from_utf8(&entry[..split])?.parse::<u64>()?;
        let path = windows_artifact_relative_path(std::str::from_utf8(&entry[split + 1..])?)?;
        if includes.iter().any(|filter| filter.matches(path, false))
            && !excludes.iter().any(|filter| filter.excludes(path))
        {
            anyhow::ensure!(
                selected.insert(path.to_string(), size).is_none(),
                "duplicate Windows artifact inventory member {path:?}"
            );
        }
    }
    Ok(selected)
}

fn windows_artifact_archive_script(
    remote_path: &str,
    selected: &std::collections::BTreeMap<String, u64>,
) -> String {
    let mut script = format!("set -eu\ncd {}\n{{\n", escape(Cow::from(remote_path)));
    for path in selected.keys() {
        script.push_str(&format!(
            "printf '%s\\0' {}\n",
            escape(Cow::Owned(format!("./{path}")))
        ));
    }
    // --null makes each pathname literal, including names beginning with '-'.
    // GNU-only --hard-dereference is unavailable in Windows' native bsdtar;
    // validated links to earlier selected files are preserved on extraction.
    script.push_str("} | tar -czf - --no-recursion --null -T -\n");
    script
}

fn windows_artifact_inventory_script(remote_path: &str) -> String {
    format!(
        "set -eu\ncd {}\n/usr/bin/find . -type f -printf '%s %p\\0'\n",
        escape(Cow::from(remote_path))
    )
}

struct WindowsArtifactReader<R> {
    reader: R,
    remaining: u64,
    deadline: TokioInstant,
    cancel: Option<tokio::sync::watch::Receiver<bool>>,
}

impl<R: std::io::Read> std::io::Read for WindowsArtifactReader<R> {
    fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
        if self.cancel.as_ref().is_some_and(|cancel| *cancel.borrow()) {
            return Err(std::io::Error::other(
                "Windows artifact extraction cancelled",
            ));
        }
        if TokioInstant::now() >= self.deadline {
            return Err(std::io::Error::other(
                "Windows artifact extraction deadline exceeded",
            ));
        }
        if self.remaining == 0 {
            return Err(std::io::Error::other(
                "Windows artifact archive exceeded its expanded size bound",
            ));
        }
        let limit = buffer
            .len()
            .min(32 * 1024)
            .min(usize::try_from(self.remaining).unwrap_or(usize::MAX));
        let count = self.reader.read(&mut buffer[..limit])?;
        self.remaining -= count as u64;
        Ok(count)
    }
}

/// Validate the complete downloaded archive before writing any caller output.
/// Both the inventory and tar are untrusted pathname inputs; only selected
/// regular files of the advertised size may cross this boundary.
fn unpack_windows_artifact_archive(
    archive_path: &Path,
    destination: &Path,
    selected: &std::collections::BTreeMap<String, u64>,
    deadline: TokioInstant,
    cancel: Option<tokio::sync::watch::Receiver<bool>>,
) -> Result<()> {
    let limit = selected.values().fold(1024 * 1024_u64, |total, size| {
        total.saturating_add(*size).saturating_add(8192)
    });
    let open = || -> Result<_> {
        Ok(tar::Archive::new(WindowsArtifactReader {
            reader: flate2::read::GzDecoder::new(std::fs::File::open(archive_path)?),
            remaining: limit,
            deadline,
            cancel: cancel.clone(),
        }))
    };
    let mut archive = open()?;
    let mut seen = BTreeSet::new();
    for entry in archive.entries()? {
        let mut entry = entry?;
        let path = entry.path()?.into_owned();
        let path =
            windows_artifact_relative_path(path.to_str().context("non-UTF-8 artifact path")?)?;
        let expected_size = selected
            .get(path)
            .context("unselected Windows artifact archive member")?;
        if entry.header().entry_type().is_file() {
            anyhow::ensure!(
                *expected_size == entry.size(),
                "Windows artifact size changed for {path:?}"
            );
        } else if entry.header().entry_type().is_hard_link() {
            let target = entry
                .link_name()?
                .context("artifact hardlink without target")?;
            let target = windows_artifact_relative_path(
                target.to_str().context("non-UTF-8 artifact link")?,
            )?;
            anyhow::ensure!(
                entry.size() == 0
                    && seen.contains(target)
                    && selected.get(target) == Some(expected_size),
                "Windows artifact hardlink does not reference a prior selected output"
            );
        } else {
            anyhow::bail!("non-regular Windows artifact archive member");
        }
        anyhow::ensure!(
            seen.insert(path.to_string()),
            "duplicate Windows artifact archive member {path:?}"
        );
        std::io::copy(&mut entry, &mut std::io::sink())?;
    }
    // tar stops at its terminator; drain gzip as well to verify its checksum.
    std::io::copy(&mut archive.into_inner(), &mut std::io::sink())?;
    anyhow::ensure!(
        seen.len() == selected.len(),
        "Windows artifact archive omitted selected outputs"
    );
    for path in selected.keys() {
        let mut target = destination.to_path_buf();
        for component in path.split('/') {
            target.push(component);
            match std::fs::symlink_metadata(&target) {
                Ok(metadata) => anyhow::ensure!(
                    !metadata.file_type().is_symlink(),
                    "symlink in artifact destination {}",
                    target.display()
                ),
                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
                Err(error) => return Err(error.into()),
            }
        }
    }
    std::fs::create_dir_all(destination)?;
    for entry in open()?.entries()? {
        anyhow::ensure!(
            entry?.unpack_in(destination)?,
            "artifact escaped destination"
        );
    }
    Ok(())
}

async fn windows_artifact_cancelled(cancel: &mut Option<tokio::sync::watch::Receiver<bool>>) {
    if let Some(cancel) = cancel {
        loop {
            if *cancel.borrow() {
                return;
            }
            if cancel.changed().await.is_err() {
                break;
            }
        }
    }
    std::future::pending::<()>().await;
}

/// The script, stdout receiver, stderr drain and child share one deadline.
/// No detached pump can keep an SSH pipe alive after cancellation returns.
async fn run_windows_artifact_process<W: tokio::io::AsyncWrite + Unpin>(
    mut command: Command,
    script: &str,
    sink: &mut W,
    output_limit: u64,
    deadline: TokioInstant,
    mut cancel: Option<tokio::sync::watch::Receiver<bool>>,
    on_line: &mut impl FnMut(&str),
) -> Result<u64> {
    if cancel.as_ref().is_some_and(|cancel| *cancel.borrow()) {
        return Err(RetrievalCancelled.into());
    }
    anyhow::ensure!(
        TokioInstant::now() < deadline,
        "Windows artifact retrieval deadline exceeded"
    );
    command
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .kill_on_drop(true);
    let mut child = command
        .spawn()
        .context("spawn Windows artifact transport")?;
    let mut stdin = child.stdin.take().context("artifact script stdin")?;
    let mut stdout = child.stdout.take().context("artifact transport stdout")?;
    let mut stderr = child.stderr.take().context("artifact transport stderr")?;
    let mut diagnostics = Vec::new();
    let mut bytes = 0_u64;
    let mut reported_bytes = 0_u64;
    let result = {
        let operation = async {
            let send = async move {
                let written = stdin.write_all(script.as_bytes()).await;
                // ChildStdin::shutdown does not close a Unix pipe. The shell needs
                // EOF to finish reading `sh -s`; retain no writer while the
                // receive future awaits its stdout/stderr and exit status.
                drop(stdin);
                written
            };
            let receive = async {
                let mut out_buffer = [0_u8; 32 * 1024];
                let mut err_buffer = [0_u8; 8192];
                let mut out_open = true;
                let mut err_open = true;
                while out_open || err_open {
                    tokio::select! {
                        read = stdout.read(&mut out_buffer), if out_open => {
                            let count = read?;
                            out_open = count != 0;
                            bytes = bytes.checked_add(count as u64).context("artifact stream size overflow")?;
                            anyhow::ensure!(bytes <= output_limit, "Windows artifact stream exceeded its size bound");
                            sink.write_all(&out_buffer[..count]).await?;
                            if bytes.saturating_sub(reported_bytes) >= 1024 * 1024 {
                                on_line(&format!("Windows artifact transfer: {bytes} bytes received"));
                                reported_bytes = bytes;
                            }
                        },
                        read = stderr.read(&mut err_buffer), if err_open => {
                            let count = read?;
                            err_open = count != 0;
                            on_line(&String::from_utf8_lossy(&err_buffer[..count]));
                            let retained = count.min((1024 * 1024_usize).saturating_sub(diagnostics.len()));
                            diagnostics.extend_from_slice(&err_buffer[..retained]);
                        },
                    }
                }
                sink.flush().await?;
                Ok::<_, anyhow::Error>(child.wait().await?)
            };
            let (sent, received) = tokio::join!(send, receive);
            let status = received?;
            anyhow::ensure!(
                status.success(),
                "Windows artifact transport failed (exit {:?}): {}",
                status.code(),
                String::from_utf8_lossy(&diagnostics).trim()
            );
            sent.context("send Windows artifact script")?;
            Ok::<_, anyhow::Error>(())
        };
        tokio::select! {
            result = operation => Some(result),
            () = windows_artifact_cancelled(&mut cancel) => None,
            () = tokio::time::sleep_until(deadline) => None,
        }
    };
    match result {
        Some(Ok(())) => Ok(bytes),
        other => {
            if child.try_wait()?.is_none() {
                child
                    .kill()
                    .await
                    .context("stop and reap Windows artifact transport")?;
            }
            if let Some(Err(error)) = other {
                return Err(error);
            }
            if cancel.as_ref().is_some_and(|cancel| *cancel.borrow()) {
                return Err(RetrievalCancelled.into());
            }
            anyhow::bail!("Windows artifact retrieval deadline exceeded");
        }
    }
}

const PROJECT_HASH_CONTENT_LIMIT_BYTES: u64 = 2 * 1024 * 1024;
const MAX_SOURCE_CONTENT_RSYNC_OUTPUT_BYTES: usize = 32 * 1024 * 1024;
const PROJECT_HASH_KEY_FILES: &[&str] = &[
    // Rust project files
    "Cargo.toml",
    "Cargo.lock",
    // Bun/Node.js project files
    "package.json",
    "bun.lockb",
    "package-lock.json",
    "pnpm-lock.yaml",
    "yarn.lock",
    // TypeScript config
    "tsconfig.json",
    // Bun config
    "bunfig.toml",
];
const REMOTE_RUNTIME_EXCLUDE_PATTERNS: &[&str] = &[
    ".rch-target/",
    ".rch-target-*/",
    ".rch-tmp/",
    // Go build/module/GOPATH caches injected into the managed remote build zone
    // (see FORCED_MANAGED_ENV_KEYS). Protect them from rsync --delete for the
    // same reason as .rch-target/.rch-tmp: a concurrent build to the same remote
    // root must not wipe another build's in-flight Go cache.
    ".rch-go/",
    ".franken_whisper/tools/ffmpeg/",
];
// Host-local language environments are neither source nor portable build
// inputs.  They commonly contain absolute symlinks (for example
// `.venv-*/lib64`) that rsync would otherwise copy into a foreign workspace
// proof root.  Keep these exclusions unconditional even when a user replaces
// the configurable default list.
const SOURCE_EPHEMERAL_EXCLUDE_PATTERNS: &[&str] =
    &[".venv/", ".venv-*/", "venv/", "venv-*/", "__pycache__/"];

/// Age floors (minutes) for worker-side runtime state reaped at transfer
/// start (bd-wfumv). TMPDIR scratch under `.rch-tmp/` goes stale within a
/// day; durable per-worker Cargo caches (issue #42) keep their top-level
/// mtime refreshed by every job that uses them (`add_cargo_isolation`
/// touches the cache dir), so a three-day floor only reaps caches whose
/// project has been idle far longer than any active session — an idle
/// return pays one amortized re-fetch, never a per-job one.
///
/// Pooled target stores (`.rch-target-*-pool-*`) are NOT on this constant:
/// their retention is the configured
/// `[remediation.pooled_target] reaper_pooled_idle_hours` (issue #53), so the
/// transfer-start janitor, the daemon reaper, `rch gc`, and `rch cache
/// status` all agree on one lifetime. See
/// [`TransferPipeline::with_pooled_target_prune_idle_hours`].
const WORKER_TMP_PRUNE_MAX_AGE_MINS: u64 = 24 * 60;
const WORKER_DURABLE_CACHE_PRUNE_MAX_AGE_MINS: u64 = 3 * 24 * 60;

/// Minutes after which the transfer-start janitor prunes an idle pooled
/// target store, from the configured retention hours. `None` disables the
/// pooled sweep (hours == 0, the same "never" the reaper honours); otherwise
/// the value is floored at the reaper's own 24 h defence so a misconfigured
/// short TTL cannot turn warm caches into per-job churn.
fn pooled_target_prune_max_age_mins(idle_hours: u32) -> Option<u64> {
    (idle_hours != 0).then(|| {
        (u64::from(idle_hours) * 60).max(rch_common::stale_target_reap::MIN_POOLED_IDLE_MINUTES)
    })
}

/// Shell fragment executed by the remote rsync-path wrapper BEFORE rsync
/// starts: reaps stale worker-side runtime state so abandoned `.rch-tmp/*`
/// scratch, orphaned per-worker Cargo caches, and superseded
/// `.rch-target-*-pool-*` stores cannot grow without bound or feed stale
/// manifests into later builds (bd-wfumv). All sweeps are age-bounded
/// `-maxdepth 1` finds: anything in active use is minutes old and can never
/// match (per-worker Cargo caches additionally refresh their own mtime on
/// every job via `add_cargo_isolation`). Failures are swallowed — pruning is
/// opportunistic hygiene and must never fail a transfer.
fn worker_cache_prune_rsync_path_prefix(
    escaped_remote_path: &str,
    pooled_target_idle_hours: u32,
    escaped_stable_pool_parent: Option<&str>,
) -> String {
    let tmp_mins = WORKER_TMP_PRUNE_MAX_AGE_MINS;
    let durable_mins = WORKER_DURABLE_CACHE_PRUNE_MAX_AGE_MINS;
    let mut prefix = format!(
        "find {p}/.rch-tmp -mindepth 1 -maxdepth 1 ! -name 'rch-cargo-cache-*' -mmin +{tmp_mins} -exec rm -rf -- '{{}}' + 2>/dev/null; \
         find {p}/.rch-tmp -mindepth 1 -maxdepth 1 -name 'rch-cargo-cache-*' -mmin +{durable_mins} -exec rm -rf -- '{{}}' + 2>/dev/null; ",
        p = escaped_remote_path
    );
    if let Some(pooled_mins) = pooled_target_prune_max_age_mins(pooled_target_idle_hours) {
        prefix.push_str(&format!(
            "find {p} -mindepth 1 -maxdepth 1 -type d -name '.rch-target-*-pool-*' -mmin +{pooled_mins} -exec rm -rf -- '{{}}' + 2>/dev/null; ",
            p = escaped_remote_path
        ));
        // Issue #60: a clean-overlay run relocates its pooled target store to a
        // stable parent OUTSIDE the (nonce-unique, teardown-reaped) remote
        // root. Sweep that parent on the same configured retention so
        // relocated pools age out exactly like in-root ones.
        if let Some(pool_parent) = escaped_stable_pool_parent
            && pool_parent != escaped_remote_path
        {
            prefix.push_str(&format!(
                "find {pool_parent} -mindepth 1 -maxdepth 1 -type d -name '.rch-target-*-pool-*' -mmin +{pooled_mins} -exec rm -rf -- '{{}}' + 2>/dev/null; ",
            ));
        }
    }
    prefix
}

/// Wall-clock budget for the whole post-timeout remote group-kill attempt
/// (issue #62): SSH connect + kill + up-to-10s verification loop.
const REMOTE_TIMEOUT_KILL_BUDGET: std::time::Duration = std::time::Duration::from_secs(45);

/// Outcome of the best-effort remote process-group kill issued when the
/// client-side SSH command timeout expires (issue #62).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RemoteTimeoutCleanup {
    /// The recorded remote process group is verified dead (killed now, or
    /// already gone), so the project's Cargo build-directory lock is free.
    Verified,
    /// This execution records no remote pgid (no build id, or a Windows
    /// worker): cleanup cannot be attempted.
    NotAttempted,
    /// The kill could not be confirmed — the remote group may still be alive,
    /// holding the project's Cargo build-directory lock. Callers should treat
    /// the worker's project lock as suspect (see hook quarantine handling).
    Unverified,
}

impl std::fmt::Display for RemoteTimeoutCleanup {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let text = match self {
            Self::Verified => "remote process group verified dead",
            Self::NotAttempted => "remote cleanup not attempted",
            Self::Unverified => "remote process group NOT verified dead",
        };
        f.write_str(text)
    }
}

/// Typed client-side command-timeout error (the RCH-E104 surface, issue #62).
///
/// The `Display` text MUST keep the `SSH command timed out after` prefix: the
/// hook's fail-closed classifier (`is_ssh_command_timeout_error`) matches on
/// it. The structured `cleanup` field lets the hook additionally quarantine
/// the worker when the orphaned remote group could not be verified dead.
#[derive(Debug)]
pub struct SshCommandTimedOut {
    /// The client-side command timeout that expired.
    pub timeout: std::time::Duration,
    /// Outcome of the best-effort remote process-group kill.
    pub cleanup: RemoteTimeoutCleanup,
    /// Human-readable detail for logs/summaries.
    pub detail: String,
    /// Where the build's process-group record lives, so the daemon can re-run
    /// the kill probe later and clear an E104 quarantine on verified death
    /// (bd-g8m4g). `None` when no record exists (Windows, mock, no build id).
    pub evidence: Option<rch_common::orphan_quarantine::QuarantineEvidence>,
}

impl std::fmt::Display for SshCommandTimedOut {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "SSH command timed out after {:?}; {} ({})",
            self.timeout, self.cleanup, self.detail
        )
    }
}

impl std::error::Error for SshCommandTimedOut {}

/// A pre-workload refusal, distinct from a compiler returning the same status.
#[derive(Debug, thiserror::Error)]
#[error("remote worker process identity setup is unavailable; workload was not started")]
pub(crate) struct RemoteProcessSetupUnavailable;

/// Failed completion/ownership evidence must never authorize replay elsewhere.
#[derive(Debug, thiserror::Error)]
#[error("remote execution completion is unconfirmed; automatic replay is unsafe")]
pub(crate) struct RemoteExecutionUnconfirmed;

/// Typed source-sync stall error (issue #59): the transfer produced NO output
/// at all — no rsync progress refresh, stats, or itemized line — for the
/// configured silence window. Deliberately distinct from the wall-clock
/// attempt timeout: silence means a dead channel or wedged rsync, so the hook
/// releases the worker's reservation and re-enters selection EXCLUDING this
/// worker instead of waiting out a cap that scales to an hour.
///
/// The `Display` text must NEVER contain `SSH command timed out after` or
/// `Command timed out after`: those phrases route into the E104 fail-closed
/// classifier (`is_ssh_command_timeout_error`), and a stalled sync is a
/// worker-path fault that must stay eligible for failover to another worker.
#[derive(Debug, Clone)]
pub struct SourceSyncStalled {
    /// The worker whose sync stalled.
    pub worker_id: String,
    /// The transfer phase that stalled (currently always `source_sync`).
    pub phase: &'static str,
    /// The silence window that expired.
    pub silence: std::time::Duration,
    /// Human-readable detail for logs/summaries.
    pub detail: String,
}

impl std::fmt::Display for SourceSyncStalled {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "source sync stalled on worker {}: no forward progress for {}s (phase {}; {})",
            self.worker_id,
            self.silence.as_secs(),
            self.phase,
            self.detail
        )
    }
}

impl std::error::Error for SourceSyncStalled {}

/// Silence policy for a streaming transfer (issue #59): abort the attempt when
/// the child emits nothing on stdout OR stderr for `limit`. Carried alongside
/// the wall-clock attempt timeout so the resulting [`SourceSyncStalled`] can
/// name the worker and phase that stalled.
#[derive(Debug, Clone)]
pub(crate) struct SyncSilencePolicy {
    pub(crate) limit: std::time::Duration,
    pub(crate) worker_id: String,
    pub(crate) phase: &'static str,
}

/// POSIX `sh` script that SIGKILLs the process group recorded in `pgid_file`
/// and verifies it is gone (issue #62). Emits exactly one
/// `RCH_E104_KILL=<verdict>` marker line on stdout and always exits 0 so a
/// non-zero exit means the CHANNEL failed, not the probe.
///
/// Group kill is `kill -KILL -PGID` with NO `--`: dash's kill builtin
/// mishandles `kill -KILL -- -PGID` (same constraint as the in-session
/// watchdog and the daemon kill path in `rchd::cancellation`).
pub(crate) fn remote_timeout_kill_script(pgid_file: &str, build_id: u64) -> String {
    // Shared with rchd, which re-runs the same probe to clear a quarantine.
    rch_common::orphan_quarantine::kill_probe_script(pgid_file, build_id)
}

/// Removes every file the durable supervisor writes beside `path`; the claim
/// is an empty `mkdir` directory. Succeeds when nothing is left.
fn recovery_completion_cleanup_script(path: &str) -> String {
    let quote = |value: String| escape(Cow::from(value)).into_owned();
    format!(
        "rm -f -- {done} {pending} {out} {err} && {{ rmdir -- {claim} 2>/dev/null || [ ! -e {claim} ]; }}",
        done = quote(path.to_owned()),
        pending = quote(format!("{path}.pending")),
        out = quote(format!("{path}.stdout")),
        err = quote(format!("{path}.stderr")),
        claim = quote(format!("{path}.started")),
    )
}

/// The same identity publisher and verifier used by daemon crash recovery.
/// Arguments: record path, timeout seconds, deadline marker, build ID, command.
fn remote_build_watchdog_script() -> String {
    format!(
        "{}\n{}",
        rch_common::REMOTE_PROCESS_IDENTITY_SCRIPT,
        r#"rch_remote_record "$1" "$4" || { printf "\n%s_IDENTITY_UNAVAILABLE\n" "$3" >&2; exit 125; }
__p=$$; __t="$2"; __m="$3"; shift 4
__cancelled=0; __w=; __watch_start=
trap '__cancelled=1' TERM
"$@" 3>&- & __c=$!
if [ "$__t" -gt 0 ] 2>/dev/null; then (
    __timer_cancelled=0
    trap '__timer_cancelled=1' TERM
    sleep "$__t" 3>&- & __sleep=$!
    rch_read_process "$__sleep" || exit 0
    __sleep_start=$rch_observed_start
    if [ "$__timer_cancelled" -eq 0 ]; then wait "$__sleep"; fi
    if [ "$__timer_cancelled" -eq 1 ]; then
        if rch_read_process "$__sleep" && [ "$rch_observed_boot" = "$rch_boot" ] &&
            [ "$rch_observed_start" = "$__sleep_start" ]; then kill -TERM "$__sleep" 2>/dev/null; fi
        wait "$__sleep" 2>/dev/null
        exit 0
    fi
    rch_remote_leader_matches || exit 0
    # fd 3 is the session's stderr pipe. If the dispatcher stopped draining it,
    # a blocking printf here would wedge forever and the deadline would never
    # fire (a 24h-stuck cargo test on vmi1153651, 2026-09-25). Emit the marker
    # from a child and give it a bounded grace; the group kill reaps it.
    printf "\n%s\n" "$__m" >&3 & __mp=$!
    __i=0
    while kill -0 "$__mp" 2>/dev/null && [ "$__i" -lt 50 ]; do
        sleep 0.1 3>&-; __i=$((__i + 1))
    done
    rch_remote_leader_matches && kill -KILL -"$__p" 2>/dev/null
) >/dev/null 2>&1 </dev/null & __w=$!; fi
if [ -n "$__w" ] && rch_read_process "$__w"; then __watch_start=$rch_observed_start; fi
wait "$__c"; __s=$?
if [ "$__cancelled" -eq 1 ]; then
    rch_remote_leader_matches && kill -KILL -"$__p" 2>/dev/null
    exit 143
fi
if [ -n "$__w" ]; then
    if rch_read_process "$__w" && [ "$rch_observed_boot" = "$rch_boot" ] &&
        [ "$rch_observed_start" = "$__watch_start" ]; then kill -TERM "$__w" 2>/dev/null; fi
    wait "$__w" 2>/dev/null
fi
exit "$__s""#,
    )
}

/// Parse the `RCH_E104_KILL=<verdict>` marker from the kill probe's stdout.
/// `None` means the probe produced no verdict (treated as unverified).
pub(crate) fn parse_remote_timeout_kill_output(stdout: &str) -> Option<RemoteTimeoutCleanup> {
    for line in stdout.lines().rev() {
        match line.trim() {
            "RCH_E104_KILL=verified_dead" => return Some(RemoteTimeoutCleanup::Verified),
            // A missing record cannot exclude a delayed, still-starting job.
            "RCH_E104_KILL=no_pgid_file" => return Some(RemoteTimeoutCleanup::Unverified),
            "RCH_E104_KILL=still_alive" => return Some(RemoteTimeoutCleanup::Unverified),
            _ => {}
        }
    }
    None
}

/// RAM budgeted per concurrent rustc when `compilation.remote_build_jobs =
/// "auto"` derives `CARGO_BUILD_JOBS` on the worker (issue #49). A large
/// crate's rustc peaks around 3 GiB, so 8 GiB per job leaves headroom for two
/// to three concurrent rch jobs on the same box before it starts paging.
const REMOTE_BUILD_JOBS_GIB_PER_JOB: u64 = 8;
/// Floor for the derived job count: below two, cargo loses pipelining
/// between codegen units and the build serialises for no memory benefit.
const REMOTE_BUILD_JOBS_AUTO_MIN: u64 = 2;
/// Ceiling for the derived job count. Above eight jobs per rch job the
/// worker's slot accounting, not per-job parallelism, is the right lever.
const REMOTE_BUILD_JOBS_AUTO_MAX: u64 = 8;
/// Linux total-memory source for the derived job count.
const LINUX_MEMINFO_PATH: &str = "/proc/meminfo";

/// POSIX `sh` fragment (dash-safe, trailing `; `) that exports
/// `CARGO_BUILD_JOBS` in the remote session according to `policy`, unless the
/// session already carries one (issue #49).
///
/// The fragment runs in the OUTER `sh -s` script, before the `sh -lc` that
/// executes the build, so:
/// - a worker-side `CARGO_BUILD_JOBS` (pam `/etc/environment`) is already
///   visible and wins via the `-z` guard;
/// - a value forwarded through the env allowlist or written inline in the
///   command (`CARGO_BUILD_JOBS=N cargo …`) sits on the inner command line and
///   overrides the exported default by ordinary shell precedence;
/// - a `-j N` flag beats the env var by cargo's own precedence.
///
/// `auto` computes `clamp(mem_gib / 8, 2, min(nproc, 8))` from live worker
/// facts. Every probe is fail-open: if either cores or memory cannot be read
/// as a number the fragment exports nothing and cargo keeps its `nproc`
/// default. `meminfo_path` is the Linux `/proc/meminfo` location (overridable
/// for tests); macOS falls back to `sysctl hw.memsize`.
fn remote_build_jobs_fragment(policy: RemoteBuildJobs, meminfo_path: &str) -> String {
    match policy {
        RemoteBuildJobs::Off => String::new(),
        RemoteBuildJobs::Fixed(n) => format!(
            "if [ -z \"${{CARGO_BUILD_JOBS:-}}\" ]; then CARGO_BUILD_JOBS={n}; export CARGO_BUILD_JOBS; fi; "
        ),
        RemoteBuildJobs::Auto => {
            let escaped_meminfo = escape(Cow::from(meminfo_path));
            format!(
                "if [ -z \"${{CARGO_BUILD_JOBS:-}}\" ]; then \
__rch_c=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null); \
__rch_m=$(awk '/^MemTotal:/ {{ print int($2 / 1048576) }}' {escaped_meminfo} 2>/dev/null); \
[ -n \"$__rch_m\" ] || __rch_m=$(sysctl -n hw.memsize 2>/dev/null | awk '{{ print int($1 / 1073741824) }}'); \
case \"$__rch_c\" in ''|*[!0-9]*) __rch_c=;; esac; \
case \"$__rch_m\" in ''|*[!0-9]*) __rch_m=;; esac; \
if [ -n \"$__rch_c\" ] && [ -n \"$__rch_m\" ]; then \
__rch_j=$((__rch_m / {gib})); \
if [ \"$__rch_j\" -lt {min} ]; then __rch_j={min}; fi; \
__rch_x=$__rch_c; if [ \"$__rch_x\" -gt {max} ]; then __rch_x={max}; fi; \
if [ \"$__rch_j\" -gt \"$__rch_x\" ]; then __rch_j=$__rch_x; fi; \
CARGO_BUILD_JOBS=$__rch_j; export CARGO_BUILD_JOBS; \
fi; fi; ",
                gib = REMOTE_BUILD_JOBS_GIB_PER_JOB,
                min = REMOTE_BUILD_JOBS_AUTO_MIN,
                max = REMOTE_BUILD_JOBS_AUTO_MAX,
            )
        }
    }
}

/// Whether the project pins cargo's parallelism itself via `[build] jobs` in
/// `.cargo/config.toml` (or legacy `.cargo/config`). An env `CARGO_BUILD_JOBS`
/// outranks that file in cargo's precedence, so rch must not inject one over a
/// project that made an explicit choice. Unreadable or unparsable files count
/// as "not declared": the injected cap is a safety default, not a contract.
fn project_declares_cargo_build_jobs(project_root: &Path) -> bool {
    [".cargo/config.toml", ".cargo/config"]
        .iter()
        .map(|rel| project_root.join(rel))
        .filter_map(|path| std::fs::read_to_string(path).ok())
        // A whole document must go through `from_str::<Table>`: `str::parse::<Value>`
        // parses a single TOML *value* and rejects any file with a table header.
        .filter_map(|text| toml::from_str::<toml::Table>(&text).ok())
        .any(|table| {
            table
                .get("build")
                .and_then(|build| build.get("jobs"))
                .is_some()
        })
}

const DEFAULT_REMOTE_CARGO_TARGET_DIR_NAME: &str = ".rch-target";

/// Environment variables whose values we ALWAYS rewrite to a managed,
/// worker-scoped path under the synchronized remote project root — regardless of
/// whether the local process forwarded them. These control where build
/// tools drop large artifacts/scratch; if they inherit a host-only absolute path
/// (e.g. a `.cargo/config.toml` `target-dir` of `/root/cass-ft-target`, or a
/// `TMPDIR` on a volatile host mount) the remote build writes outside the managed
/// `/data/tmp` zone the disk reapers watch, or fails outright. Forcing them into
/// `<remote_path>/.rch-*` makes placement deterministic and reclaimable, and —
/// because env precedence beats a `.cargo/config.toml` `target-dir` — covers
/// wrapper/unclassified builds that never went through target-dir rewriting.
const FORCED_MANAGED_ENV_KEYS: &[&str] = &[
    "CARGO_TARGET_DIR",
    "TMPDIR",
    "GOCACHE",
    "GOMODCACHE",
    "GOPATH",
];
/// Remote loop-break. A command we are *already* executing on a worker must
/// compile there — it must never re-enter the offload path and try to hand the
/// work to yet another machine.
///
/// The worker's own `cargo` is shim-wrapped (`rch shim install` self-heals onto
/// every host), so without this the worker's shim sees the caller's fail-closed
/// intent (`RCH_REQUIRE_REMOTE`) and tries to offload again. On a pure worker
/// — no workers of its own — that refuses local fallback and the job dies with
/// exit 103; on a dispatcher it "succeeds" by bouncing the build to a third
/// host, paying a second sync for no reason.
///
/// This mirrors the local-fallback loop-break in [`crate::hook`], which sets the
/// same variable on rch's own local re-exec. The shim checks it before anything
/// else, so it short-circuits regardless of how the remote-required intent
/// reached the worker.
const REMOTE_LOOP_BREAK_ENV: (&str, &str) = ("RCH_CARGO_WRAPPER_BYPASS", "1");
const CONFIG_EXCLUDE_REWRITES: &[(&str, &str)] = &[
    ("core.*", "core.[0-9]*"),
    (".core.*", ".core.[0-9]*"),
    (".git/objects/", ".git/"),
];

fn normalize_config_exclude_pattern(pattern: &str) -> &str {
    CONFIG_EXCLUDE_REWRITES
        .iter()
        .find_map(|(legacy, replacement)| (*legacy == pattern).then_some(*replacement))
        .unwrap_or(pattern)
}

/// A *linked* git worktree records its git state indirection in a `.git` FILE
/// (not a `.git` directory) whose sole meaningful line is
/// `gitdir: <parent-repo>/.git/worktrees/<name>`.
///
/// That absolute path points back into the PARENT repository's `.git`, which
/// does NOT exist on a remote worker after the rsync. Shipping the `.git` file
/// verbatim therefore leaves a dangling pointer: any git/cargo step that tries
/// to resolve the repository on the worker fails, and the hook fails open with
/// `[RCH] local (remote execution failed)` — silently changing which machine/OS
/// actually ran the build. See [`git_worktree_upload_exclude`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct LinkedWorktreeGitPointer {
    /// The `gitdir` target parsed from the `.git` file, verbatim as written
    /// (normally an absolute path into the parent repo's `.git/worktrees/…`).
    pub(crate) gitdir: String,
}

/// Parse the `gitdir:` pointer out of a linked-worktree `.git` file's contents.
///
/// Git writes exactly one `gitdir: <path>` line; we tolerate leading/trailing
/// whitespace, blank lines, and CRLF. Returns `None` for a normal (non-pointer)
/// `.git` file body.
pub(crate) fn parse_worktree_gitdir_pointer(contents: &str) -> Option<LinkedWorktreeGitPointer> {
    for line in contents.lines() {
        if let Some(rest) = line.trim().strip_prefix("gitdir:") {
            let gitdir = rest.trim();
            if !gitdir.is_empty() {
                return Some(LinkedWorktreeGitPointer {
                    gitdir: gitdir.to_string(),
                });
            }
        }
    }
    None
}

/// Detect whether `project_root` is a *linked git worktree* — i.e. its `.git`
/// entry is a regular FILE holding a `gitdir:` pointer, rather than a real
/// `.git` directory (normal repo) or absent (no VCS).
///
/// Returns the parsed pointer when so, else `None`.
pub(crate) fn detect_linked_worktree_git_pointer(
    project_root: &Path,
) -> Option<LinkedWorktreeGitPointer> {
    let git_path = project_root.join(".git");
    // `symlink_metadata` so a `.git` symlink is not transparently followed to a
    // directory — we only treat a genuine regular file as a worktree pointer.
    let metadata = std::fs::symlink_metadata(&git_path).ok()?;
    if !metadata.file_type().is_file() {
        // A real `.git/` directory (normal repo) or a symlink — leave the
        // existing `.git/` directory exclusion to handle it.
        return None;
    }
    let contents = std::fs::read_to_string(&git_path).ok()?;
    parse_worktree_gitdir_pointer(&contents)
}

/// If `project_root` is a linked git worktree, return the anchored rsync exclude
/// (`/.git`) that keeps its dangling `.git` FILE out of the upload, paired with
/// the parsed pointer (for diagnostics).
///
/// The default upload excludes already drop the whole `.git/` *directory* for
/// normal repos, and builds succeed on the worker with no git metadata at all.
/// The `.git/` pattern (trailing slash) matches only directories, though, so a
/// worktree's `.git` FILE would otherwise slip through and be synced as a
/// dangling pointer. Excluding it yields exactly the same git-free remote source
/// tree a normal repo already produces, so the build proceeds instead of failing
/// open to local. The leading `/` anchors the rule to the transfer root so it
/// only affects the project root's own `.git`, never a nested one.
pub(crate) fn git_worktree_upload_exclude(
    project_root: &Path,
) -> Option<(String, LinkedWorktreeGitPointer)> {
    detect_linked_worktree_git_pointer(project_root).map(|pointer| ("/.git".to_string(), pointer))
}

fn add_portable_rsync_archive_args(cmd: &mut Command) {
    // `-a` includes owner/group preservation. Across independently provisioned
    // workers those metadata IDs are not portable and can turn an otherwise
    // writable sync into a fail-open chgrp failure.
    cmd.arg("--no-owner").arg("--no-group");
}

/// Anchor an artifact retrieval pattern so rsync only matches it at the
/// transfer source root, NOT at any arbitrary depth in the source tree.
/// Closes RCH bug `d7xc3` ("Artifact retrieval must not dirty local source
/// checkout"). Unanchored patterns like `target/debug/**` would match
/// `<root>/target/debug/foo` AND `<root>/anything/target/debug/foo` — the
/// second branch lets a hostile or stale remote layout (e.g., another
/// agent's build tree at `<root>/some-crate/target/...`) drift into the
/// retrieval set and overwrite a local file.
///
/// Rules (rsync filter semantics):
///   * pattern already starts with `/`  → leave as-is (already anchored)
///   * pattern starts with `**/`        → leave as-is (explicit recursion)
///   * otherwise                        → prepend `/` to anchor
///
/// Empty / whitespace-only patterns are returned unchanged so the caller
/// can decide whether to drop them; we do not silently mutate junk input.
fn anchor_retrieval_pattern(pattern: &str) -> String {
    let trimmed = pattern.trim_start();
    if trimmed.is_empty() {
        return pattern.to_string();
    }
    if trimmed.starts_with('/') || trimmed.starts_with("**/") {
        return pattern.to_string();
    }
    format!("/{}", pattern)
}

/// An artifact-pattern entry that begins with the rsync-style `- ` marker is an
/// EXCLUDE rule, not an include. Returns the exclude payload (the text after the
/// marker) so the retrieve builders can emit it as `--exclude` BEFORE the include
/// rules — rsync evaluates filter rules in order with first-match-wins, so an
/// up-front exclude (e.g. cargo's `incremental/`/`.fingerprint/`/`build/` cache
/// trees, `*.d` dep files) keeps those bytes from ever transferring even though a
/// broad include like `debug/**` would otherwise match them. See
/// `hook::artifact_patterns::CARGO_TARGET_CACHE_EXCLUDES`.
fn artifact_pattern_exclude(pattern: &str) -> Option<&str> {
    pattern.strip_prefix("- ")
}

/// Partition an artifact-pattern list into `(excludes, includes)`. Exclude
/// entries (`- <pat>`) yield their bare payload (the `- ` marker stripped);
/// everything else is an include pattern, preserved verbatim and in order. The
/// include list is what the existing root/source-integrity helpers consume, so
/// they never mistake an exclude marker for an artifact root.
fn partition_artifact_filters(artifact_patterns: &[String]) -> (Vec<String>, Vec<String>) {
    let mut excludes = Vec::new();
    let mut includes = Vec::new();
    for pattern in artifact_patterns {
        match artifact_pattern_exclude(pattern) {
            Some(payload) => excludes.push(payload.to_string()),
            None => includes.push(pattern.clone()),
        }
    }
    (excludes, includes)
}

/// Compute the set of "allowed top-level roots" implied by anchored
/// artifact patterns. Used to build the `--exclude` belt-and-suspenders
/// for retrieval: anything at the rsync transfer root that ISN'T in this
/// set gets explicitly excluded, regardless of pattern matching quirks.
///
/// For each anchored pattern, the first path component after the leading
/// `/` is the implied root. Glob metacharacters in that component (e.g.,
/// `*.tsbuildinfo`) disable the implication for that pattern — we can't
/// safely derive a single root from a glob, so we conservatively allow
/// the rsync source root to be scanned for that pattern.
fn allowed_artifact_roots(artifact_patterns: &[String]) -> std::collections::BTreeSet<String> {
    let mut roots = std::collections::BTreeSet::new();
    for pattern in artifact_patterns {
        let anchored = anchor_retrieval_pattern(pattern);
        let after_slash = anchored.trim_start_matches('/');
        let first = after_slash.split('/').next().unwrap_or("");
        if first.is_empty() {
            continue;
        }
        if has_rsync_glob_meta(first) {
            // Top-level glob (e.g., `*.tsbuildinfo`) — can't derive a
            // single allowed root. The caller's `--include` rule will
            // accept matching files at the source root; we don't add a
            // root exclusion that would block them.
            continue;
        }
        roots.insert(first.to_string());
    }
    roots
}

fn has_rsync_glob_meta(pattern: &str) -> bool {
    pattern
        .chars()
        .any(|ch| matches!(ch, '*' | '?' | '[' | ']'))
}

fn top_level_artifact_pattern_matches_entry(pattern: &str, entry_name: &str) -> bool {
    let anchored = anchor_retrieval_pattern(pattern);
    if anchored.starts_with("**/") {
        return false;
    }

    let after_slash = anchored.trim_start_matches('/');
    let first = after_slash.split('/').next().unwrap_or("");
    if first.is_empty() {
        return false;
    }
    if first == "*" && after_slash == "*" {
        // A bare `*` (the C/C++ defaults' best-effort root-level fetch) is too
        // broad to prove that an existing local top-level entry is an artifact,
        // so it must not disable the source-integrity exclude guard. A deeper
        // `*/...` prefix (e.g. `*/release/**` for cargo --target triple dirs)
        // is a directory-scoped artifact pattern and MUST allow descent: with
        // the old blanket `first == "*"` refusal, a pre-existing local triple
        // dir got excluded at the transfer root and every cross-target binary
        // silently vanished from sync-down (hfdt release leg, 2026-08-06).
        return false;
    }
    if first == entry_name {
        return true;
    }
    has_rsync_glob_meta(first)
        && Pattern::new(first)
            .map(|pattern| pattern.matches(entry_name))
            .unwrap_or(false)
}

fn escape_rsync_filter_literal_component(name: &str) -> Cow<'_, str> {
    if !has_rsync_glob_meta(name) {
        return Cow::Borrowed(name);
    }

    let mut escaped = String::with_capacity(name.len());
    for ch in name.chars() {
        match ch {
            '\\' => escaped.push_str(r"\\"),
            '*' | '?' | '[' => {
                escaped.push('\\');
                escaped.push(ch);
            }
            _ => escaped.push(ch),
        }
    }
    Cow::Owned(escaped)
}

/// Build anchored, literal rsync include filters for clean-overlay paths.
///
/// Every ancestor directory is included so rsync can descend through the final
/// catch-all exclude. Glob metacharacters in real file names are escaped, which
/// prevents a selected path such as `src/star*.rs` from widening the upload.
pub(crate) fn clean_overlay_include_patterns(
    project_root: &Path,
    overlay_paths: &[PathBuf],
) -> Result<Vec<String>> {
    let mut patterns = BTreeSet::new();
    for relative in overlay_paths {
        let mut escaped_parts = Vec::new();
        for component in relative.components() {
            match component {
                Component::Normal(part) => {
                    let part = part.to_str().ok_or_else(|| {
                        anyhow::anyhow!(
                            "clean-overlay path is not valid UTF-8: {}",
                            relative.display()
                        )
                    })?;
                    escaped_parts.push(escape_rsync_filter_literal_component(part).into_owned());
                }
                Component::CurDir => {}
                Component::ParentDir | Component::RootDir | Component::Prefix(_) => {
                    anyhow::bail!(
                        "clean-overlay path is not repository-relative: {}",
                        relative.display()
                    );
                }
            }
        }
        if escaped_parts.is_empty() {
            anyhow::bail!("clean-overlay path must not select the repository root");
        }

        for end in 1..escaped_parts.len() {
            patterns.insert(format!("/{}/", escaped_parts[..end].join("/")));
        }

        let escaped_path = escaped_parts.join("/");
        let metadata = std::fs::symlink_metadata(project_root.join(relative))
            .with_context(|| format!("inspect clean-overlay path {}", relative.display()))?;
        if metadata.file_type().is_dir() {
            patterns.insert(format!("/{escaped_path}/"));
            patterns.insert(format!("/{escaped_path}/***"));
        } else {
            patterns.insert(format!("/{escaped_path}"));
        }
    }
    Ok(patterns.into_iter().collect())
}

fn artifact_patterns_allow_top_level_entry(artifact_patterns: &[String], entry_name: &str) -> bool {
    artifact_patterns
        .iter()
        .any(|pattern| top_level_artifact_pattern_matches_entry(pattern, entry_name))
}

fn first_path_component(pattern: &str) -> Option<&str> {
    let trimmed = pattern.trim_start_matches('/');
    let trimmed = trimmed.trim_end_matches('/');
    if trimmed.is_empty() {
        None
    } else {
        Some(trimmed.split('/').next().unwrap_or(trimmed))
    }
}

fn retrieval_exclude_can_block_artifacts(pattern: &str, artifact_patterns: &[String]) -> bool {
    let trimmed = pattern.trim();
    if !trimmed.ends_with('/') {
        return true;
    }

    let Some(exclude_root) = first_path_component(trimmed) else {
        return true;
    };
    if has_rsync_glob_meta(exclude_root) {
        return true;
    }

    artifact_patterns.iter().any(|artifact| {
        first_path_component(artifact)
            .map(|artifact_root| artifact_root == exclude_root)
            .unwrap_or(false)
    })
}

fn path_matches_transfer_exclude(relative: &Path, is_dir: bool, excludes: &[String]) -> bool {
    let relative_text = relative.to_string_lossy().replace('\\', "/");
    let basename = relative
        .file_name()
        .map(|name| name.to_string_lossy())
        .unwrap_or_default();

    excludes.iter().any(|raw_pattern| {
        let normalized = normalize_config_exclude_pattern(raw_pattern);
        let directory_only = normalized.ends_with('/');
        if directory_only && !is_dir {
            return false;
        }
        let pattern_text = normalized.trim_start_matches('/').trim_end_matches('/');
        if pattern_text.is_empty() {
            return false;
        }
        Pattern::new(pattern_text)
            .is_ok_and(|pattern| pattern.matches(&relative_text) || pattern.matches(&basename))
    })
}

/// Conservative local upper bound for an ordinary rsync payload.
///
/// This intentionally counts sparse files by logical length and stops at the
/// amount needed to reach the one-hour timeout cap. It skips configured rsync
/// exclusions and never follows symlinks. An unreadable entry is ignored; the
/// resulting 30-second floor remains the fail-open fallback.
fn local_sync_payload_upper_bound(project_root: &Path, excludes: &[String]) -> u64 {
    const PAYLOAD_BYTES_AT_TIMEOUT_CAP: u64 = (TransferConfig::MAX_SYNC_TIMEOUT_MS
        - TransferConfig::DEFAULT_SYNC_TIMEOUT_FLOOR_MS)
        / 1_000
        * 1024
        * 1024;

    let mut total = 0_u64;
    let mut pending = vec![project_root.to_path_buf()];
    while let Some(directory) = pending.pop() {
        let Ok(entries) = std::fs::read_dir(&directory) else {
            continue;
        };
        for entry in entries.flatten() {
            let path = entry.path();
            let Ok(relative) = path.strip_prefix(project_root) else {
                continue;
            };
            let Ok(metadata) = std::fs::symlink_metadata(&path) else {
                continue;
            };
            let file_type = metadata.file_type();
            if path_matches_transfer_exclude(relative, file_type.is_dir(), excludes) {
                continue;
            }
            if file_type.is_dir() {
                pending.push(path);
            } else if file_type.is_file() {
                total = total.saturating_add(metadata.len());
                if total >= PAYLOAD_BYTES_AT_TIMEOUT_CAP {
                    return PAYLOAD_BYTES_AT_TIMEOUT_CAP;
                }
            }
        }
    }
    total
}

/// One deterministic source-transfer attempt retained for hook diagnostics.
///
/// These records are unit-observable evidence about the retry state machine;
/// they are not represented as live-network proof by tests.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct TransferAttemptDiagnostic {
    pub(crate) attempt: u32,
    pub(crate) max_attempts: u32,
    pub(crate) outcome: &'static str,
    pub(crate) detail: String,
}

#[derive(Debug)]
pub(crate) struct CleanOverlayMaterialization {
    pub(crate) sync_result: SyncResult,
    pub(crate) attempts: Vec<TransferAttemptDiagnostic>,
}

#[derive(Debug)]
pub(crate) struct TransferAttemptsExhausted {
    pub(crate) attempts: Vec<TransferAttemptDiagnostic>,
    last_error: String,
}

impl std::fmt::Display for TransferAttemptsExhausted {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            formatter,
            "source transfer failed before remote Cargo execution after {} attempt(s): {}",
            self.attempts.len(),
            self.last_error
        )
    }
}

impl std::error::Error for TransferAttemptsExhausted {}

/// Run a source-transfer operation with a full timeout budget for each attempt.
///
/// The older generic transfer retry path treats `total_timeout_ms` as one shared
/// wall-clock budget, so an attempt that consumes the whole timeout necessarily
/// prevents attempt two. Cold pinned trees need different semantics: each
/// retry gets the configured source-sync timeout and reuses the same partial
/// remote base. The injected operation makes the state machine deterministic in
/// unit tests without presenting the injection as live network evidence.
async fn run_source_transfer_attempts<T, F, Fut>(
    retry: &RetryConfig,
    attempt_timeout: std::time::Duration,
    operation_name: &str,
    mut operation: F,
) -> std::result::Result<(T, Vec<TransferAttemptDiagnostic>), TransferAttemptsExhausted>
where
    F: FnMut(u32) -> Fut,
    Fut: Future<Output = anyhow::Result<T>>,
{
    let max_attempts = retry.max_attempts.max(1);
    let mut attempts = Vec::with_capacity(max_attempts as usize);

    for attempt in 1..=max_attempts {
        if attempt > 1 {
            sleep(retry.delay_for_attempt(attempt - 1)).await;
        }

        let result = match tokio::time::timeout(attempt_timeout, operation(attempt)).await {
            Ok(result) => result,
            Err(_) => Err(anyhow::anyhow!(
                "{operation_name}: timed out after {}ms",
                attempt_timeout.as_millis()
            )),
        };

        match result {
            Ok(value) => {
                attempts.push(TransferAttemptDiagnostic {
                    attempt,
                    max_attempts,
                    outcome: "succeeded",
                    detail: format!("{operation_name} completed"),
                });
                return Ok((value, attempts));
            }
            Err(error) => {
                let retryable = is_retryable_transport_error(&error);
                let detail = source_transfer_error_detail(&error);
                attempts.push(TransferAttemptDiagnostic {
                    attempt,
                    max_attempts,
                    outcome: if retryable { "retryable" } else { "fatal" },
                    detail: detail.clone(),
                });
                warn!(
                    "{}: {} error on attempt {}/{}: {}",
                    operation_name,
                    if retryable { "retryable" } else { "fatal" },
                    attempt,
                    max_attempts,
                    error
                );

                if !retryable || attempt == max_attempts {
                    return Err(TransferAttemptsExhausted {
                        attempts,
                        last_error: detail,
                    });
                }
            }
        }
    }

    unreachable!("source transfer loop always executes at least once")
}

// =============================================================================
// Retry Logic (bd-x1ek)
// =============================================================================

/// Execute an async operation with retry and exponential backoff.
///
/// Only retries on transient transport errors (connection timeout, reset, etc.).
/// Non-retryable errors (auth failure, host key issues) fail immediately.
///
/// Returns the result of the first successful attempt or the last error.
///
/// This variant works with operations that return `anyhow::Result<T>`.
async fn retry_with_backoff<T, F, Fut>(
    config: &RetryConfig,
    operation_name: &str,
    mut operation: F,
) -> Result<T>
where
    F: FnMut() -> Fut,
    Fut: Future<Output = anyhow::Result<T>>,
{
    let start = std::time::Instant::now();
    let mut last_error: Option<anyhow::Error> = None;

    for attempt in 0..config.max_attempts {
        // Check total timeout before attempting
        if attempt > 0 && !config.should_retry(attempt, start.elapsed()) {
            debug!(
                "{}: total timeout exceeded after {} attempts",
                operation_name, attempt
            );
            break;
        }

        // Apply delay (exponential backoff with jitter) for retries
        if attempt > 0 {
            let delay = config.delay_for_attempt(attempt);
            debug!(
                "{}: attempt {}/{} after {}ms delay",
                operation_name,
                attempt + 1,
                config.max_attempts,
                delay.as_millis()
            );
            sleep(delay).await;
        }

        let elapsed_ms = start.elapsed().as_millis();
        let remaining_ms = if elapsed_ms >= config.total_timeout_ms as u128 {
            0
        } else {
            config.total_timeout_ms - elapsed_ms as u64
        };
        let attempt_timeout = std::time::Duration::from_millis(remaining_ms.max(1));

        let operation_result = match tokio::time::timeout(attempt_timeout, operation()).await {
            Ok(result) => result,
            Err(_) => Err(anyhow::anyhow!(
                "{}: timed out after {}ms",
                operation_name,
                config.total_timeout_ms
            )),
        };

        match operation_result {
            Ok(result) => {
                if attempt > 0 {
                    info!(
                        "{}: succeeded on attempt {}/{}",
                        operation_name,
                        attempt + 1,
                        config.max_attempts
                    );
                }
                return Ok(result);
            }
            Err(err) => {
                // Check if error is retryable
                if !is_retryable_transport_error(&err) {
                    debug!(
                        "{}: non-retryable error on attempt {}: {}",
                        operation_name,
                        attempt + 1,
                        err
                    );
                    return Err(err);
                }

                warn!(
                    "{}: retryable error on attempt {}/{}: {}",
                    operation_name,
                    attempt + 1,
                    config.max_attempts,
                    err
                );
                last_error = Some(err);
            }
        }
    }

    // All retries exhausted
    Err(last_error.unwrap_or_else(|| anyhow::anyhow!("{}: all retries exhausted", operation_name)))
}

/// Execute a tokio Command with retry logic.
///
/// Wraps the command execution in retry_with_backoff, retrying on transient
/// rsync/SSH errors. The retry timeout bounds each attempt; timed-out child
/// processes are killed on drop so rsync/SSH cannot keep running in the
/// background.
async fn execute_rsync_with_retry(
    config: &RetryConfig,
    operation_name: &str,
    build_command: impl Fn() -> Command,
) -> Result<std::process::Output> {
    retry_with_backoff(config, operation_name, || {
        execute_rsync_attempt(build_command())
    })
    .await
}

async fn execute_rsync_attempt(mut cmd: Command) -> Result<std::process::Output> {
    cmd.kill_on_drop(true);
    let child = cmd
        .spawn()
        .map_err(|e| anyhow::anyhow!("rsync I/O error: {}", e))?;
    let output = child
        .wait_with_output()
        .await
        .map_err(|e| anyhow::anyhow!("rsync I/O error: {}", e))?;
    // A process that RAN but exited non-zero on a transient transport error
    // must surface as Err so the retry layer can classify it. Non-transport
    // exits stay as Output for the caller's structured rsync failure handling.
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        if is_retryable_transport_error_text(&stderr) {
            return Err(anyhow::anyhow!(
                "rsync transport error (exit {}): {}",
                output
                    .status
                    .code()
                    .map_or_else(|| "signal".to_string(), |c| c.to_string()),
                stderr
            ));
        }
    }
    Ok(output)
}

async fn execute_source_rsync_attempt(cmd: Command) -> Result<std::process::Output> {
    let output = execute_rsync_attempt(cmd).await?;
    if output.status.success() {
        return Ok(output);
    }

    let stderr = String::from_utf8_lossy(&output.stderr).to_string();
    Err(TransferError::SyncFailed {
        reason: "rsync failed".to_string(),
        exit_code: output.status.code(),
        stderr,
    }
    .into())
}

fn use_mock_transport(worker: &WorkerConfig) -> bool {
    mock::is_mock_enabled() || mock::is_mock_worker(worker)
}

/// Parse a .rchignore file and return patterns.
///
/// Format is similar to .gitignore:
/// - One pattern per line
/// - Lines starting with # are comments
/// - Empty lines and whitespace-only lines are ignored
/// - Leading/trailing whitespace is trimmed from patterns
///
/// Note: Unlike .gitignore, negation patterns (starting with !) are not
/// supported and will be treated as literal patterns.
pub fn parse_rchignore(path: &Path) -> std::io::Result<Vec<String>> {
    let content = std::fs::read_to_string(path)?;
    Ok(parse_rchignore_content(&content))
}

/// Parse .rchignore content (for testing).
pub fn parse_rchignore_content(content: &str) -> Vec<String> {
    content
        .lines()
        .map(|line| line.trim())
        .filter(|line| !line.is_empty() && !line.starts_with('#'))
        .map(|line| line.to_string())
        .collect()
}

/// Isolate Git reads used to construct a clean-overlay receipt from ambient
/// repository-selection variables and configuration that can rewrite objects
/// or archive contents.
pub(crate) fn configure_clean_git_command(command: &mut Command) {
    const REPOSITORY_ENV: &[&str] = &[
        "GIT_DIR",
        "GIT_WORK_TREE",
        "GIT_COMMON_DIR",
        "GIT_OBJECT_DIRECTORY",
        "GIT_ALTERNATE_OBJECT_DIRECTORIES",
        "GIT_INDEX_FILE",
        "GIT_NAMESPACE",
        "GIT_CEILING_DIRECTORIES",
        "GIT_DISCOVERY_ACROSS_FILESYSTEM",
        "GIT_CONFIG_PARAMETERS",
        "GIT_CONFIG_COUNT",
    ];
    for key in REPOSITORY_ENV {
        command.env_remove(key);
    }
    for (key, _) in std::env::vars_os() {
        if key.to_string_lossy().starts_with("GIT_") {
            command.env_remove(key);
        }
    }
    command
        .env("GIT_NO_REPLACE_OBJECTS", "1")
        .env("GIT_ATTR_NOSYSTEM", "1")
        .env("GIT_CONFIG_NOSYSTEM", "1")
        .arg("-c")
        .arg("core.attributesFile=/dev/null")
        .arg("-c")
        .arg("tar.umask=0022");
}

/// Transfer pipeline for remote compilation.
#[derive(Clone)]
pub struct TransferPipeline {
    /// Correlates deadline enforcement with this execution, independently of exit 137.
    deadline_marker: String,
    /// Local project root.
    project_root: PathBuf,
    /// Caller-owned tree used to distinguish source from artifact roots. A
    /// private retrieval staging directory must not replace this evidence.
    retrieval_reference_root: PathBuf,
    /// Project identifier (usually directory name).
    project_id: String,
    /// Project hash for cache invalidation.
    project_hash: String,
    /// Transfer configuration.
    transfer_config: TransferConfig,
    /// SSH options.
    #[cfg(unix)]
    ssh_options: SshOptions,
    /// Color mode for remote command output.
    color_mode: ColorMode,
    /// Environment variables to forward to workers.
    env_allowlist: Vec<String>,
    /// Layer 0 configuration-pack env pairs (bd-bqu38): config-resolved
    /// `CARGO_PROFILE_*` assignments forced onto the remote build, bypassing
    /// the ambient-environment allowlist lookup entirely.
    layer0_env: Vec<(String, String)>,
    /// Optional environment overrides for testing.
    env_overrides: Option<HashMap<String, String>>,
    /// Compilation kind for command-specific handling.
    ///
    /// Used to apply appropriate timeouts and wrappers (e.g., external timeout
    /// for bun test to protect against known CPU hang issues).
    compilation_kind: Option<CompilationKind>,
    /// Compilation configuration for timeouts and other settings.
    ///
    /// Provides configurable external timeout values per command type and
    /// the ability to enable/disable timeout wrapping entirely.
    compilation_config: rch_common::CompilationConfig,
    /// Optional estimated transfer size (bytes) for adaptive compression.
    ///
    /// Populated by `should_skip_transfer` when estimation is performed.
    estimated_transfer_bytes: Option<u64>,
    /// Optional explicit remote path override.
    ///
    /// When set, transfer and execution use this path directly instead of
    /// deriving `<remote_base>/<project_id>/<project_hash>`.
    remote_path_override: Option<String>,
    /// Remote Cargo target directory basename.
    ///
    /// Defaults to `.rch-target` for direct `TransferPipeline` users. The rch
    /// hook sets a unique value per remote job to prevent parallel Cargo runs
    /// in the same synchronized project from contending on Cargo's artifact
    /// directory lock.
    remote_cargo_target_dir_name: String,
    /// Optional ABSOLUTE remote Cargo target directory override (issue #60).
    ///
    /// When set, the remote `CARGO_TARGET_DIR` uses this exact path instead of
    /// `<remote_path>/<remote_cargo_target_dir_name>`. Clean-overlay runs hash
    /// a per-command job nonce into their remote root (an overlap-safety
    /// invariant) and reap that root at teardown, so a pooled target dir placed
    /// UNDER it can never be reused across jobs. This override relocates the
    /// pooled store to a stable per-project path OUTSIDE the throwaway root.
    remote_cargo_target_dir_override: Option<String>,
    /// Optional include-only patterns for sync-to-remote uploads.
    sync_include_patterns: Option<Vec<String>>,
    /// Whether sync-to-remote should delete extraneous files remotely.
    sync_delete: bool,
    /// Whether uploads must compare file contents instead of size and mtime.
    sync_checksum: bool,
    /// Build ID for tracking and cancellation.
    build_id: Option<u64>,
    /// Remote worker OS family. Selects transport (rsync vs tar) and path
    /// conventions. Defaults to `Posix`; set per-worker at construction via
    /// [`Self::with_worker_platform`]. Left at the default, behaviour is
    /// identical to before this field existed.
    worker_platform: WorkerPlatform,
    /// Idle retention (hours) for pooled target stores pruned by the
    /// transfer-start janitor; `0` disables that sweep. Mirrors
    /// `[remediation.pooled_target] reaper_pooled_idle_hours` so every
    /// surface reports the same lifetime (issue #53).
    pooled_target_prune_idle_hours: u32,
    /// Pinned rsync binary + flavour (issue #66). `None` resolves lazily via
    /// [`rch_common::rsync_flavor::resolve_rsync_cached`] using
    /// `transfer_config.rsync_bin`; tests pin a flavour to assert argv.
    rsync_override: Option<ResolvedRsync>,
    /// Non-replayable, identity-bound remote completion evidence.
    recovery_completion: Option<(String, String)>,
    /// Worker-side activity lease bound to the persisted source grant. Every
    /// source operation checks the grant after joining its drain barrier.
    source_authority_prefix: Option<String>,
    /// Cooperative collection-only cancellation; the sender never drops a live child future.
    retrieval_control: Option<(tokio::sync::watch::Receiver<bool>, TokioInstant)>,
}

/// Only emitted after the interrupted local receiver has been killed and reaped.
#[derive(Debug, thiserror::Error)]
#[error("artifact collection interrupted for same-job recovery")]
pub(crate) struct RetrievalCancelled;

/// Validate a project hash for safe use in file paths.
///
/// The hash should be a hex string (output of BLAKE3). This function
/// validates that it contains only hex digits to prevent path traversal
/// or shell injection attacks.
fn validate_project_hash(hash: &str) -> String {
    // Hash should be hex digits only
    if hash.is_empty() {
        return "0000000000000000".to_string();
    }

    // Reject if it contains anything other than hex digits
    if !hash.chars().all(|c| c.is_ascii_hexdigit()) {
        warn!(
            "Project hash contains non-hex characters, sanitizing: {:?}",
            hash
        );
        // Filter to only hex characters
        let sanitized: String = hash.chars().filter(|c| c.is_ascii_hexdigit()).collect();
        if sanitized.is_empty() {
            return "0000000000000000".to_string();
        }
        return sanitized;
    }

    hash.to_string()
}

/// Remote environment application plan.
#[derive(Debug, Clone, PartialEq, Eq)]
struct RemoteEnvPlan {
    /// Shell-safe environment variable prefix.
    env_prefix: EnvPrefix,
    /// Directories that must exist before command execution.
    ensure_dirs: Vec<String>,
    /// Managed temp directories (`TMPDIR`/`TMP`/`TEMP` values) that are
    /// chmodded to `1700` right after creation so a permissive worker umask
    /// (e.g. `002`) can never leave them group-writable without the sticky
    /// bit. Consumers such as atomic-write durability code validate ancestor
    /// permissions and refuse group-writable, non-sticky parents.
    restricted_dirs: Vec<String>,
}

impl TransferPipeline {
    /// Create a new transfer pipeline.
    ///
    /// The project_id and project_hash are sanitized to prevent path traversal
    /// and shell injection attacks.
    pub fn new(
        project_root: PathBuf,
        project_id: String,
        project_hash: String,
        transfer_config: TransferConfig,
    ) -> Self {
        // Sanitize inputs to prevent path traversal and injection attacks
        let safe_project_id = sanitize_project_id(&project_id);
        let safe_project_hash = validate_project_hash(&project_hash);

        if safe_project_id != project_id {
            warn!(
                "Project ID sanitized: {:?} -> {:?}",
                project_id, safe_project_id
            );
        }

        #[cfg(unix)]
        let ssh_options = SshOptions {
            server_alive_interval: Some(std::time::Duration::from_secs(
                transfer_config.ssh_server_alive_interval_secs.unwrap_or(15),
            )),
            control_persist_idle: transfer_config
                .ssh_control_persist_secs
                .map(std::time::Duration::from_secs),
            control_master: transfer_config.ssh_control_persist_secs.is_some(),
            ..Default::default()
        };

        Self {
            deadline_marker: format!("RCH_EXTERNAL_DEADLINE:{}", uuid::Uuid::new_v4()),
            retrieval_reference_root: project_root.clone(),
            project_root,
            project_id: safe_project_id,
            project_hash: safe_project_hash,
            transfer_config,
            #[cfg(unix)]
            ssh_options,
            color_mode: ColorMode::default(),
            env_allowlist: Vec::new(),
            layer0_env: Vec::new(),
            env_overrides: None,
            compilation_kind: None,
            compilation_config: rch_common::CompilationConfig::default(),
            estimated_transfer_bytes: None,
            remote_path_override: None,
            remote_cargo_target_dir_name: DEFAULT_REMOTE_CARGO_TARGET_DIR_NAME.to_string(),
            remote_cargo_target_dir_override: None,
            sync_include_patterns: None,
            sync_delete: true,
            sync_checksum: false,
            build_id: None,
            worker_platform: WorkerPlatform::Posix,
            pooled_target_prune_idle_hours:
                rch_common::remediation_config::DEFAULT_POOLED_REAPER_POOLED_IDLE_HOURS,
            rsync_override: None,
            recovery_completion: None,
            source_authority_prefix: None,
            retrieval_control: None,
        }
    }

    pub(crate) fn with_retrieval_control(
        mut self,
        cancel: tokio::sync::watch::Receiver<bool>,
        started: TokioInstant,
    ) -> Self {
        self.retrieval_control = Some((cancel, started));
        self
    }

    async fn execute_retrieval_rsync(
        &self,
        config: &RetryConfig,
        operation: &str,
        build: impl Fn() -> Command,
        mut on_line: impl FnMut(&str),
    ) -> Result<std::process::Output> {
        let Some((cancel, started)) = self.retrieval_control.as_ref() else {
            return execute_rsync_with_retry(config, operation, build).await;
        };
        let deadline = *started + Duration::from_millis(config.total_timeout_ms);
        let mut cancel = cancel.clone();
        for attempt in 0..config.max_attempts.max(1) {
            if *cancel.borrow() {
                return Err(RetrievalCancelled.into());
            }
            anyhow::ensure!(
                TokioInstant::now() < deadline,
                "{operation}: retrieval deadline exceeded"
            );
            if attempt > 0 {
                tokio::select! {
                    _ = tokio::time::sleep_until(deadline) => anyhow::bail!("{operation}: retrieval deadline exceeded"),
                    _ = cancel.wait_for(|requested| *requested) => return Err(RetrievalCancelled.into()),
                    _ = sleep(config.delay_for_attempt(attempt)) => {},
                }
            }
            if *cancel.borrow() {
                return Err(RetrievalCancelled.into());
            }
            anyhow::ensure!(
                TokioInstant::now() < deadline,
                "{operation}: retrieval deadline exceeded"
            );
            let mut cmd = build();
            cmd.kill_on_drop(true);
            let mut child = cmd.spawn().with_context(|| format!("spawn {operation}"))?;
            let mut stdout = child.stdout.take().context("retrieval stdout missing")?;
            let mut stderr = child.stderr.take().context("retrieval stderr missing")?;
            let mut out = Vec::new();
            let mut err = Vec::new();
            let mut out_buffer = [0_u8; 8192];
            let mut err_buffer = [0_u8; 8192];
            let mut out_open = true;
            let mut err_open = true;
            let collected = async {
                while out_open || err_open {
                    tokio::select! {
                        read = stdout.read(&mut out_buffer), if out_open => {
                            let n = read?;
                            out_open = n != 0;
                            on_line(&String::from_utf8_lossy(&out_buffer[..n]));
                            if out.len() < 10 * 1024 * 1024 { out.extend_from_slice(&out_buffer[..n]); }
                        },
                        read = stderr.read(&mut err_buffer), if err_open => {
                            let n = read?;
                            err_open = n != 0;
                            on_line(&String::from_utf8_lossy(&err_buffer[..n]));
                            if err.len() < 10 * 1024 * 1024 { err.extend_from_slice(&err_buffer[..n]); }
                        },
                    }
                }
                child.wait().await
            };
            let status = tokio::select! {
                biased;
                result = collected => Some(result),
                _ = cancel.wait_for(|requested| *requested) => None,
                _ = tokio::time::sleep_until(deadline) => None,
            };
            let status = match status {
                Some(Ok(status)) => status,
                other => {
                    // kill() awaits wait(); a retry must never overlap a receiver.
                    child
                        .kill()
                        .await
                        .context("stop and reap interrupted retrieval")?;
                    if let Some(Err(error)) = other {
                        return Err(error.into());
                    }
                    if *cancel.borrow() {
                        return Err(RetrievalCancelled.into());
                    }
                    anyhow::bail!("{operation}: retrieval deadline exceeded");
                }
            };
            if !status.success()
                && is_retryable_transport_error_text(&String::from_utf8_lossy(&err))
                && attempt + 1 < config.max_attempts.max(1)
            {
                continue;
            }
            return Ok(std::process::Output {
                status,
                stdout: out,
                stderr: err,
            });
        }
        unreachable!("at least one retrieval attempt is required")
    }

    pub(crate) fn with_recovery_completion(mut self, path: String, identity: String) -> Self {
        self.recovery_completion = Some((path, identity));
        self
    }

    #[cfg(unix)]
    pub(crate) fn with_source_authority(mut self, identity: String) -> Result<Self> {
        self.source_authority_prefix =
            Some(crate::hook::source_authority_activity_prefix(&identity)?);
        Ok(self)
    }

    /// Cleanup is admitted only while the exact active grant is cancelling.
    /// Final cancellation drains these operations before releasing its roots.
    #[cfg(unix)]
    pub(crate) fn with_source_authority_cleanup(mut self, identity: String) -> Result<Self> {
        self.source_authority_prefix =
            Some(crate::hook::source_authority_cleanup_prefix(&identity)?);
        Ok(self)
    }

    fn source_activity_command(&self, command: &str) -> String {
        match &self.source_authority_prefix {
            Some(prefix) => format!("{prefix} sh -c {}", escape(Cow::from(command))),
            None => command.to_owned(),
        }
    }

    /// rsync appends its server argv to this shell fragment. Forward those
    /// arguments inside the lease, after any destination setup has completed.
    fn source_rsync_path(&self, command: String) -> String {
        match &self.source_authority_prefix {
            Some(prefix) => {
                let script = format!("{command} \"$@\"");
                format!(
                    "{prefix} sh -c {} rch-source-rsync",
                    escape(Cow::from(script))
                )
            }
            None => command,
        }
    }

    fn append_source_rsync_path(&self, command: &mut Command) {
        if self.source_authority_prefix.is_some() {
            command
                .arg("--rsync-path")
                .arg(self.source_rsync_path("rsync".to_owned()));
        }
    }

    pub(crate) fn with_local_root(mut self, root: PathBuf) -> Self {
        self.project_root = root;
        self
    }

    pub(crate) fn with_retrieval_reference_root(mut self, root: PathBuf) -> Self {
        self.retrieval_reference_root = root;
        self
    }

    fn artifact_retrieval_filters(
        &self,
        artifact_patterns: &[String],
    ) -> Result<(Vec<WindowsArtifactFilter>, Vec<WindowsArtifactFilter>)> {
        let (caller_excludes, includes) = partition_artifact_filters(artifact_patterns);
        let mut excludes = self.get_retrieval_excludes(&includes);
        excludes.extend(caller_excludes);
        excludes.extend(
            self.local_source_roots_to_exclude(&allowed_artifact_roots(&includes), &includes),
        );
        Ok((
            includes
                .iter()
                .map(|pattern| WindowsArtifactFilter::new(pattern, true))
                .collect::<Result<Vec<_>>>()?,
            excludes
                .iter()
                .map(|pattern| WindowsArtifactFilter::new(pattern, false))
                .collect::<Result<Vec<_>>>()?,
        ))
    }

    /// Retained staging bytes may predate the current transfer filters. Check
    /// them again before publication, including any interrupted pending write.
    pub(crate) fn validate_staged_artifact_paths(
        &self,
        paths: &[PathBuf],
        artifact_patterns: &[String],
    ) -> Result<()> {
        let (includes, excludes) = self.artifact_retrieval_filters(artifact_patterns)?;
        for path in paths {
            let name = path.to_str().context("non-UTF-8 staged artifact path")?;
            anyhow::ensure!(
                !name.is_empty()
                    && path
                        .components()
                        .all(|component| matches!(component, Component::Normal(_)))
                    && includes.iter().any(|filter| filter.matches(name, false))
                    && !excludes.iter().any(|filter| filter.excludes(name)),
                "staged file is outside the caller's artifact policy: {}",
                path.display()
            );
        }
        Ok(())
    }

    /// The supervisor owns all output descriptors, so loss of the streaming
    /// channel cannot suppress its terminal record or execute the command twice.
    fn durable_execution_command(&self, command: String) -> String {
        let Some((path, identity)) = &self.recovery_completion else {
            return command;
        };
        let quote = |value: &str| escape(Cow::from(value)).into_owned();
        // The receipts stay private, but the workload runs under the caller's
        // umask: a leaked 077 made every remote output 0600/0700, and rsync -a
        // carried those modes home.
        let supervisor = format!(
            "trap '' HUP; ( {command}\n); s=$?; (umask 077; printf '%s %s\\n' {identity} \"$s\" > {pending}) && sync -f {pending} && mv -f -- {pending} {done} && sync -f {directory}; exit \"$s\"",
            identity = quote(identity),
            pending = quote(&format!("{path}.pending")),
            done = quote(path),
            directory = quote(Path::new(path).parent().unwrap().to_str().unwrap()),
        );
        // GNU `tail --pid` exits only after a final read once the supervisor
        // is gone, so the stream ends with the last byte of output. It needs
        // the supervisor reaped first (a zombie still answers `kill -0`). A
        // 120s backstop bounds the wait should the reaped PID be reused
        // before tail notices. Without `--pid`, the old bounded
        // `sleep 1; kill` can still cut a slow tail. Follow by polling, not
        // inotify: bytes written between tail's first read and its watch
        // raise no event, and once the writer is gone an inotify tail exits
        // without a final read. With a slow reader that cut ~5-8% of streams
        // on uutils tail; a polling tail reads to EOF before it gives up.
        format!(
            "set -e; rch_umask=$(umask); umask 077; mkdir -p -- {directory}; mkdir {claim}; : > {out}; : > {err}; umask \"$rch_umask\"; \
             nohup sh -c {supervisor} </dev/null >{out} 2>{err} & \
             p=$!; follow=; if tail --pid=\"$p\" -c 0 /dev/null >/dev/null 2>&1; then follow=--pid=$p; \
               if tail ---disable-inotify -c 0 /dev/null >/dev/null 2>&1; then follow=\"$follow ---disable-inotify\"; fi; fi; \
             tail $follow -c +1 -f {out} & a=$!; tail $follow -c +1 -f {err} >&2 & b=$!; \
             trap 'kill \"$a\" \"$b\" 2>/dev/null || :' EXIT; \
             while [ ! -f {done} ]; do sleep 1; done; \
             wait \"$p\" || :; \
             if [ -n \"$follow\" ]; then \
               ( trap 'kill \"$s\" 2>/dev/null; exit 0' TERM; sleep 120 </dev/null >/dev/null 2>&1 & s=$!; wait \"$s\"; kill \"$a\" \"$b\" 2>/dev/null ) & k=$!; \
               wait \"$a\" || :; a=; wait \"$b\" || :; b=; kill \"$k\" 2>/dev/null || :; \
             else sleep 1; kill \"$a\" \"$b\" 2>/dev/null || :; a=; b=; fi; \
             read -r identity status < {done}; [ \"$identity\" = {identity} ]; exit \"$status\"",
            claim = quote(&format!("{path}.started")),
            out = quote(&format!("{path}.stdout")),
            err = quote(&format!("{path}.stderr")),
            supervisor = quote(&supervisor),
            done = quote(path),
            identity = quote(identity),
            directory = quote(Path::new(path).parent().unwrap().to_str().unwrap()),
        )
    }

    pub(crate) async fn read_recovery_completion(
        &self,
        worker: &WorkerConfig,
    ) -> Result<Option<i32>> {
        let Some((path, identity)) = &self.recovery_completion else {
            return Ok(None);
        };
        let script = format!(
            "if [ -f {p} ] && [ ! -L {p} ]; then cat -- {p}; fi",
            p = escape(Cow::from(path.as_str()))
        );
        let output = tokio::time::timeout(
            Duration::from_secs(10),
            // This immutable identity-bound receipt is outside source-tree
            // authority. Its probe must not wait behind the running command's
            // exclusive activity lease and stall diagnostic streaming.
            self.worker_ssh_command_with_activity(
                worker,
                &["sh", "-c", &escape(Cow::from(script.as_str()))],
                false,
            )
            .output(),
        )
        .await??;
        anyhow::ensure!(output.status.success(), "completion probe SSH failed");
        let text = std::str::from_utf8(&output.stdout)?.trim();
        if text.is_empty() {
            return Ok(None);
        }
        let (observed, status) = text
            .split_once(' ')
            .context("malformed remote completion")?;
        anyhow::ensure!(observed == identity, "remote completion identity mismatch");
        let status: i32 = status.parse()?;
        anyhow::ensure!(
            (0..=255).contains(&status),
            "invalid remote completion status"
        );
        Ok(Some(status))
    }

    /// Delete the supervisor's claim, logs, and completion receipt. Call only
    /// once the local recipe is durably retired: until then the receipt is the
    /// sole completion proof and the claim is what forbids a second launch.
    pub(crate) async fn discard_recovery_completion(&self, worker: &WorkerConfig) -> Result<()> {
        let Some((path, _)) = &self.recovery_completion else {
            return Ok(());
        };
        let script = recovery_completion_cleanup_script(path);
        let mut command = self.worker_ssh_command_with_activity(
            worker,
            &["sh", "-c", &escape(Cow::from(script.as_str()))],
            false,
        );
        // Best-effort garbage removal after the result is already delivered:
        // bound what an unresponsive worker can add, and never leave the ssh
        // client running past the deadline.
        command.kill_on_drop(true);
        let output = tokio::time::timeout(Duration::from_secs(3), command.output()).await??;
        anyhow::ensure!(
            output.status.success(),
            "completion receipt cleanup failed: {}",
            String::from_utf8_lossy(&output.stderr).trim()
        );
        Ok(())
    }

    /// Pin the rsync binary and flavour instead of probing (issue #66).
    #[must_use]
    #[allow(dead_code)] // Argv tests pin flavours; embedders may pin a binary.
    pub fn with_rsync(mut self, resolved: ResolvedRsync) -> Self {
        self.rsync_override = Some(resolved);
        self
    }

    /// The rsync binary this pipeline execs, with its probed flavour.
    ///
    /// Resolution failures are not fatal here: the builders must still hand
    /// back a `Command`, so a missing binary degrades to a bare `rsync` (or
    /// the configured override) with the modern argv, and the spawn error
    /// names the program. `rch doctor` reports the same failure with the
    /// exact remedy.
    fn resolved_rsync(&self) -> ResolvedRsync {
        if let Some(resolved) = &self.rsync_override {
            return resolved.clone();
        }
        let configured = self.transfer_config.rsync_bin.as_deref();
        match rch_common::rsync_flavor::resolve_rsync_cached(configured) {
            Ok(resolved) => resolved,
            Err(error) => {
                warn!(
                    "rsync resolution failed ({error}); falling back to PATH rsync with the \
                     rsync 3.x argv — run `rch doctor` for the remedy"
                );
                ResolvedRsync {
                    path: PathBuf::from(configured.unwrap_or("rsync")),
                    flavor: RsyncFlavor::Unknown,
                    version_line: String::new(),
                    source: if configured.is_some() {
                        RsyncSource::Config
                    } else {
                        RsyncSource::Path
                    },
                    shadowed: None,
                }
            }
        }
    }

    /// A `Command` for the resolved rsync binary (C locale for stable output
    /// parsing) plus the argv capabilities the caller must honour.
    fn rsync_command(&self) -> (Command, RsyncCapabilities) {
        let resolved = self.resolved_rsync();
        let capabilities = resolved.capabilities();
        if capabilities.is_compatibility_mode() {
            debug!(
                rsync = %resolved.describe(),
                "driving rsync with the openrsync/2.6.9-compatible argv (issue #66)"
            );
        }
        let mut cmd = Command::new(&resolved.path); // ubs:ignore — trusted local rsync configuration/PATH selection, not remote input
        // Force C locale for consistent output parsing
        cmd.env("LC_ALL", "C");
        (cmd, capabilities)
    }

    /// Append `--compress-choice=zstd --compress-level=N` (or the legacy
    /// zlib-capped `--compress-level`) for the transfer's compression level.
    fn append_compression_args(&self, cmd: &mut Command, capabilities: &RsyncCapabilities) {
        cmd.args(capabilities.compression_args(self.compression_level_for_transfer()));
    }

    /// Set the idle retention (hours) after which the transfer-start janitor
    /// prunes a pooled target store on the worker (`0` = never). Callers pass
    /// the configured `[remediation.pooled_target] reaper_pooled_idle_hours`
    /// so the janitor and the reaper never disagree (issue #53).
    pub fn with_pooled_target_prune_idle_hours(mut self, hours: u32) -> Self {
        self.pooled_target_prune_idle_hours = hours;
        self
    }

    /// Set build id for remote execution.
    pub fn with_build_id(mut self, build_id: Option<u64>) -> Self {
        self.build_id = build_id;
        self
    }

    /// Set the remote worker's OS platform. Callers that know the selected
    /// worker pass `WorkerPlatform::from_worker(worker)` so transport and path
    /// construction match the worker. Omitting it keeps the Posix default.
    #[must_use]
    pub fn with_worker_platform(mut self, platform: WorkerPlatform) -> Self {
        self.worker_platform = platform;
        self
    }

    /// Set custom SSH options.
    #[cfg(unix)]
    #[allow(dead_code)] // Reserved for future CLI/config support
    pub fn with_ssh_options(mut self, options: SshOptions) -> Self {
        self.ssh_options = options;
        self
    }

    /// Set color mode for remote command output.
    #[allow(dead_code)] // Reserved for future CLI/config support
    pub fn with_color_mode(mut self, color_mode: ColorMode) -> Self {
        self.color_mode = color_mode;
        self
    }

    /// Set environment allowlist for remote execution.
    pub fn with_env_allowlist(mut self, allowlist: Vec<String>) -> Self {
        self.env_allowlist = allowlist;
        self
    }

    /// Force Layer 0 configuration-pack env pairs (bd-bqu38).
    ///
    /// These are resolved from the `[layer0]` config knobs by the caller and
    /// applied verbatim on the remote side — they do NOT consult the ambient
    /// local environment, so a knob takes effect even when the invoking shell
    /// has no such variable set.
    #[must_use]
    pub fn with_layer0_env(mut self, layer0_env: Vec<(String, String)>) -> Self {
        self.layer0_env = layer0_env;
        self
    }

    /// Restrict sync-to-remote uploads to a small include-only set.
    pub fn with_sync_include_patterns(mut self, patterns: Vec<String>) -> Self {
        self.sync_include_patterns = Some(patterns);
        self
    }

    /// Control whether sync-to-remote deletes extraneous remote files.
    pub fn with_sync_delete(mut self, delete: bool) -> Self {
        self.sync_delete = delete;
        self
    }

    /// Force content checks for sync-to-remote uploads.
    pub fn with_sync_checksum(mut self, checksum: bool) -> Self {
        self.sync_checksum = checksum;
        self
    }

    pub fn with_env_overrides(mut self, overrides: HashMap<String, String>) -> Self {
        self.env_overrides = Some(overrides);
        self
    }

    /// Set command timeout for remote execution.
    ///
    /// Different command types may need different timeouts. For example,
    /// test commands often need longer timeouts than build commands.
    #[cfg(unix)]
    #[allow(dead_code)] // Reserved for future CLI/config support
    pub fn with_command_timeout(mut self, timeout: std::time::Duration) -> Self {
        self.ssh_options.command_timeout = timeout;
        self
    }

    /// Set the compilation kind for command-specific handling.
    ///
    /// This enables the pipeline to apply appropriate wrappers for specific
    /// command types. For example, bun test commands are wrapped with an
    /// external timeout to protect against known CPU hang issues.
    pub fn with_compilation_kind(mut self, kind: Option<CompilationKind>) -> Self {
        self.compilation_kind = kind;
        self
    }

    /// Set the compilation configuration for timeout settings.
    ///
    /// This allows customizing external timeout durations for different command
    /// types and enables/disables timeout wrapping entirely.
    pub fn with_compilation_config(mut self, config: rch_common::CompilationConfig) -> Self {
        self.compilation_config = config;
        self
    }

    #[cfg(test)]
    pub fn with_estimated_transfer_bytes(mut self, bytes: Option<u64>) -> Self {
        self.estimated_transfer_bytes = bytes;
        self
    }

    fn effective_rsync_retry_config(&self) -> RetryConfig {
        let mut retry = self.transfer_config.retry.clone();
        if let Some(max_transfer_time_ms) = self.transfer_config.max_transfer_time_ms
            && max_transfer_time_ms > 0
        {
            retry.total_timeout_ms = max_transfer_time_ms;
        }
        retry
    }

    fn source_retry_config(&self, mut retry: RetryConfig) -> RetryConfig {
        if self.source_authority_prefix.is_some() {
            // A timed-out remote mutator may still arrive or run. Retire this
            // token through preparation cancellation before retrying a job;
            // otherwise an older upload could start after source verification.
            retry.max_attempts = 1;
        }
        retry
    }

    fn artifact_retry_config_for_size(&self, bytes: u64) -> RetryConfig {
        let mut retry = self.effective_rsync_retry_config();
        if self
            .transfer_config
            .max_transfer_time_ms
            .is_some_and(|ms| ms > 0)
        {
            return retry;
        }
        // Budget the uncompressed selected files, not dry-run protocol bytes.
        // A configured bandwidth cap can be slower than the conservative default.
        let bytes_per_second = self
            .transfer_config
            .bwlimit_kbps
            .filter(|limit| *limit > 0)
            .map_or(1024 * 1024, |limit| {
                limit.saturating_mul(1024).min(1024 * 1024)
            });
        let seconds = bytes.div_ceil(bytes_per_second);
        retry.total_timeout_ms = retry.total_timeout_ms.max(
            30_000_u64
                .saturating_add(seconds.saturating_mul(1000))
                .min(TransferConfig::MAX_SYNC_TIMEOUT_MS),
        );
        retry
    }

    async fn artifact_retry_config(
        &self,
        worker: &WorkerConfig,
        escaped_remote_path: &str,
        patterns: &[String],
    ) -> RetryConfig {
        let fallback = self.effective_rsync_retry_config();
        // An explicit operator ceiling remains the whole retrieval budget:
        // do not spend an additional planning window before starting it.
        if self
            .transfer_config
            .max_transfer_time_ms
            .is_some_and(|ms| ms > 0)
        {
            return fallback;
        }
        let estimate = self
            .execute_retrieval_rsync(
                &fallback,
                "estimate_artifact_retrieval",
                || {
                    let mut cmd =
                        self.build_retrieve_command(worker, escaped_remote_path, patterns);
                    cmd.arg("--dry-run");
                    cmd
                },
                |_| {},
            )
            .await;
        match estimate {
            Ok(output) if output.status.success() => {
                if let Some(bytes) =
                    parse_rsync_selected_size(&String::from_utf8_lossy(&output.stdout))
                {
                    let retry = self.artifact_retry_config_for_size(bytes);
                    info!(
                        selected_bytes = bytes,
                        total_timeout_ms = retry.total_timeout_ms,
                        "Artifact retrieval budget from selected remote files"
                    );
                    return retry;
                }
                warn!("Artifact size estimate missing; retaining configured retrieval budget");
            }
            Ok(output) => warn!(
                "Artifact size estimate failed: {}",
                String::from_utf8_lossy(&output.stderr)
            ),
            Err(error) => warn!(
                "Artifact size estimate failed: {error}; retaining configured retrieval budget"
            ),
        }
        fallback
    }

    fn source_sync_attempt_timeout(&self, effective_excludes: &[String]) -> std::time::Duration {
        if self.transfer_config.sync_timeout_ms.is_some() {
            return self.transfer_config.sync_timeout_for_payload(0);
        }
        let payload_bytes = self.estimated_transfer_bytes.unwrap_or_else(|| {
            local_sync_payload_upper_bound(&self.project_root, effective_excludes)
        });
        self.transfer_config.sync_timeout_for_payload(payload_bytes)
    }

    /// Silence policy for the streaming source sync (issue #59): abort when
    /// rsync emits no output at all for the configured window. `None` when the
    /// operator disabled silence detection (`source_sync_silence_timeout_secs
    /// = 0`).
    fn source_sync_silence_policy(&self, worker: &WorkerConfig) -> Option<SyncSilencePolicy> {
        let secs = self.transfer_config.source_sync_silence_timeout_secs;
        (secs > 0).then(|| SyncSilencePolicy {
            limit: std::time::Duration::from_secs(secs),
            worker_id: worker.id.to_string(),
            phase: "source_sync",
        })
    }

    /// Override the remote project path used for sync and command execution.
    ///
    /// Intended for canonical multi-repo layouts where the remote path must
    /// match deterministic host topology (for example `/data/projects/<repo>`).
    pub fn with_remote_path_override(mut self, remote_path: impl Into<String>) -> Self {
        let remote_path = remote_path.into();
        let trimmed = remote_path.trim();
        if trimmed.is_empty() {
            warn!("Ignoring empty remote path override");
            return self;
        }
        if !trimmed.starts_with('/') && !is_windows_drive_abs_path(trimmed) {
            warn!(
                "Ignoring remote path override that is not absolute: {}",
                trimmed
            );
            return self;
        }
        if trimmed.contains('\n') || trimmed.contains('\r') || trimmed.contains('\0') {
            warn!("Ignoring unsafe remote path override containing control characters");
            return self;
        }

        let normalized = if trimmed.len() > 1 {
            trimmed.trim_end_matches('/').to_string()
        } else {
            "/".to_string()
        };
        self.remote_path_override = Some(normalized);
        self
    }

    /// Set the remote Cargo target directory basename.
    ///
    /// The value must be a single relative path segment because it is appended
    /// to the worker-side project root. Invalid names are ignored so callers
    /// fail closed to the default path rather than creating surprising remote
    /// paths.
    pub fn with_remote_cargo_target_dir_name(mut self, name: impl Into<String>) -> Self {
        let name = name.into();
        let trimmed = name.trim();
        if trimmed.is_empty()
            || trimmed == "."
            || trimmed == ".."
            || trimmed.contains('/')
            || trimmed.contains('\\')
            || trimmed.contains('\n')
            || trimmed.contains('\r')
            || trimmed.contains('\0')
        {
            warn!(
                "Ignoring invalid remote Cargo target directory name: {:?}",
                name
            );
            return self;
        }

        self.remote_cargo_target_dir_name = trimmed.to_string();
        self
    }

    /// Set an ABSOLUTE remote Cargo target directory that outlives the
    /// per-command remote root (issue #60).
    ///
    /// Used by clean-overlay runs: their remote project root embeds a
    /// per-command job nonce and is reaped at teardown, so the pooled
    /// `.rch-target-…-pool-…` store must live OUTSIDE it to be reusable.
    /// The path must be absolute (POSIX or Windows drive-letter); invalid
    /// values are ignored so callers fail closed to the derived per-root
    /// default rather than producing surprising remote paths.
    pub fn with_remote_cargo_target_dir_override(mut self, path: impl Into<String>) -> Self {
        let path = path.into();
        let trimmed = path.trim();
        if trimmed.is_empty()
            || (!trimmed.starts_with('/') && !is_windows_drive_abs_path(trimmed))
            || trimmed.contains('\n')
            || trimmed.contains('\r')
            || trimmed.contains('\0')
            || trimmed
                .split(['/', '\\'])
                .any(|segment| segment == "." || segment == "..")
        {
            warn!(
                "Ignoring invalid remote Cargo target directory override: {:?}",
                path
            );
            return self;
        }

        self.remote_cargo_target_dir_override = Some(trimmed.trim_end_matches('/').to_string());
        self
    }

    fn remote_cargo_target_dir_for_remote_path(&self, remote_path: &str) -> String {
        if let Some(override_path) = &self.remote_cargo_target_dir_override {
            return override_path.clone();
        }
        format!(
            "{}/{}",
            remote_path.trim_end_matches('/'),
            self.remote_cargo_target_dir_name
        )
    }

    /// Parent directory of the pooled target-dir override, shell-escaped, for
    /// the transfer-start janitor's pooled sweep. `None` when no override is
    /// set or a durable source grant is active. That grant owns this pool,
    /// not its siblings: age cannot release another job's retained outputs.
    fn escaped_pooled_target_override_parent(&self) -> Option<String> {
        if self.source_authority_prefix.is_some() {
            return None;
        }
        let override_path = self.remote_cargo_target_dir_override.as_deref()?;
        let (parent, _basename) = override_path.rsplit_once('/')?;
        if parent.is_empty() {
            return None;
        }
        Some(escape(Cow::from(parent.to_string())).to_string())
    }

    fn env_value(&self, key: &str) -> Option<String> {
        if let Some(ref overrides) = self.env_overrides
            && let Some(value) = overrides.get(key)
        {
            return Some(value.clone());
        }
        std::env::var(key).ok()
    }

    /// Compute the managed, worker-scoped value (and the directory that must
    /// exist for it) for an env var that we pin into the remote build zone.
    ///
    /// Returns `Some((value, ensure_dir))` for a managed key, or `None` for any
    /// other key. This is the single source of truth for managed placement,
    /// shared by [`Self::rewrite_remote_env_value`] (rewrites a forwarded value)
    /// and [`Self::build_remote_env_plan`] (injects the key unconditionally). All
    /// managed paths sit under `<remote_path>/.rch-*` so they land in the
    /// managed `/data/tmp` build zone and are reclaimable.
    fn managed_remote_env_value(&self, key: &str, remote_path: &str) -> Option<(String, String)> {
        let go_base = format!("{remote_path}/.rch-go");
        match key {
            // Absolute or host-specific target directories are brittle on workers.
            // Force a remote-scoped target dir rooted in the synchronized project.
            "CARGO_TARGET_DIR" => {
                let target_dir = self.remote_cargo_target_dir_for_remote_path(remote_path);
                Some((target_dir.clone(), target_dir))
            }
            // Temporary directories may point to host-only volatile mounts.
            // Keep temp files project-scoped on the worker for stability.
            "TMPDIR" | "TMP" | "TEMP" => {
                let temp_dir = format!("{remote_path}/.rch-tmp");
                Some((temp_dir.clone(), temp_dir))
            }
            // Go build cache, module cache, and GOPATH — pin under the managed
            // build zone so `go`/`bun`-orchestrated Go builds don't scatter large
            // caches into host-global locations (~/.cache/go-build, ~/go).
            "GOCACHE" => {
                let dir = format!("{go_base}/cache");
                Some((dir.clone(), dir))
            }
            "GOMODCACHE" => {
                let dir = format!("{go_base}/mod");
                Some((dir.clone(), dir))
            }
            "GOPATH" => {
                let dir = format!("{go_base}/path");
                Some((dir.clone(), dir))
            }
            _ => None,
        }
    }

    fn rewrite_remote_env_value(
        &self,
        key: &str,
        value: &str,
        remote_path: &str,
    ) -> (String, Option<String>, bool) {
        // Delegate to the managed-placement helper so a forwarded ABSOLUTE value
        // (e.g. an absolute `.cargo/config.toml` target-dir) is rewritten to the
        // managed worker-scoped path. Non-managed keys pass through unchanged.
        match self.managed_remote_env_value(key, remote_path) {
            Some((managed_value, ensure_dir)) => {
                let rewritten = managed_value != value;
                (managed_value, Some(ensure_dir), rewritten)
            }
            None => (value.to_string(), None, false),
        }
    }

    /// Format one env assignment for the remote command prefix.
    ///
    /// Managed directory values are resolved to their PHYSICAL path on the
    /// worker (`pwd -P`) at execution time. Worker mirror layouts route the
    /// synced project root through symlinks (e.g. `/Users/... -> /data/...`),
    /// and consumers that walk TMPDIR ancestors with `O_NOFOLLOW` hardening
    /// (pi's auth storage does, and refuses even trusted symlinks when the
    /// job runs as root) fail with ENOTDIR on such a path. `cd` runs after
    /// the `mkdir -p` ensure-dirs step, so the directory exists; if it still
    /// fails, fall back to the literal managed path.
    fn format_env_assignment(&self, key: &str, escaped: &str, managed_dir: bool) -> String {
        if managed_dir && !self.worker_platform.is_windows() {
            format!("{key}=\"$(cd {escaped} 2>/dev/null && pwd -P || printf %s {escaped})\"")
        } else {
            format!("{key}={escaped}")
        }
    }

    fn build_remote_env_plan(&self, remote_path: &str) -> RemoteEnvPlan {
        let mut parts = Vec::new();
        let mut applied = Vec::new();
        let mut rejected = Vec::new();
        let mut ensure_dirs = Vec::new();
        let mut restricted_dirs = Vec::new();

        for raw_key in &self.env_allowlist {
            let key = raw_key.trim();
            if key.is_empty() {
                continue;
            }
            if !is_valid_env_key(key) {
                info!(
                    "Rejecting env var '{}': invalid key name (must start with letter/underscore, contain only alphanumeric/underscore)",
                    key
                );
                rejected.push(key.to_string());
                continue;
            }

            let Some(original_value) = self.env_value(key) else {
                continue;
            };

            let (effective_value, ensure_dir, rewritten) =
                self.rewrite_remote_env_value(key, &original_value, remote_path);
            if rewritten {
                info!(
                    "Rewriting {} for remote execution (worker-scoped path): {} -> {}",
                    key, original_value, effective_value
                );
            }

            let Some(escaped) = shell_escape_value(&effective_value) else {
                info!(
                    "Rejecting env var '{}': value contains unsafe characters (newline, carriage return, or NUL)",
                    key
                );
                rejected.push(key.to_string());
                continue;
            };

            let managed_dir = ensure_dir.is_some();
            if let Some(dir) = ensure_dir {
                if !ensure_dirs.iter().any(|existing| existing == &dir) {
                    ensure_dirs.push(dir.clone());
                }
                if matches!(key, "TMPDIR" | "TMP" | "TEMP")
                    && !restricted_dirs.iter().any(|existing| existing == &dir)
                {
                    restricted_dirs.push(dir);
                }
            }

            parts.push(self.format_env_assignment(key, &escaped, managed_dir));
            applied.push(key.to_string());
        }

        // Unconditionally inject the managed build-artifact env vars, regardless
        // of what the local environment forwarded. Any of these already handled
        // above (present in the env AND allowlisted) were rewritten to the same
        // managed value, so skip them here to avoid a duplicate assignment; the
        // rest are injected fresh. This is what forces wrapper/unclassified
        // builds — which may never forward CARGO_TARGET_DIR/GOCACHE/etc. — to
        // still drop artifacts inside the managed `/data/tmp` zone.
        for &key in FORCED_MANAGED_ENV_KEYS {
            if applied.iter().any(|k| k == key) {
                continue;
            }
            let Some((managed_value, ensure_dir)) = self.managed_remote_env_value(key, remote_path)
            else {
                continue;
            };
            let Some(escaped) = shell_escape_value(&managed_value) else {
                continue;
            };
            if !ensure_dirs.iter().any(|existing| existing == &ensure_dir) {
                ensure_dirs.push(ensure_dir.clone());
            }
            if matches!(key, "TMPDIR" | "TMP" | "TEMP")
                && !restricted_dirs
                    .iter()
                    .any(|existing| existing == &ensure_dir)
            {
                restricted_dirs.push(ensure_dir);
            }
            parts.push(self.format_env_assignment(key, &escaped, true));
            applied.push(key.to_string());
        }

        // Remote loop-break (see REMOTE_LOOP_BREAK_ENV): stop the worker's own
        // cargo shim from offloading the job onward. Injected unconditionally
        // and last-wins over a forwarded value, because a caller that exported
        // RCH_CARGO_WRAPPER_BYPASS=0 locally must not be able to re-arm the
        // shim on the worker and reintroduce the bounce.
        {
            let (key, value) = REMOTE_LOOP_BREAK_ENV;
            if let Some(escaped) = shell_escape_value(value) {
                let assignment_prefix = format!("{key}=");
                parts.retain(|part| !part.starts_with(&assignment_prefix));
                applied.retain(|applied_key| applied_key != key);
                parts.push(self.format_env_assignment(key, &escaped, false));
                applied.push(key.to_string());
            }
        }

        // Layer 0 configuration pack (bd-bqu38): explicit config-resolved
        // assignments (e.g. CARGO_PROFILE_RELEASE_LTO=thin). These never come
        // from the ambient local environment — they are resolved from the
        // `[layer0]` knobs — so they bypass the allowlist lookup and apply
        // even when the caller's environment is empty. Skip any key already
        // applied above so an operator-forwarded duplicate cannot disagree
        // with the pack.
        for (key, value) in &self.layer0_env {
            if applied.iter().any(|k| k == key) {
                continue;
            }
            let Some(escaped) = shell_escape_value(value) else {
                warn!("Rejecting Layer 0 env var '{key}': unsafe characters in value");
                continue;
            };
            parts.push(self.format_env_assignment(key, &escaped, false));
            applied.push(key.to_string());
        }

        let prefix = if parts.is_empty() {
            String::new()
        } else {
            format!("{} ", parts.join(" "))
        };

        RemoteEnvPlan {
            env_prefix: EnvPrefix {
                prefix,
                applied,
                rejected,
            },
            ensure_dirs,
            restricted_dirs,
        }
    }

    #[cfg(test)]
    fn build_env_prefix(&self) -> EnvPrefix {
        self.build_remote_env_plan(&self.remote_path()).env_prefix
    }

    /// Get the effective exclude patterns by merging config defaults with .rchignore.
    ///
    /// Merge order (deterministic):
    /// 1. Default exclude patterns (from config)
    /// 2. User config exclude patterns (already in transfer_config)
    /// 3. Project-local .rchignore patterns (if present)
    fn get_effective_excludes(&self) -> Vec<String> {
        let mut excludes = Vec::new();
        for pattern in &self.transfer_config.exclude_patterns {
            let normalized = normalize_config_exclude_pattern(pattern);
            if normalized != pattern {
                debug!(
                    "Rewriting legacy broad exclude pattern '{}' to '{}'",
                    pattern, normalized
                );
            }
            if !excludes.iter().any(|existing| existing == normalized) {
                excludes.push(normalized.to_string());
            }
        }

        // Always protect remote-only runtime scratch/output directories from rsync --delete.
        // Without this, concurrent builds targeting the same remote root can remove each
        // other's in-flight compiler state (e.g. incremental dep-graph.part files).
        for pattern in REMOTE_RUNTIME_EXCLUDE_PATTERNS {
            if !excludes.iter().any(|existing| existing == pattern) {
                excludes.push((*pattern).to_string());
            }
        }
        for pattern in SOURCE_EPHEMERAL_EXCLUDE_PATTERNS {
            if !excludes.iter().any(|existing| existing == pattern) {
                excludes.push((*pattern).to_string());
            }
        }

        // Linked git worktree: its `.git` is a FILE pointing back into the parent
        // repo (`gitdir: …/.git/worktrees/<name>`). That path does not exist on
        // the worker, so syncing the file verbatim leaves a dangling pointer that
        // breaks remote git/cargo resolution and forces a fail-open to local. The
        // `.git/` directory exclude above only matches directories, so the FILE
        // slips through unless we exclude it explicitly. Normal repos already sync
        // with NO `.git` at all and build fine, so this just gives the worktree the
        // same git-free remote source tree.
        if let Some((worktree_git_exclude, pointer)) =
            git_worktree_upload_exclude(&self.project_root)
            && !excludes.contains(&worktree_git_exclude)
        {
            info!(
                "Linked git worktree detected at {} (gitdir: {}); excluding dangling '.git' file from upload so the remote build resolves git-free like a normal repo",
                self.project_root.display(),
                pointer.gitdir
            );
            excludes.push(worktree_git_exclude);
        }

        // Read and merge .rchignore if present
        let rchignore_path = self.project_root.join(".rchignore");
        if let Ok(patterns) = parse_rchignore(&rchignore_path) {
            let original_count = excludes.len();
            for pattern in patterns {
                if !excludes.contains(&pattern) {
                    excludes.push(pattern);
                }
            }
            let added = excludes.len() - original_count;
            if added > 0 {
                info!(
                    "Loaded {} pattern(s) from .rchignore (total: {})",
                    added,
                    excludes.len()
                );
            }
        }

        excludes
    }

    /// Return the exact filter policy used by source uploads.
    ///
    /// Source-content receipts bind these values alongside the file manifest so
    /// a verifier can distinguish a byte-identical manifest produced under a
    /// different include/exclude policy.
    pub(crate) fn source_content_filter_policy(
        &self,
    ) -> (Option<Vec<String>>, Vec<String>, bool, bool) {
        (
            self.sync_include_patterns.clone(),
            self.get_effective_excludes(),
            self.sync_delete,
            self.sync_checksum,
        )
    }

    fn append_sync_filter_args(&self, cmd: &mut Command, effective_excludes: &[String]) {
        if let Some(include_patterns) = &self.sync_include_patterns {
            cmd.arg("--prune-empty-dirs");
            for pattern in include_patterns {
                cmd.arg("--include").arg(pattern);
            }
            cmd.arg("--exclude").arg("*");
        } else {
            for pattern in effective_excludes {
                cmd.arg("--exclude").arg(pattern);
            }
        }
    }

    /// Enumerate the exact regular-file universe selected by the upload's
    /// rsync filters without creating or changing a destination tree.
    ///
    /// Reusing rsync for selection is deliberate: a second glob interpreter
    /// would eventually drift from rsync's first-match-wins semantics and let a
    /// receipt authenticate a different set of files than the transfer itself.
    pub(crate) async fn enumerate_source_content_files(&self) -> Result<Vec<PathBuf>> {
        if self.worker_platform.is_windows() {
            anyhow::bail!("source-content receipts require the rsync transport");
        }
        self.enumerate_source_upload_files().await
    }

    // Selection runs entirely locally. Windows include-only tar uploads must
    // use the same rsync filters as POSIX uploads, without enabling the separate
    // remote source-content receipt protocol on Windows.
    async fn enumerate_source_upload_files(&self) -> Result<Vec<PathBuf>> {
        let effective_excludes = self.get_effective_excludes();
        let nonexistent_destination = std::env::temp_dir().join(format!(
            "rch-source-content-enumeration-{}",
            uuid::Uuid::new_v4()
        ));
        let (mut cmd, capabilities) = self.rsync_command();
        cmd.arg("-arni")
            .arg("--out-format=%i\t%n")
            .args(capabilities.no_motd_args());
        add_portable_rsync_archive_args(&mut cmd);
        self.append_sync_filter_args(&mut cmd, &effective_excludes);
        cmd.arg(format!("{}/", self.project_root.display()))
            .arg(format!("{}/", nonexistent_destination.display()));
        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());

        let output = run_source_content_rsync_capture(
            cmd,
            "source-content transfer-universe enumeration",
            self.source_sync_attempt_timeout(&effective_excludes),
        )
        .await?;
        if !output.status.success() {
            anyhow::bail!(
                "source-content enumeration failed (exit {:?}): {}",
                output.status.code(),
                String::from_utf8_lossy(&output.stderr).trim()
            );
        }
        if !output.stderr.is_empty() {
            anyhow::bail!(
                "source-content enumeration produced stderr: {}",
                String::from_utf8_lossy(&output.stderr).trim()
            );
        }

        let stdout = std::str::from_utf8(&output.stdout)
            .context("source-content enumeration output was not UTF-8")?;
        let mut paths = Vec::new();
        for line in stdout.lines() {
            if line.is_empty() || line.starts_with("created directory ") {
                continue;
            }
            let (itemized, raw_path) = line.split_once('\t').ok_or_else(|| {
                anyhow::anyhow!("unexpected source-content enumeration line: {line:?}")
            })?;
            if itemized.as_bytes().get(1) == Some(&b'd') {
                continue;
            }
            if itemized.as_bytes().get(1) != Some(&b'f') {
                anyhow::bail!(
                    "source-content proof refuses non-regular rsync item {itemized:?} at {raw_path:?}"
                );
            }
            let relative = raw_path.strip_prefix("./").unwrap_or(raw_path);
            if relative.is_empty()
                || relative.chars().any(char::is_control)
                || Path::new(relative).components().any(|component| {
                    matches!(
                        component,
                        Component::ParentDir | Component::RootDir | Component::Prefix(_)
                    )
                })
            {
                anyhow::bail!("unsafe source-content path emitted by rsync: {raw_path:?}");
            }
            paths.push(PathBuf::from(relative));
        }
        paths.sort();
        if paths.windows(2).any(|pair| pair[0] == pair[1]) {
            anyhow::bail!("source-content enumeration emitted duplicate paths");
        }
        Ok(paths)
    }

    /// Prove that a checksum-aware rsync would transfer no selected file and
    /// delete no selected remote entry. Directory metadata differences are not
    /// source bytes and are intentionally ignored here; the per-file remote
    /// verifier independently checks type, length, mode, and SHA-256.
    pub(crate) async fn verify_source_content_rsync_barrier(
        &self,
        worker: &WorkerConfig,
    ) -> Result<()> {
        if self.worker_platform.is_windows() {
            anyhow::bail!("source-content receipts require the rsync transport");
        }

        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));
        let destination = format!("{}@{}:{}", worker.user, worker.host, escaped_remote_path);
        let effective_excludes = self.get_effective_excludes();
        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));
        let ssh_command = self.build_rsync_ssh_command(escaped_identity.as_ref());

        let (mut cmd, capabilities) = self.rsync_command();
        cmd.arg("-azn")
            .arg("--checksum")
            .arg("--itemize-changes")
            .arg("--out-format=%i\t%n")
            .args(capabilities.no_motd_args())
            .arg("-e")
            .arg(ssh_command)
            .arg("--rsync-path")
            .arg(self.source_rsync_path(format!("mkdir -p {} && rsync", escaped_remote_path)));
        add_portable_rsync_archive_args(&mut cmd);
        if self.sync_delete {
            cmd.arg("--delete");
        }
        self.append_sync_filter_args(&mut cmd, &effective_excludes);
        cmd.arg(format!("{}/", self.project_root.display()))
            .arg(destination);
        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());

        let output = run_source_content_rsync_capture(
            cmd,
            "source-content rsync no-delta barrier",
            self.source_sync_attempt_timeout(&effective_excludes),
        )
        .await?;
        if !output.status.success() {
            anyhow::bail!(
                "source-content rsync barrier failed (exit {:?}): {}",
                output.status.code(),
                String::from_utf8_lossy(&output.stderr).trim()
            );
        }
        if !output.stderr.is_empty() {
            anyhow::bail!(
                "source-content rsync barrier produced stderr: {}",
                String::from_utf8_lossy(&output.stderr).trim()
            );
        }
        let stdout = std::str::from_utf8(&output.stdout)
            .context("source-content rsync barrier output was not UTF-8")?;
        let changed = stdout
            .lines()
            .filter(|line| !line.is_empty())
            .filter(|line| {
                !line
                    .split_once('\t')
                    .is_some_and(|(itemized, _)| itemized.as_bytes().get(1) == Some(&b'd'))
            })
            .collect::<Vec<_>>();
        if !changed.is_empty() {
            let preview = changed
                .iter()
                .take(8)
                .copied()
                .collect::<Vec<_>>()
                .join(" | ");
            anyhow::bail!(
                "source-content rsync barrier detected {} remote delta(s): {}",
                changed.len(),
                preview
            );
        }
        Ok(())
    }

    /// Belt-and-suspenders source-integrity guard for retrieval (RCH bug
    /// `d7xc3`). Scans the LOCAL project root's top-level entries and
    /// returns a list of explicit `--exclude /<entry>` rules for every
    /// entry that ISN'T in the allowed-artifact-roots set. Rsync's filter
    /// semantics: anchored excludes match only at the rsync transfer
    /// root, so this rules out an entire class of source-overwrite bugs
    /// (unanchored artifact patterns, malformed includes, hostile remote
    /// layouts) by stopping rsync from even descending into known-source
    /// top-level directories like `rch/`, `rch-common/`, etc.
    ///
    /// Why scan LOCAL not remote: the local project root mirrors the
    /// remote source tree (we uploaded it). Listing local is fast (one
    /// `read_dir`) and avoids an extra SSH round-trip. The threat model
    /// is "stale or hostile remote tree dirties local source"; if our
    /// LOCAL layout is being tampered with, the operator has bigger
    /// problems than this retrieve.
    fn local_source_roots_to_exclude(
        &self,
        allowed_roots: &BTreeSet<String>,
        artifact_patterns: &[String],
    ) -> Vec<String> {
        let Ok(entries) = std::fs::read_dir(&self.retrieval_reference_root) else {
            // Project root unreadable → nothing to exclude here. Other
            // retrieval excludes (REMOTE_RUNTIME_EXCLUDE_PATTERNS, the
            // final `--exclude "*"`) still apply, so retrieval remains
            // safe; we just lose the explicit belt-and-suspenders layer.
            return Vec::new();
        };
        let mut excludes: Vec<String> = Vec::new();
        for entry in entries.flatten() {
            let file_name = entry.file_name();
            let Some(name) = file_name.to_str() else {
                continue;
            };
            if name.is_empty() || name == "." || name == ".." {
                continue;
            }
            if allowed_roots.contains(name) {
                // Permit descent into this artifact root.
                continue;
            }
            if artifact_patterns_allow_top_level_entry(artifact_patterns, name) {
                // Permit exact top-level artifact files and top-level artifact
                // globs such as `*.tsbuildinfo`. Otherwise an existing local
                // artifact file would be excluded before the later include rule
                // can refresh it from the worker.
                continue;
            }
            // Anchor with leading `/` so the exclude matches ONLY at the
            // rsync transfer root, not at any nested depth.
            let is_dir = entry.file_type().map(|t| t.is_dir()).unwrap_or(false);
            let name = escape_rsync_filter_literal_component(name);
            let exclude = if is_dir {
                format!("/{name}/")
            } else {
                format!("/{name}")
            };
            excludes.push(exclude);
        }
        // Deterministic order so test assertions and tracing logs are stable.
        excludes.sort();
        excludes
    }

    /// Retrieval-side excludes used when pulling artifacts back from the worker.
    ///
    /// This is intentionally narrower than upload filtering. Upload wants broad
    /// project hygiene exclusions (`target/`, `dist/`, coverage caches, etc.),
    /// but retrieval still needs to descend into artifact roots like `target/`
    /// and `build/`. The retrieval pass applies runtime scratch guards plus
    /// project-local `.rchignore` directory patterns that cannot hide the
    /// requested artifacts. File globs and artifact-root directories are skipped
    /// because rsync evaluates these excludes before the artifact includes.
    fn get_retrieval_excludes(&self, artifact_patterns: &[String]) -> Vec<String> {
        let mut excludes = Vec::new();

        for pattern in REMOTE_RUNTIME_EXCLUDE_PATTERNS {
            if !excludes.iter().any(|existing| existing == pattern) {
                excludes.push((*pattern).to_string());
            }
        }

        let rchignore_path = self.retrieval_reference_root.join(".rchignore");
        if let Ok(patterns) = parse_rchignore(&rchignore_path) {
            let original_count = excludes.len();
            for pattern in patterns {
                if retrieval_exclude_can_block_artifacts(&pattern, artifact_patterns) {
                    debug!(
                        "Skipping retrieval exclude pattern '{}' because it may hide requested artifacts",
                        pattern
                    );
                    continue;
                }
                if !excludes.contains(&pattern) {
                    excludes.push(pattern);
                }
            }
            let added = excludes.len() - original_count;
            if added > 0 {
                info!(
                    "Loaded {} retrieval exclude pattern(s) from .rchignore (total: {})",
                    added,
                    excludes.len()
                );
            }
        }

        excludes
    }

    fn compression_level_for_transfer(&self) -> u32 {
        self.transfer_config
            .select_compression_level(self.estimated_transfer_bytes)
    }

    /// Get the remote project path on the worker.
    pub fn remote_path(&self) -> String {
        if let Some(remote_path) = &self.remote_path_override {
            return remote_path.clone();
        }
        // A Windows worker uses `C:/rch` regardless of the (Unix) global
        // `remote_base`, since `/data/tmp/rch` does not exist there and cargo
        // needs a drive-letter path. Everything else uses the configured base.
        let base = if self.worker_platform.is_windows() {
            WINDOWS_DEFAULT_REMOTE_BASE
        } else {
            self.transfer_config.remote_base.trim_end_matches('/')
        };
        format!("{}/{}/{}", base, self.project_id, self.project_hash)
    }

    /// Get the remote Cargo target directory path on the worker.
    pub fn remote_cargo_target_dir(&self) -> String {
        self.remote_cargo_target_dir_for_remote_path(&self.remote_path())
    }

    #[cfg(test)]
    pub fn remote_pgid_file_path(&self) -> Option<String> {
        self.build_id
            .map(|build_id| Self::remote_pgid_file_path_for_root(&self.remote_path(), build_id))
    }

    pub fn remote_run_dir_for_root(remote_root: &str) -> String {
        let project_id = project_id_from_path(Path::new(remote_root));
        let hash = blake3::hash(remote_root.as_bytes()).to_hex();
        format!("/tmp/rch-run/{}-{}", project_id, &hash[..16])
    }

    pub fn remote_pgid_file_path_for_root(remote_root: &str, build_id: u64) -> String {
        format!(
            "{}/{build_id}.pgid",
            Self::remote_run_dir_for_root(remote_root)
        )
    }

    /// Build the full remote command string with all wrappers.
    fn build_remote_command(&self, command: &str, toolchain: Option<&ToolchainInfo>) -> String {
        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));
        let toolchain_command = wrap_command_with_toolchain(command, toolchain);

        let env_plan = self.build_remote_env_plan(&remote_path);
        if !env_plan.env_prefix.applied.is_empty() {
            debug!("Forwarding env vars: {:?}", env_plan.env_prefix.applied);
        }
        if !env_plan.env_prefix.rejected.is_empty() {
            warn!("Skipping env vars: {:?}", env_plan.env_prefix.rejected);
        }
        let env_command = if env_plan.env_prefix.prefix.is_empty() {
            toolchain_command
        } else {
            format!("{}{}", env_plan.env_prefix.prefix, toolchain_command)
        };

        // Apply color mode environment variables
        let colored_command = wrap_command_with_color(&env_command, self.color_mode);

        // Apply external process timeout wrapper for commands known to hang.
        // Bun tests have known issues where they can hang at 100% CPU indefinitely:
        // - https://github.com/oven-sh/bun/issues/21277 (sync loops block timeout)
        // - https://github.com/oven-sh/bun/issues/6751 (multiple test files cause hangs)
        // The `timeout` command provides a hard kill that works even for CPU-bound loops.
        let timeout_wrapped_command = self.wrap_with_external_timeout(&colored_command);
        // Wall-clock cap in seconds for the pgid-tracked path's watchdog (0 = disabled).
        // Same source of truth as `wrap_with_external_timeout`, applied via an
        // in-session group-kill watchdog instead of `timeout(1)` (see build_id branch).
        let external_timeout_secs = if self.compilation_config.external_timeout_enabled() {
            self.compilation_config
                .timeout_for_kind(self.compilation_kind)
                .as_secs()
        } else {
            0
        };

        // Ensure remote-scoped env directories exist before build execution.
        let ensure_dirs_command = if env_plan.ensure_dirs.is_empty() {
            String::new()
        } else {
            let escaped_dirs = env_plan
                .ensure_dirs
                .iter()
                .map(|dir| escape(Cow::from(dir.as_str())).to_string())
                .collect::<Vec<_>>()
                .join(" ");
            let mut command = format!("mkdir -p {} && ", escaped_dirs);
            if !env_plan.restricted_dirs.is_empty() && !self.worker_platform.is_windows() {
                let escaped_restricted = env_plan
                    .restricted_dirs
                    .iter()
                    .map(|dir| escape(Cow::from(dir.as_str())).to_string())
                    .collect::<Vec<_>>()
                    .join(" ");
                command.push_str(&format!("chmod 1700 {} && ", escaped_restricted));
            }
            command
        };

        // Force LC_ALL=C to ensure English output for error parsing.
        // Touching the remote root refreshes directory mtime so age-based cleanup
        // treats actively used caches as hot.
        // Wrap command to run in project directory.
        //
        // The pgid-watchdog path is Unix-only: it relies on `setsid`, process
        // groups, and `kill -KILL -$pgid`, none of which exist under Git's `sh`
        // on a Windows worker. Worse, its backgrounded timer subshell keeps the
        // SSH channel's stdio open there, so sshd never sees EOF and every
        // *successful* build hangs until the command timeout (the #20 failure
        // mode, re-triggered on Windows). Windows workers therefore run the
        // command directly (the `else` branch). The tradeoff: the daemon's
        // stuck-detector reaps a runaway by SIGKILLing the remote process GROUP
        // via the `.pgid` file the watchdog writes, so it does NOT cover Windows
        // builds (no pgid file, no process groups). Bounding there is only the
        // SSH ConnectTimeout and the in-loop `command_timeout`, which kill the
        // local `ssh` on expiry; a detached `cargo.exe`/`rustc.exe` can still
        // outlive the channel close. Acceptable for v1 single-Windows-worker use.
        let execution_command =
            if let Some(build_id) = self.build_id.filter(|_| !self.worker_platform.is_windows()) {
                let remote_pgid_file = Self::remote_pgid_file_path_for_root(&remote_path, build_id);
                let remote_run_dir = Self::remote_run_dir_for_root(&remote_path);
                let escaped_pgid_file = escape(Cow::from(remote_pgid_file));
                let escaped_run_dir = escape(Cow::from(remote_run_dir));
                // For the pgid-tracked path we do NOT use the `timeout(1)` wrapper:
                // `timeout --foreground` only signals its direct child, so a livelocked
                // test binary (and its fixtures) that the test harness spawned survive
                // the cap and reparent to init as 20-45h PPID-1 orphans. Instead we run
                // the raw command and arm an in-session watchdog that, at the wall-clock
                // cap, SIGKILLs the whole process group (`-$pgid`) — the SAME group the
                // daemon's stuck-detector kills (`cancellation.rs`). One group, both
                // reapers, entire tree. Killing the group includes the leader `sh -c`,
                // but that is a child of the ssh `sh -s`, so the outer shell still
                // reports 137 (128+SIGKILL) for clean timeout exit semantics.
                let escaped_command = escape(Cow::from(colored_command.as_str()));
                // The watchdog publishes boot UUID + leader start ticks + build ID
                // before launching the workload, so daemon recovery rejects an
                // observed reboot or reused leader before signalling. An abnormal
                // external leader exit can still race the final proc read and kill.
                // NOTE: group kill is `kill -KILL -PGID` with NO `--`. dash's (/bin/sh)
                // kill builtin mishandles `kill -KILL -- -PGID` (the `--` makes it a
                // no-op), so `--` would silently fail to reap on the Ubuntu fleet. The
                // `-PGID` form works in both dash and bash.
                // The timer detaches standard I/O and its sleep closes fd 3.
                // Normal completion stops the timer and waits while it reaps
                // that sleep, leaving neither an open SSH pipe nor an orphan
                // group member that would obstruct later recovery. Only the
                // timer retains fd 3 for its deadline marker before group kill.
                let watchdog = escape(Cow::Owned(remote_build_watchdog_script()));

                format!(
                    "if ! command -v setsid >/dev/null 2>&1; then \
printf '\\n%s\\n' {} >&2; exit 125; fi; \
mkdir -p {} && rm -f {} && \
setsid sh -c {} rch-build {} {} {} {} sh -lc {} 3>&2",
                    self.remote_process_setup_marker(),
                    escaped_run_dir,
                    escaped_pgid_file,
                    watchdog,
                    escaped_pgid_file,
                    external_timeout_secs,
                    self.deadline_marker,
                    build_id,
                    escaped_command,
                )
            } else {
                timeout_wrapped_command
            };

        // Per-job rustc cap (issue #49). Exported in the outer session so
        // explicit worker/command/project settings all still win; see
        // `remote_build_jobs_fragment`. Windows workers run Git's sh without
        // nproc/meminfo/sysctl and are skipped.
        let build_jobs_fragment = self.remote_build_jobs_fragment();
        let cargo_home_base = if command.contains("cargo") {
            let base_var = rch_common::RCH_CARGO_HOME_BASE_VAR;
            let physical_base = if self.worker_platform.is_windows() {
                String::new()
            } else {
                format!(
                    "case \"${{{base_var}}}\" in /*) ;; *) {base_var}=\"./${{{base_var}}}\" ;; esac; {base_var}=\"$(CDPATH= cd \"${{{base_var}}}\" && pwd -P)\" || exit $?; "
                )
            };
            format!(
                "{}; {physical_base}export {base_var}; ",
                rch_common::remote_cargo_home_base_prelude()
            )
        } else {
            String::new()
        };

        format!(
            "export LC_ALL=C; {}{}touch {} && cd {} && {}{}{}",
            cargo_home_base,
            build_jobs_fragment,
            escaped_remote_path,
            escaped_remote_path,
            self.node_modules_bootstrap(),
            ensure_dirs_command,
            execution_command
        )
    }

    /// The `CARGO_BUILD_JOBS` fragment for this pipeline's worker and project,
    /// or empty when the policy is `off`, the worker is Windows, or the project
    /// pins `[build] jobs` itself.
    fn remote_build_jobs_fragment(&self) -> String {
        let policy = self.compilation_config.remote_build_jobs;
        if policy == RemoteBuildJobs::Off || self.worker_platform.is_windows() {
            return String::new();
        }
        if project_declares_cargo_build_jobs(&self.project_root) {
            debug!(
                "Not injecting CARGO_BUILD_JOBS: {} declares [build] jobs in .cargo/config",
                self.project_root.display()
            );
            return String::new();
        }
        debug!("Remote build jobs policy: {policy} (CARGO_BUILD_JOBS exported unless already set)");
        remote_build_jobs_fragment(policy, LINUX_MEMINFO_PATH)
    }

    /// Provision `node_modules` on the worker for TypeScript kinds.
    ///
    /// The project sync deliberately EXCLUDES `node_modules/` (see
    /// `default_excludes`), and that exclusion is correct: node_modules holds
    /// platform-native binaries, so rsyncing a macOS tree onto a Linux worker
    /// would ship unusable `.node` files. rch-wkr has a `prepare()` hook that
    /// installs dependencies — but the hook's build path executes the command
    /// over plain SSH and never invokes `rch-wkr execute`, so on this path
    /// nothing provisions them.
    ///
    /// Without this, `tsc --noEmit` on a worker finds no project-local
    /// TypeScript, and `npx` silently downloads the unrelated `tsc` stub package
    /// from the registry — which prints "This is not the tsc command you are
    /// looking for" and exits 1, turning a passing local typecheck into a failing
    /// remote build.
    ///
    /// Install only when `node_modules` is absent. The worker's project directory
    /// persists between builds, so this is a one-time cost per project. `npm ci`
    /// is preferred (lockfile-exact, so the worker typechecks with the SAME
    /// TypeScript version as local); `npm install` is the fallback for projects
    /// with no lockfile. Installer output is routed to stderr so it can never
    /// pollute the command's stdout.
    ///
    /// Deliberately scoped to `Tsc` only: Bun kinds keep their existing behavior.
    fn node_modules_bootstrap(&self) -> &'static str {
        match self.compilation_kind {
            Some(CompilationKind::Tsc) => {
                "if [ -f package.json ] && [ ! -d node_modules ]; then \
                 (npm ci --no-audit --no-fund --loglevel=error || \
                  npm install --no-audit --no-fund --loglevel=error) 1>&2 || exit 1; \
                 fi && "
            }
            _ => "",
        }
    }

    /// Wrap a command with an external timeout to prevent zombie/stuck processes.
    ///
    /// All remote commands are wrapped with the `timeout` command to ensure they
    /// don't run indefinitely. Timeouts are configurable per command type via
    /// CompilationConfig:
    /// - Bun commands: bun_timeout_sec (default 600s = 10 min) - known hang issues
    /// - Test commands: test_timeout_sec (default 1800s = 30 min)
    /// - Build/other: build_timeout_sec (default 300s = 5 min)
    ///
    /// The timeout wrapper can be disabled entirely via `external_timeout_enabled`.
    ///
    /// Returns the original command unchanged if timeout wrapping is disabled.
    fn wrap_with_external_timeout(&self, command: &str) -> String {
        // On a Windows worker, `timeout` under Git's sh resolves to Windows
        // `timeout.exe` (a pause utility), NOT GNU coreutils timeout — wrapping
        // with it would corrupt the command. Skip it. Note the Windows path also
        // skips the pgid watchdog (see `build_remote_command`), so runaways are
        // bounded only by the SSH ConnectTimeout / `command_timeout` killing the
        // local ssh — not by the daemon's process-group stuck-detector.
        if self.worker_platform.is_windows() {
            return command.to_string();
        }

        // Check if external timeout protection is enabled
        if !self.compilation_config.external_timeout_enabled() {
            debug!("External timeout protection disabled by config");
            return command.to_string();
        }

        // Get the appropriate timeout for this command type
        let timeout_duration = self
            .compilation_config
            .timeout_for_kind(self.compilation_kind);
        let timeout_secs = timeout_duration.as_secs();

        // Log the timeout being applied
        let kind_name = match self.compilation_kind {
            Some(CompilationKind::BunTest) => "bun test",
            Some(CompilationKind::BunTypecheck) => "bun typecheck",
            Some(CompilationKind::CargoTest) => "cargo test",
            Some(CompilationKind::CargoNextest) => "cargo nextest",
            Some(CompilationKind::CargoBuild) => "cargo build",
            Some(CompilationKind::CargoCheck) => "cargo check",
            Some(CompilationKind::CargoClippy) => "cargo clippy",
            Some(CompilationKind::CargoDoc) => "cargo doc",
            Some(CompilationKind::CargoBench) => "cargo bench",
            Some(CompilationKind::CargoZigbuild) => "cargo zigbuild",
            Some(CompilationKind::Rustc) => "rustc",
            Some(CompilationKind::Gcc) => "gcc",
            Some(CompilationKind::Gpp) => "g++",
            Some(CompilationKind::Clang) => "clang",
            Some(CompilationKind::Clangpp) => "clang++",
            Some(CompilationKind::Make) => "make",
            Some(CompilationKind::CmakeBuild) => "cmake build",
            Some(CompilationKind::Ninja) => "ninja",
            Some(CompilationKind::Meson) => "meson",
            Some(CompilationKind::NixBuild) => "nix build",
            Some(CompilationKind::GoBuild) => "go build",
            Some(CompilationKind::GoTest) => "go test",
            Some(CompilationKind::GoVet) => "go vet",
            Some(CompilationKind::Tsc) => "tsc",
            Some(CompilationKind::Job) => "job",
            None => "unknown",
        };

        info!(
            kind = %kind_name,
            timeout_secs = %timeout_secs,
            "Wrapping command with external timeout protection"
        );

        // Exit 137 alone is ambiguous. Give the utility a private diagnostic
        // pipe while forwarding the workload's stdout/stderr through fd 4/3.
        // Only timeout's own KILL diagnostic can establish deadline enforcement.
        // The final reader preserves the producer status (POSIX pipelines use
        // the last command's status), and consumes through EOF before returning.
        let status_marker = format!("{}_STATUS=", self.deadline_marker);
        format!(
            "{{ {{ LC_ALL=C timeout --verbose --signal=KILL --foreground --preserve-status {timeout_secs} \
sh -c 'exec \"$@\" 2>&3 3>&- 4>&-' rch-timeout env {command} 2>&1 1>&4; \
printf '{status_marker}%s\\n' \"$?\"; }} | \
{{ __s=125; __seen=0; __bad=0; __deadline=0; \
while IFS= read -r __line || [ -n \"$__line\" ]; do \
case \"$__line\" in \
{status_marker}*) __seen=$((__seen + 1)); __value=${{__line#{status_marker}}}; \
case \"$__value\" in ''|*[!0-9]*) __bad=1;; *) \
if [ \"${{#__value}}\" -le 3 ] && [ \"$__value\" -le 255 ]; then __s=$__value; else __bad=1; fi;; esac;; \
\"timeout: sending signal KILL to command 'sh'\") __deadline=1; printf '%s\\n' \"$__line\" >&2;; \
*) printf '%s\\n' \"$__line\" >&2;; esac; done; \
if [ \"$__seen\" -ne 1 ] || [ \"$__bad\" -ne 0 ]; then exit 125; fi; \
if [ \"$__s\" -eq 137 ] && [ \"$__deadline\" -eq 1 ]; then printf '\\n{marker}\\n' >&2; fi; \
exit \"$__s\"; }}; }} 3>&2 4>&1",
            marker = self.deadline_marker,
        )
    }

    /// Match before bounded stderr capture: a noisy workload must not hide a
    /// deadline receipt at the end of its output. Exit status is checked later.
    pub(crate) fn is_deadline_marker(&self, line: &str) -> bool {
        line.trim_end_matches(['\r', '\n']) == self.deadline_marker
    }

    pub(crate) fn remote_process_setup_marker(&self) -> String {
        format!("{}_IDENTITY_UNAVAILABLE", self.deadline_marker)
    }

    /// Invoke after source-lock and durable-completion checks. Only an exact
    /// attempt marker plus setup status proves safe failover before workload.
    pub(crate) fn ensure_remote_process_setup(&self, result: &CommandResult) -> Result<()> {
        let marker = self.remote_process_setup_marker();
        let mut matching_lines = 0;
        let mut complete_lines = 0;
        for line in result.stderr.split_inclusive('\n') {
            let body = line.strip_suffix('\n').unwrap_or(line);
            let body = body.strip_suffix('\r').unwrap_or(body);
            if body == marker {
                matching_lines += 1;
                complete_lines += usize::from(line.ends_with('\n'));
            }
        }
        if result.exit_code == 125 && matching_lines == 1 && complete_lines == 1 {
            return Err(RemoteProcessSetupUnavailable.into());
        }
        Ok(())
    }

    // =========================================================================
    // Transfer Size Estimation (bd-3hho)
    // =========================================================================

    /// Wall-clock budget for the pre-upload `rsync --dry-run --stats`
    /// estimate (issue #74).
    ///
    /// The dry-run prints nothing until it finishes, so silence and wall-clock
    /// are the same thing here: the estimate is bounded by the source-sync
    /// silence window, and never outlives an explicit `sync_timeout_ms`. With
    /// both disabled/unset it uses the default silence window. A stalled
    /// estimate then fails open into the bounded upload instead of holding the
    /// worker reservation indefinitely.
    fn transfer_estimate_timeout(&self) -> std::time::Duration {
        const FALLBACK: std::time::Duration = std::time::Duration::from_secs(120);
        let silence = match self.transfer_config.source_sync_silence_timeout_secs {
            0 => None,
            secs => Some(std::time::Duration::from_secs(secs)),
        };
        let explicit = self
            .transfer_config
            .sync_timeout_ms
            .filter(|ms| TransferConfig::valid_sync_timeout_ms(*ms))
            .map(std::time::Duration::from_millis);
        match (silence, explicit) {
            (Some(silence), Some(explicit)) => silence.min(explicit),
            (Some(bound), None) | (None, Some(bound)) => bound,
            (None, None) => FALLBACK,
        }
    }

    /// Build the `rsync --dry-run --stats` command behind
    /// [`Self::estimate_transfer_size`]. It shares the upload's ssh transport
    /// (`build_rsync_ssh_command`), so `ServerAliveInterval` and connection
    /// multiplexing reach the estimate as well (issue #74).
    fn build_estimate_command(&self, worker: &WorkerConfig) -> Command {
        let effective_excludes = self.get_effective_excludes();
        let (mut cmd, _capabilities) = self.rsync_command();

        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));
        let ssh_command = format!(
            "{} -o ConnectTimeout=5",
            self.build_rsync_ssh_command(escaped_identity.as_ref())
        );

        cmd.arg("-az");
        add_portable_rsync_archive_args(&mut cmd);
        cmd.arg("--dry-run")
            .arg("--stats")
            .arg("-e")
            .arg(ssh_command);
        self.append_source_rsync_path(&mut cmd);

        for pattern in &effective_excludes {
            cmd.arg("--exclude").arg(pattern);
        }

        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));
        let destination = format!("{}@{}:{}", worker.user, worker.host, escaped_remote_path);

        cmd.arg(format!("{}/", self.project_root.display()))
            .arg(&destination);

        cmd.stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true);
        cmd
    }

    /// Estimate transfer size using rsync dry-run.
    ///
    /// Returns `None` if estimation fails (e.g., rsync unavailable) or does not
    /// finish within [`Self::transfer_estimate_timeout`]. Fail-open: if
    /// estimation fails, proceed with transfer rather than blocking.
    #[allow(dead_code)]
    pub async fn estimate_transfer_size(&self, worker: &WorkerConfig) -> Option<TransferEstimate> {
        let start = std::time::Instant::now();
        let mut cmd = self.build_estimate_command(worker);
        let budget = self.transfer_estimate_timeout();

        // `kill_on_drop` kills the stalled rsync when the timeout drops the
        // pending `output()` future; its ssh then loses its peer and exits.
        let output = match tokio::time::timeout(budget, cmd.output()).await {
            Ok(Ok(output)) => output,
            Ok(Err(e)) => {
                debug!("Transfer estimation failed (rsync error): {}", e);
                return None;
            }
            Err(_) => {
                warn!(
                    worker = %worker.id,
                    timeout_secs = budget.as_secs_f64(),
                    "Transfer size estimate produced no result in time; proceeding with the \
                     bounded upload (fail-open)"
                );
                return None;
            }
        };

        let estimation_ms = start.elapsed().as_millis() as u64;
        let stdout = String::from_utf8_lossy(&output.stdout);

        if !output.status.success() {
            debug!(
                "Transfer estimation failed (exit {}): {}",
                output.status.code().unwrap_or(-1),
                String::from_utf8_lossy(&output.stderr)
            );
            return None;
        }

        let bytes = crate::transfer::parse_rsync_total_size(&stdout).unwrap_or(0);
        let files = crate::transfer::parse_rsync_total_files(&stdout).unwrap_or(0);

        // Calculate estimated transfer time using configured or default bandwidth
        // Default: 10 MB/s (reasonable for local network)
        let bandwidth_bps = self
            .transfer_config
            .estimated_bandwidth_bps
            .unwrap_or(10 * 1024 * 1024);

        let estimated_time_ms = if bandwidth_bps > 0 {
            (bytes as f64 / bandwidth_bps as f64 * 1000.0).round() as u64
        } else {
            0
        };

        Some(TransferEstimate {
            bytes,
            files,
            estimated_time_ms,
            estimation_ms,
        })
    }

    /// Check if transfer should be skipped based on size/time thresholds.
    ///
    /// Returns `Some(reason)` if transfer should be skipped, `None` if it should proceed.
    #[allow(dead_code)]
    pub async fn should_skip_transfer(&mut self, worker: &WorkerConfig) -> Option<String> {
        // Check if any thresholds are configured
        let max_mb = self.transfer_config.max_transfer_mb;
        let max_time_ms = self.transfer_config.max_transfer_time_ms;

        let needs_estimate =
            self.transfer_config.adaptive_compression || max_mb.is_some() || max_time_ms.is_some();

        if !needs_estimate {
            return None; // No thresholds configured
        }

        // Run estimation
        let estimate = match self.estimate_transfer_size(worker).await {
            Some(e) => e,
            None => {
                self.estimated_transfer_bytes = None;
                debug!("Transfer estimation failed, proceeding with transfer (fail-open)");
                return None;
            }
        };
        self.estimated_transfer_bytes = Some(estimate.bytes);

        // Check size threshold
        if let Some(max_mb) = max_mb {
            let max_bytes = max_mb.saturating_mul(1024 * 1024);
            if estimate.bytes > max_bytes {
                let estimated_mb = estimate.bytes as f64 / (1024.0 * 1024.0);
                return Some(format!(
                    "Transfer size ({:.2} MB) exceeds threshold ({:.2} MB)",
                    estimated_mb, max_mb as f64
                ));
            }
        }

        // Check time threshold
        if let Some(max_time) = max_time_ms
            && estimate.estimated_time_ms > max_time
        {
            return Some(format!(
                "Estimated transfer time ({} ms) exceeds threshold ({} ms)",
                estimate.estimated_time_ms, max_time
            ));
        }

        None
    }

    /// Build rsync command for sync_to_remote.
    fn build_sync_command(
        &self,
        worker: &WorkerConfig,
        destination: &str,
        escaped_remote_path: &str,
        effective_excludes: &[String],
    ) -> Command {
        let (mut cmd, capabilities) = self.rsync_command();

        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));
        let ssh_command = self.build_rsync_ssh_command(escaped_identity.as_ref());

        cmd.arg("-az"); // Archive mode + compression
        add_portable_rsync_archive_args(&mut cmd);
        cmd.arg("--partial")
            .arg("--partial-dir=.rch-partial")
            .arg("--stats") // Structured output for parse_rsync_bytes/files
            .arg("-e")
            .arg(ssh_command);

        if self.sync_delete {
            cmd.arg("--delete"); // Remove extraneous files from destination
        }
        if self.sync_checksum {
            cmd.arg("--checksum");
        }

        // Create remote directory implicitly using rsync-path wrapper
        // This saves a separate SSH handshake for 'mkdir -p'. The same remote
        // shell invocation also reaps stale worker-side runtime state
        // (bd-wfumv) — see worker_cache_prune_rsync_path_prefix.
        cmd.arg("--rsync-path").arg(self.source_rsync_path(format!(
            "{}mkdir -p {} && rsync",
            worker_cache_prune_rsync_path_prefix(
                escaped_remote_path,
                self.pooled_target_prune_idle_hours,
                self.escaped_pooled_target_override_parent().as_deref()
            ),
            escaped_remote_path
        )));

        self.append_sync_filter_args(&mut cmd, effective_excludes);

        // zstd compression where the binary supports it (rsync 3.2+); zlib
        // via `-z` otherwise (issue #66).
        self.append_compression_args(&mut cmd, &capabilities);

        // Add bandwidth limit if configured (bd-3hho)
        if let Some(bwlimit) = self.transfer_config.bwlimit_kbps
            && bwlimit > 0
        {
            cmd.arg(format!("--bwlimit={}", bwlimit));
        }

        // Source and destination
        cmd.arg(format!("{}/", self.project_root.display())) // Trailing slash = contents only
            .arg(destination);

        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
        cmd
    }

    /// Build rsync command for sync_to_remote_streaming.
    fn build_sync_streaming_command(
        &self,
        worker: &WorkerConfig,
        destination: &str,
        escaped_remote_path: &str,
        effective_excludes: &[String],
    ) -> Command {
        let (mut cmd, capabilities) = self.rsync_command();

        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));
        let ssh_command = self.build_rsync_ssh_command(escaped_identity.as_ref());

        cmd.arg("-az"); // Archive mode + compression
        add_portable_rsync_archive_args(&mut cmd);
        // `--info=progress2 --info=stats2` on rsync 3.1+; `--progress --stats`
        // on openrsync / 2.6.9, which reject every `--info=` value (issue #66).
        cmd.arg("--partial")
            .arg("--partial-dir=.rch-partial")
            .args(capabilities.progress_args())
            .args(capabilities.stats_args())
            .arg("-e")
            .arg(ssh_command);

        if self.sync_delete {
            cmd.arg("--delete"); // Remove extraneous files from destination
        }
        if self.sync_checksum {
            cmd.arg("--checksum");
        }

        // Create remote directory implicitly using rsync-path wrapper; the
        // same remote shell invocation reaps stale worker-side runtime state
        // (bd-wfumv) — see worker_cache_prune_rsync_path_prefix.
        cmd.arg("--rsync-path").arg(self.source_rsync_path(format!(
            "{}mkdir -p {} && rsync",
            worker_cache_prune_rsync_path_prefix(
                escaped_remote_path,
                self.pooled_target_prune_idle_hours,
                self.escaped_pooled_target_override_parent().as_deref()
            ),
            escaped_remote_path
        )));

        self.append_sync_filter_args(&mut cmd, effective_excludes);

        // zstd compression where the binary supports it (rsync 3.2+); zlib
        // via `-z` otherwise (issue #66).
        self.append_compression_args(&mut cmd, &capabilities);

        // Add bandwidth limit if configured (bd-3hho)
        if let Some(bwlimit) = self.transfer_config.bwlimit_kbps
            && bwlimit > 0
        {
            cmd.arg(format!("--bwlimit={}", bwlimit));
        }

        // Source and destination
        cmd.arg(format!("{}/", self.project_root.display())) // Trailing slash = contents only
            .arg(destination);

        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
        cmd
    }

    #[cfg(unix)]
    async fn create_git_archive(
        git_root: &Path,
        base_commit: &str,
    ) -> Result<tempfile::NamedTempFile> {
        if !matches!(base_commit.len(), 40 | 64)
            || !base_commit.chars().all(|ch| ch.is_ascii_hexdigit())
        {
            anyhow::bail!("clean-overlay base must be a full hexadecimal commit object ID");
        }

        // Materialize the immutable archive once locally. Retrying the network
        // transfer then uses the exact same byte sequence and lets rsync resume
        // the stable remote partial file with --append-verify.
        let archive_file = tempfile::Builder::new()
            .prefix("rch-clean-overlay-")
            .suffix(".tar")
            .tempfile()
            .context("create clean-overlay archive file")?;
        let mut archive = Command::new("git");
        configure_clean_git_command(&mut archive);
        archive
            .current_dir(git_root)
            .arg("archive")
            .arg("--format=tar")
            .arg("--output")
            .arg(archive_file.path())
            .arg(base_commit)
            .stdin(Stdio::null())
            .stdout(Stdio::null())
            .stderr(Stdio::piped())
            .kill_on_drop(true);
        let archive_output = archive
            .output()
            .await
            .with_context(|| format!("run git archive for {base_commit}"))?;
        if !archive_output.status.success() {
            anyhow::bail!(
                "git archive failed for {base_commit}: {}",
                String::from_utf8_lossy(&archive_output.stderr).trim()
            );
        }
        Ok(archive_file)
    }

    /// Materialize an immutable Git commit directly into this pipeline's fresh
    /// remote project root without creating a local branch, worktree, clone, or
    /// staging directory. Windows streams the archive to tar over SSH; POSIX
    /// workers retain the resumable rsync archive transport.
    #[cfg(unix)]
    pub async fn materialize_git_archive(
        &self,
        worker: &WorkerConfig,
        git_root: &Path,
        base_commit: &str,
    ) -> Result<CleanOverlayMaterialization> {
        if use_mock_transport(worker) {
            return Ok(CleanOverlayMaterialization {
                sync_result: SyncResult {
                    bytes_transferred: 0,
                    files_transferred: 0,
                    duration_ms: 0,
                },
                attempts: vec![TransferAttemptDiagnostic {
                    attempt: 1,
                    max_attempts: self.transfer_config.retry.max_attempts.max(1),
                    outcome: "succeeded",
                    detail: "mock clean-overlay source transfer completed (unit transport)"
                        .to_string(),
                }],
            });
        }
        let archive_file = Self::create_git_archive(git_root, base_commit).await?;
        let archive_path = archive_file.path().to_path_buf();
        let payload_bytes = archive_file
            .as_file()
            .metadata()
            .context("stat clean-overlay archive")?
            .len();
        let attempt_timeout = self.transfer_config.sync_timeout_for_payload(payload_bytes);
        let remote_path = self.remote_path();
        if self.worker_platform.is_windows() {
            let start = std::time::Instant::now();
            let ((), attempts) = run_source_transfer_attempts(
                &self.source_retry_config(self.transfer_config.retry.clone()),
                attempt_timeout,
                "clean_overlay_base_sync",
                |_attempt| async {
                    self.upload_windows_archive(worker, &archive_path, &remote_path)
                        .await?;
                    Ok(())
                },
            )
            .await
            .map_err(anyhow::Error::new)?;
            return Ok(CleanOverlayMaterialization {
                sync_result: SyncResult {
                    bytes_transferred: payload_bytes,
                    files_transferred: 0,
                    duration_ms: u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
                },
                attempts,
            });
        }
        let escaped_remote_path = escape(Cow::from(remote_path.as_str()));
        // The remote root includes the clean-overlay job nonce, so it is unique
        // to this execution. Keeping this filename stable across attempts is
        // what allows rsync to validate and continue the partial payload.
        let remote_archive_path = format!("{remote_path}/.rch-clean-overlay-base.tar");
        let escaped_remote_archive = escape(Cow::from(remote_archive_path.as_str()));
        let destination = format!("{}@{}:{}", worker.user, worker.host, escaped_remote_archive);
        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));
        let ssh_command = self.build_rsync_ssh_command(escaped_identity.as_ref());
        let extraction_script = format!(
            "umask 0022\nmkdir -p {path}\nTAR_OPTIONS='' tar -xf {archive} -C {path}",
            path = escaped_remote_path,
            archive = escaped_remote_archive
        );
        let resolved_rsync = self.resolved_rsync();
        let rsync_path = resolved_rsync.path.clone();
        let resume_args = resolved_rsync.capabilities().resume_args();
        let start = std::time::Instant::now();
        let (((), attempts), duration_ms) = {
            let result = run_source_transfer_attempts(
                &self.source_retry_config(self.transfer_config.retry.clone()),
                attempt_timeout,
                "clean_overlay_base_sync",
                |_attempt| {
                    let destination = destination.clone();
                    let ssh_command = ssh_command.clone();
                    let extraction_script = extraction_script.clone();
                    let archive_path = archive_path.clone();
                    let escaped_remote_path = escaped_remote_path.clone();
                    let rsync_path = rsync_path.clone();
                    async move {
                        let mut rsync = Command::new(&rsync_path); // ubs:ignore — same trusted rsync resolver as rsync_command; payload remains argv
                        rsync
                            .env("LC_ALL", "C")
                            .arg("-a")
                            .arg("--no-owner")
                            .arg("--no-group")
                            // `--partial --append-verify` on rsync 3.x; bare
                            // `--partial` (delta-resumed) on openrsync/2.6.9,
                            // which rejects `--append-verify` (issue #66).
                            .args(resume_args)
                            .arg("-e")
                            .arg(ssh_command)
                            .arg("--rsync-path")
                            .arg(self.source_rsync_path(format!(
                                "mkdir -p {escaped_remote_path} && rsync"
                            )))
                            .arg(archive_path)
                            .arg(destination)
                            .stdout(Stdio::piped())
                            .stderr(Stdio::piped())
                            .kill_on_drop(true);
                        let output = rsync
                            .output()
                            .await
                            .context("run resumable clean-overlay base rsync")?;
                        if !output.status.success() {
                            let stderr = String::from_utf8_lossy(&output.stderr).to_string();
                            if is_retryable_transport_error_text(&stderr) {
                                anyhow::bail!(
                                    "rsync transport error (exit {:?}): {}",
                                    output.status.code(),
                                    stderr.trim()
                                );
                            }
                            return Err(TransferError::SyncFailed {
                                reason: "clean-overlay base rsync failed".to_string(),
                                exit_code: output.status.code(),
                                stderr,
                            }
                            .into());
                        }
                        self.run_remote_sh(worker, &extraction_script)
                            .await
                            .context("extract clean-overlay base archive")?;
                        Ok(())
                    }
                },
            )
            .await
            .map_err(anyhow::Error::new)?;
            (result, start.elapsed().as_millis() as u64)
        };

        Ok(CleanOverlayMaterialization {
            sync_result: SyncResult {
                bytes_transferred: payload_bytes,
                files_transferred: 0,
                duration_ms,
            },
            attempts,
        })
    }

    /// Synchronize local project to remote worker.
    ///
    /// Uses retry logic with exponential backoff for transient network errors.
    /// Build an `ssh` command carrying this pipeline's connection options and
    /// ready to run `remote_args` on the worker. The Windows tar transport uses
    /// this instead of rsync.
    fn worker_ssh_command(&self, worker: &WorkerConfig, remote_args: &[&str]) -> Command {
        self.worker_ssh_command_with_activity(worker, remote_args, true)
    }

    fn worker_ssh_command_with_activity(
        &self,
        worker: &WorkerConfig,
        remote_args: &[&str],
        bind_activity: bool,
    ) -> Command {
        let identity_file = shellexpand::tilde(&worker.identity_file);
        let mut cmd = Command::new("ssh");
        cmd.arg("-o").arg("BatchMode=yes");
        cmd.arg("-o").arg("StrictHostKeyChecking=accept-new");
        cmd.arg("-o").arg(format!(
            "ConnectTimeout={}",
            self.ssh_options.connect_timeout.as_secs().max(1)
        ));
        cmd.arg("-i").arg(identity_file.as_ref());
        if let Some(interval) = self.ssh_options.server_alive_interval {
            let secs = interval.as_secs();
            if secs > 0 {
                cmd.arg("-o").arg(format!("ServerAliveInterval={secs}"));
            }
        }
        cmd.arg(format!("{}@{}", worker.user, worker.host));
        if bind_activity && let Some(prefix) = &self.source_authority_prefix {
            cmd.arg(prefix);
        }
        for a in remote_args {
            cmd.arg(a);
        }
        // Kill the local ssh (and thus close the channel) if this Command's
        // Child is dropped — e.g. when a dispatch-level timeout drops the
        // transfer future. Harmless for the awaited-to-completion callers.
        cmd.kill_on_drop(true);
        cmd
    }

    /// Run a POSIX script on the worker via `ssh <host> sh -s` (script on stdin).
    ///
    /// Deliberately NOT `ssh <host> sh -c '<script>'`: a Windows worker's sshd
    /// default shell is `cmd.exe`, which re-parses and mangles quoted arguments
    /// (`mkdir: missing operand`). Feeding the script on stdin to `sh -s` — Git's
    /// POSIX shell — sidesteps that entirely, exactly as the exec path does.
    async fn run_remote_sh(&self, worker: &WorkerConfig, script: &str) -> Result<()> {
        let mut cmd = self.worker_ssh_command(worker, &["sh", "-s"]);
        cmd.stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());
        let mut child = cmd.spawn().context("spawn ssh sh -s")?;
        if let Some(mut stdin) = child.stdin.take() {
            stdin.write_all(script.as_bytes()).await?;
            stdin.write_all(b"\n").await?;
            drop(stdin);
        }
        let out = child.wait_with_output().await.context("ssh sh -s wait")?;
        if !out.status.success() {
            return Err(anyhow::anyhow!(
                "remote sh script failed (exit {:?}): {}",
                out.status.code(),
                String::from_utf8_lossy(&out.stderr)
            ));
        }
        Ok(())
    }

    /// OpenSSH joins remote argv before cmd.exe parses it. A POSIX single quote
    /// is literal there, and percent/exclamation expansion also occurs inside
    /// double quotes. Admit the drive-path alphabet we can pass literally and
    /// refuse other paths before creating a directory or sending source bytes.
    fn windows_archive_destination(remote_path: &str) -> Result<String> {
        let normalized = remote_path.replace('\\', "/");
        if !is_windows_drive_abs_path(&normalized)
            || !normalized[2..]
                .chars()
                .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '/' | ' ' | '-' | '_' | '.'))
            || normalized[3..].split('/').any(|part| {
                part == "." || part == ".." || part.ends_with('.') || part.ends_with(' ')
            })
        {
            anyhow::bail!("Windows archive destination is not a safe literal drive path");
        }
        Ok(format!("\"{normalized}\""))
    }

    /// Send an already materialized archive. Both the immutable Git base and
    /// the selected overlay use this transport; neither sends the working tree
    /// recursively or invokes rsync on a Windows drive-letter destination.
    async fn upload_windows_archive(
        &self,
        worker: &WorkerConfig,
        archive_path: &Path,
        remote_path: &str,
    ) -> Result<u64> {
        let destination = Self::windows_archive_destination(remote_path)?;
        self.run_remote_sh(
            worker,
            &format!(
                "mkdir -p {}",
                escape(Cow::from(remote_path.replace('\\', "/")))
            ),
        )
        .await?;
        let mut archive = tokio::fs::File::open(archive_path)
            .await
            .context("open Windows source archive")?;
        let mut ssh = self.worker_ssh_command(worker, &["tar", "xf", "-", "-C", &destination]);
        ssh.stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());
        let mut child = ssh.spawn().context("spawn ssh tar for source archive")?;
        let mut stdin = child.stdin.take().context("source archive ssh stdin")?;
        // Keep both futures scoped to this transfer: cancellation drops the
        // SSH child and the pump, and stderr is drained while bytes are sent.
        let (copied, output) = tokio::join!(
            async move {
                let result = tokio::io::copy(&mut archive, &mut stdin).await;
                drop(stdin);
                result
            },
            child.wait_with_output(),
        );
        let output = output.context("wait for Windows source archive extraction")?;
        if !output.status.success() {
            anyhow::bail!(
                "remote tar extract failed (exit {:?}): {}",
                output.status.code(),
                String::from_utf8_lossy(&output.stderr).trim()
            );
        }
        copied.context("stream Windows source archive")
    }

    async fn selected_source_archive(&self) -> Result<(tempfile::NamedTempFile, u32)> {
        let paths = self.enumerate_source_upload_files().await?;
        let files_transferred = u32::try_from(paths.len())
            .context("selected source archive exceeds file-count limit")?;
        let project_root = self.project_root.clone();
        tokio::task::spawn_blocking(move || {
            let mut archive = tempfile::Builder::new()
                .prefix("rch-selected-overlay-")
                .suffix(".tar")
                .tempfile()
                .context("create selected source archive")?;
            let mut builder = tar::Builder::new(archive.as_file_mut());
            for relative in &paths {
                let path = project_root.join(relative);
                if !std::fs::symlink_metadata(&path)?.is_file() {
                    anyhow::bail!(
                        "selected source is no longer a regular file: {}",
                        relative.display()
                    );
                }
                let mut file = std::fs::File::open(&path)?;
                if !file.metadata()?.is_file() {
                    anyhow::bail!(
                        "selected source is no longer a regular file: {}",
                        relative.display()
                    );
                }
                // append_file never recurses and preserves the exact relative
                // name, including spaces and literal glob characters.
                builder.append_file(relative, &mut file)?;
            }
            builder.finish().context("finish selected source archive")?;
            drop(builder);
            Ok((archive, files_transferred))
        })
        .await
        .context("join selected source archive writer")?
    }

    /// Windows source sync: create the remote dir, then `tar czf -` the project
    /// locally (honouring excludes) and pipe it into `tar xzf -` on the worker.
    /// Native `tar` on both ends — no rsync, no `C:`-colon path ambiguity.
    async fn sync_to_remote_windows(
        &self,
        worker: &WorkerConfig,
        remote_path: &str,
        excludes: &[String],
    ) -> Result<SyncResult> {
        let start = std::time::Instant::now();
        info!(
            "Syncing (windows/tar) {} -> {} on {}",
            self.project_root.display(),
            remote_path,
            worker.id
        );

        if self.sync_include_patterns.is_some() {
            let (archive, files_transferred) = self.selected_source_archive().await?;
            let bytes_transferred = self
                .upload_windows_archive(worker, archive.path(), remote_path)
                .await?;
            return Ok(SyncResult {
                bytes_transferred,
                files_transferred,
                duration_ms: u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
            });
        }

        // 1. Ensure the remote dir exists. `mkdir -p` under Git's sh accepts the
        //    C:/ form. Escape defensively even though our paths have no metachars.
        self.run_remote_sh(
            worker,
            &format!("mkdir -p {}", escape(Cow::from(remote_path))),
        )
        .await?;

        // 2. local `tar c` | remote `tar x`.
        let mut tar = Command::new("tar");
        tar.arg("czf").arg("-").arg("-C").arg(&self.project_root);
        for ex in excludes {
            tar.arg(format!("--exclude={ex}"));
        }
        tar.arg(".");
        tar.stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true);

        let mut ssh = self.worker_ssh_command(worker, &["tar", "xzf", "-", "-C", remote_path]);
        ssh.stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());

        let mut tar_child = tar.spawn().context("spawn local tar (windows sync)")?;
        let mut ssh_child = ssh.spawn().context("spawn ssh tar x (windows sync)")?;
        let mut tar_out = tar_child.stdout.take().context("local tar stdout")?;
        let mut tar_err = tar_child.stderr.take();
        let mut ssh_in = ssh_child.stdin.take().context("ssh stdin")?;
        let pump = tokio::spawn(async move {
            let _ = tokio::io::copy(&mut tar_out, &mut ssh_in).await;
            drop(ssh_in);
        });
        // Drain the local tar's stderr concurrently. `ssh_child.wait_with_output`
        // blocks until the remote extract finishes; if a chatty tar (many
        // per-file warnings) fills its undrained stderr pipe in the meantime it
        // stops writing stdout, the pump starves the ssh stdin, and the whole
        // transfer deadlocks. We only need the exit status, so discard the text.
        let tar_err_drain = tokio::spawn(async move {
            if let Some(err) = tar_err.as_mut() {
                let _ = tokio::io::copy(err, &mut tokio::io::sink()).await;
            }
        });
        let ssh_out = ssh_child
            .wait_with_output()
            .await
            .context("ssh tar x wait")?;
        let tar_status = tar_child.wait().await.context("local tar wait")?;
        let _ = pump.await;
        let _ = tar_err_drain.await;

        if !tar_status.success() {
            return Err(anyhow::anyhow!("local tar failed during windows sync"));
        }
        if !ssh_out.status.success() {
            return Err(anyhow::anyhow!(
                "remote tar extract failed (exit {:?}): {}",
                ssh_out.status.code(),
                String::from_utf8_lossy(&ssh_out.stderr)
            ));
        }
        Ok(SyncResult {
            // tar does not report per-file byte/count stats the way rsync does;
            // these are used only for logging, so 0 is acceptable here.
            bytes_transferred: 0,
            files_transferred: 0,
            duration_ms: u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
        })
    }

    /// Select exact regular files with the same filters as rsync, then archive
    /// only those names. A complete, validated download precedes local writes.
    async fn retrieve_artifacts_windows(
        &self,
        worker: &WorkerConfig,
        remote_path: &str,
        artifact_patterns: &[String],
        on_line: &mut impl FnMut(&str),
    ) -> Result<ArtifactRetrieval> {
        let start = TokioInstant::now();
        let (cancel, started) = self
            .retrieval_control
            .as_ref()
            .map_or((None, start), |(cancel, started)| {
                (Some(cancel.clone()), *started)
            });
        let (includes, excludes) = self.artifact_retrieval_filters(artifact_patterns)?;
        if includes.is_empty() {
            return Ok(ArtifactRetrieval {
                stats: SyncResult {
                    bytes_transferred: 0,
                    files_transferred: 0,
                    duration_ms: 0,
                },
                manifest_regular_files: Vec::new(),
                matched_regular_files: Some(0),
            });
        }
        info!(
            "Retrieving (windows/tar) artifacts from {} on {}",
            remote_path, worker.id
        );
        // Git for Windows supplies GNU find alongside the POSIX shell already
        // required for execution. -P (the default) does not traverse symlinks.
        // A NUL-delimited inventory preserves spaces and shell metacharacters.
        let inventory_script = windows_artifact_inventory_script(remote_path);
        let mut inventory = Vec::new();
        run_windows_artifact_process(
            self.worker_ssh_command(worker, &["sh", "-s"]),
            &inventory_script,
            &mut inventory,
            MAX_SOURCE_CONTENT_RSYNC_OUTPUT_BYTES as u64,
            started + Duration::from_millis(self.effective_rsync_retry_config().total_timeout_ms),
            cancel.clone(),
            on_line,
        )
        .await?;
        let selected = windows_artifact_selection(&inventory, &includes, &excludes)?;
        let count = u32::try_from(selected.len()).context("too many Windows artifacts")?;
        let mut bytes = 0;
        if !selected.is_empty() {
            let selected_size = selected
                .values()
                .fold(0_u64, |total, size| total.saturating_add(*size));
            let retry = self.artifact_retry_config_for_size(selected_size);
            let deadline = started + Duration::from_millis(retry.total_timeout_ms);
            let archive = tempfile::Builder::new()
                .prefix("rch-windows-artifacts-")
                .suffix(".tar.gz")
                .tempfile()?;
            let mut output = tokio::fs::File::from_std(archive.reopen()?);
            bytes = run_windows_artifact_process(
                self.worker_ssh_command(worker, &["sh", "-s"]),
                &windows_artifact_archive_script(remote_path, &selected),
                &mut output,
                selected_size
                    .saturating_add(u64::from(count).saturating_mul(8192))
                    .saturating_add(1024 * 1024),
                deadline,
                cancel.clone(),
                on_line,
            )
            .await?;
            drop(output);
            if cancel.as_ref().is_some_and(|cancel| *cancel.borrow()) {
                return Err(RetrievalCancelled.into());
            }
            anyhow::ensure!(
                TokioInstant::now() < deadline,
                "Windows artifact retrieval deadline exceeded"
            );
            // Await validation/publication to completion: a detached blocking
            // task must never continue overwriting outputs after recovery starts.
            let destination = self.project_root.clone();
            let expected = selected.clone();
            let extraction_cancel = cancel.clone();
            let extracted = tokio::task::spawn_blocking(move || {
                unpack_windows_artifact_archive(
                    archive.path(),
                    &destination,
                    &expected,
                    deadline,
                    extraction_cancel,
                )
            })
            .await?;
            if cancel.as_ref().is_some_and(|cancel| *cancel.borrow()) {
                return Err(RetrievalCancelled.into());
            }
            extracted?;
        }
        on_line(&format!(
            "Retrieved {count} Windows artifact files ({bytes} archive bytes)"
        ));
        Ok(ArtifactRetrieval {
            stats: SyncResult {
                bytes_transferred: bytes,
                files_transferred: count,
                duration_ms: u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
            },
            manifest_regular_files: selected.into_keys().collect(),
            matched_regular_files: Some(count),
        })
    }

    /// Mock-transport source sync shared by the streaming and non-streaming
    /// paths, WITH the same retry behavior as a real transfer so mock-based
    /// tests observe identical attempt histories (TransferAttemptsExhausted)
    /// regardless of which sync path the orchestration chooses.
    async fn mock_sync_to_remote_with_retries(
        &self,
        destination: &str,
        effective_excludes: &[String],
    ) -> Result<SyncResult> {
        // Create MockRsync ONCE and share via Arc so failure counters persist across retries
        let rsync = std::sync::Arc::new(MockRsync::new(MockRsyncConfig::from_env()));
        let project_root_str = self.project_root.display().to_string();
        let retry_config = self.source_retry_config(self.transfer_config.retry.clone());
        let attempt_timeout = self.source_sync_attempt_timeout(effective_excludes);
        let (result, _attempts) = run_source_transfer_attempts(
            &retry_config,
            attempt_timeout,
            "mock_sync_to_remote",
            |_attempt| {
                let rsync = rsync.clone();
                let project_root = project_root_str.clone();
                let dest = destination.to_string();
                let excludes = effective_excludes.to_vec();
                async move { rsync.sync_to_remote(&project_root, &dest, &excludes).await }
            },
        )
        .await
        .map_err(anyhow::Error::new)?;
        Ok(SyncResult {
            bytes_transferred: result.bytes_transferred,
            files_transferred: result.files_transferred,
            duration_ms: result.duration_ms,
        })
    }

    pub async fn sync_to_remote(&self, worker: &WorkerConfig) -> Result<SyncResult> {
        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));
        let destination = format!("{}@{}:{}", worker.user, worker.host, escaped_remote_path);

        // Get effective excludes (config defaults + .rchignore)
        let effective_excludes = self.get_effective_excludes();

        if use_mock_transport(worker) {
            return self
                .mock_sync_to_remote_with_retries(&destination, &effective_excludes)
                .await;
        }

        // A Windows worker has no rsync; use tar-over-ssh instead. Gated on the
        // worker platform, so this branch is unreachable for linux/darwin. Bounded
        // by `command_timeout` because — unlike the rsync path — the tar transport
        // has no retry/timeout of its own; on expiry the transfer future drops and
        // the `kill_on_drop` children are reaped.
        if self.worker_platform.is_windows() {
            let timeout = self.ssh_options.command_timeout;
            return match tokio::time::timeout(
                timeout,
                self.sync_to_remote_windows(worker, &remote_path, &effective_excludes),
            )
            .await
            {
                Ok(result) => result,
                Err(_) => Err(anyhow::anyhow!(
                    "windows tar sync to {} timed out after {:?}",
                    worker.id,
                    timeout
                )),
            };
        }

        info!(
            "Syncing {} -> {} on {}",
            self.project_root.display(),
            remote_path,
            worker.id
        );

        debug!("Effective exclude patterns: {:?}", effective_excludes);

        let start = std::time::Instant::now();

        // Source uploads receive a full payload-aware timeout on every attempt.
        // This is intentionally independent of the remote Cargo command timeout
        // and artifact-return retry budget.
        let retry_config = self.source_retry_config(self.transfer_config.retry.clone());
        let attempt_timeout = self.source_sync_attempt_timeout(&effective_excludes);
        let (output, _attempts) = run_source_transfer_attempts(
            &retry_config,
            attempt_timeout,
            "sync_to_remote",
            |_attempt| {
                execute_source_rsync_attempt(self.build_sync_command(
                    worker,
                    &destination,
                    &escaped_remote_path,
                    &effective_excludes,
                ))
            },
        )
        .await
        .map_err(anyhow::Error::new)?;

        let duration = start.elapsed();
        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
        let stderr = String::from_utf8_lossy(&output.stderr).to_string();

        if !output.status.success() {
            // Check if the failure is retryable (it wasn't if we got here)
            if is_retryable_transport_error(&anyhow::anyhow!("{}", stderr)) {
                warn!(
                    "rsync failed with retryable error (retries exhausted): {}",
                    stderr
                );
            } else {
                warn!("rsync failed: {}", stderr);
            }
            return Err(TransferError::SyncFailed {
                reason: "rsync failed".to_string(),
                exit_code: output.status.code(),
                stderr: stderr.to_string(),
            }
            .into());
        }

        // rsync can exit 0 even when interrupted mid-file in edge cases. Treat a
        // partial-transfer indicator on a "successful" sync as a failure rather
        // than warning and returning Ok: otherwise the hook reports success, the
        // remote source tree is incomplete, and the remote build compiles
        // stale/partial sources, returning a trusted-but-wrong result.
        if let Some(indicator) = detect_partial_transfer(&stderr) {
            warn!(
                "rsync exited 0 but reported a partial transfer (matched '{}'): {}",
                indicator,
                stderr.lines().next().unwrap_or(&stderr)
            );
            return Err(TransferError::SyncFailed {
                reason: format!("partial transfer despite exit 0 ({indicator})"),
                exit_code: output.status.code(),
                stderr: stderr.to_string(),
            }
            .into());
        }

        info!("Sync completed in {}ms", duration.as_millis());

        Ok(SyncResult {
            bytes_transferred: parse_rsync_bytes(&stdout, RsyncTransferDirection::Upload),
            files_transferred: parse_rsync_files(&stdout),
            duration_ms: duration.as_millis() as u64,
        })
    }

    /// Synchronize local project to remote worker with streaming output.
    ///
    /// The `on_line` callback receives rsync progress lines for UI rendering.
    pub async fn sync_to_remote_streaming<F>(
        &self,
        worker: &WorkerConfig,
        mut on_line: F,
    ) -> Result<SyncResult>
    where
        F: FnMut(&str),
    {
        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));
        let destination = format!("{}@{}:{}", worker.user, worker.host, escaped_remote_path);

        // Get effective excludes (config defaults + .rchignore)
        let effective_excludes = self.get_effective_excludes();

        if use_mock_transport(worker) {
            // Same retry-aware mock path as sync_to_remote: since issue #59
            // routed every non-Windows sync through the streaming variant,
            // mock-based tests must see identical attempt histories here.
            return self
                .mock_sync_to_remote_with_retries(&destination, &effective_excludes)
                .await;
        }

        info!(
            "Syncing {} -> {} on {} (streaming)",
            self.project_root.display(),
            remote_path,
            worker.id
        );

        debug!("Effective exclude patterns: {:?}", effective_excludes);

        // Rebuilt per retry attempt: rsync consumes its `Command`, and a
        // transient SSH/rsync drop requires a fresh command. Durable source
        // grants instead leave retries to the job's cancellation boundary.
        let build_cmd = || {
            self.build_sync_streaming_command(
                worker,
                &destination,
                &escaped_remote_path,
                &effective_excludes,
            )
        };

        debug!(
            "Running (streaming): rsync {:?}",
            build_cmd().as_std().get_args().collect::<Vec<_>>()
        );

        let retry_config = self.source_retry_config(self.effective_rsync_retry_config());
        let attempt_timeout = self.source_sync_attempt_timeout(&effective_excludes);
        // Issue #59: silence-based stall detection rides alongside the
        // wall-clock attempt timeout. Every rsync output segment (including
        // bare-`\r` progress2 refreshes) counts as forward progress, so a
        // large-but-moving transfer is never killed while a dead channel is
        // detected within the silence window instead of the 1-hour cap.
        let silence_policy = self.source_sync_silence_policy(worker);
        let (output, duration_ms) = run_command_streaming_with_retry(
            &retry_config,
            "sync_to_remote_streaming",
            Some(attempt_timeout),
            silence_policy.as_ref(),
            build_cmd,
            |line| {
                on_line(line);
            },
        )
        .await?;

        // Same exit-0-but-incomplete guard as the non-streaming sync_to_remote:
        // run_command_streaming returns the combined stdout+stderr, so scan it for
        // partial-transfer indicators and fail rather than report a success that
        // would feed the remote build stale/partial sources.
        if let Some(indicator) = detect_partial_transfer(&output) {
            warn!(
                "streaming rsync exited 0 but reported a partial transfer (matched '{}')",
                indicator
            );
            return Err(TransferError::SyncFailed {
                reason: format!("partial transfer despite exit 0 ({indicator})"),
                exit_code: None,
                stderr: output,
            }
            .into());
        }

        Ok(SyncResult {
            bytes_transferred: parse_rsync_bytes(&output, RsyncTransferDirection::Upload),
            files_transferred: parse_rsync_files(&output),
            duration_ms,
        })
    }

    /// Execute a compilation command on the remote worker.
    ///
    /// If `toolchain` is provided, the command will be wrapped with `rustup run <toolchain>`.
    /// Color-forcing environment variables are applied based on the configured color mode.
    #[allow(dead_code)] // Reserved for future usage
    pub async fn execute_remote(
        &self,
        worker: &WorkerConfig,
        command: &str,
        toolchain: Option<&ToolchainInfo>,
    ) -> Result<CommandResult> {
        let wrapped_command = self.build_remote_command(command, toolchain);

        if use_mock_transport(worker) {
            let mut client = MockSshClient::new(worker.clone(), MockConfig::from_env());
            client.connect().await?;
            let result = client.execute(&wrapped_command).await;
            if let Err(e) = client.disconnect().await {
                warn!("Failed to disconnect mock SSH client: {}", e);
            }
            return result;
        }

        // Mask sensitive data (API keys, tokens) before logging
        info!(
            "Executing on {}: {}",
            worker.id,
            rch_common::util::mask_sensitive_command(command)
        );

        #[cfg(not(unix))]
        {
            return Err(crate::error::PlatformError::UnixOnly {
                feature: "SSH remote execution".to_string(),
            }
            .into());
        }

        #[cfg(unix)]
        {
            let result = self
                .execute_over_ssh_streaming(worker, &wrapped_command, |_| {}, |_| {})
                .await?;
            self.ensure_remote_process_setup(&result)?;

            if result.success() {
                info!("Command succeeded in {}ms", result.duration_ms);
            } else {
                warn!(
                    "Command failed (exit={}) in {}ms",
                    result.exit_code, result.duration_ms
                );
            }

            Ok(result)
        }
    }

    /// Best-effort SIGKILL + verification of the remote process group after a
    /// client-side command timeout (issue #62). Uses the pgid recorded by the
    /// in-session watchdog (`<run-dir>/<build_id>.pgid`) — the SAME group the
    /// daemon's stuck-detector kills — over a fresh short-lived SSH channel.
    ///
    /// Returns the cleanup verdict plus a human-readable detail. Never fails:
    /// any channel/probe error degrades to `Unverified` so callers can flag
    /// the worker's project lock as suspect instead of silently re-admitting
    /// the next job onto a held lock.
    #[cfg(unix)]
    async fn kill_remote_process_group_after_timeout(
        &self,
        worker: &WorkerConfig,
    ) -> (RemoteTimeoutCleanup, String) {
        if self.worker_platform.is_windows() {
            return (
                RemoteTimeoutCleanup::NotAttempted,
                "windows workers record no remote pgid".to_string(),
            );
        }
        let Some(build_id) = self.build_id else {
            return (
                RemoteTimeoutCleanup::NotAttempted,
                "no build id: no remote pgid recorded".to_string(),
            );
        };
        if use_mock_transport(worker) {
            return (
                RemoteTimeoutCleanup::NotAttempted,
                "mock transport: no remote process group".to_string(),
            );
        }

        let pgid_file = Self::remote_pgid_file_path_for_root(&self.remote_path(), build_id);
        let script = remote_timeout_kill_script(&pgid_file, build_id);

        let destination = format!("{}@{}", worker.user, worker.host);
        let identity_file = shellexpand::tilde(&worker.identity_file);
        let mut cmd = Command::new("ssh");
        cmd.arg("-o").arg("BatchMode=yes");
        cmd.arg("-o").arg("StrictHostKeyChecking=accept-new");
        cmd.arg("-o").arg(format!(
            "ConnectTimeout={}",
            self.ssh_options.connect_timeout.as_secs().max(1)
        ));
        cmd.arg("-i").arg(identity_file.as_ref());
        cmd.arg(&destination).arg("sh").arg("-s");
        cmd.kill_on_drop(true);
        cmd.stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());

        let attempt = async {
            let mut child = cmd
                .spawn()
                .map_err(|e| format!("failed to spawn kill ssh: {e}"))?;
            if let Some(mut stdin) = child.stdin.take() {
                stdin
                    .write_all(script.as_bytes())
                    .await
                    .map_err(|e| format!("failed to write kill script: {e}"))?;
                drop(stdin);
            }
            let output = child
                .wait_with_output()
                .await
                .map_err(|e| format!("failed to await kill ssh: {e}"))?;
            if !output.status.success() {
                return Err(format!(
                    "kill channel exited with {}: {}",
                    output.status,
                    String::from_utf8_lossy(&output.stderr).trim()
                ));
            }
            Ok(String::from_utf8_lossy(&output.stdout).to_string())
        };

        match tokio::time::timeout(REMOTE_TIMEOUT_KILL_BUDGET, attempt).await {
            Ok(Ok(stdout)) => match parse_remote_timeout_kill_output(&stdout) {
                Some(RemoteTimeoutCleanup::Verified) => (
                    RemoteTimeoutCleanup::Verified,
                    "remote process group SIGKILLed and verified dead".to_string(),
                ),
                Some(RemoteTimeoutCleanup::Unverified)
                | Some(RemoteTimeoutCleanup::NotAttempted) => (
                    RemoteTimeoutCleanup::Unverified,
                    "remote process group identity or termination could not be verified"
                        .to_string(),
                ),
                None => (
                    RemoteTimeoutCleanup::Unverified,
                    format!("kill probe produced no verdict: {}", stdout.trim()),
                ),
            },
            Ok(Err(reason)) => (RemoteTimeoutCleanup::Unverified, reason),
            Err(_) => (
                RemoteTimeoutCleanup::Unverified,
                format!(
                    "kill probe exceeded {}s budget",
                    REMOTE_TIMEOUT_KILL_BUDGET.as_secs()
                ),
            ),
        }
    }

    #[cfg(unix)]
    async fn execute_over_ssh_streaming<F, G>(
        &self,
        worker: &WorkerConfig,
        remote_script: &str,
        mut on_stdout: F,
        mut on_stderr: G,
    ) -> Result<CommandResult>
    where
        F: FnMut(&str),
        G: FnMut(&str),
    {
        let destination = format!("{}@{}", worker.user, worker.host);
        let identity_file = shellexpand::tilde(&worker.identity_file);

        let mut cmd = Command::new("ssh");
        cmd.arg("-o").arg("BatchMode=yes");
        cmd.arg("-o").arg("StrictHostKeyChecking=accept-new");
        cmd.arg("-o").arg(format!(
            "ConnectTimeout={}",
            self.ssh_options.connect_timeout.as_secs().max(1)
        ));
        cmd.arg("-i").arg(identity_file.as_ref());

        if let Some(interval) = self.ssh_options.server_alive_interval {
            let secs = interval.as_secs();
            if secs > 0 {
                cmd.arg("-o").arg(format!("ServerAliveInterval={secs}"));
            }
        }

        // IMPORTANT: pass the script via stdin (`sh -s`) to avoid quoting issues with
        // newlines/comments in `remote_script`. This preserves the prior mux behavior
        // where the script is an argv payload, not re-parsed by the user's login shell.
        cmd.arg(&destination);
        if let Some(prefix) = &self.source_authority_prefix {
            cmd.arg(prefix);
        }
        cmd.arg("sh").arg("-s");

        cmd.stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());

        let start = std::time::Instant::now();
        let mut child = cmd
            .spawn()
            .with_context(|| format!("Failed to spawn ssh to {}", destination))?;

        if let Some(mut stdin) = child.stdin.take() {
            // Feed the script, then close stdin so `sh -s` begins execution.
            stdin
                .write_all(remote_script.as_bytes())
                .await
                .context("Failed to write remote script to ssh stdin")?;
            stdin
                .write_all(b"\n")
                .await
                .context("Failed to finalize remote script")?;
            drop(stdin);
        }

        let stdout = child.stdout.take();
        let stderr = child.stderr.take();

        let (tx, mut rx) = tokio::sync::mpsc::channel(100);

        enum StreamEvent {
            Stdout(String),
            Stderr(String),
        }

        // Spawn stdout reader
        if let Some(out) = stdout {
            let tx = tx.clone();
            tokio::spawn(async move {
                let mut reader = BufReader::new(out);
                let mut line = String::new();
                loop {
                    line.clear();
                    match reader.read_line(&mut line).await {
                        Ok(0) => break,
                        Ok(_) => {
                            if tx.send(StreamEvent::Stdout(line.clone())).await.is_err() {
                                break;
                            }
                        }
                        Err(_) => break,
                    }
                }
            });
        }

        // Spawn stderr reader
        if let Some(err) = stderr {
            let tx = tx.clone();
            tokio::spawn(async move {
                let mut reader = BufReader::new(err);
                let mut line = String::new();
                loop {
                    line.clear();
                    match reader.read_line(&mut line).await {
                        Ok(0) => break,
                        Ok(_) => {
                            if tx.send(StreamEvent::Stderr(line.clone())).await.is_err() {
                                break;
                            }
                        }
                        Err(_) => break,
                    }
                }
            });
        }

        drop(tx);

        let mut stdout_acc = String::new();
        let mut stderr_acc = String::new();

        let command_timeout = self.ssh_options.command_timeout;
        const MAX_OUTPUT_SIZE: usize = 10 * 1024 * 1024;
        let mut completion_tick = tokio::time::interval(Duration::from_secs(3));
        completion_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
        let mut durable_status = None;
        // Seeing the completion receipt means the workload finished, not that
        // its output has arrived: the remote wrapper is still streaming the
        // tail. Keep draining until the channel closes; kill only a channel
        // that goes quiet for the idle grace, or outlives the cap (the hung
        // case the probe exists for). A large final burst over a slow link
        // keeps extending the idle deadline.
        const COMPLETION_DRAIN_IDLE: Duration = Duration::from_secs(15);
        const COMPLETION_DRAIN_CAP: Duration = Duration::from_secs(120);
        let mut drain_deadline: Option<tokio::time::Instant> = None;
        let mut drain_cap: Option<tokio::time::Instant> = None;

        let status = match tokio::time::timeout(command_timeout, async {
            loop {
                let drain_expired = async move {
                    match drain_deadline {
                        Some(deadline) => tokio::time::sleep_until(deadline).await,
                        None => std::future::pending().await,
                    }
                };
                let event = tokio::select! {
                    event = rx.recv() => match event { Some(event) => event, None => break },
                    _ = completion_tick.tick(), if self.recovery_completion.is_some() && durable_status.is_none() => {
                        if let Ok(Some(status)) = self.read_recovery_completion(worker).await {
                            durable_status = Some(status);
                            let now = tokio::time::Instant::now();
                            drain_cap = Some(now + COMPLETION_DRAIN_CAP);
                            drain_deadline = Some(now + COMPLETION_DRAIN_IDLE);
                        }
                        continue;
                    }
                    () = drain_expired => {
                        let _ = child.kill().await;
                        break;
                    }
                };
                if let Some(cap) = drain_cap {
                    drain_deadline = Some((tokio::time::Instant::now() + COMPLETION_DRAIN_IDLE).min(cap));
                }
                match event {
                    StreamEvent::Stdout(line) => {
                        on_stdout(&line);
                        if stdout_acc.len() < MAX_OUTPUT_SIZE {
                            stdout_acc.push_str(&line);
                            if stdout_acc.len() >= MAX_OUTPUT_SIZE {
                                stdout_acc.push_str("\n...[output truncated]...\n");
                            }
                        }
                    }
                    StreamEvent::Stderr(line) => {
                        on_stderr(&line);
                        if stderr_acc.len() < MAX_OUTPUT_SIZE {
                            stderr_acc.push_str(&line);
                            if stderr_acc.len() >= MAX_OUTPUT_SIZE {
                                stderr_acc.push_str("\n...[output truncated]...\n");
                            }
                        }
                    }
                }
            }

            child.wait().await.context("Failed to wait for ssh command")
        })
        .await
        {
            Ok(status) => status?,
            Err(_) => {
                // Best-effort: kill local ssh process first so the channel is closed.
                let _ = child.kill().await;
                // Issue #62: killing the LOCAL ssh does NOT stop the remote
                // cargo — the orphan keeps building and retains the project's
                // Cargo build-directory lock, deadlocking the next job that
                // selects this worker. Before surfacing E104, SIGKILL the
                // recorded remote process group over a fresh SSH channel (the
                // same group the in-session watchdog and the daemon kill path
                // target) and verify it is gone.
                let (cleanup, detail) = self.kill_remote_process_group_after_timeout(worker).await;
                warn!(
                    "SSH command timed out after {:?} on {}; remote cleanup: {} ({})",
                    command_timeout, worker.id, cleanup, detail
                );
                let evidence = (!self.worker_platform.is_windows()
                    && !use_mock_transport(worker))
                .then(|| {
                    self.build_id.map(|build_id| {
                        rch_common::orphan_quarantine::QuarantineEvidence {
                            build_id,
                            pgid_file: Self::remote_pgid_file_path_for_root(
                                &self.remote_path(),
                                build_id,
                            ),
                        }
                    })
                })
                .flatten();
                return Err(SshCommandTimedOut {
                    timeout: command_timeout,
                    cleanup,
                    detail,
                    evidence,
                }
                .into());
            }
        };

        let duration = start.elapsed();
        Ok(CommandResult {
            exit_code: durable_status.unwrap_or_else(|| status.code().unwrap_or(-1)),
            stdout: stdout_acc,
            stderr: stderr_acc,
            duration_ms: duration.as_millis() as u64,
        })
    }

    /// Execute a command and stream output in real-time.
    ///
    /// If `toolchain` is provided, the command will be wrapped with `rustup run <toolchain>`.
    /// Color-forcing environment variables are applied based on the configured color mode
    /// to preserve ANSI colors in the streamed output.
    pub async fn execute_remote_streaming<F, G>(
        &self,
        worker: &WorkerConfig,
        command: &str,
        toolchain: Option<&ToolchainInfo>,
        on_stdout: F,
        on_stderr: G,
    ) -> Result<CommandResult>
    where
        F: FnMut(&str),
        G: FnMut(&str),
    {
        let wrapped_command =
            self.durable_execution_command(self.build_remote_command(command, toolchain));

        if use_mock_transport(worker) {
            let mut client = MockSshClient::new(worker.clone(), MockConfig::from_env());
            client.connect().await?;
            let result = client
                .execute_streaming(&wrapped_command, on_stdout, on_stderr)
                .await;
            if let Err(e) = client.disconnect().await {
                warn!("Failed to disconnect mock SSH client: {}", e);
            }
            return result;
        }

        #[cfg(not(unix))]
        {
            return Err(crate::error::PlatformError::UnixOnly {
                feature: "SSH remote streaming".to_string(),
            }
            .into());
        }

        #[cfg(unix)]
        {
            self.execute_over_ssh_streaming(worker, &wrapped_command, on_stdout, on_stderr)
                .await
        }
    }

    /// Build rsync command for retrieve_artifacts.
    fn build_retrieve_command(
        &self,
        worker: &WorkerConfig,
        escaped_remote_path: &str,
        artifact_patterns: &[String],
    ) -> Command {
        let (mut cmd, capabilities) = self.rsync_command();

        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));
        let ssh_command = self.build_rsync_ssh_command(escaped_identity.as_ref());

        // Use --safe-links to prevent symlink traversal attacks from malicious workers.
        // --stats is required so parse_rsync_bytes/parse_rsync_files can read transfer
        // counts from stdout; without it rsync produces no output and the parsers
        // return 0, causing a false "No artifacts retrieved" warning.
        // --info=name2 + --out-format itemize EVERY matched regular file —
        // transferred (`>f…`) AND verified up-to-date (`.f`) — which is what the
        // zero-build-output detector (bd-mpbav) reads to prove a sync-back that
        // matched only non-output files left the local artifacts stale. name2
        // (the modern form of -vv's unchanged-file listing) is what surfaces the
        // `.f` lines; without it an up-to-date no-op rebuild would look
        // indistinguishable from a zero-output miss. On openrsync / rsync 2.6.9
        // the `-vv` spelling is used instead (issue #66).
        cmd.arg("-az");
        add_portable_rsync_archive_args(&mut cmd);
        cmd.arg("--stats")
            .arg("--timeout=30")
            .args(capabilities.name_listing_args())
            .arg("--out-format=%i %n")
            .arg("--safe-links")
            .arg("-e")
            .arg(ssh_command);
        self.append_source_rsync_path(&mut cmd);

        // Add zstd compression (zlib on a legacy binary; issue #66)
        self.append_compression_args(&mut cmd, &capabilities);

        // Add bandwidth limit if configured (bd-3hho)
        if let Some(bwlimit) = self.transfer_config.bwlimit_kbps
            && bwlimit > 0
        {
            cmd.arg(format!("--bwlimit={}", bwlimit));
        }

        // Prune empty directories to prevent cluttering local project with
        // empty parents of excluded files (side effect of --include="*/")
        cmd.arg("--prune-empty-dirs");

        // Split caller-supplied EXCLUDE rules (`- <pat>`) from INCLUDE patterns.
        // Excludes (e.g. cargo cache trees in a custom-target sync) must be emitted
        // first so rsync's first-match-wins ordering keeps them from transferring;
        // only the include patterns feed the root/source-integrity helpers below.
        let (caller_excludes, include_patterns) = partition_artifact_filters(artifact_patterns);

        // Apply retrieval-safe excludes before the directory include so rsync
        // never descends into known junk trees like `.beads/recovery_*` on the
        // worker, while still allowing traversal into declared artifact roots.
        for pattern in self.get_retrieval_excludes(&include_patterns) {
            cmd.arg("--exclude").arg(pattern);
        }

        // Caller-supplied excludes (cargo `incremental/`, `.fingerprint/`,
        // `build/`, `*.d`, …) — emitted before the includes so a broad output
        // include like `debug/**` cannot drag the cache trees back.
        for pattern in &caller_excludes {
            cmd.arg("--exclude").arg(pattern);
        }

        // Source-integrity guard (RCH bug d7xc3): explicitly exclude every
        // top-level entry in the local project root that ISN'T an allowed
        // artifact root. Defends against unanchored pattern matching, malformed
        // includes, or a stale remote tree pulling source files into the local
        // checkout. The excludes are emitted BEFORE the directory include so
        // rsync evaluates them first and refuses to descend into source dirs.
        let allowed_roots = allowed_artifact_roots(&include_patterns);
        for exclude in self.local_source_roots_to_exclude(&allowed_roots, &include_patterns) {
            cmd.arg("--exclude").arg(exclude);
        }

        // Essential: Include all directories so rsync can traverse to match patterns.
        // Without this, the final --exclude "*" prevents rsync from entering directories
        // like "target/" to check for matches.
        cmd.arg("--include").arg("*/");

        // Include only specified artifact patterns, anchored at the rsync
        // transfer root via `anchor_retrieval_pattern` (RCH bug d7xc3) so
        // a pattern like `target/debug/**` cannot match `<root>/anything/
        // target/debug/...` at arbitrary depth.
        for pattern in &include_patterns {
            cmd.arg("--include").arg(anchor_retrieval_pattern(pattern));
        }
        cmd.arg("--exclude").arg("*"); // Exclude everything else

        let source = format!("{}@{}:{}/", worker.user, worker.host, escaped_remote_path);
        cmd.arg(&source)
            .arg(format!("{}/", self.project_root.display()));

        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
        cmd
    }

    /// Build rsync command for streaming artifact retrieval.
    fn build_retrieve_streaming_command(
        &self,
        worker: &WorkerConfig,
        escaped_remote_path: &str,
        artifact_patterns: &[String],
    ) -> Command {
        let (mut cmd, capabilities) = self.rsync_command();

        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));

        let ssh_command = self.build_rsync_ssh_command(escaped_identity.as_ref());

        cmd.arg("-az");
        add_portable_rsync_archive_args(&mut cmd);
        // Flavour-specific spellings of progress / stats / per-file listing
        // (issue #66); see RsyncCapabilities for the mapping.
        cmd.args(capabilities.progress_args())
            .arg("--timeout=30")
            .args(capabilities.stats_args())
            // name2 + itemized out-format feed the zero-build-output detector
            // (bd-mpbav); see build_retrieve_command for the rationale.
            .args(capabilities.name_listing_args())
            .arg("--out-format=%i %n")
            .arg("--safe-links")
            .arg("-e")
            .arg(ssh_command);
        self.append_source_rsync_path(&mut cmd);

        // Add zstd compression (zlib on a legacy binary; issue #66)
        self.append_compression_args(&mut cmd, &capabilities);

        // Add bandwidth limit if configured (bd-3hho)
        if let Some(bwlimit) = self.transfer_config.bwlimit_kbps
            && bwlimit > 0
        {
            cmd.arg(format!("--bwlimit={}", bwlimit));
        }

        // Prune empty directories to prevent cluttering local project
        cmd.arg("--prune-empty-dirs");

        // Split caller-supplied EXCLUDE rules (`- <pat>`) from INCLUDE patterns;
        // see build_retrieve_command for the rationale (first-match-wins ordering).
        let (caller_excludes, include_patterns) = partition_artifact_filters(artifact_patterns);

        // Reuse the retrieval-safe excludes so streaming downloads skip stale
        // worker-local junk trees without excluding legitimate artifact roots.
        for pattern in self.get_retrieval_excludes(&include_patterns) {
            cmd.arg("--exclude").arg(pattern);
        }

        // Caller-supplied excludes (cargo cache trees, `*.d`, …) emitted before
        // the includes so a broad output include cannot drag them back.
        for pattern in &caller_excludes {
            cmd.arg("--exclude").arg(pattern);
        }

        // Source-integrity guard (RCH bug d7xc3): see build_retrieve_command.
        // Same belt-and-suspenders defense applied to the streaming variant.
        let allowed_roots = allowed_artifact_roots(&include_patterns);
        for exclude in self.local_source_roots_to_exclude(&allowed_roots, &include_patterns) {
            cmd.arg("--exclude").arg(exclude);
        }

        // Essential: Include all directories so rsync can traverse to match patterns.
        cmd.arg("--include").arg("*/");

        // Artifact include patterns are anchored (RCH bug d7xc3) so they can
        // only match at the rsync transfer root.
        for pattern in &include_patterns {
            cmd.arg("--include").arg(anchor_retrieval_pattern(pattern));
        }
        cmd.arg("--exclude").arg("*");

        let source = format!("{}@{}:{}/", worker.user, worker.host, escaped_remote_path);
        cmd.arg(&source)
            .arg(format!("{}/", self.project_root.display()));

        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
        cmd
    }

    fn build_rsync_ssh_command(&self, escaped_identity: &str) -> String {
        let mut command = format!(
            "ssh -i {} -o StrictHostKeyChecking=accept-new -o BatchMode=yes",
            escaped_identity
        );

        #[cfg(unix)]
        {
            if let Some(interval) = self.ssh_options.server_alive_interval {
                let secs = interval.as_secs();
                if secs > 0 {
                    command.push_str(&format!(" -o ServerAliveInterval={secs}"));
                }
            }

            if self.ssh_options.control_master
                && let Some(idle) = self.ssh_options.control_persist_idle
            {
                let control_dir = self.rsync_control_dir();
                if let Err(e) = std::fs::create_dir_all(&control_dir) {
                    warn!(
                        "Failed to create rsync SSH control dir {:?}: {}",
                        control_dir, e
                    );
                } else {
                    // Set restrictive permissions (0700) to prevent symlink attacks
                    // and unauthorized access to SSH control sockets
                    use std::os::unix::fs::PermissionsExt;
                    if let Err(e) = std::fs::set_permissions(
                        &control_dir,
                        std::fs::Permissions::from_mode(0o700),
                    ) {
                        warn!(
                            "Failed to set permissions on rsync SSH control dir {:?}: {}",
                            control_dir, e
                        );
                    }
                }

                let control_path = control_dir.join("rch-rsync-%C");
                let escaped_control_path = escape(control_path.to_string_lossy());
                command.push_str(" -o ControlMaster=auto");
                command.push_str(&format!(" -o ControlPath={}", escaped_control_path));

                if idle.is_zero() {
                    command.push_str(" -o ControlPersist=no");
                } else {
                    command.push_str(&format!(" -o ControlPersist={}s", idle.as_secs()));
                }
            }
        }

        command
    }

    fn rsync_control_dir(&self) -> PathBuf {
        // Prefer ~/.ssh/rch to avoid exceeding the Unix socket path limit
        // (104 bytes on macOS). See rch-common/src/ssh.rs for rationale.
        if let Some(home) = dirs::home_dir() {
            home.join(".ssh").join("rch")
        } else if let Some(runtime_dir) = dirs::runtime_dir() {
            runtime_dir.join("rch")
        } else {
            let username = std::env::var("USER").unwrap_or_else(|_| "unknown".to_string());
            std::env::temp_dir().join(format!("rch-ssh-{}", username))
        }
    }

    /// Retrieve build artifacts from the remote worker.
    ///
    /// Uses retry logic with exponential backoff for transient network errors.
    /// Returns the transfer [`SyncResult`] plus the per-file manifest the
    /// zero-build-output detector consumes (bd-mpbav); see [`ArtifactRetrieval`].
    pub async fn retrieve_artifacts(
        &self,
        worker: &WorkerConfig,
        artifact_patterns: &[String],
    ) -> Result<ArtifactRetrieval> {
        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));

        if use_mock_transport(worker) {
            // Mock path also uses retry logic for consistent behavior
            // Create MockRsync ONCE and share via Arc so failure counters persist across retries
            let rsync = std::sync::Arc::new(MockRsync::new(MockRsyncConfig::from_env()));
            let source = format!("{}@{}:{}/", worker.user, worker.host, escaped_remote_path);
            let project_root_str = self.project_root.display().to_string();
            let patterns = artifact_patterns.to_vec();
            let retry_config = self.transfer_config.retry.clone();
            let result = retry_with_backoff(&retry_config, "mock_retrieve_artifacts", || {
                let rsync = rsync.clone();
                let src = source.clone();
                let dest = project_root_str.clone();
                let pats = patterns.clone();
                async move { rsync.retrieve_artifacts(&src, &dest, &pats).await }
            })
            .await?;
            return Ok(ArtifactRetrieval::from_stats(SyncResult {
                bytes_transferred: result.bytes_transferred,
                files_transferred: result.files_transferred,
                duration_ms: result.duration_ms,
            }));
        }

        // The Windows transport enforces its deadline and cancellation while
        // owning/reaping SSH children, then verifies and publishes the archive.
        if self.worker_platform.is_windows() {
            return self
                .retrieve_artifacts_windows(worker, &remote_path, artifact_patterns, &mut |_| {})
                .await;
        }

        info!("Retrieving artifacts from {} on {}", remote_path, worker.id);

        let start = std::time::Instant::now();

        // Execute rsync with retry logic for transient errors
        let retry_config = self
            .artifact_retry_config(worker, &escaped_remote_path, artifact_patterns)
            .await;
        let output = self
            .execute_retrieval_rsync(
                &retry_config,
                "retrieve_artifacts",
                || self.build_retrieve_command(worker, &escaped_remote_path, artifact_patterns),
                |_| {},
            )
            .await?;

        let duration = start.elapsed();
        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
        let stderr = String::from_utf8_lossy(&output.stderr).to_string();

        if !output.status.success() {
            warn!("Artifact retrieval failed: {}", stderr);
            return Err(TransferError::SyncFailed {
                reason: "rsync artifact retrieval failed".to_string(),
                exit_code: output.status.code(),
                stderr: stderr.clone(),
            }
            .into());
        }

        // An exit-0 partial download leaves the local artifact tree incomplete;
        // fail rather than report success (see sync_to_remote).
        if let Some(indicator) = detect_partial_transfer(&stderr) {
            warn!(
                "rsync exited 0 but reported a partial artifact retrieval (matched '{}'): {}",
                indicator,
                stderr.lines().next().unwrap_or(&stderr)
            );
            return Err(TransferError::SyncFailed {
                reason: format!("partial artifact retrieval despite exit 0 ({indicator})"),
                exit_code: output.status.code(),
                stderr: stderr.clone(),
            }
            .into());
        }

        let bytes_transferred = parse_rsync_bytes(&stdout, RsyncTransferDirection::Download);
        let files_transferred = parse_rsync_files(&stdout);

        // Warn if no artifacts were retrieved - this may indicate a build failure
        // or misconfigured artifact patterns. We don't fail here because some
        // commands (e.g., cargo check) don't produce artifacts.
        if files_transferred == 0 && bytes_transferred == 0 {
            warn!(
                "No artifacts retrieved from {} - build may have failed or artifact patterns may be misconfigured",
                worker.id
            );
            debug!("Artifact patterns used: {:?}", artifact_patterns);
        }

        info!(
            "Artifacts retrieved in {}ms ({} files, {} bytes)",
            duration.as_millis(),
            files_transferred,
            bytes_transferred
        );

        Ok(ArtifactRetrieval::from_rsync_output(
            SyncResult {
                bytes_transferred,
                files_transferred,
                duration_ms: duration.as_millis() as u64,
            },
            &stdout,
        ))
    }

    /// Build the rsync command that pulls one declared job result directory.
    ///
    /// Explicit source path instead of the pattern machinery used by
    /// [`Self::build_retrieve_command`]: rsync itself fails when the source
    /// does not exist on the worker, which is precisely the loud
    /// missing-declared-output semantic bd-p0yoo requires.
    fn build_result_dir_retrieve_command(
        &self,
        worker: &WorkerConfig,
        escaped_remote_path: &str,
        rel: &Path,
    ) -> Command {
        let (mut cmd, capabilities) = self.rsync_command();

        let identity_file = shellexpand::tilde(&worker.identity_file);
        let escaped_identity = escape(Cow::from(identity_file.as_ref()));
        let ssh_command = self.build_rsync_ssh_command(escaped_identity.as_ref());

        // Same transport hardening as artifact retrieval: --safe-links blocks
        // symlink traversal out of the declared tree.
        cmd.arg("-az");
        add_portable_rsync_archive_args(&mut cmd);
        cmd.arg("--stats")
            .arg("--safe-links")
            .arg("-e")
            .arg(ssh_command);
        self.append_source_rsync_path(&mut cmd);

        self.append_compression_args(&mut cmd, &capabilities);
        if let Some(bwlimit) = self.transfer_config.bwlimit_kbps
            && bwlimit > 0
        {
            cmd.arg(format!("--bwlimit={}", bwlimit));
        }

        // Trailing slash on BOTH sides copies the CONTENTS of `<remote>/<rel>/`
        // into `<project_root>/<rel>/`, so declared paths materialize at their
        // identical repository-relative location locally.
        let rel_str = rel.as_os_str().to_string_lossy();
        let escaped_rel = escape(Cow::from(rel_str.as_ref()));
        let source = format!(
            "{}@{}:{}/{}/",
            worker.user, worker.host, escaped_remote_path, escaped_rel
        );
        let local_dest = format!("{}/{}/", self.project_root.display(), rel.display());
        cmd.arg(&source).arg(local_dest);

        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
        cmd
    }

    /// Apply a content epoch while the source-pair lease is held, after complete
    /// fresh-root materialization and overlay verification. Identical selected
    /// bytes reuse their timestamp; changed bytes must be newer than artifacts
    /// even when their archived mtimes are old. This never reuses source files.
    pub async fn refresh_clean_overlay_source(
        &self,
        worker: &WorkerConfig,
        identity: &str,
        retirement_root: &str,
    ) -> Result<()> {
        if use_mock_transport(worker) {
            return Ok(());
        }
        tokio::time::timeout(
            Duration::from_secs(120),
            self.run_remote_sh(
                worker,
                &self.clean_overlay_source_refresh_command_at(identity, retirement_root)?,
            ),
        )
        .await
        .context("timed out proving clean-overlay source freshness")?
    }

    #[cfg(test)]
    fn clean_overlay_source_refresh_command(&self, identity: &str) -> Result<String> {
        self.clean_overlay_source_refresh_command_at(identity, &self.remote_path())
    }

    fn clean_overlay_source_refresh_command_at(
        &self,
        identity: &str,
        retirement_root: &str,
    ) -> Result<String> {
        if identity.len() != 64 || !identity.bytes().all(|byte| byte.is_ascii_hexdigit()) {
            anyhow::bail!("invalid clean-overlay freshness identity");
        }
        let root = self.remote_path();
        let retirement_root = retirement_root.trim_end_matches('/');
        anyhow::ensure!(
            !retirement_root.is_empty()
                && (root == retirement_root || root.starts_with(&format!("{retirement_root}/"))),
            "freshness source is outside its owned retirement container"
        );
        let pool = self.remote_cargo_target_dir();
        let quote = |value: &str| escape(Cow::from(value)).into_owned();
        // Selected sibling roots are retired together. Controls must remain
        // outside that entire container, with disjoint exact-root identities.
        let control = if root == retirement_root {
            root.clone()
        } else {
            format!(
                "{retirement_root}.source-{}",
                blake3::hash(root.as_bytes()).to_hex()
            )
        };
        let anchor = format!("{control}.freshness-anchor");
        let epoch = format!("{control}.freshness-epoch-v1");
        let conservative = format!(
            "set -eu; root={root}; pool={pool}; anchor={anchor}; epoch={epoch}; identity={identity}; \
             [ ! -L \"$anchor\" ] && [ ! -L \"$epoch\" ] && [ ! -L \"$pool\" ]; \
             [ ! -e \"$epoch\" ] || [ -f \"$epoch\" ]; \
             if [ -f \"$epoch\" ] && [ \"$(wc -c < \"$epoch\" | tr -d ' ')\" = 65 ] && \
             [ \"$(cat \"$epoch\")\" = \"$identity\" ]; then \
             find \"$root\" \\( -type f -o -type d \\) -exec touch -r \"$epoch\" {{}} +; exit 0; fi; \
             stamp=$(mktemp \"$epoch.pending.XXXXXX\"); \
             printf '%s\\n' \"$identity\" > \"$stamp\"; touch \"$anchor\"; \
             if [ -d \"$pool\" ]; then \
             find \"$pool\" -type f -exec sh -c \
             'anchor=$1; shift; for file do if [ \"$file\" -nt \"$anchor\" ]; then touch -r \"$file\" \"$anchor\" || exit; fi; done' \
             sh \"$anchor\" {{}} +; fi; \
             attempts=0; touch \"$stamp\"; \
             until [ \"$stamp\" -nt \"$anchor\" ]; do \
             attempts=$((attempts + 1)); if [ \"$attempts\" -gt 5 ]; then \
             echo 'RCH: worker clock cannot advance beyond cached artifacts; source freshness unproved' >&2; exit 1; fi; \
             sleep 1; touch \"$stamp\"; done; \
             find \"$root\" \\( -type f -o -type d \\) -exec touch -r \"$stamp\" {{}} +; \
             mv -f \"$stamp\" \"$epoch\"",
            root = quote(&root),
            pool = quote(&pool),
            anchor = quote(&anchor),
            epoch = quote(&epoch),
            identity = quote(identity),
        );
        Ok(format!(
            "if command -v python3 >/dev/null 2>&1; then python3 -c {script} {root} {pool} {epoch} {identity}; \
             else printf '%s\\n' 'RCH: python3 unavailable; using conservative whole-source freshness' >&2; \
             {conservative}; fi",
            script = quote(Self::clean_overlay_freshness_ledger_script()),
            root = quote(&root),
            pool = quote(&pool),
            epoch = quote(&epoch),
            identity = quote(identity),
        ))
    }

    /// Metadata only: source bytes are always rematerialized under the existing
    /// source-pair lease. Compare against the immediate predecessor, never a
    /// historical content-to-time cache (which would make A -> B -> A stale).
    fn clean_overlay_freshness_ledger_script() -> &'static str {
        r#"import hashlib, json, os, stat, sys, tempfile, time

root, pool, epoch, identity = sys.argv[1:]
ledger = epoch + '.files-v2'
MAX_ENTRIES = 200000
MAX_BYTES = 8 * 1024**3
MAX_LEDGER = 64 * 1024**2

def require(condition, reason):
    if not condition:
        raise RuntimeError('RCH: source freshness unproved: ' + reason)

def canonical(value):
    return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode('ascii')

def digest(value):
    return hashlib.sha256(canonical(value)).hexdigest()

def regular_control(path):
    try:
        info = os.lstat(path)
    except FileNotFoundError:
        return None
    require(stat.S_ISREG(info.st_mode), 'nonregular freshness control')
    return info

require(stat.S_ISDIR(os.lstat(root).st_mode), 'source root is not a real directory')
try:
    pool_info = os.lstat(pool)
except FileNotFoundError:
    pool_info = None
require(pool_info is None or stat.S_ISDIR(pool_info.st_mode), 'target pool is not a real directory')
root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
root_identity = os.fstat(root_fd)

def parent_fd(relative):
    descriptor = os.dup(root_fd)
    parts = relative.split('/') if relative else ['.']
    try:
        for part in parts[:-1]:
            next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor)
            os.close(descriptor)
            descriptor = next_fd
        return descriptor, parts[-1]
    except BaseException:
        os.close(descriptor)
        raise

epoch_info = regular_control(epoch)
ledger_info = regular_control(ledger)
previous = {}
if epoch_info and ledger_info and ledger_info.st_size <= MAX_LEDGER:
    try:
        with os.fdopen(os.open(ledger, os.O_RDONLY | os.O_NOFOLLOW), 'rb') as stream:
            encoded = stream.read(MAX_LEDGER + 1)
        require(len(encoded) <= MAX_LEDGER, 'freshness ledger grew beyond cap')
        stored = json.loads(encoded)
        payload = stored['payload']
        with os.fdopen(os.open(epoch, os.O_RDONLY | os.O_NOFOLLOW), 'rb') as stream:
            marker = stream.read(66)
        valid = (stored['sha256'] == digest(payload)
                 and payload['schema'] == 2
                 and marker == (payload['identity'] + '\n').encode('ascii')
                 and epoch_info.st_mtime_ns == payload['epoch_ns']
                 and isinstance(payload['entries'], dict)
                 and len(payload['entries']) <= MAX_ENTRIES)
        for path, entry in payload['entries'].items():
            valid = valid and (path == '' or (not path.startswith('/')
                and all(part not in ('', '.', '..') for part in path.split('/'))))
            valid = valid and (set(entry) == {'fingerprint', 'mtime_ns'}
                and isinstance(entry['fingerprint'], str) and len(entry['fingerprint']) == 64
                and all(c in '0123456789abcdef' for c in entry['fingerprint'])
                and type(entry['mtime_ns']) is int and 0 <= entry['mtime_ns'] <= payload['epoch_ns'])
        if valid:
            previous = payload['entries']
    except (ValueError, KeyError, TypeError, AttributeError, UnicodeError):
        pass

def inventory():
    entries = {}
    total = 0
    def visit(relative, depth):
        nonlocal total
        require(depth <= 128 and len(entries) < MAX_ENTRIES, 'source inventory cap exceeded')
        directory_fd, name = parent_fd(relative)
        before = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
        mode = stat.S_IMODE(before.st_mode)
        entries[relative] = None
        if stat.S_ISREG(before.st_mode):
            total += before.st_size
            require(total <= MAX_BYTES, 'source byte cap exceeded')
            hasher = hashlib.sha256()
            # Reject a symlink substitution between lstat and open.
            descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd)
            with os.fdopen(descriptor, 'rb') as stream:
                opened = os.fstat(stream.fileno())
                require((opened.st_dev, opened.st_ino) == (before.st_dev, before.st_ino), 'source replaced while hashing')
                read_bytes = 0
                for block in iter(lambda: stream.read(1024 * 1024), b''):
                    read_bytes += len(block)
                    require(read_bytes <= before.st_size, 'source grew while hashing')
                    hasher.update(block)
            value = ['file', mode, before.st_size, hasher.hexdigest()]
        elif stat.S_ISLNK(before.st_mode):
            value = ['symlink', mode, os.readlink(name, dir_fd=directory_fd)]
        elif stat.S_ISDIR(before.st_mode):
            child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory_fd)
            try:
                children = []
                with os.scandir(child_fd) as listing:
                    for child in listing:
                        require(len(children) + len(entries) < MAX_ENTRIES, 'source directory cap exceeded')
                        children.append(child.name)
                children.sort()
            finally:
                os.close(child_fd)
            value = ['directory', mode, [[name, visit(os.path.join(relative, name), depth + 1)] for name in children]]
        else:
            raise RuntimeError('RCH: unsupported source entry type')
        after = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
        os.close(directory_fd)
        require((before.st_dev, before.st_ino, before.st_mode, before.st_size, before.st_mtime_ns, before.st_ctime_ns)
                == (after.st_dev, after.st_ino, after.st_mode, after.st_size, after.st_mtime_ns, after.st_ctime_ns),
                'source changed while hashing')
        fingerprint = digest(value)
        entries[relative] = {'fingerprint': fingerprint}
        return fingerprint
    visit('', 0)
    named_root = os.lstat(root)
    require((named_root.st_dev, named_root.st_ino) == (root_identity.st_dev, root_identity.st_ino), 'source root replaced')
    return entries

current = inventory()
changed = [path for path, entry in current.items()
           if previous.get(path, {}).get('fingerprint') != entry['fingerprint']]
# A new metadata seal can retain source timestamps even when only the selected
# commit changed. Changed inputs must still be newer than ALL cached artifacts.
anchor_ns = max([0] + [entry['mtime_ns'] for entry in previous.values()])
def traversal_error(error):
    raise error

if changed and pool_info is not None:
    for directory, directories, files in os.walk(pool, followlinks=False, onerror=traversal_error):
        directories[:] = [name for name in directories if not os.path.islink(os.path.join(directory, name))]
        for name in files:
            info = os.lstat(os.path.join(directory, name))
            if stat.S_ISREG(info.st_mode):
                anchor_ns = max(anchor_ns, info.st_mtime_ns)
deadline = time.monotonic() + 5
stamp_ns = time.time_ns()
while stamp_ns <= anchor_ns:
    require(time.monotonic() < deadline, 'worker clock cannot advance beyond cached artifacts')
    time.sleep(0.01)
    stamp_ns = time.time_ns()

for relative, entry in sorted(current.items(), key=lambda item: item[0].count('/'), reverse=True):
    prior = previous.get(relative)
    modified = prior['mtime_ns'] if prior and prior['fingerprint'] == entry['fingerprint'] else stamp_ns
    directory_fd, name = parent_fd(relative)
    info = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
    os.utime(name, ns=(info.st_atime_ns, modified), dir_fd=directory_fd, follow_symlinks=False)
    require(os.stat(name, dir_fd=directory_fd, follow_symlinks=False).st_mtime_ns == modified, 'source timestamp precision changed')
    os.close(directory_fd)
    entry['mtime_ns'] = modified
verified = inventory()
require({path: entry['fingerprint'] for path, entry in current.items()}
        == {path: entry['fingerprint'] for path, entry in verified.items()}, 'source changed during freshness refresh')

payload = {'schema': 2, 'identity': identity, 'epoch_ns': stamp_ns, 'entries': current}
encoded = canonical({'payload': payload, 'sha256': digest(payload)})
require(len(encoded) <= MAX_LEDGER, 'freshness ledger cap exceeded')
def stage(path, data):
    descriptor, temporary = tempfile.mkstemp(prefix=os.path.basename(path) + '.pending.', dir=os.path.dirname(path))
    with os.fdopen(descriptor, 'wb') as stream:
        stream.write(data)
        stream.flush()
        os.fsync(stream.fileno())
    return temporary

pending_ledger = stage(ledger, encoded)
pending_epoch = stage(epoch, (identity + '\n').encode('ascii'))
os.utime(pending_epoch, ns=(stamp_ns, stamp_ns), follow_symlinks=False)
require(os.lstat(pending_epoch).st_mtime_ns == stamp_ns, 'epoch timestamp precision changed')
# The epoch is the commit marker. A crash between these replacements leaves a
# mismatched pair, causing conservative freshness on the next invocation.
os.replace(pending_ledger, ledger)
os.replace(pending_epoch, epoch)
directory_fd = os.open(os.path.dirname(epoch), os.O_RDONLY | os.O_DIRECTORY)
try:
    os.fsync(directory_fd)
finally:
    os.close(directory_fd)
print('RCH_SOURCE_FRESHNESS_V2 unchanged=%d changed=%d' % (len(current) - len(changed), len(changed)))
"#
    }

    /// Best-effort removal of an isolated remote tree (bd-p1vlb).
    ///
    /// Unpooled clean-overlay roots are per-run. A pooled source path may be
    /// reused only after retirement, under its full-lifecycle source lease —
    /// but each holds a full materialized snapshot. Without explicit reaping,
    /// every overlay run leaks hundreds of MB to GBs on the worker's staging
    /// base (observed: a 37G `/tmp/rch-sync` tmpfs pile within days,
    /// bd-lvbax). Unpooled retirement is best-effort. Paired source reuse
    /// requires successful retirement before its owner can release the lease.
    pub async fn reap_remote_tree(&self, worker: &WorkerConfig, root: &str) -> Result<()> {
        if self.worker_platform.is_windows() {
            // Overlay transport is rsync/ssh-only today; nothing to reap.
            return Ok(());
        }
        // The source can already have been retired by its owner. Avoid even
        // invoking removal in that case. A bounded owned SSH child also keeps
        // cleanup from holding a source-pair lease indefinitely on a lost link.
        tokio::time::timeout(
            Duration::from_secs(120),
            self.run_remote_sh(worker, &Self::remote_tree_retirement_command(root)),
        )
        .await
        .context("timed out retiring clean-overlay source")?
    }

    fn remote_tree_retirement_command(root: &str) -> String {
        let root = escape(Cow::from(root));
        format!("if [ -e {root} ] || [ -L {root} ]; then rm -rf -- {root}; fi")
    }

    /// Retrieve one declared job result directory (bd-p0yoo).
    ///
    /// Runs its own rsync per directory with the directory as an explicit
    /// source, so a directory the job never created is a hard rsync error
    /// rather than a silent zero-file success. Any error means the invocation's
    /// declared outputs are INCOMPLETE and callers must fail loudly.
    pub async fn retrieve_result_dir(
        &self,
        worker: &WorkerConfig,
        rel: &Path,
    ) -> Result<SyncResult> {
        if self.worker_platform.is_windows() {
            return Err(anyhow::anyhow!(
                "result-dir retrieval requires the Unix rsync transport; worker {} is Windows",
                worker.id
            ));
        }

        // Defense in depth: re-validate here rather than trusting the caller.
        // `rel` is interpolated into a remote rsync source spec AND joined
        // into the local project root below, so the repository-relative rules
        // (no traversal, no absolute, ASCII-only) must hold at this boundary.
        let rel =
            &crate::hook::normalize_repository_relative_path("--result-dir", rel).map_err(|e| {
                TransferError::SyncFailed {
                    reason: e.to_string(),
                    exit_code: None,
                    stderr: String::new(),
                }
            })?;

        // Materialize only the PARENT tree of the local destination so nested
        // relative paths (`out/shards/a`) work; rsync itself creates the leaf
        // on success. Pre-creating the leaf would leave a misleading empty
        // directory behind whenever retrieval fails (bd-p0yoo loud-failure
        // semantics: nothing should look materialized when it is not).
        let local_dest = self.project_root.join(rel);
        let dest_parent = local_dest.parent().unwrap_or(self.project_root.as_path());
        std::fs::create_dir_all(dest_parent).map_err(|e| TransferError::SyncFailed {
            reason: format!(
                "failed to create parent directories for local result dir {}",
                rel.display()
            ),
            exit_code: None,
            stderr: e.to_string(),
        })?;

        let remote_path = self.remote_path();

        if use_mock_transport(worker) {
            // Mock transports have no existence model; exercise the same
            // pattern-based plumbing the mock artifact path uses so mock-based
            // tests cover the call wiring end to end.
            let patterns = [format!("{}/**", rel.display())];
            return self
                .retrieve_artifacts(worker, &patterns)
                .await
                .map(|retrieval| retrieval.stats);
        }

        let escaped_remote_path = escape(Cow::from(&remote_path));
        let start = std::time::Instant::now();
        let retry_config = self.effective_rsync_retry_config();
        let output = self
            .execute_retrieval_rsync(
                &retry_config,
                "retrieve_result_dir",
                || self.build_result_dir_retrieve_command(worker, &escaped_remote_path, rel),
                |_| {},
            )
            .await?;

        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
        let stderr = String::from_utf8_lossy(&output.stderr).to_string();

        if !output.status.success() {
            warn!(
                "Declared result dir '{}' retrieval failed from {}: {}",
                rel.display(),
                worker.id,
                stderr
            );
            return Err(TransferError::SyncFailed {
                reason: format!(
                    "declared result dir '{}' missing or unreadable on worker",
                    rel.display()
                ),
                exit_code: output.status.code(),
                stderr: stderr.clone(),
            }
            .into());
        }

        // Exit-0-but-incomplete guard, same as artifact retrieval.
        if let Some(indicator) = detect_partial_transfer(&stderr) {
            warn!(
                "rsync exited 0 but reported a partial result-dir retrieval for '{}' (matched '{}')",
                rel.display(),
                indicator
            );
            return Err(TransferError::SyncFailed {
                reason: format!(
                    "partial retrieval of declared result dir '{}' despite exit 0 ({indicator})",
                    rel.display()
                ),
                exit_code: output.status.code(),
                stderr: stderr.clone(),
            }
            .into());
        }

        let bytes_transferred = parse_rsync_bytes(&stdout, RsyncTransferDirection::Download);
        let files_transferred = parse_rsync_files(&stdout);
        let duration_ms = start.elapsed().as_millis() as u64;
        info!(
            "Result dir '{}' retrieved in {}ms ({} files, {} bytes)",
            rel.display(),
            duration_ms,
            files_transferred,
            bytes_transferred
        );
        Ok(SyncResult {
            bytes_transferred,
            files_transferred,
            duration_ms,
        })
    }

    /// Retrieve build artifacts with streaming progress output.
    ///
    /// Returns the same manifest-carrying result as [`Self::retrieve_artifacts`]
    /// (bd-mpbav); only the progress reporting differs.
    pub async fn retrieve_artifacts_streaming<F>(
        &self,
        worker: &WorkerConfig,
        artifact_patterns: &[String],
        mut on_line: F,
    ) -> Result<ArtifactRetrieval>
    where
        F: FnMut(&str),
    {
        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));

        if use_mock_transport(worker) {
            let rsync = MockRsync::new(MockRsyncConfig::from_env());
            let result = rsync
                .retrieve_artifacts(
                    &format!("{}@{}:{}/", worker.user, worker.host, escaped_remote_path),
                    &self.project_root.display().to_string(),
                    artifact_patterns,
                )
                .await?;
            return Ok(ArtifactRetrieval::from_stats(SyncResult {
                bytes_transferred: result.bytes_transferred,
                files_transferred: result.files_transferred,
                duration_ms: result.duration_ms,
            }));
        }

        if self.worker_platform.is_windows() {
            return self
                .retrieve_artifacts_windows(worker, &remote_path, artifact_patterns, &mut on_line)
                .await;
        }

        info!(
            "Retrieving artifacts from {} on {} (streaming)",
            remote_path, worker.id
        );

        // Rebuilt per retry attempt (see `sync_to_remote_streaming`): a transient
        // transport drop while pulling artifacts must reconnect and retry instead
        // of failing the build's artifact return outright.
        let build_cmd = || {
            self.build_retrieve_streaming_command(worker, &escaped_remote_path, artifact_patterns)
        };

        debug!(
            "Running artifact retrieval (streaming): rsync {:?}",
            build_cmd().as_std().get_args().collect::<Vec<_>>()
        );

        let retrieval_start = std::time::Instant::now();
        let retry_config = self
            .artifact_retry_config(worker, &escaped_remote_path, artifact_patterns)
            .await;
        let output = if self.retrieval_control.is_some() {
            let output = self
                .execute_retrieval_rsync(
                    &retry_config,
                    "retrieve_artifacts_streaming",
                    build_cmd,
                    &mut on_line,
                )
                .await?;
            let combined = format!(
                "{}\n{}",
                String::from_utf8_lossy(&output.stdout),
                String::from_utf8_lossy(&output.stderr)
            );
            if !output.status.success() {
                return Err(TransferError::SyncFailed {
                    reason: "rsync artifact retrieval failed".to_owned(),
                    exit_code: output.status.code(),
                    stderr: combined,
                }
                .into());
            }
            combined
        } else {
            run_command_streaming_with_retry(
                &retry_config,
                "retrieve_artifacts_streaming",
                None,
                None,
                build_cmd,
                &mut on_line,
            )
            .await?
            .0
        };
        let duration_ms = retrieval_start.elapsed().as_millis() as u64;

        // An exit-0 partial download leaves the local artifact tree incomplete;
        // fail rather than report success (see retrieve_artifacts).
        if let Some(indicator) = detect_partial_transfer(&output) {
            warn!(
                "streaming rsync exited 0 but reported a partial artifact retrieval (matched '{}')",
                indicator
            );
            return Err(TransferError::SyncFailed {
                reason: format!("partial artifact retrieval despite exit 0 ({indicator})"),
                exit_code: None,
                stderr: output,
            }
            .into());
        }

        Ok(ArtifactRetrieval::from_rsync_output(
            SyncResult {
                bytes_transferred: parse_rsync_bytes(&output, RsyncTransferDirection::Download),
                files_transferred: parse_rsync_files(&output),
                duration_ms,
            },
            &output,
        ))
    }

    /// Clean up remote project directory.
    #[allow(dead_code)] // Reserved for future cleanup routines
    pub async fn cleanup_remote(&self, worker: &WorkerConfig) -> Result<()> {
        let remote_path = self.remote_path();
        let escaped_remote_path = escape(Cow::from(&remote_path));

        if use_mock_transport(worker) {
            debug!("Mock cleanup of {} on {}", remote_path, worker.id);
            return Ok(());
        }

        info!("Cleaning up {} on {}", remote_path, worker.id);

        #[cfg(not(unix))]
        {
            return Err(crate::error::PlatformError::UnixOnly {
                feature: "SSH remote cleanup".to_string(),
            }
            .into());
        }

        #[cfg(unix)]
        {
            let mut client = SshClient::new(worker.clone(), self.ssh_options.clone());
            client.connect().await?;

            let result = client
                .execute(&self.source_activity_command(&format!("rm -rf {}", escaped_remote_path)))
                .await;

            if let Err(e) = client.disconnect().await {
                warn!("Failed to disconnect SSH client after cleanup: {}", e);
            }

            let result = result?;

            if !result.success() {
                warn!("Cleanup failed: {}", result.stderr);
            }

            Ok(())
        }
    }

    /// Best-effort reaping of *stale* sibling per-job target dirs for this
    /// project on the worker.
    ///
    /// rch gives every forwarded-`CARGO_TARGET_DIR` build a per-job target dir
    /// (`.rch-target-<worker>-job-<id>-<ts>-<seq>`). Such a dir can stay in active
    /// use far beyond a single command — a long-running build keeps writing into
    /// it, and one was observed accumulating ~11.5h of build artifacts. So a
    /// per-job dir must *never* be removed merely because some build finished; that
    /// could clip a build still in flight. Instead we remove only dirs that
    /// have seen **no file activity for `idle_hours`** — i.e. finished/abandoned
    /// ones. A dir idle that long cannot be a live job (an active build touches its
    /// dir continuously), so this never races a concurrent build on the same
    /// project, even when multiple agents build it on the same worker at once.
    ///
    /// The sweep is confined to the *single current project dir* (`remote_path()`),
    /// reaping only its abandoned sibling per-job dirs. The expensive cross-project
    /// full-tree scan has moved OFF this per-dispatch path into the durable
    /// daemon-side worker sweep (`rchd::stale_target_reap`), which scans every
    /// project under the worker's `remote_base` on a background interval. Both
    /// share the idle predicate via `rch_common::stale_target_reap` so they cannot
    /// drift; this orchestrator side stays cheap (one `cd` + a two-glob loop).
    ///
    /// The staleness check looks at the dir itself *and* any descendant (file or
    /// subdir): a recent deep file means an active build (a top-dir-mtime-only
    /// check would miss it, because the top dir mtime can go stale while deep
    /// incremental artifacts keep changing), while a recent *dir* mtime means a
    /// freshly-created target — e.g. a concurrent build that has `mkdir`'d its dir
    /// but not yet written a file (a files-only check would wrongly reap it). The
    /// removal *itself* is detached on the worker (a backgrounded `rm`), so the
    /// potentially-large reclaim runs concurrently with the build — only a quick
    /// SSH dispatch is awaited here. Failures are swallowed — reaping is
    /// opportunistic, never load-bearing.
    pub async fn reap_stale_sibling_per_job_target_dirs(
        &self,
        worker: &WorkerConfig,
        idle_hours: u32,
    ) {
        if self.source_authority_prefix.is_some() {
            // Detached preparation work can start after final verification.
            // The independent daemon GC understands durable retained grants.
            return;
        }
        let project_dir = self.remote_path();
        let current = self.remote_cargo_target_dir_name.clone();

        // Hard safety guards. Both values are rch-generated and should be simple
        // path tokens; refuse anything that could escape the intended
        // `<project_dir>/.rch-target-*` scope or inject shell syntax. The reap
        // script embeds these unescaped (inside double quotes), so this guard is
        // the security boundary. The predicate + safety checks are shared with the
        // daemon-side worker sweep (`rchd::stale_target_reap`) via
        // `rch_common::stale_target_reap` so the two can't drift.
        if !rch_common::stale_target_reap::is_safe_reap_path(&project_dir)
            || !rch_common::stale_target_reap::is_safe_reap_token(&current)
        {
            warn!(
                "stale-target reap: refusing unsafe inputs (project_dir={:?}, current={:?})",
                project_dir, current
            );
            return;
        }
        // Never below a 1h floor, no matter how the threshold was configured.
        let idle_minutes = rch_common::stale_target_reap::idle_minutes_from_hours(idle_hours);

        if use_mock_transport(worker) {
            debug!(
                "Mock stale-target reap in {} on {} (idle>{}h)",
                project_dir, worker.id, idle_hours
            );
            return;
        }

        #[cfg(not(unix))]
        {
            let _ = (worker, idle_minutes);
        }

        #[cfg(unix)]
        {
            // For each per-job sibling dir apply the SHARED reap predicate
            // (`rch_common::stale_target_reap::reap_loop_body`): keep it if the dir
            // OR any descendant was modified within the idle window (an active or
            // just-created build); otherwise remove it. This job's own dir is
            // always excluded. The glob list is the shared `REAP_GLOBS`.
            let globs = rch_common::stale_target_reap::REAP_GLOBS.join(" ");
            let loop_body =
                rch_common::stale_target_reap::reap_loop_body(idle_minutes, Some(&current), "", "");
            let script = format!(
                "cd \"{project_dir}\" 2>/dev/null || exit 0; \
                 for d in {globs}; do {loop_body} done"
            );
            // Detach on the worker so a large reclaim runs concurrently with the
            // build rather than blocking it. The detached operation joins the
            // source activity barrier itself, so release either drains it or
            // fences it before the first filesystem effect.
            let protected_script = self.source_activity_command(&script);
            let remote_command = format!(
                "nohup sh -c {} >/dev/null 2>&1 &",
                escape(Cow::from(protected_script))
            );

            let mut client = SshClient::new(worker.clone(), self.ssh_options.clone());
            if let Err(e) = client.connect().await {
                debug!(
                    "stale-target reap skipped (ssh connect failed on {}): {}",
                    worker.id, e
                );
                return;
            }
            if let Err(e) = client.execute(&remote_command).await {
                debug!("stale-target reap dispatch failed on {}: {}", worker.id, e);
            }
            if let Err(e) = client.disconnect().await {
                debug!(
                    "stale-target reap: ssh disconnect warning on {}: {}",
                    worker.id, e
                );
            }
        }
    }
}

// The stale-target reap safety predicates now live in
// `rch_common::stale_target_reap` so the orchestrator reaper (here) and the
// daemon-side worker sweep (`rchd::stale_target_reap`) share a single source of
// truth and cannot drift. These thin wrappers preserve the local test surface.

/// See [`rch_common::stale_target_reap::is_safe_reap_path`].
#[cfg(test)]
fn is_safe_reap_path(s: &str) -> bool {
    rch_common::stale_target_reap::is_safe_reap_path(s)
}

/// See [`rch_common::stale_target_reap::is_safe_reap_token`].
#[cfg(test)]
fn is_safe_reap_token(s: &str) -> bool {
    rch_common::stale_target_reap::is_safe_reap_token(s)
}

/// Result of a file synchronization operation.
#[derive(Debug, Clone)]
pub struct SyncResult {
    /// Bytes transferred.
    pub bytes_transferred: u64,
    /// Number of files transferred.
    pub files_transferred: u32,
    /// Duration in milliseconds.
    pub duration_ms: u64,
}

/// Result of an artifact sync-back, extending the raw [`SyncResult`] counts
/// with the per-file evidence the zero-build-output detector needs (bd-mpbav).
///
/// - `manifest_regular_files` lists every REGULAR FILE rsync had in its
///   matched file list — transferred items (`>f…`) AND verified-up-to-date
///   items (`.f`), parsed from the retrieval's `--out-format='%i %n'` +
///   `--info=name2` output. Including up-to-date items is what lets the
///   detector distinguish "outputs already current locally" (a legitimate
///   no-op rebuild) from "outputs never matched" (the stale-local hazard).
/// - `matched_regular_files` is the regular-file count from rsync's `--stats`
///   `Number of files: N (reg: R, dir: D)` line, `None` when unparseable. It
///   is the completeness cross-check: if the manifest lists fewer files than
///   rsync matched, the manifest is incomplete and the detector must decline
///   to fire (fail-open).
///
/// Transports that cannot produce a manifest (mock rsync) return an empty
/// manifest with `matched_regular_files:
/// None`, which the detector treats as "no proof" and never fires on.
#[derive(Debug, Clone)]
pub struct ArtifactRetrieval {
    /// Transfer counts (bytes/files/duration), as historically returned.
    pub stats: SyncResult,
    /// Every matched regular file, transferred or verified up-to-date.
    pub manifest_regular_files: Vec<String>,
    /// Regular-file count from `--stats`, when parseable.
    pub matched_regular_files: Option<u32>,
}

impl ArtifactRetrieval {
    /// A manifest-less retrieval (mock transport).
    fn from_stats(stats: SyncResult) -> Self {
        Self {
            stats,
            manifest_regular_files: Vec::new(),
            matched_regular_files: None,
        }
    }

    /// A real rsync retrieval: parse the manifest and matched-file count out
    /// of the captured rsync stdout.
    fn from_rsync_output(stats: SyncResult, output: &str) -> Self {
        Self {
            stats,
            manifest_regular_files: parse_rsync_itemized_regular_files(output),
            matched_regular_files: parse_rsync_matched_regular_files(output),
        }
    }
}

/// Parse the itemized per-file manifest out of rsync output produced with
/// `--info=name2 --out-format='%i %n'` (the flags `build_retrieve_command` /
/// `build_retrieve_streaming_command` pass).
///
/// Recognized lines look like:
///
/// ```text
/// >f++++++++++ release-perf/mybin          (transferred regular file)
/// .f           release-perf/mybin          (matched, already up-to-date)
/// cd++++++++++ release-perf/               (directory — skipped)
/// ```
///
/// The item code is `<update-flag><item-type><flags…>` followed by whitespace
/// and the relative path; only regular files (`f` as the second byte) are
/// returned, directories and symlinks are skipped. The item string contains
/// no spaces, so everything after the first space run is the path (paths may
/// themselves contain spaces).
fn parse_rsync_itemized_regular_files(output: &str) -> Vec<String> {
    output
        .lines()
        .filter_map(|line| {
            let bytes = line.as_bytes();
            // Need at least a 2-byte item code, a separator, and a name.
            if bytes.len() < 4 {
                return None;
            }
            // First byte: the update flag (`>` received, `<` sent, `c` created,
            // `.` unchanged, `h` hardlinked, `*` special). Second byte: item
            // type — only `f` (regular file) belongs in the manifest.
            let update_flag = matches!(bytes[0], b'<' | b'>' | b'c' | b'.' | b'h' | b'*');
            if !update_flag || bytes[1] != b'f' {
                return None;
            }
            // The separator must sit past the item code itself.
            let separator = line.find(' ').filter(|pos| *pos >= 2)?;
            let name = line[separator..].trim_start();
            (!name.is_empty()).then(|| name.to_string())
        })
        .collect()
}

/// Parse the regular-file count from rsync `--stats` output
/// (`Number of files: 7 (reg: 4, dir: 3)`). This counts every regular file in
/// the matched file list — transferred AND up-to-date — unlike
/// [`parse_rsync_files`], which counts only transferred files. Returns `None`
/// when the stats line is missing or malformed.
fn parse_rsync_matched_regular_files(output: &str) -> Option<u32> {
    output.lines().find_map(|line| {
        let rest = line.strip_prefix("Number of files:")?.trim();
        // Rsync omits `reg:` when only directories matched. A complete,
        // directory-only breakdown proves zero regular files; missing or
        // inconsistent accounting must remain unknown.
        if let Some((total, directories)) = rest.split_once(" (dir:") {
            let parse_count = |value: &str| {
                let value = value.trim();
                let grouped = value.contains(',');
                if !value.split(',').enumerate().all(|(index, group)| {
                    !group.is_empty()
                        && group.bytes().all(|byte| byte.is_ascii_digit())
                        && (!grouped
                            || if index == 0 {
                                group.len() <= 3
                            } else {
                                group.len() == 3
                            })
                }) {
                    return None;
                }
                value.replace(',', "").parse::<u32>().ok()
            };
            let total = parse_count(total)?;
            let directories = parse_count(directories.strip_suffix(')')?)?;
            return (total == directories).then_some(0);
        }
        // The first whitespace-delimited token after "(reg:" is the regular-
        // file count; strip thousands commas before parsing.
        let count = rest
            .split("(reg:")
            .nth(1)?
            .split_whitespace()
            .next()?
            .replace(',', "");
        count.parse().ok()
    })
}

/// Estimate of transfer size from rsync dry-run (bd-3hho).
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct TransferEstimate {
    /// Total bytes that would be transferred.
    pub bytes: u64,
    /// Total files that would be transferred.
    pub files: u32,
    /// Estimated transfer time in milliseconds (based on configured bandwidth).
    pub estimated_time_ms: u64,
    /// Time taken to run the estimation in milliseconds.
    pub estimation_ms: u64,
}

#[derive(Debug, Clone, Copy)]
enum RsyncTransferDirection {
    Upload,
    Download,
}

/// Parse rsync protocol bytes in the payload direction for this attempt.
/// Downloads receive the payload; sent bytes are mostly requests/checksums.
/// These counters include protocol overhead, not logical file sizes.
fn parse_rsync_bytes(output: &str, direction: RsyncTransferDirection) -> u64 {
    let (stats_prefix, summary_key) = match direction {
        RsyncTransferDirection::Upload => ("Total bytes sent:", "sent"),
        RsyncTransferDirection::Download => ("Total bytes received:", "received"),
    };
    let mut summary_bytes = None;
    for line in output.lines() {
        let line = line.trim_start();
        if let Some(rest) = line.strip_prefix(stats_prefix)
            && let Some(bytes_str) = rest.split_whitespace().next()
            && let Ok(bytes) = bytes_str.replace(',', "").parse()
        {
            return bytes;
        }

        // Require the two complete counter fields, so filenames or diagnostics
        // mentioning "sent" cannot become transfer measurements. Keep this as
        // a fallback: structured stats take precedence wherever they appear.
        if summary_bytes.is_none() && (line.starts_with("sent ") || line.starts_with("received ")) {
            let mut fields = line.split_whitespace();
            let first_key = fields.next();
            let first_bytes = fields.next();
            let first_unit = fields.next();
            let second_key = fields.next();
            let second_bytes = fields.next();
            let second_unit = fields.next();
            if matches!(
                (first_key, second_key),
                (Some("sent"), Some("received")) | (Some("received"), Some("sent"))
            ) && first_unit == Some("bytes")
                && second_unit == Some("bytes")
            {
                let bytes = if first_key == Some(summary_key) {
                    first_bytes
                } else {
                    second_bytes
                };
                summary_bytes = bytes.and_then(|value| value.replace(',', "").parse().ok());
            }
        }
    }
    summary_bytes.unwrap_or(0)
}

/// Parse files transferred from rsync output.
fn parse_rsync_files(output: &str) -> u32 {
    let mut total_files = None;

    for line in output.lines() {
        let line = line.trim_start();
        if let Some(rest) = line
            .strip_prefix("Number of regular files transferred:")
            .or_else(|| line.strip_prefix("Number of files transferred:"))
            && let Some(count) = rest.split_whitespace().next()
            && let Ok(parsed) = count.replace(',', "").parse::<u32>()
        {
            return parsed;
        }
        if let Some(rest) = line.strip_prefix("Number of files:")
            && let Some(count) = rest.split_whitespace().next()
            && let Ok(parsed) = count.replace(',', "").parse::<u32>()
        {
            total_files = Some(parsed);
        }
    }

    // Older output with no transferred counter retains the total-tree estimate.
    // A parsed transferred count of zero already returned above: unchanged
    // files must not turn a no-op transfer into the total tree count.
    total_files.unwrap_or(0)
}

// =============================================================================
// Transfer Estimation Parsers (bd-3hho)
// =============================================================================

/// Parse total file size from rsync --dry-run --stats output.
/// Artifact planning requires the full selected size, even on a no-op copy.
/// Delta-only statistics cannot establish that bound.
fn parse_rsync_selected_size(output: &str) -> Option<u64> {
    output.lines().find_map(|line| {
        line.strip_prefix("Total file size:")?
            .split_whitespace()
            .next()?
            .replace(',', "")
            .parse()
            .ok()
    })
}

/// Parse total file size from rsync --dry-run --stats output.
///
/// Looks for "Total file size:" line which shows the total bytes that would
/// be transferred (not the delta, but the full file size).
#[allow(dead_code)]
fn parse_rsync_total_size(output: &str) -> Option<u64> {
    for line in output.lines() {
        // "Total file size: 1,234,567 bytes"
        if let Some(rest) = line.strip_prefix("Total file size:") {
            let cleaned = rest.trim().replace(',', "");
            if let Some(bytes_str) = cleaned.split_whitespace().next() {
                return bytes_str.parse().ok();
            }
        }
        // Also check "Total transferred file size:" for delta transfers
        if let Some(rest) = line.strip_prefix("Total transferred file size:") {
            let cleaned = rest.trim().replace(',', "");
            if let Some(bytes_str) = cleaned.split_whitespace().next() {
                return bytes_str.parse().ok();
            }
        }
    }
    None
}

/// Parse total file count from rsync --dry-run --stats output.
///
/// Looks for "Number of files:" or "Number of regular files:" line.
#[allow(dead_code)]
fn parse_rsync_total_files(output: &str) -> Option<u32> {
    for line in output.lines() {
        // "Number of files: 1,234 (reg: 1,000, dir: 234)"
        if let Some(rest) = line.strip_prefix("Number of files:") {
            let cleaned = rest.trim().replace(',', "");
            if let Some(count_str) = cleaned.split_whitespace().next() {
                return count_str.parse().ok();
            }
        }
        // "Number of regular files transferred: 500"
        if let Some(rest) = line.strip_prefix("Number of regular files transferred:") {
            let cleaned = rest.trim().replace(',', "");
            if let Some(count_str) = cleaned.split_whitespace().next() {
                return count_str.parse().ok();
            }
        }
    }
    None
}

/// Pump a child stream into the segment channel, splitting on BOTH `\n` and
/// `\r`.
///
/// rsync's `--info=progress2` refreshes a single progress line with bare
/// carriage returns and writes a newline only at step boundaries, so a
/// newline-only reader can observe NOTHING for the entire duration of a large
/// transfer. Splitting on `\r` turns every progress refresh into a
/// forward-progress event — exactly what the silence-based stall detector
/// (issue #59) and the sync heartbeat need to distinguish "large but moving"
/// from "dead". Empty segments (e.g. the gap inside `\r\n`) are dropped.
async fn pump_stream_segments<R>(stream: R, tx: tokio::sync::mpsc::Sender<String>)
where
    R: tokio::io::AsyncRead + Unpin,
{
    let mut reader = BufReader::new(stream);
    let mut pending: Vec<u8> = Vec::new();
    let mut chunk = [0u8; 8192];
    loop {
        let read = match reader.read(&mut chunk).await {
            Ok(0) | Err(_) => break,
            Ok(read) => read,
        };
        for &byte in &chunk[..read] {
            if byte == b'\n' || byte == b'\r' {
                if !pending.is_empty() {
                    let segment = String::from_utf8_lossy(&pending).into_owned();
                    pending.clear();
                    if tx.send(segment).await.is_err() {
                        return;
                    }
                }
            } else {
                pending.push(byte);
            }
        }
    }
    if !pending.is_empty() {
        let _ = tx
            .send(String::from_utf8_lossy(&pending).into_owned())
            .await;
    }
}

async fn run_command_streaming<F>(
    mut cmd: Command,
    operation_name: &str,
    operation_timeout: std::time::Duration,
    silence: Option<&SyncSilencePolicy>,
    mut on_line: F,
) -> Result<(String, u64)>
where
    F: FnMut(&str),
{
    let start = TokioInstant::now();
    cmd.kill_on_drop(true);
    let program = cmd.as_std().get_program().to_string_lossy().into_owned();
    let mut child = cmd
        .spawn()
        .with_context(|| format!("Failed to execute rsync ({program})"))?;

    let stdout = child.stdout.take().context("Failed to capture stdout")?;
    let stderr = child.stderr.take().context("Failed to capture stderr")?;

    // Use a channel to aggregate segments from both streams
    // Capacity 100 ensures we don't consume too much memory if on_line is slow,
    // but allows some buffering.
    let (tx, mut rx) = tokio::sync::mpsc::channel(100);
    let tx_stderr = tx.clone();
    let tx_stdout = tx.clone();

    tokio::spawn(pump_stream_segments(stdout, tx_stdout));
    tokio::spawn(pump_stream_segments(stderr, tx_stderr));

    // Drop the original tx so rx will close when both tasks are done
    drop(tx);

    // Distinguishes "streams closed and child exited" from "silence window
    // expired with the child still running" inside the wall-clock guard.
    enum StreamEnd {
        Completed(std::process::ExitStatus),
        Silent,
    }

    let effective_silence = silence.filter(|policy| !policy.limit.is_zero());
    let mut combined = String::new();
    const MAX_RSYNC_OUTPUT: usize = 10 * 1024 * 1024;
    let stream_end = match tokio::time::timeout(operation_timeout, async {
        loop {
            let received = match effective_silence {
                Some(policy) => match tokio::time::timeout(policy.limit, rx.recv()).await {
                    Ok(received) => received,
                    // Issue #59: no output from either stream for the whole
                    // silence window — a dead channel or wedged rsync, not a
                    // large-but-progressing transfer (progress2 refreshes
                    // count as segments).
                    Err(_) => return Ok(StreamEnd::Silent),
                },
                None => rx.recv().await,
            };
            let Some(text) = received else { break };
            on_line(&text);
            if combined.len() < MAX_RSYNC_OUTPUT {
                combined.push_str(&text);
                combined.push('\n');
                if combined.len() >= MAX_RSYNC_OUTPUT {
                    combined.push_str("...[output truncated]...\n");
                }
            }
        }

        child
            .wait()
            .await
            .context("Failed to wait on rsync")
            .map(StreamEnd::Completed)
    })
    .await
    {
        Ok(end) => end?,
        Err(_) => {
            let _ = child.kill().await;
            anyhow::bail!(
                "{}: timed out after {}ms",
                operation_name,
                operation_timeout.as_millis()
            );
        }
    };
    let status = match stream_end {
        StreamEnd::Completed(status) => status,
        StreamEnd::Silent => {
            let policy = effective_silence.expect("silent stream end requires a silence policy");
            let _ = child.kill().await;
            return Err(SourceSyncStalled {
                worker_id: policy.worker_id.clone(),
                phase: policy.phase,
                silence: policy.limit,
                detail: format!(
                    "{operation_name}: no rsync output for {}s (wall-clock cap {}s)",
                    policy.limit.as_secs(),
                    operation_timeout.as_secs()
                ),
            }
            .into());
        }
    };
    if !status.success() {
        return Err(TransferError::SyncFailed {
            reason: "rsync failed".to_string(),
            exit_code: status.code(),
            stderr: combined.trim().to_string(),
        }
        .into());
    }

    Ok((combined, start.elapsed().as_millis() as u64))
}

/// Scan rsync output for indicators that a transfer was incomplete despite a
/// zero exit code.
///
/// rsync usually exits non-zero on a partial transfer, but it can exit 0 while
/// interrupted mid-file in edge cases. Trusting that success is dangerous: for
/// an upload the remote build then compiles stale/partial sources and returns a
/// trusted-but-wrong result; for a download the local artifact tree is silently
/// incomplete. Returns the first matched indicator, if any, so the caller can
/// fail the transfer instead of reporting success.
///
/// Note: callers on the streaming path pass the combined stdout+stderr (rsync
/// runs without `-v`, so it emits progress/stats — not a per-file listing — and
/// these phrases do not appear benignly).
fn detect_partial_transfer(output: &str) -> Option<&'static str> {
    if output.is_empty() {
        return None;
    }
    const PARTIAL_INDICATORS: [&str; 5] = [
        "partial transfer",
        "connection unexpectedly closed",
        "write error",
        "read error",
        "truncated file",
    ];
    let lower = output.to_lowercase();
    PARTIAL_INDICATORS
        .into_iter()
        .find(|ind| lower.contains(ind))
}

/// Decide whether a [`run_command_streaming`] failure is a transient transport
/// error worth retrying.
///
/// `run_command_streaming` reports a non-zero rsync exit as
/// `TransferError::SyncFailed { stderr, .. }`, whose `Display` is only
/// `"Project sync failed: rsync failed"` — the transport signature lives in the
/// captured `stderr`, NOT in the error chain. So the generic
/// `is_retryable_transport_error` (which walks `err.chain()` strings) would
/// classify every streaming rsync drop as fatal. We therefore inspect the
/// captured `stderr` directly for `SyncFailed`, and fall back to the error-chain
/// classifier for everything else (spawn I/O errors, the streaming-timeout
/// `bail!`, etc.).
fn streaming_error_is_retryable(err: &anyhow::Error) -> bool {
    // Issue #59: a silence-detected stall is never retried IN PLACE. Retrying
    // on the same worker re-runs the same dead channel for another full
    // silence window each attempt; the failover contract is release the
    // reservation and re-enter selection excluding this worker (hook side).
    if find_source_sync_stall(err).is_some() {
        return false;
    }
    if let Some(TransferError::SyncFailed { stderr, .. }) = err.downcast_ref::<TransferError>() {
        return is_retryable_transport_error_text(stderr);
    }
    is_retryable_transport_error(err)
}

/// Extract the typed source-sync stall (issue #59) from anywhere in an error's
/// context chain.
pub fn find_source_sync_stall(error: &anyhow::Error) -> Option<&SourceSyncStalled> {
    error
        .chain()
        .find_map(|cause| cause.downcast_ref::<SourceSyncStalled>())
}

fn source_transfer_error_detail(err: &anyhow::Error) -> String {
    if let Some(TransferError::SyncFailed { stderr, .. }) = err.downcast_ref::<TransferError>()
        && !stderr.is_empty()
    {
        format!("{err}: {stderr}")
    } else {
        err.to_string()
    }
}

/// Streaming counterpart of [`execute_rsync_with_retry`].
///
/// `run_command_streaming` consumes its `Command` (so it cannot be retried in
/// place) and surfaces transport failures inside a `TransferError::SyncFailed`
/// whose `Display` omits the stderr — which is why this loop is hand-rolled
/// rather than delegating to `retry_with_backoff`. Each attempt rebuilds the
/// command via `build_command`, and retries transient transport failures up to
/// `max_attempts`. Source uploads pass `Some(timeout)` so every attempt receives
/// the full payload-aware source-sync timeout. Artifact retrieval passes `None`
/// and retains the existing shared `RetryConfig::total_timeout_ms` budget.
///
/// `on_line` is re-invoked from scratch on every attempt (rsync restarts from
/// the beginning). All call sites use it purely for progress/heartbeat display,
/// which tolerates repetition; do not route side-effecting work through it.
///
/// `silence` (issue #59) applies per attempt; a silence-detected stall is
/// returned immediately WITHOUT further attempts (see
/// `streaming_error_is_retryable`) and keeps its typed [`SourceSyncStalled`]
/// for the hook's worker-failover downcast.
async fn run_command_streaming_with_retry<F>(
    config: &RetryConfig,
    operation_name: &str,
    source_attempt_timeout: Option<std::time::Duration>,
    silence: Option<&SyncSilencePolicy>,
    build_command: impl Fn() -> Command,
    mut on_line: F,
) -> Result<(String, u64)>
where
    F: FnMut(&str),
{
    let start = std::time::Instant::now();
    let mut last_error: Option<anyhow::Error> = None;
    let max_attempts = config.max_attempts.max(1);
    let mut source_attempts = Vec::with_capacity(max_attempts as usize);

    for attempt in 0..max_attempts {
        // Artifact retrieval retains the shared total budget. Source uploads
        // deliberately do not: each configured attempt gets its full timeout.
        if source_attempt_timeout.is_none()
            && attempt > 0
            && !config.should_retry(attempt, start.elapsed())
        {
            debug!(
                "{}: total timeout exceeded after {} attempts",
                operation_name, attempt
            );
            break;
        }

        // Exponential backoff with jitter for retries only.
        if attempt > 0 {
            let delay = config.delay_for_attempt(attempt);
            debug!(
                "{}: attempt {}/{} after {}ms delay",
                operation_name,
                attempt + 1,
                config.max_attempts,
                delay.as_millis()
            );
            sleep(delay).await;
        }

        let attempt_timeout = source_attempt_timeout.unwrap_or_else(|| {
            let elapsed_ms = start.elapsed().as_millis();
            let remaining_ms = if elapsed_ms >= config.total_timeout_ms as u128 {
                1
            } else {
                (config.total_timeout_ms - elapsed_ms as u64).max(1)
            };
            std::time::Duration::from_millis(remaining_ms)
        });

        let cmd = build_command();
        match run_command_streaming(cmd, operation_name, attempt_timeout, silence, &mut on_line)
            .await
        {
            Ok(result) => {
                if source_attempt_timeout.is_some() {
                    source_attempts.push(TransferAttemptDiagnostic {
                        attempt: attempt + 1,
                        max_attempts,
                        outcome: "succeeded",
                        detail: format!("{operation_name} completed"),
                    });
                }
                if attempt > 0 {
                    info!(
                        "{}: succeeded on attempt {}/{}",
                        operation_name,
                        attempt + 1,
                        config.max_attempts
                    );
                }
                return Ok(result);
            }
            Err(err) => {
                let retryable = streaming_error_is_retryable(&err);
                let detail = source_transfer_error_detail(&err);
                if source_attempt_timeout.is_some() {
                    source_attempts.push(TransferAttemptDiagnostic {
                        attempt: attempt + 1,
                        max_attempts,
                        outcome: if retryable { "retryable" } else { "fatal" },
                        detail: detail.clone(),
                    });
                }
                if !retryable {
                    debug!(
                        "{}: non-retryable error on attempt {}: {}",
                        operation_name,
                        attempt + 1,
                        err
                    );
                    // Issue #59: a silence-detected stall must stay
                    // DOWNCASTABLE for the hook's failover arm (release the
                    // reservation, reselect excluding this worker), so it is
                    // returned as-is instead of being flattened into
                    // TransferAttemptsExhausted's string-only history. The
                    // stall aborts on its first attempt by design, so no
                    // multi-attempt history is lost.
                    if find_source_sync_stall(&err).is_some() {
                        return Err(err);
                    }
                    if source_attempt_timeout.is_some() {
                        return Err(anyhow::Error::new(TransferAttemptsExhausted {
                            attempts: source_attempts,
                            last_error: detail,
                        }));
                    }
                    return Err(err);
                }
                warn!(
                    "{}: retryable error on attempt {}/{}: {}",
                    operation_name,
                    attempt + 1,
                    config.max_attempts,
                    err
                );
                last_error = Some(err);
            }
        }
    }

    if source_attempt_timeout.is_some() {
        let last_error = source_attempts
            .last()
            .map(|attempt| attempt.detail.clone())
            .unwrap_or_else(|| format!("{operation_name}: all retries exhausted"));
        return Err(anyhow::Error::new(TransferAttemptsExhausted {
            attempts: source_attempts,
            last_error,
        }));
    }

    Err(last_error.unwrap_or_else(|| anyhow::anyhow!("{}: all retries exhausted", operation_name)))
}

fn normalize_hash_root(path: &Path, policy: &PathTopologyPolicy) -> PathBuf {
    normalize_project_path_with_policy(path, policy)
        .map(|normalized| normalized.canonical_path().to_path_buf())
        .or_else(|_| std::fs::canonicalize(path))
        .unwrap_or_else(|_| path.to_path_buf())
}

fn update_hasher_with_file_fingerprint(hasher: &mut blake3::Hasher, file_path: &Path, label: &str) {
    let Ok(metadata) = std::fs::metadata(file_path) else {
        return;
    };

    hasher.update(label.as_bytes());
    hasher.update(&metadata.len().to_le_bytes());
    if let Ok(modified) = metadata.modified()
        && let Ok(duration) = modified.duration_since(std::time::UNIX_EPOCH)
    {
        hasher.update(&duration.as_nanos().to_le_bytes());
    }

    if metadata.len() <= PROJECT_HASH_CONTENT_LIMIT_BYTES
        && let Ok(bytes) = std::fs::read(file_path)
    {
        hasher.update(&(bytes.len() as u64).to_le_bytes());
        hasher.update(&bytes);
    }
}

fn collect_hash_roots(
    project_path: &Path,
    dependency_roots: &[PathBuf],
    policy: &PathTopologyPolicy,
) -> Vec<PathBuf> {
    let mut roots = BTreeSet::new();
    roots.insert(normalize_hash_root(project_path, policy));
    for root in dependency_roots {
        roots.insert(normalize_hash_root(root, policy));
    }
    roots.into_iter().collect()
}

/// Compute a project hash that includes dependency-closure fingerprints.
///
/// The resulting value is deterministic across `/dp` vs `/data/projects` alias
/// forms and changes when any tracked key file for any closure member changes.
///
/// Convenience wrapper using the default topology policy; production code
/// should prefer [`compute_project_hash_with_dependency_roots_and_policy`].
#[cfg(test)]
pub fn compute_project_hash_with_dependency_roots(
    project_path: &Path,
    dependency_roots: &[PathBuf],
) -> String {
    compute_project_hash_with_dependency_roots_and_policy(
        project_path,
        dependency_roots,
        &PathTopologyPolicy::default(),
    )
}

/// Compute a project hash using an explicit topology policy.
pub fn compute_project_hash_with_dependency_roots_and_policy(
    project_path: &Path,
    dependency_roots: &[PathBuf],
    policy: &PathTopologyPolicy,
) -> String {
    let mut hasher = blake3::Hasher::new();
    hasher.update(b"rch-project-hash-v2");

    for root in collect_hash_roots(project_path, dependency_roots, policy) {
        hasher.update(b"\0root\0");
        hasher.update(root.to_string_lossy().as_bytes());
        for filename in PROJECT_HASH_KEY_FILES {
            update_hasher_with_file_fingerprint(&mut hasher, &root.join(filename), filename);
        }
    }

    hasher.finalize().to_hex()[..16].to_string()
}

/// Compute a hash of the project for cache invalidation.
///
/// Convenience wrapper using the default topology policy; production code
/// should prefer [`compute_project_hash_with_dependency_roots_and_policy`].
#[cfg(test)]
pub fn compute_project_hash(project_path: &Path) -> String {
    compute_project_hash_with_dependency_roots(project_path, &[])
}

/// Validate a project identifier for safe use in file paths.
///
/// Rejects:
/// - Path traversal sequences (.., ./)
/// - Null bytes
/// - Shell metacharacters that could cause injection
/// - Names starting with hyphen (could be interpreted as flags)
///
/// Returns the sanitized name or "unknown" if invalid.
fn sanitize_project_id(name: &str) -> String {
    // Reject obviously dangerous patterns
    if name.is_empty()
        || name == "."
        || name == ".."
        || name.contains("..")
        || name.contains('/')
        || name.contains('\\')
        || name.contains('\0')
        || name.starts_with('-')
    {
        return "unknown".to_string();
    }

    // Reject shell metacharacters that could cause injection
    // Allow: alphanumeric, underscore, hyphen, dot (but not leading dot for hidden files)
    let is_safe = name
        .chars()
        .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')
        && !name.starts_with('.');

    if is_safe {
        name.to_string()
    } else {
        // Replace unsafe characters with underscores
        let sanitized: String = name
            .chars()
            .map(|c| {
                if c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.' {
                    c
                } else {
                    '_'
                }
            })
            .collect();

        // Remove leading dots after sanitization
        let result = sanitized.trim_start_matches('.');
        // If result is empty after trimming, return "unknown"
        if result.is_empty() {
            "unknown".to_string()
        } else {
            result.to_string()
        }
    }
}

/// Get the project identifier from a path.
///
/// Extracts the directory name and sanitizes it for safe use in remote paths.
/// Returns "unknown" if the path is invalid or the name contains dangerous characters.
pub fn project_id_from_path(path: &Path) -> String {
    let name = path
        .file_name()
        .and_then(|n| n.to_str())
        .unwrap_or("unknown");

    sanitize_project_id(name)
}

/// Default artifact patterns for Rust projects.
///
/// Includes the triple-aware `target/*/<profile>/**` globs: an ordinary
/// `cargo build --target <triple>` (not just zigbuild) writes its outputs
/// under `target/<triple>/<profile>/`, and the anchored `target/<profile>/**`
/// patterns alone silently exclude every final binary from artifact
/// sync-down. Found 2026-08-05 when an HFDT release leg completed remotely
/// but synced back 92 metadata files (2.1 KB) and no binaries (hfdt-elh1t).
pub fn default_rust_artifact_patterns() -> Vec<String> {
    vec![
        "target/debug/**".to_string(),
        "target/release/**".to_string(),
        // Explicit --target outputs: target/<triple>/<profile>/**
        "target/*/debug/**".to_string(),
        "target/*/release/**".to_string(),
        "target/doc/**".to_string(),
        "target/.rustc_info.json".to_string(),
        "target/CACHEDIR.TAG".to_string(),
    ]
}

/// Default artifact patterns for `cargo zigbuild` cross-compiles.
///
/// cargo-zigbuild always builds for an explicit `--target <triple>`, so cargo
/// writes its outputs under `target/<triple>/<profile>/` rather than the plain
/// `target/<profile>/` that [`default_rust_artifact_patterns`] captures. The
/// `target/*/…` globs (one `*` = the triple component) bring the cross-compiled
/// binary/libs home; the non-cross paths are kept too so a `--target` matching
/// the host still syncs, and for the odd zigbuild without `--target`.
pub fn default_zigbuild_artifact_patterns() -> Vec<String> {
    vec![
        // Cross-compile outputs: target/<triple>/<profile>/**
        "target/*/debug/**".to_string(),
        "target/*/release/**".to_string(),
        // Non-cross (host-target) outputs, same as a plain cargo build.
        "target/debug/**".to_string(),
        "target/release/**".to_string(),
        "target/.rustc_info.json".to_string(),
        "target/CACHEDIR.TAG".to_string(),
    ]
}

/// Minimal artifact patterns for Rust test-only commands.
///
/// Test runs stream their output via stdout/stderr and don't need the full
/// target/ directory returned. This function returns only patterns for:
/// - Coverage reports (when using cargo-llvm-cov, tarpaulin, etc.)
/// - Nextest archive/junit artifacts
/// - Benchmark results
///
/// This dramatically reduces artifact transfer time for test commands,
/// especially on large projects where target/ can be several gigabytes.
#[allow(dead_code)] // Reserved for future test-only artifact optimization
pub fn default_rust_test_artifact_patterns() -> Vec<String> {
    vec![
        // cargo-llvm-cov coverage data
        "target/llvm-cov-target/**".to_string(),
        // Alternative coverage output locations
        "target/coverage/**".to_string(),
        // Tarpaulin coverage reports
        "tarpaulin-report.html".to_string(),
        "tarpaulin-report.json".to_string(),
        "cobertura.xml".to_string(),
        // cargo-nextest artifacts
        "target/nextest/**".to_string(),
        // JUnit test result format (common CI integration)
        "junit.xml".to_string(),
        "test-results.xml".to_string(),
        // Criterion benchmark results
        "target/criterion/**".to_string(),
    ]
}

/// Default artifact patterns for Bun/Node.js projects.
///
/// These patterns retrieve test results and coverage reports generated
/// during `bun test` and `bun typecheck` execution.
pub fn default_bun_artifact_patterns() -> Vec<String> {
    vec![
        // Coverage reports (generated by bun test --coverage)
        "coverage/**".to_string(),
        // TypeScript incremental build info (speeds up subsequent typechecks)
        "*.tsbuildinfo".to_string(),
        "tsconfig.tsbuildinfo".to_string(),
        // Common test result formats
        "test-results/**".to_string(),
        "junit.xml".to_string(),
        "test-report.json".to_string(),
        // NYC (Istanbul) coverage output
        ".nyc_output/**".to_string(),
    ]
}

/// Default artifact patterns for C/C++ projects.
pub fn default_c_cpp_artifact_patterns() -> Vec<String> {
    vec![
        // Common build directories
        "build/**".to_string(),
        "bin/**".to_string(),
        "out/**".to_string(),
        ".libs/**".to_string(),
        // Object files
        "*.o".to_string(),
        "*.obj".to_string(),
        // Libraries
        "*.a".to_string(),
        "*.so".to_string(),
        "*.so.*".to_string(),
        "*.dylib".to_string(),
        "*.dll".to_string(),
        "*.lib".to_string(),
        // Executables (Windows)
        "*.exe".to_string(),
        // Best-effort root-level outputs (like `a.out`) when they are newly
        // created remotely. Existing local top-level entries stay protected by
        // the retrieval source-integrity guard.
        "*".to_string(),
    ]
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The wrapper's stream must end with the command's last byte, however
    /// large the final burst, and carry its exit status.
    #[test]
    fn durable_execution_streams_all_output_and_status() {
        let directory = tempfile::tempdir().unwrap();
        let receipt = directory
            .path()
            .join("recovery-9-id")
            .to_str()
            .unwrap()
            .to_owned();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "myproject".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_recovery_completion(receipt.clone(), "id".to_string());
        let command = pipeline.durable_execution_command(
            "i=0; while [ $i -lt 20000 ]; do echo line-$i; i=$((i + 1)); done; echo last >&2; exit 3"
                .to_string(),
        );
        let output = std::process::Command::new("sh") // ubs:ignore — fixed wrapper over this test's temporary receipt
            .arg("-c")
            .arg(&command)
            .output()
            .expect("run durable wrapper");
        let stdout = String::from_utf8(output.stdout).unwrap();
        assert_eq!(output.status.code(), Some(3));
        assert_eq!(stdout.lines().count(), 20_000);
        assert!(stdout.ends_with("line-19999\n"), "tail was cut");
        assert_eq!(String::from_utf8_lossy(&output.stderr), "last\n");
        assert!(std::path::Path::new(&receipt).is_file());
    }

    /// The build must create its outputs under the caller's umask; only the
    /// wrapper's own receipt is private.
    #[cfg(unix)]
    #[test]
    fn durable_execution_runs_the_workload_under_the_callers_umask() {
        use std::os::unix::fs::PermissionsExt;
        let directory = tempfile::tempdir().unwrap();
        let receipt = directory
            .path()
            .join("recovery-9-id")
            .to_str()
            .unwrap()
            .to_owned();
        let output_file = directory.path().join("built-output");
        let pipeline = TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "myproject".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_recovery_completion(receipt.clone(), "id".to_string());
        let command = pipeline.durable_execution_command(format!(
            "umask; : > {}",
            escape(Cow::from(output_file.to_str().unwrap()))
        ));
        let output = std::process::Command::new("sh") // ubs:ignore — fixed wrapper over this test's temporary receipt
            .arg("-c")
            .arg(format!("umask 0022; {command}"))
            .output()
            .expect("run durable wrapper");
        assert!(output.status.success(), "{output:?}");
        assert_eq!(String::from_utf8_lossy(&output.stdout), "0022\n");
        let mode =
            |path: &std::path::Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777;
        assert_eq!(mode(&output_file), 0o644);
        assert_eq!(mode(std::path::Path::new(&receipt)) & 0o077, 0);
    }

    /// A reader that lags behind the wrapper (a slow SSH link) leaves tail
    /// blocked on a full pipe when the command finishes; the stream must
    /// still end with the last line rather than being cut by a timer. An
    /// inotify tail also lost bytes written before its watch existed, in
    /// ~5-8% of runs, so eight wrappers race at once to make that visible.
    #[cfg(target_os = "linux")]
    #[test]
    fn durable_execution_waits_for_a_slow_reader() {
        let runs: Vec<_> = (0..8)
            .map(|_| {
                std::thread::spawn(|| {
                    let directory = tempfile::tempdir().unwrap();
                    let receipt = directory
                        .path()
                        .join("recovery-9-id")
                        .to_str()
                        .unwrap()
                        .to_owned();
                    let pipeline = TransferPipeline::new(
                        PathBuf::from("/home/user/project"),
                        "myproject".to_string(),
                        "abc123".to_string(),
                        TransferConfig::default(),
                    )
                    .with_recovery_completion(receipt, "id".to_string());
                    let command = pipeline.durable_execution_command(
                        "i=0; while [ $i -lt 20000 ]; do echo line-$i; i=$((i + 1)); done"
                            .to_string(),
                    );
                    let output = std::process::Command::new("sh") // ubs:ignore — fixed wrapper piped into a delayed reader
                        .arg("-c")
                        .arg(format!("( {command} ) | ( sleep 3; cat )"))
                        .output()
                        .expect("run durable wrapper");
                    String::from_utf8(output.stdout).unwrap()
                })
            })
            .collect();
        for run in runs {
            let stdout = run.join().unwrap();
            assert_eq!(stdout.lines().count(), 20_000, "tail was cut");
            assert!(stdout.ends_with("line-19999\n"), "tail was cut");
        }
    }

    #[test]
    fn recovery_completion_cleanup_removes_only_its_own_receipts() {
        let directory = tempfile::tempdir().unwrap();
        let base = directory.path().join("rch base");
        std::fs::create_dir(&base).unwrap();
        let path = base.join("recovery-7-id").to_str().unwrap().to_owned();
        let neighbour = base.join("recovery-8-id.done");
        std::fs::create_dir(format!("{path}.started")).unwrap();
        for suffix in ["", ".stdout", ".stderr"] {
            std::fs::write(format!("{path}{suffix}"), b"x").unwrap();
        }
        std::fs::write(&neighbour, b"x").unwrap();
        let script = recovery_completion_cleanup_script(&path);
        for _ in 0..2 {
            let status = std::process::Command::new("sh") // ubs:ignore — fixed cleanup script over this test's temporary tree
                .arg("-c")
                .arg(&script)
                .status()
                .expect("run cleanup script");
            assert!(status.success(), "cleanup must succeed, also when rerun");
        }
        let left: Vec<_> = std::fs::read_dir(&base)
            .unwrap()
            .map(|entry| entry.unwrap().file_name())
            .collect();
        assert_eq!(left, vec![std::ffi::OsString::from("recovery-8-id.done")]);
    }

    #[tokio::test]
    async fn bounded_output_stream_refuses_one_byte_over_cap() {
        assert_eq!(
            read_bounded_output_stream(&b"abcd"[..], 4)
                .await
                .expect("exact-cap output"),
            b"abcd"
        );
        let error = read_bounded_output_stream(&b"abcde"[..], 4)
            .await
            .expect_err("one byte over cap must refuse");
        assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
        assert_eq!(error.to_string(), "command output exceeded 4 bytes");
    }
    use rch_common::WorkerId;
    use rch_common::mock::Phase;
    use rch_common::test_guard;
    use serial_test::serial;

    fn arg_pair_position(args: &[String], flag: &str, value: &str) -> Option<usize> {
        args.windows(2).position(|window| {
            matches!(window, [observed_flag, observed_value]
                if observed_flag.as_str() == flag && observed_value.as_str() == value)
        })
    }

    fn worker_with_os(os: Option<&str>) -> WorkerConfig {
        WorkerConfig {
            id: WorkerId::new("w"),
            host: "h".to_string(),
            user: "u".to_string(),
            identity_file: "~/.ssh/id".to_string(),
            total_slots: 4,
            priority: 100,
            tags: os.map(rch_common::os_tag).into_iter().collect(),
            tools: Vec::new(),
        }
    }

    #[test]
    fn worker_platform_only_windows_selects_windows() {
        assert_eq!(
            WorkerPlatform::from_worker(&worker_with_os(Some("windows"))),
            WorkerPlatform::Windows
        );
        // Everything else — including an undeclared OS — is Posix, so the
        // linux/darwin fleet is untouched.
        assert_eq!(
            WorkerPlatform::from_worker(&worker_with_os(None)),
            WorkerPlatform::Posix
        );
        assert_eq!(
            WorkerPlatform::from_worker(&worker_with_os(Some("linux"))),
            WorkerPlatform::Posix
        );
        assert_eq!(
            WorkerPlatform::from_worker(&worker_with_os(Some("darwin"))),
            WorkerPlatform::Posix
        );
    }

    #[test]
    fn windows_platform_uses_c_drive_remote_base() {
        let base = TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "myproject".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        // Default (Posix) is unchanged.
        assert_eq!(base.remote_path(), "/data/tmp/rch/myproject/abc123");

        // Windows worker gets the C:/ base cargo.exe + Git sh both accept.
        let win = TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "myproject".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_worker_platform(WorkerPlatform::Windows);
        assert_eq!(win.remote_path(), "C:/rch/myproject/abc123");
    }

    #[test]
    fn windows_platform_skips_external_timeout_wrapper() {
        // Windows `timeout.exe` is a pause utility, not GNU timeout; wrapping
        // with it would corrupt the command, so the wrapper must be a no-op.
        let win = TransferPipeline::new(
            PathBuf::from("/p"),
            "p".to_string(),
            "abc".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::CargoBuild))
        .with_worker_platform(WorkerPlatform::Windows);
        assert_eq!(win.wrap_with_external_timeout("cargo build"), "cargo build");

        // A Posix worker still gets wrapped (sanity that the guard is platform-
        // specific, not a blanket disable).
        let posix = TransferPipeline::new(
            PathBuf::from("/p"),
            "p".to_string(),
            "abc".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::CargoBuild));
        assert!(
            posix
                .wrap_with_external_timeout("cargo build")
                .contains("timeout")
        );
    }

    fn command_args(cmd: &Command) -> Vec<String> {
        cmd.as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect()
    }

    fn assert_portable_rsync_archive_args(args: &[String]) {
        assert!(
            args.iter().any(|arg| arg == "--no-owner"),
            "rsync archive mode must not preserve owner metadata across workers"
        );
        assert!(
            args.iter().any(|arg| arg == "--no-group"),
            "rsync archive mode must not preserve group metadata across workers"
        );
    }

    #[test]
    fn test_remote_path() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "myproject".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );

        assert_eq!(pipeline.remote_path(), "/data/tmp/rch/myproject/abc123");
    }

    #[test]
    fn test_is_safe_reap_path_accepts_real_project_dirs() {
        // The two shapes remote_path() actually produces.
        assert!(is_safe_reap_path("/data/tmp/rch/myproject/abc123"));
        assert!(is_safe_reap_path(
            "/data/projects/coding_agent_session_search/9f8e7d6c"
        ));
    }

    #[test]
    fn test_is_safe_reap_path_rejects_dangerous_inputs() {
        assert!(!is_safe_reap_path(""));
        assert!(!is_safe_reap_path("/"));
        assert!(!is_safe_reap_path("relative/path")); // not absolute
        assert!(!is_safe_reap_path("/toplevel")); // < 2 segments
        assert!(!is_safe_reap_path("/data/../etc")); // parent traversal
        // Shell metacharacters / quotes / globs / spaces must all be rejected so
        // the unescaped embedding in the reap script cannot be subverted.
        for bad in [
            "/data/projects/a b",
            "/data/projects/a'b",
            "/data/projects/a\"b",
            "/data/projects/a$b",
            "/data/projects/a`b",
            "/data/projects/a;b",
            "/data/projects/a|b",
            "/data/projects/a*b",
            "/data/projects/a&b",
        ] {
            assert!(!is_safe_reap_path(bad), "must reject {bad:?}");
        }
    }

    #[test]
    fn test_is_safe_reap_token_guards_basenames() {
        // A real per-job dir basename is accepted.
        assert!(is_safe_reap_token(
            ".rch-target-ts2-job-29863360510034113-1780109474952075077-0"
        ));
        // Path separators, traversal, and shell metacharacters are rejected.
        assert!(!is_safe_reap_token(""));
        assert!(!is_safe_reap_token("."));
        assert!(!is_safe_reap_token(".."));
        assert!(!is_safe_reap_token("a/b"));
        assert!(!is_safe_reap_token("a b"));
        assert!(!is_safe_reap_token("a'b"));
        assert!(!is_safe_reap_token("a$b"));
        assert!(!is_safe_reap_token("a*b"));
    }

    /// End-to-end behavioral test for the cheap CURRENT-PROJECT-ONLY orchestrator
    /// reaper: actually run the generated script (under POSIX `sh`, single-quote
    /// wrapped exactly like the dispatch site) against a fake repo dir holding
    /// idle, live, current-job, and empty sibling per-job dirs and assert which
    /// survive. The expensive cross-project full-tree sweep moved to the daemon;
    /// the orchestrator now only `cd`s into the one repo dir and globs its
    /// siblings, always excluding the current job's own dir.
    #[cfg(unix)]
    #[test]
    fn test_current_project_reap_script_reaps_idle_keeps_live_current_and_empty() {
        use std::fs;
        use std::process::Command;
        use tempfile::tempdir;

        let tmp = tempdir().expect("create repo root");
        // The single repo dir the orchestrator `cd`s into (>=2 segments deep so it
        // passes is_safe_reap_path; the script itself just `cd`s into it verbatim).
        let project_dir = tmp.path().join("repo");
        fs::create_dir_all(&project_dir).expect("mkdir repo dir");

        // Helper: a per-job sibling dir with one artifact, optionally aged past the
        // idle window via `touch -t` (portable GNU/BSD).
        let make = |name: &str, aged: bool| -> std::path::PathBuf {
            let d = project_dir.join(name);
            fs::create_dir_all(d.join("deps")).expect("mkdir per-job dir");
            fs::write(d.join("deps/a.rlib"), b"x").expect("write artifact");
            if aged {
                let ok = Command::new("find")
                    .arg(&d)
                    .args(["-exec", "touch", "-t", "202601010000", "{}", ";"])
                    .status()
                    .map(|s| s.success())
                    .unwrap_or(false);
                assert!(ok, "aging {name} should succeed");
            }
            d
        };

        let current = ".rch-target-host-job-9-999-1";
        let idle = make(".rch-target-host-job-1-111-0", true);
        let idle_pid = make(".rch-target-host-pid-2-222-0", true);
        let live = make(".rch-target-host-job-3-333-0", false);
        let current_dir = make(current, true); // aged but must be EXCLUDED by name
        // Empty just-created dir (mkdir, no first write) must be kept.
        let empty = project_dir.join(".rch-target-host-job-4-444-0");
        fs::create_dir_all(&empty).expect("mkdir empty dir");
        // A non-rch sibling must never match the glob.
        let bystander = project_dir.join("target");
        fs::create_dir_all(&bystander).expect("mkdir bystander");

        // Build the script EXACTLY as the reaper does (shared globs + loop body,
        // excluding the current job dir), then run it single-quote wrapped.
        let globs = rch_common::stale_target_reap::REAP_GLOBS.join(" ");
        let loop_body = rch_common::stale_target_reap::reap_loop_body(720, Some(current), "", "");
        let project_dir_str = project_dir.to_str().unwrap();
        let script = format!(
            "cd \"{project_dir_str}\" 2>/dev/null || exit 0; \
             for d in {globs}; do {loop_body} done"
        );
        // Guard: no single quote of its own (it is single-quote wrapped at dispatch
        // — `reap_loop_body`'s `awk '{{print $1}}'` is only emitted in the METRICS
        // variant; the orchestrator passes empty counters so no awk/quotes appear).
        assert!(
            !script.contains('\''),
            "orchestrator script must contain no single quotes: {script}"
        );
        let status = Command::new("sh") // ubs:ignore — fixed reaper script over this test's generated temporary tree
            .arg("-c")
            .arg(format!("sh -c '{script}'"))
            .status()
            .expect("run reap script");
        assert!(status.success(), "reap script should exit 0");

        assert!(!idle.exists(), "idle -job- sibling must be reaped");
        assert!(!idle_pid.exists(), "idle -pid- sibling must be reaped");
        assert!(live.exists(), "freshly-touched sibling must be kept");
        assert!(
            current_dir.exists(),
            "the current job's own dir must NEVER be reaped (excluded by name)"
        );
        assert!(empty.exists(), "empty just-created dir must be kept");
        assert!(bystander.exists(), "non-rch `target` must never be touched");
    }

    /// The orchestrator `cd`s into the repo dir verbatim, so a SYMLINKED project
    /// dir is followed transparently by `cd` (no `pwd -P` needed) and its idle
    /// siblings are still reaped while the live one survives.
    #[cfg(unix)]
    #[test]
    fn test_current_project_reap_script_follows_symlinked_project_dir() {
        use std::fs;
        use std::os::unix::fs::symlink;
        use std::process::Command;
        use tempfile::tempdir;

        let base = tempdir().expect("create base");
        let physical = base.path().join("data_repo");
        fs::create_dir_all(&physical).expect("mkdir physical repo dir");
        let link = base.path().join("home_repo");
        symlink(&physical, &link).expect("create symlink to physical repo dir");

        let make = |name: &str, aged: bool| -> std::path::PathBuf {
            let d = physical.join(name);
            fs::create_dir_all(d.join("deps")).expect("mkdir per-job dir");
            fs::write(d.join("deps/a.rlib"), b"x").expect("write artifact");
            if aged {
                let ok = Command::new("find")
                    .arg(&d)
                    .args(["-exec", "touch", "-t", "202601010000", "{}", ";"])
                    .status()
                    .map(|s| s.success())
                    .unwrap_or(false);
                assert!(ok, "aging {name} should succeed");
            }
            d
        };

        let idle = make(".rch-target-host-job-1-111-0", true);
        let live = make(".rch-target-host-job-2-222-1", false);

        // Pass the SYMLINK path as the project dir — `cd <symlink>` follows it.
        let link_str = link.to_str().unwrap();
        let globs = rch_common::stale_target_reap::REAP_GLOBS.join(" ");
        let loop_body = rch_common::stale_target_reap::reap_loop_body(720, None, "", "");
        let script = format!(
            "cd \"{link_str}\" 2>/dev/null || exit 0; \
             for d in {globs}; do {loop_body} done"
        );
        let status = Command::new("sh") // ubs:ignore — fixed reaper script over this test's generated symlink fixture
            .arg("-c")
            .arg(format!("sh -c '{script}'"))
            .status()
            .expect("run reap script");
        assert!(status.success(), "reap script should exit 0");

        assert!(
            !idle.exists(),
            "idle sibling behind a SYMLINKED project dir must be reaped"
        );
        assert!(
            live.exists(),
            "live sibling must be preserved via the symlinked project dir"
        );
    }

    #[test]
    fn test_project_id_from_path() {
        let _guard = test_guard!();
        assert_eq!(
            project_id_from_path(Path::new("/home/user/my-project")),
            "my-project"
        );
        assert_eq!(
            project_id_from_path(Path::new("/workspace/remote_compilation_helper")),
            "remote_compilation_helper"
        );
    }

    #[test]
    fn test_parse_rsync_bytes() {
        let _guard = test_guard!();
        let output = "sent 1,234 bytes  received 567 bytes  1800.50 bytes/sec";
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Upload),
            1234
        );
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Download),
            567
        );

        let empty = "";
        assert_eq!(parse_rsync_bytes(empty, RsyncTransferDirection::Upload), 0);
        assert_eq!(
            parse_rsync_bytes(empty, RsyncTransferDirection::Download),
            0
        );
    }

    #[test]
    fn test_parse_rsync_bytes_total_format() {
        let _guard = test_guard!();
        // Test "Total bytes sent:" format (newer rsync versions)
        let output = "Total bytes sent: 45,678\nTotal bytes received: 123";
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Upload),
            45678
        );
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Download),
            123
        );
    }

    #[test]
    fn test_parse_rsync_bytes_no_commas() {
        let _guard = test_guard!();
        let output = "sent 999 bytes  received 100 bytes  1000.00 bytes/sec";
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Upload),
            999
        );
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Download),
            100
        );
    }

    #[test]
    fn test_parse_rsync_bytes_large_number() {
        let _guard = test_guard!();
        let output = "sent 1,234,567,890 bytes  received 100 bytes  total";
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Upload),
            1234567890
        );
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Download),
            100
        );
    }

    #[test]
    fn test_parse_rsync_bytes_structured_stats_override_summary() {
        let _guard = test_guard!();
        let summary = "received 12,345 bytes  sent 67 bytes  1000 bytes/sec";
        assert_eq!(
            parse_rsync_bytes(summary, RsyncTransferDirection::Upload),
            67
        );
        assert_eq!(
            parse_rsync_bytes(summary, RsyncTransferDirection::Download),
            12345
        );
        let stats = "  Total bytes received: 98,765\nTotal bytes sent: 43";
        for output in [format!("{summary}\n{stats}"), format!("{stats}\n{summary}")] {
            assert_eq!(
                parse_rsync_bytes(&output, RsyncTransferDirection::Upload),
                43
            );
            assert_eq!(
                parse_rsync_bytes(&output, RsyncTransferDirection::Download),
                98765
            );
        }
    }

    #[test]
    fn test_parse_rsync_bytes_rejects_missing_malformed_and_unrelated_counters() {
        let _guard = test_guard!();
        for output in [
            "diagnostic 123 sent bytes",
            "sent 123 bytes in a filename",
            "sent 123 items received 456 items",
            "Total bytes sent: invalid\nTotal bytes received: invalid",
            "Total bytes sent: 18446744073709551616\nTotal bytes received: 18446744073709551616",
            "sent invalid bytes received invalid bytes",
        ] {
            for direction in [
                RsyncTransferDirection::Upload,
                RsyncTransferDirection::Download,
            ] {
                assert_eq!(parse_rsync_bytes(output, direction), 0, "{output}");
            }
        }
        // A missing direction cannot borrow the other direction's counter.
        assert_eq!(
            parse_rsync_bytes("Total bytes sent: 999", RsyncTransferDirection::Download),
            0
        );
        assert_eq!(
            parse_rsync_bytes("Total bytes received: 999", RsyncTransferDirection::Upload),
            0
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_rsync_download_statistics_match_real_payload_noop_and_update() {
        let _guard = test_guard!();
        let source = tempfile::tempdir().unwrap();
        let destination = tempfile::tempdir().unwrap();
        let nested = source.path().join("one/two");
        std::fs::create_dir_all(&nested).unwrap();
        let payload = vec![0x5a; 1024 * 1024];
        std::fs::write(nested.join("payload.bin"), &payload).unwrap();
        std::fs::write(source.path().join("message.txt"), b"first\n").unwrap();

        async fn pull(source: &Path, destination: &Path) -> String {
            // Both rsync endpoints are real processes. This fixed shell adapter
            // discards the dummy hostname and starts the actual peer over pipes;
            // it supplies no simulated protocol, statistics or file contents.
            let mut command = Command::new("rsync");
            command
                .args(["-a", "--checksum", "--stats", "-e"])
                .arg(r#"sh -c 'shift; exec "$@"' rch-local-peer"#)
                .arg("--")
                .arg(format!("local-peer:{}/", source.display()))
                .arg(format!("{}/", destination.display()))
                .env("LC_ALL", "C")
                // Pin the working directory to one this test owns. Every path
                // here is absolute, so this changes nothing about the transfer
                // — but rsync calls getcwd() at startup, and inheriting the
                // process CWD made this test fail with
                // `getcwd(): No such file or directory` whenever a CONCURRENT
                // test removed the directory the process happened to be in
                // (bd-es64k). A test should not depend on state no other test
                // agreed to leave alone.
                .current_dir(destination)
                .kill_on_drop(true);
            let output = tokio::time::timeout(std::time::Duration::from_secs(15), command.output())
                .await
                .expect("real rsync pull must finish within its test deadline")
                .expect("start real rsync pull");
            let stdout = String::from_utf8(output.stdout).unwrap();
            let stderr = String::from_utf8_lossy(&output.stderr);
            eprintln!("real rsync pull: {}\n{stdout}\n{stderr}", output.status);
            assert!(output.status.success(), "{stdout}\n{stderr}");
            stdout
        }

        let first = pull(source.path(), destination.path()).await;
        assert_eq!(
            std::fs::read(destination.path().join("one/two/payload.bin")).unwrap(),
            payload
        );
        assert_eq!(
            std::fs::read(destination.path().join("message.txt")).unwrap(),
            b"first\n"
        );
        assert_eq!(parse_rsync_files(&first), 2);
        // Uncompressed first-copy payload dominates the received direction;
        // request traffic must never masquerade as the downloaded megabyte.
        let received = parse_rsync_bytes(&first, RsyncTransferDirection::Download);
        let sent = parse_rsync_bytes(&first, RsyncTransferDirection::Upload);
        assert!(received >= 1024 * 1024, "{first}");
        assert!(sent < 8192, "{first}");

        let unchanged = pull(source.path(), destination.path()).await;
        assert_eq!(parse_rsync_files(&unchanged), 0, "{unchanged}");
        assert!(parse_rsync_bytes(&unchanged, RsyncTransferDirection::Download) > 0);

        std::fs::write(source.path().join("message.txt"), b"other\n").unwrap();
        let changed = pull(source.path(), destination.path()).await;
        assert_eq!(parse_rsync_files(&changed), 1, "{changed}");
        assert_eq!(
            std::fs::read(destination.path().join("message.txt")).unwrap(),
            b"other\n"
        );
        assert_eq!(
            std::fs::read(destination.path().join("one/two/payload.bin")).unwrap(),
            payload
        );
    }

    #[test]
    fn test_parse_rsync_files() {
        let _guard = test_guard!();
        // Test "Number of files transferred:" format
        let output = "Number of files transferred: 42";
        assert_eq!(parse_rsync_files(output), 42);
    }

    #[test]
    fn test_parse_rsync_files_with_comma() {
        let _guard = test_guard!();
        let output = "Number of files transferred: 1,234";
        assert_eq!(parse_rsync_files(output), 1234);
    }

    #[test]
    fn test_parse_rsync_files_number_of_files_format() {
        let _guard = test_guard!();
        // Test "Number of files:" format (alternate rsync output)
        let output = "Number of files: 100\nsome other line";
        assert_eq!(parse_rsync_files(output), 100);
    }

    #[test]
    fn test_parse_rsync_files_prefers_transferred_count_over_total_tree_count() {
        let _guard = test_guard!();
        for counter in [
            "Number of files transferred:",
            "Number of regular files transferred:",
        ] {
            for count in [0, 42] {
                let output = format!(
                    "Number of files: 28,779 (reg: 20,000, dir: 8,779)\n{counter} {count}\nTotal bytes sent: 1,271,299"
                );
                assert_eq!(parse_rsync_files(&output), count);
            }
        }
    }

    #[test]
    fn test_parse_rsync_files_empty() {
        let _guard = test_guard!();
        let empty = "";
        assert_eq!(parse_rsync_files(empty), 0);
    }

    #[test]
    fn test_parse_rsync_files_no_structured_stats_returns_zero() {
        let _guard = test_guard!();
        // When no "Number of files" line exists, return 0 rather than guessing.
        let output = "file1.txt\nfile2.txt\nfile3.txt";
        assert_eq!(parse_rsync_files(output), 0);
    }

    #[test]
    fn test_parse_rsync_files_no_structured_stats_ignores_sent_line() {
        let _guard = test_guard!();
        // No structured stats: still return 0, even if "sent" appears.
        let output = "file1.txt\nfile2.txt\nsent 100 bytes";
        assert_eq!(parse_rsync_files(output), 0);
    }

    #[test]
    fn test_parse_rsync_itemized_regular_files_transfer_run() {
        let _guard = test_guard!();
        // Captured shape of a real `rsync -az --stats --info=name2
        // --out-format='%i %n'` run that transferred everything (bd-mpbav).
        let output = "\
sending incremental file list
.d..t....... ./
>f++++++++++ .rustc_info.json
>f++++++++++ CACHEDIR.TAG
cd++++++++++ release-perf/
>f++++++++++ release-perf/mybin
cd++++++++++ release-perf/deps/
>f++++++++++ release-perf/deps/lib.rlib

Number of files: 7 (reg: 4, dir: 3)
Number of regular files transferred: 4
";
        let files = parse_rsync_itemized_regular_files(output);
        // Directories (`d` item type) are excluded; regular files kept in order.
        assert_eq!(
            files,
            vec![
                ".rustc_info.json".to_string(),
                "CACHEDIR.TAG".to_string(),
                "release-perf/mybin".to_string(),
                "release-perf/deps/lib.rlib".to_string(),
            ]
        );
        assert_eq!(parse_rsync_matched_regular_files(output), Some(4));
    }

    #[test]
    fn test_parse_rsync_itemized_regular_files_uptodate_run() {
        let _guard = test_guard!();
        // The repeat run: nothing transfers, every matched file is listed as
        // up-to-date (`.f` padded items). These entries are exactly what keeps
        // an already-current no-op rebuild from being misread as a zero-output
        // miss by the bd-mpbav gate.
        let output = "\
.d           ./
.f           .rustc_info.json
.f           CACHEDIR.TAG
.d           release-perf/
.f           release-perf/mybin
.d           release-perf/deps/
.f           release-perf/deps/lib.rlib

Number of files: 7 (reg: 4, dir: 3)
Number of regular files transferred: 0
";
        let files = parse_rsync_itemized_regular_files(output);
        assert_eq!(
            files,
            vec![
                ".rustc_info.json".to_string(),
                "CACHEDIR.TAG".to_string(),
                "release-perf/mybin".to_string(),
                "release-perf/deps/lib.rlib".to_string(),
            ]
        );
        assert_eq!(parse_rsync_matched_regular_files(output), Some(4));
    }

    #[test]
    fn test_parse_rsync_itemized_regular_files_rejects_noise() {
        let _guard = test_guard!();
        // Non-itemized noise (progress lines, stats text, sender chatter,
        // filenames that merely look like flags) must never enter the manifest.
        let output = "\
          1,234    99%    0.00kB/s    0:00:00
[sender] showing file release-perf/mybin because of pattern release-perf/**
total: matches=0  hash_hits=0  false_alarms=0 data=54
Number of files transferred: 42
--pretend-not-a-file
";
        assert!(parse_rsync_itemized_regular_files(output).is_empty());
        // No stats "Number of files:" line here: matched count is unknown.
        assert_eq!(parse_rsync_matched_regular_files(output), None);
    }

    #[test]
    fn test_parse_rsync_matched_regular_files_comma_forms() {
        let _guard = test_guard!();
        assert_eq!(
            parse_rsync_matched_regular_files("Number of files: 28,779 (reg: 20,000, dir: 8,779)"),
            Some(20_000)
        );
        // "Number of files transferred:" must NOT satisfy the prefix match.
        assert_eq!(
            parse_rsync_matched_regular_files("Number of files transferred: 42"),
            None
        );
        assert_eq!(parse_rsync_matched_regular_files(""), None);
    }

    #[test]
    fn test_cargo_package_verification_directory_only_rsync_count() {
        let _guard = test_guard!();
        // Captured from a real read-only rsync dry-run with the archive glob
        // that missed Cargo's workspace tmp-registry directory.
        let output = "Number of files: 2 (dir: 2)\n\
                      Number of created files: 1 (dir: 1)\n\
                      Number of regular files transferred: 0\n";
        assert_eq!(parse_rsync_matched_regular_files(output), Some(0));
        assert_eq!(
            parse_rsync_matched_regular_files("Number of files: 1,234 (dir: 1,234)"),
            Some(0)
        );
        for malformed_or_unknown in [
            "Number of files: 2 (dir: 1)",
            "Number of files: 2 (dir: 1, link: 1)",
            "Number of files: unknown (dir: 2)",
            "Number of files: 2 (dir: unknown)",
            "Number of files: 2 (dir: 2",
            "Number of files: 2 (dir: 2) extra",
            "Number of files: 1,2 (dir: 12)",
            "Number of files: 2 (dir: 2,)",
            "Number of files: 2",
            "Number of regular files transferred: 0",
        ] {
            assert_eq!(
                parse_rsync_matched_regular_files(malformed_or_unknown),
                None,
                "unproven regular-file count: {malformed_or_unknown}"
            );
        }
    }

    #[test]
    fn test_default_artifact_patterns() {
        let _guard = test_guard!();
        let patterns = default_rust_artifact_patterns();
        assert!(!patterns.is_empty());
        assert!(patterns.iter().any(|p| p.contains("debug")));
        assert!(patterns.iter().any(|p| p.contains("release")));
    }

    #[test]
    fn test_default_bun_artifact_patterns() {
        let _guard = test_guard!();
        let patterns = default_bun_artifact_patterns();
        assert!(!patterns.is_empty());
        assert!(patterns.iter().any(|p| p.contains("coverage")));
        assert!(patterns.iter().any(|p| p.contains("tsbuildinfo")));
    }

    #[test]
    fn test_default_rust_test_artifact_patterns() {
        let _guard = test_guard!();
        let patterns = default_rust_test_artifact_patterns();
        // Test patterns should be non-empty but minimal
        assert!(!patterns.is_empty());

        // Should include coverage-related patterns
        assert!(patterns.iter().any(|p| p.contains("llvm-cov")));
        assert!(patterns.iter().any(|p| p.contains("coverage")));

        // Should include nextest artifacts
        assert!(patterns.iter().any(|p| p.contains("nextest")));

        // Should NOT include full debug/release directories (that's the point!)
        assert!(!patterns.iter().any(|p| p == "target/debug/**"));
        assert!(!patterns.iter().any(|p| p == "target/release/**"));
    }

    #[test]
    fn test_rust_test_patterns_vs_full_patterns() {
        let _guard = test_guard!();
        let test_patterns = default_rust_test_artifact_patterns();
        let full_patterns = default_rust_artifact_patterns();

        // Full patterns should include debug/release (the heavy directories)
        assert!(full_patterns.iter().any(|p| p.contains("debug")));
        assert!(full_patterns.iter().any(|p| p.contains("release")));

        // Test patterns should NOT include debug/release directories
        // (This is the key optimization - avoiding GB of data transfer)
        assert!(!test_patterns.iter().any(|p| p.contains("debug")));
        assert!(!test_patterns.iter().any(|p| p.contains("release")));

        // Test patterns focus on results/coverage, not build artifacts
        assert!(test_patterns.iter().any(|p| p.contains("coverage")));
    }

    #[test]
    fn test_compute_project_hash_basic() {
        let _guard = test_guard!();
        use std::fs;
        use tempfile::tempdir;

        let dir = tempdir().expect("create temp dir");
        let path = dir.path();

        // Create a Cargo.toml file
        fs::write(path.join("Cargo.toml"), "[package]\nname = \"test\"").expect("write cargo");

        let hash1 = compute_project_hash(path);
        assert!(!hash1.is_empty());
        assert_eq!(hash1.len(), 16); // Should be 16 hex chars
    }

    #[test]
    fn test_compute_project_hash_different_paths() {
        let _guard = test_guard!();
        use tempfile::tempdir;

        let dir1 = tempdir().expect("create temp dir 1");
        let dir2 = tempdir().expect("create temp dir 2");

        let hash1 = compute_project_hash(dir1.path());
        let hash2 = compute_project_hash(dir2.path());

        // Different paths should produce different hashes
        assert_ne!(hash1, hash2);
    }

    #[test]
    fn test_compute_project_hash_includes_key_files() {
        let _guard = test_guard!();
        use std::fs;
        use std::thread::sleep;
        use std::time::Duration;
        use tempfile::tempdir;

        let dir = tempdir().expect("create temp dir");
        let path = dir.path();

        let hash_before = compute_project_hash(path);

        // Add a key file (Cargo.toml)
        sleep(Duration::from_millis(10)); // Ensure mtime differs
        fs::write(path.join("Cargo.toml"), "[package]\nname = \"test\"").expect("write cargo");

        let hash_after = compute_project_hash(path);

        // Hash should change when key file is added
        assert_ne!(hash_before, hash_after);
    }

    #[test]
    fn test_compute_project_hash_with_dependency_roots_changes_on_dependency_manifest_change() {
        let _guard = test_guard!();
        use std::fs;
        use tempfile::tempdir;

        let root = tempdir().expect("create root dir");
        let dep = tempdir().expect("create dep dir");
        fs::write(root.path().join("Cargo.toml"), "[package]\nname = \"root\"")
            .expect("write root cargo");
        fs::write(dep.path().join("Cargo.toml"), "[package]\nname = \"dep\"")
            .expect("write dep cargo");

        let hash_before =
            compute_project_hash_with_dependency_roots(root.path(), &[dep.path().to_path_buf()]);

        fs::write(
            dep.path().join("Cargo.toml"),
            "[package]\nname = \"dep\"\nversion = \"0.2.0\"",
        )
        .expect("rewrite dep cargo");
        let hash_after =
            compute_project_hash_with_dependency_roots(root.path(), &[dep.path().to_path_buf()]);

        assert_ne!(
            hash_before, hash_after,
            "dependency manifest changes must invalidate closure hash"
        );
    }

    #[test]
    fn test_compute_project_hash_with_dependency_roots_ignores_non_key_noise_changes() {
        let _guard = test_guard!();
        use std::fs;
        use tempfile::tempdir;

        let root = tempdir().expect("create root dir");
        let dep = tempdir().expect("create dep dir");
        fs::write(root.path().join("Cargo.toml"), "[package]\nname = \"root\"")
            .expect("write root cargo");
        fs::write(dep.path().join("Cargo.toml"), "[package]\nname = \"dep\"")
            .expect("write dep cargo");

        let hash_before =
            compute_project_hash_with_dependency_roots(root.path(), &[dep.path().to_path_buf()]);

        fs::create_dir_all(dep.path().join("src")).expect("create dep src");
        fs::write(
            dep.path().join("src/lib.rs"),
            "pub fn unchanged_policy() {}",
        )
        .expect("write dep source");
        let hash_after =
            compute_project_hash_with_dependency_roots(root.path(), &[dep.path().to_path_buf()]);

        assert_eq!(
            hash_before, hash_after,
            "non-key file noise should not perturb closure hash policy"
        );
    }

    #[cfg(unix)]
    #[test]
    fn test_compute_project_hash_with_dependency_roots_normalizes_alias_equivalence() {
        let _guard = test_guard!();
        use std::fs;
        use tempfile::tempdir;

        let base = tempdir().expect("create temp base");
        let root = base.path().join("root");
        let dep = base.path().join("dep");
        let dep_alias = base.path().join("dep_alias");
        fs::create_dir_all(&root).expect("create root");
        fs::create_dir_all(&dep).expect("create dep");
        fs::write(root.join("Cargo.toml"), "[package]\nname = \"root\"").expect("write root cargo");
        fs::write(dep.join("Cargo.toml"), "[package]\nname = \"dep\"").expect("write dep cargo");
        std::os::unix::fs::symlink(&dep, &dep_alias).expect("create dep alias symlink");

        let canonical_hash =
            compute_project_hash_with_dependency_roots(&root, std::slice::from_ref(&dep));
        let alias_hash = compute_project_hash_with_dependency_roots(&root, &[dep_alias]);

        assert_eq!(
            canonical_hash, alias_hash,
            "alias and canonical dependency roots must produce identical closure hash"
        );
    }

    #[test]
    fn test_compute_project_hash_with_dependency_roots_perf_budget_smoke() {
        let _guard = test_guard!();
        use std::fs;
        use std::time::{Duration, Instant};
        use tempfile::tempdir;

        let base = tempdir().expect("create perf temp base");
        let root = base.path().join("root");
        fs::create_dir_all(&root).expect("create root");
        fs::write(root.join("Cargo.toml"), "[package]\nname = \"root\"").expect("write root cargo");

        let mut deps = Vec::new();
        for idx in 0..4 {
            let dep = base.path().join(format!("dep-{idx}"));
            fs::create_dir_all(&dep).expect("create dep root");
            fs::write(
                dep.join("Cargo.toml"),
                format!("[package]\nname = \"dep-{idx}\"\nversion = \"0.1.{idx}\""),
            )
            .expect("write dep cargo");
            deps.push(dep);
        }

        let start = Instant::now();
        for _ in 0..100 {
            let _ = compute_project_hash_with_dependency_roots(&root, &deps);
        }
        let elapsed = start.elapsed();

        assert!(
            elapsed < Duration::from_secs(2),
            "closure hash computation too slow: {:?}",
            elapsed
        );
    }

    #[test]
    fn test_project_id_from_path_root() {
        let _guard = test_guard!();
        // Test with root path - falls back to "unknown" since "/" has no file_name
        assert_eq!(project_id_from_path(Path::new("/")), "unknown");
    }

    #[test]
    fn test_project_id_from_path_with_special_chars() {
        let _guard = test_guard!();
        // Test with path containing underscores and dashes
        assert_eq!(
            project_id_from_path(Path::new("/home/user/my_project-v2")),
            "my_project-v2"
        );
    }

    #[test]
    fn test_default_c_cpp_artifact_patterns() {
        let _guard = test_guard!();
        let patterns = default_c_cpp_artifact_patterns();
        assert!(!patterns.is_empty());
        assert!(patterns.iter().any(|p| p.contains("build")));
        assert!(patterns.iter().any(|p| p.contains(".o")));
        assert!(patterns.iter().any(|p| p.contains(".so")));
    }

    #[test]
    fn test_transfer_pipeline_builder_methods() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_color_mode(ColorMode::Always)
        .with_env_allowlist(vec!["RUSTFLAGS".to_string(), "CC".to_string()]);

        assert_eq!(pipeline.remote_path(), "/data/tmp/rch/test-project/abc123");
    }

    #[test]
    fn test_transfer_pipeline_with_ssh_options() {
        let _guard = test_guard!();
        let custom_options = SshOptions {
            connect_timeout: std::time::Duration::from_secs(30),
            command_timeout: std::time::Duration::from_secs(120),
            ..Default::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_ssh_options(custom_options)
        .with_command_timeout(std::time::Duration::from_secs(300));

        // Just verify it builds without panic
        assert_eq!(pipeline.remote_path(), "/data/tmp/rch/test-project/abc123");
    }

    #[test]
    fn test_transfer_pipeline_defaults_to_plain_ssh_sessions() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );

        assert!(!pipeline.ssh_options.control_master);
        assert!(pipeline.ssh_options.control_persist_idle.is_none());
    }

    #[test]
    fn test_transfer_pipeline_enables_control_master_when_persist_configured() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig {
                ssh_control_persist_secs: Some(60),
                ..TransferConfig::default()
            },
        );

        assert!(pipeline.ssh_options.control_master);
        assert_eq!(
            pipeline.ssh_options.control_persist_idle,
            Some(std::time::Duration::from_secs(60))
        );
    }

    #[test]
    fn test_bun_test_external_timeout_wrapper() {
        let _guard = test_guard!();
        // Test that BunTest commands get wrapped with timeout
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::BunTest));

        let wrapped = pipeline.wrap_with_external_timeout("bun test");
        assert!(wrapped.contains("timeout --verbose"));
        assert!(wrapped.contains("--signal=KILL"));
        assert!(wrapped.contains("--foreground"));
        assert!(wrapped.contains("600")); // Default timeout
        assert!(wrapped.contains("bun test"));
    }

    #[test]
    fn test_bun_typecheck_external_timeout_wrapper() {
        let _guard = test_guard!();
        // Test that BunTypecheck commands also get wrapped
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::BunTypecheck));

        let wrapped = pipeline.wrap_with_external_timeout("bun typecheck");
        assert!(wrapped.contains("timeout"));
        assert!(wrapped.contains("bun typecheck"));
    }

    #[test]
    fn test_cargo_build_wrapped_with_build_timeout() {
        let _guard = test_guard!();
        // All commands now wrapped with appropriate timeout (bd-1nmv)
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::CargoBuild));

        let wrapped = pipeline.wrap_with_external_timeout("cargo build");
        assert!(wrapped.contains("timeout"));
        assert!(wrapped.contains("--signal=KILL"));
        assert!(wrapped.contains("--foreground"));
        assert!(wrapped.contains("300")); // Default build_timeout_sec
        assert!(wrapped.contains("cargo build"));
    }

    #[test]
    fn test_unknown_compilation_kind_uses_build_timeout() {
        let _guard = test_guard!();
        // Commands without compilation kind use build_timeout (bd-1nmv)
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        ); // No with_compilation_kind() call

        let wrapped = pipeline.wrap_with_external_timeout("some command");
        assert!(wrapped.contains("timeout"));
        assert!(wrapped.contains("300")); // Default build_timeout_sec
        assert!(wrapped.contains("some command"));
    }

    #[test]
    fn test_cargo_test_wrapped_with_test_timeout() {
        let _guard = test_guard!();
        // Test commands use test_timeout_sec (bd-1nmv)
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::CargoTest));

        let wrapped = pipeline.wrap_with_external_timeout("cargo test");
        assert!(wrapped.contains("timeout"));
        assert!(wrapped.contains("1800")); // Default test_timeout_sec
        assert!(wrapped.contains("cargo test"));
    }

    #[test]
    fn test_external_timeout_preserves_leading_env_assignments() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::CargoTest));

        let wrapped = pipeline.wrap_with_external_timeout(
            "CARGO_TARGET_DIR='/tmp/rch target' RUSTFLAGS='-C target-cpu=native' cargo test",
        );

        assert!(wrapped.contains("timeout"));
        assert!(wrapped.contains(" env CARGO_TARGET_DIR="));
        assert!(wrapped.contains("rch-timeout env CARGO_TARGET_DIR="));
        assert!(!wrapped.contains("1800 CARGO_TARGET_DIR="));
        assert!(wrapped.contains("RUSTFLAGS='-C target-cpu=native' cargo test"));
    }

    #[test]
    fn test_external_timeout_disabled() {
        let _guard = test_guard!();
        use rch_common::CompilationConfig;

        // External timeout can be disabled via config (bd-1nmv)
        let config = CompilationConfig {
            external_timeout_enabled: false,
            ..CompilationConfig::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::BunTest))
        .with_compilation_config(config);

        let wrapped = pipeline.wrap_with_external_timeout("bun test");
        assert!(!wrapped.contains("timeout"));
        assert_eq!(wrapped, "bun test");
    }

    #[test]
    fn test_custom_timeout_values() {
        let _guard = test_guard!();
        use rch_common::CompilationConfig;

        // Custom timeout values can be configured (bd-1nmv)
        let config = CompilationConfig {
            build_timeout_sec: 120,
            test_timeout_sec: 900,
            bun_timeout_sec: 180,
            external_timeout_enabled: true,
            ..CompilationConfig::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::BunTest))
        .with_compilation_config(config);

        let wrapped = pipeline.wrap_with_external_timeout("bun test");
        assert!(wrapped.contains("180")); // Custom bun_timeout_sec
        assert!(wrapped.contains("bun test"));
    }

    #[test]
    fn test_external_timeout_marker_is_exact_and_attempt_specific() {
        let first = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test".into(),
            "hash".into(),
            TransferConfig::default(),
        );
        let second = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test".into(),
            "hash".into(),
            TransferConfig::default(),
        );
        assert!(first.is_deadline_marker(&first.deadline_marker));
        assert!(first.is_deadline_marker(&format!("{}\r\n", first.deadline_marker)));
        for unrelated in [
            second.deadline_marker,
            format!("prefix {}", first.deadline_marker),
            format!("{} suffix", first.deadline_marker),
            format!("{}_STATUS=137", first.deadline_marker),
            "timeout: sending signal KILL to command 'sh'".into(),
        ] {
            assert!(!first.is_deadline_marker(&unrelated), "{unrelated}");
        }
    }

    #[test]
    fn remote_process_setup_refusal_requires_exact_attempt_and_status() {
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test".into(),
            "hash".into(),
            TransferConfig::default(),
        );
        let other = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test".into(),
            "hash".into(),
            TransferConfig::default(),
        );
        let marker = pipeline.remote_process_setup_marker();
        for (exit_code, stderr, refused) in [
            (125, format!("{marker}\n"), true),
            (125, format!("{marker}\r\n"), true),
            (125, "compiler returned 125\n".to_string(), false),
            (127, "compiler returned 127\n".to_string(), false),
            (127, format!("{marker}\n"), false),
            (0, format!("{marker}\n"), false),
            (125, format!("prefix {marker}\n"), false),
            (125, format!("{marker} suffix\n"), false),
            (125, marker.clone(), false),
            (125, format!("{marker}\n{marker}\n"), false),
            (125, format!("{marker}\n{marker}"), false),
            (125, other.remote_process_setup_marker(), false),
        ] {
            let result = CommandResult {
                exit_code,
                stderr,
                stdout: String::new(),
                duration_ms: 0,
            };
            let checked = pipeline.ensure_remote_process_setup(&result);
            assert_eq!(checked.is_err(), refused, "{result:?}");
            if let Err(error) = checked {
                assert!(error.is::<RemoteProcessSetupUnavailable>());
            }
        }
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn remote_process_setup_missing_capability_never_starts_workload() {
        use std::process::Command;

        let dir = tempfile::tempdir().unwrap().keep();
        let bin = dir.join("bin");
        std::fs::create_dir(&bin).unwrap();
        // Real setup tools are available, while setsid is genuinely absent.
        // chmod belongs here: the generated command chmods restricted dirs
        // before the setsid capability check, so without it the fixture died
        // at exit 127 and never reached the refusal this test asserts.
        for tool in ["touch", "mkdir", "rm", "chmod"] {
            std::os::unix::fs::symlink(format!("/bin/{tool}"), bin.join(tool)).unwrap();
        }
        let pipeline = TransferPipeline::new(
            dir.clone(),
            "test".into(),
            "hash".into(),
            TransferConfig::default(),
        )
        .with_remote_path_override(dir.to_str().unwrap())
        .with_env_allowlist(Vec::new())
        .with_build_id(Some(42))
        .with_compilation_config(rch_common::CompilationConfig {
            remote_build_jobs: RemoteBuildJobs::Off,
            ..Default::default()
        });
        let workload = "printf 'started' > workload-started";
        let unavailable = Command::new("/bin/sh") // ubs:ignore — production command with isolated capability PATH
            .arg("-c")
            .arg(pipeline.build_remote_command(workload, None))
            .env("PATH", &bin)
            .output()
            .unwrap();
        // Publishing to an absent parent fails after real /proc identity reads,
        // and must give the same pre-workload refusal as a missing capability.
        let unpublished = Command::new("setsid")
            .arg("sh")
            .arg("-c")
            .arg(remote_build_watchdog_script())
            .arg("rch-build")
            .arg(dir.join("missing-parent/job.pgid"))
            .arg("0")
            .arg(&pipeline.deadline_marker)
            .arg("42")
            .arg("sh")
            .arg("-c")
            .arg(workload)
            .current_dir(&dir)
            .output()
            .unwrap();
        for output in [unavailable, unpublished] {
            let result = CommandResult {
                exit_code: output.status.code().unwrap(),
                stdout: String::from_utf8(output.stdout).unwrap(),
                stderr: String::from_utf8(output.stderr).unwrap(),
                duration_ms: 0,
            };
            assert_eq!(result.exit_code, 125, "{result:?}");
            assert!(
                pipeline
                    .ensure_remote_process_setup(&result)
                    .unwrap_err()
                    .is::<RemoteProcessSetupUnavailable>()
            );
        }
        assert!(!dir.join("workload-started").exists());
        assert!(!PathBuf::from(pipeline.remote_pgid_file_path().unwrap()).exists());
    }

    #[cfg(target_os = "linux")]
    async fn run_external_timeout_fixture(directory: &Path, command: &str) -> std::process::Output {
        // These are actual stock shell/timeout processes, never Cargo/compiler
        // invocations. Keep fixtures and the exact generated command for review.
        std::fs::write(directory.join("command.sh"), command).unwrap();
        let mut child = Command::new("/bin/sh");
        child
            .args(["-c", command])
            .current_dir(directory)
            .env_clear()
            .env("PATH", "/usr/bin:/bin")
            .env("HOME", directory)
            .env("LC_ALL", "C")
            .env("NO_COLOR", "1")
            .stdin(Stdio::null())
            .kill_on_drop(true);
        let output = tokio::time::timeout(std::time::Duration::from_secs(6), child.output())
            .await
            .expect("finite workload must finish; inherited timer pipes must not hold it open")
            .expect("run generated production timeout wrapper");
        std::fs::write(directory.join("stdout"), &output.stdout).unwrap();
        std::fs::write(directory.join("stderr"), &output.stderr).unwrap();
        std::fs::write(directory.join("status"), output.status.to_string()).unwrap();
        output
    }

    #[cfg(target_os = "linux")]
    #[tokio::test]
    async fn test_external_timeout_real_fallback_distinguishes_deadline_and_child_137() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir().unwrap().keep();
        let cases = [
            (
                "streams",
                "printf '%s' \"$MESSAGE\"; printf 'stderr-without-newline' >&2",
                0,
                false,
                15,
            ),
            (
                "exit137",
                "printf 'stdout'; printf 'stderr' >&2; exit 137",
                137,
                false,
                15,
            ),
            ("selfkill", "kill -KILL $$", 137, false, 15),
            (
                "spoof",
                "printf \"timeout: sending signal KILL to command 'sh'\\n\" >&2; exit 137",
                137,
                false,
                15,
            ),
            ("deadline", "exec sleep 4", 137, true, 1),
        ];
        for (name, script, status, deadline, seconds) in cases {
            let directory = retained.join(name);
            std::fs::create_dir(&directory).unwrap();
            let pipeline = TransferPipeline::new(
                directory.clone(),
                "test".into(),
                "hash".into(),
                TransferConfig::default(),
            )
            .with_compilation_config(rch_common::CompilationConfig {
                build_timeout_sec: seconds,
                external_timeout_enabled: true,
                ..Default::default()
            });
            let command = format!(
                "MESSAGE='space $dollar; literal' sh -c {}",
                escape(Cow::Borrowed(script)),
            );
            let output = run_external_timeout_fixture(
                &directory,
                &pipeline.wrap_with_external_timeout(&command),
            )
            .await;
            assert_eq!(output.status.code(), Some(status), "{name}: {output:?}");
            let stderr = String::from_utf8(output.stderr.clone()).unwrap();
            assert_eq!(
                stderr.lines().any(|line| pipeline.is_deadline_marker(line)),
                deadline,
                "{name}: {stderr}"
            );
            match name {
                "streams" => {
                    assert_eq!(output.stdout, b"space $dollar; literal");
                    assert_eq!(output.stderr, b"stderr-without-newline");
                }
                "exit137" => {
                    assert_eq!(output.stdout, b"stdout");
                    assert_eq!(output.stderr, b"stderr");
                }
                "spoof" => assert_eq!(
                    output.stderr,
                    b"timeout: sending signal KILL to command 'sh'\n"
                ),
                _ => {}
            }
        }
    }

    #[cfg(target_os = "linux")]
    #[tokio::test]
    async fn test_external_timeout_real_build_id_watchdog_and_pipe_lifetime() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir().unwrap().keep();
        for (name, script, status, deadline, seconds) in [
            (
                "streams",
                "printf 'stdout'; printf 'stderr-without-newline' >&2",
                0,
                false,
                15,
            ),
            ("exit137", "exit 137", 137, false, 15),
            ("selfkill", "kill -KILL $$", 137, false, 15),
            (
                "deadline",
                "sleep 4 & printf '%s\\n' \"$!\" > descendant.pid; wait",
                137,
                true,
                1,
            ),
        ] {
            let directory = retained.join(name);
            std::fs::create_dir(&directory).unwrap();
            let pipeline = TransferPipeline::new(
                directory.clone(),
                "test".into(),
                "hash".into(),
                TransferConfig::default(),
            )
            .with_remote_path_override(directory.to_str().unwrap())
            .with_env_allowlist(Vec::new())
            .with_build_id(Some(1))
            .with_compilation_config(rch_common::CompilationConfig {
                build_timeout_sec: seconds,
                external_timeout_enabled: true,
                ..Default::default()
            });
            // Each case owns a fresh root/run identity: production's initial
            // stale-PGID unlink encounters no existing file; retain all outputs.
            let pgid_path = PathBuf::from(pipeline.remote_pgid_file_path().unwrap());
            assert!(!pgid_path.exists());
            let command = format!("sh -c {}", escape(Cow::Borrowed(script)));
            let output = run_external_timeout_fixture(
                &directory,
                &pipeline.build_remote_command(&command, None),
            )
            .await;
            assert_eq!(output.status.code(), Some(status), "{name}: {output:?}");
            let stderr = String::from_utf8(output.stderr.clone()).unwrap();
            assert_eq!(
                stderr.lines().any(|line| pipeline.is_deadline_marker(line)),
                deadline,
                "{name}: {stderr}"
            );
            assert!(recorded_remote_pgid(&std::fs::read_to_string(pgid_path).unwrap(), 1) > 1);
            if name == "streams" {
                assert_eq!(output.stdout, b"stdout");
                assert_eq!(output.stderr, b"stderr-without-newline");
            }
            if deadline {
                let descendant = std::fs::read_to_string(directory.join("descendant.pid")).unwrap();
                let stat_path = PathBuf::from(format!("/proc/{}/stat", descendant.trim()));
                tokio::time::timeout(std::time::Duration::from_secs(3), async {
                    loop {
                        match std::fs::read_to_string(&stat_path) {
                            Err(error) if error.kind() == std::io::ErrorKind::NotFound => break,
                            Ok(stat) if stat.split_whitespace().nth(2) == Some("Z") => break,
                            Ok(_) => tokio::time::sleep(std::time::Duration::from_millis(20)).await,
                            Err(error) => panic!("read owned descendant state: {error}"), // ubs:ignore — test fails on unexpected fixture I/O
                        }
                    }
                })
                .await
                .expect("group deadline must leave no running recorded descendant");
            }
        }
    }

    #[test]
    fn test_build_sync_command_includes_keepalive_and_controlpersist_when_set() {
        let _guard = test_guard!();
        let custom_options = SshOptions {
            server_alive_interval: Some(std::time::Duration::from_secs(30)),
            control_persist_idle: Some(std::time::Duration::from_secs(60)),
            control_master: true,
            ..Default::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_ssh_options(custom_options);

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        let e_index = args.iter().position(|arg| arg == "-e").expect("-e arg");
        let ssh_arg = args.get(e_index + 1).expect("ssh -e value");

        assert!(ssh_arg.contains("ServerAliveInterval=30"));
        assert!(ssh_arg.contains("ControlMaster=auto"));
        assert!(ssh_arg.contains("ControlPath="));
        assert!(ssh_arg.contains("rch-rsync-%C"));
        assert!(ssh_arg.contains("ControlPersist=60s"));
    }

    fn estimate_test_worker() -> WorkerConfig {
        WorkerConfig {
            id: WorkerId::new("estimate-worker"),
            host: "worker.example".to_string(),
            user: "ubuntu".to_string(),
            identity_file: "~/.ssh/id_ed25519".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        }
    }

    #[test]
    fn test_estimate_command_uses_configured_ssh_keepalive() {
        // Issue #74: the estimate's ssh must carry ServerAliveInterval like the
        // upload's, and keep its connect bound.
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "estimate-project".to_string(),
            "abc123".to_string(),
            TransferConfig {
                adaptive_compression: true,
                ..TransferConfig::default()
            },
        )
        .with_rsync(pinned_rsync(RsyncFlavor::Rsync {
            major: 3,
            minor: 2,
            patch: 7,
        }))
        .with_ssh_options(SshOptions {
            server_alive_interval: Some(Duration::from_secs(5)),
            control_master: false,
            ..SshOptions::default()
        });

        let cmd = pipeline.build_estimate_command(&estimate_test_worker());
        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();
        assert!(args.iter().any(|arg| arg == "--dry-run"));
        assert!(args.iter().any(|arg| arg == "--stats"));
        let e_index = args.iter().position(|arg| arg == "-e").expect("-e arg");
        let ssh_arg = &args[e_index + 1];
        assert!(ssh_arg.contains("ServerAliveInterval=5"), "{ssh_arg}");
        assert!(ssh_arg.contains("ConnectTimeout=5"), "{ssh_arg}");
        assert!(ssh_arg.contains("BatchMode=yes"), "{ssh_arg}");
    }

    #[test]
    fn test_transfer_estimate_timeout_follows_sync_bounds() {
        // Issue #74: the estimate is bounded by the silence window, never
        // outlives an explicit sync_timeout_ms, and stays bounded when silence
        // detection is disabled.
        let pipeline_with = |config: TransferConfig| {
            TransferPipeline::new(
                PathBuf::from("/home/user/project"),
                "estimate-project".to_string(),
                "abc123".to_string(),
                config,
            )
        };
        let default_budget = pipeline_with(TransferConfig::default()).transfer_estimate_timeout();
        assert_eq!(
            default_budget,
            Duration::from_secs(TransferConfig::default().source_sync_silence_timeout_secs)
        );
        let silence = pipeline_with(TransferConfig {
            source_sync_silence_timeout_secs: 10,
            sync_timeout_ms: Some(60_000),
            ..TransferConfig::default()
        });
        assert_eq!(silence.transfer_estimate_timeout(), Duration::from_secs(10));
        let tight_cap = pipeline_with(TransferConfig {
            source_sync_silence_timeout_secs: 120,
            sync_timeout_ms: Some(5_000),
            ..TransferConfig::default()
        });
        assert_eq!(
            tight_cap.transfer_estimate_timeout(),
            Duration::from_secs(5)
        );
        let explicit_only = pipeline_with(TransferConfig {
            source_sync_silence_timeout_secs: 0,
            sync_timeout_ms: Some(45_000),
            ..TransferConfig::default()
        });
        assert_eq!(
            explicit_only.transfer_estimate_timeout(),
            Duration::from_secs(45)
        );
        let unbounded_config = pipeline_with(TransferConfig {
            source_sync_silence_timeout_secs: 0,
            ..TransferConfig::default()
        });
        assert_eq!(
            unbounded_config.transfer_estimate_timeout(),
            Duration::from_secs(120),
            "the estimate stays bounded even with silence detection disabled"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_estimate_transfer_size_fails_open_when_rsync_stalls() {
        // Issue #74: an rsync that connects and then goes silent must not hang
        // the estimate; it fails open (None) within the silence window and
        // should_skip_transfer proceeds to the bounded upload.
        use std::os::unix::fs::PermissionsExt;
        let _guard = test_guard!();
        let dir = tempfile::tempdir().unwrap();
        let fake = dir.path().join("stalled-rsync");
        std::fs::write(&fake, "#!/bin/sh\nexec sleep 60\n").unwrap();
        std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap();

        let mut pipeline = TransferPipeline::new(
            dir.path().to_path_buf(),
            "estimate-project".to_string(),
            "abc123".to_string(),
            TransferConfig {
                adaptive_compression: true,
                source_sync_silence_timeout_secs: 1,
                ..TransferConfig::default()
            },
        )
        .with_rsync(ResolvedRsync {
            path: fake,
            flavor: RsyncFlavor::Rsync {
                major: 3,
                minor: 2,
                patch: 7,
            },
            version_line: String::new(),
            source: RsyncSource::Config,
            shadowed: None,
        });
        let worker = estimate_test_worker();

        let started = std::time::Instant::now();
        assert!(pipeline.estimate_transfer_size(&worker).await.is_none());
        let elapsed = started.elapsed();
        assert!(
            elapsed < Duration::from_secs(20),
            "stalled estimate must be abandoned near the 1 s window, took {elapsed:?}"
        );
        assert_eq!(pipeline.should_skip_transfer(&worker).await, None);
        assert_eq!(pipeline.estimated_transfer_bytes, None);
    }

    #[test]
    fn test_build_sync_command_adaptive_compression_uses_estimate() {
        let _guard = test_guard!();
        let transfer_config = TransferConfig {
            adaptive_compression: true,
            compression_level: 3,
            min_compression_level: 1,
            max_compression_level: 9,
            ..TransferConfig::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            transfer_config,
        )
        .with_estimated_transfer_bytes(Some(500_000_000));

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        assert!(args.iter().any(|arg| arg == "--compress-choice=zstd"));
        assert!(args.iter().any(|arg| arg == "--compress-level=7"));
    }

    #[cfg(unix)]
    #[test]
    fn source_activity_rsync_wrapper_preserves_server_argv_and_fences_setup() {
        let directory = tempfile::tempdir().unwrap();
        let destination = directory.path().join("source with spaces");
        let mut pipeline = TransferPipeline::new(
            directory.path().to_owned(),
            "activity".into(),
            "abcdef".into(),
            TransferConfig::default(),
        );
        // The production registry tests exercise the real grant. Here the
        // controlled prefix isolates shell argument forwarding and ordering.
        pipeline.source_authority_prefix = Some("env RCH_ACTIVITY_TEST=owned".into());
        let setup = format!(
            "test \"$RCH_ACTIVITY_TEST\" = owned && mkdir -p {} && printf '%s\\0'",
            escape(destination.to_string_lossy())
        );
        let server_args = [
            "--server",
            "--sender",
            "-logDtpre.iLsfxCIvu",
            ".",
            "a '$literal; b",
        ];
        let remote = format!(
            "{} {}",
            pipeline.source_rsync_path(setup),
            server_args
                .iter()
                .map(|arg| escape(Cow::from(*arg)).into_owned())
                .collect::<Vec<_>>()
                .join(" ")
        );
        let output = std::process::Command::new("sh")
            .arg("-c")
            .arg(remote)
            .output()
            .unwrap();
        assert!(output.status.success(), "{:?}", output);
        assert!(destination.is_dir());
        let expected: Vec<u8> = server_args
            .iter()
            .flat_map(|arg| arg.bytes().chain(std::iter::once(0)))
            .collect();
        assert_eq!(output.stdout, expected);

        pipeline.source_authority_prefix = Some("false".into());
        let denied = directory.path().join("must-not-exist");
        let remote = pipeline.source_rsync_path(format!(
            "mkdir -p {} && printf '%s'",
            escape(denied.to_string_lossy())
        ));
        let output = std::process::Command::new("sh")
            .arg("-c")
            .arg(remote)
            .output()
            .unwrap();
        assert!(!output.status.success());
        assert!(
            !denied.exists(),
            "a rejected lease must precede destination setup"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn owned_source_failure_does_not_retry_inside_the_same_grant() {
        use std::os::unix::fs::PermissionsExt;
        let _guard = test_guard!();
        let directory = tempfile::tempdir().unwrap();
        let counter = directory.path().join("attempts");
        let fake = directory.path().join("failed-rsync");
        std::fs::write(&fake, format!(
            "#!/bin/sh\nprintf 'attempt\\n' >> {}\nprintf 'ssh: connection reset by peer\\n' >&2\nexit 255\n",
            escape(counter.to_string_lossy())
        )).unwrap();
        std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap();
        let mut pipeline = TransferPipeline::new(
            directory.path().to_owned(),
            "activity".into(),
            "abcdef".into(),
            TransferConfig {
                retry: RetryConfig {
                    max_attempts: 3,
                    base_delay_ms: 1,
                    max_delay_ms: 1,
                    jitter_factor: 0.0,
                    total_timeout_ms: 30_000,
                },
                ..TransferConfig::default()
            },
        )
        .with_rsync(ResolvedRsync {
            path: fake,
            flavor: RsyncFlavor::Rsync {
                major: 3,
                minor: 2,
                patch: 7,
            },
            version_line: String::new(),
            source: RsyncSource::Config,
            shadowed: None,
        });
        let worker = estimate_test_worker();
        pipeline.source_authority_prefix = Some("env RCH_ACTIVITY_TEST=owned".into());
        assert!(pipeline.sync_to_remote(&worker).await.is_err());
        assert_eq!(
            std::fs::read_to_string(&counter).unwrap().lines().count(),
            1
        );
        assert!(
            pipeline
                .sync_to_remote_streaming(&worker, |_| {})
                .await
                .is_err()
        );
        assert_eq!(
            std::fs::read_to_string(&counter).unwrap().lines().count(),
            2
        );
        // Control: the ordinary transport still exercises the configured
        // three attempts. The owned failure must return for token cancellation.
        pipeline.source_authority_prefix = None;
        assert!(pipeline.sync_to_remote(&worker).await.is_err());
        assert_eq!(
            std::fs::read_to_string(&counter).unwrap().lines().count(),
            5
        );
    }

    #[cfg(unix)]
    #[test]
    fn source_activity_is_carried_by_all_rsync_transports() {
        let directory = tempfile::tempdir().unwrap();
        let mut pipeline = TransferPipeline::new(
            directory.path().to_owned(),
            "activity".into(),
            "abcdef".into(),
            TransferConfig::default(),
        );
        pipeline.source_authority_prefix = Some("env RCH_ACTIVITY_TEST=owned".into());
        let worker = estimate_test_worker();
        let commands = [
            pipeline.build_sync_command(&worker, "user@worker:/src", "/src", &[]),
            pipeline.build_sync_streaming_command(&worker, "user@worker:/src", "/src", &[]),
            pipeline.build_estimate_command(&worker),
            pipeline.build_retrieve_command(&worker, "/src", &[]),
            pipeline.build_retrieve_streaming_command(&worker, "/src", &[]),
            pipeline.build_result_dir_retrieve_command(&worker, "/src", Path::new("results")),
        ];
        for command in commands {
            let args: Vec<_> = command.as_std().get_args().collect();
            let paths: Vec<_> = args
                .windows(2)
                .filter(|pair| pair[0] == "--rsync-path")
                .collect();
            assert_eq!(paths.len(), 1, "exactly one activity wrapper: {args:?}");
            let path = paths[0][1].to_string_lossy();
            assert!(
                path.starts_with("env RCH_ACTIVITY_TEST=owned sh -c "),
                "{path}"
            );
            assert!(path.ends_with(" rch-source-rsync"), "{path}");
        }
    }

    #[cfg(unix)]
    #[test]
    fn source_grant_does_not_authorize_pruning_sibling_target_pools() {
        let directory = tempfile::tempdir().unwrap();
        let mut pipeline = TransferPipeline::new(
            directory.path().to_owned(),
            "activity".into(),
            "abcdef".into(),
            TransferConfig::default(),
        )
        .with_remote_path_override("/worker/sources/current")
        .with_remote_cargo_target_dir_override("/worker/pools/.rch-target-current-pool-abcdef");
        pipeline.source_authority_prefix = Some("env RCH_ACTIVITY_TEST=owned".into());
        assert_eq!(pipeline.escaped_pooled_target_override_parent(), None);
        let worker = estimate_test_worker();
        for command in [
            pipeline.build_sync_command(
                &worker,
                "user@worker:/worker/sources/current",
                "/worker/sources/current",
                &[],
            ),
            pipeline.build_sync_streaming_command(
                &worker,
                "user@worker:/worker/sources/current",
                "/worker/sources/current",
                &[],
            ),
        ] {
            let args: Vec<_> = command.as_std().get_args().collect();
            let wrapper = args
                .windows(2)
                .find(|pair| pair[0] == "--rsync-path")
                .unwrap()[1]
                .to_string_lossy();
            assert!(!wrapper.contains("find /worker/pools"), "{wrapper}");
            assert!(
                wrapper.contains("find /worker/sources/current"),
                "{wrapper}"
            );
        }
    }

    #[test]
    fn test_rsync_path_prefix_prunes_stale_worker_runtime_state() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let root = pipeline.remote_path();
        let cmd = pipeline.build_sync_command(
            &worker,
            &format!("mockuser@mock://worker:{root}"),
            &root,
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();
        let idx = args
            .iter()
            .position(|arg| arg == "--rsync-path")
            .expect("--rsync-path");
        let path_val = args.get(idx + 1).expect("rsync-path value");

        // bd-wfumv: every source upload reaps stale worker-side runtime state
        // ahead of the transfer, with durable caches on a much longer floor.
        assert!(path_val.starts_with("find "));
        assert!(path_val.contains(&format!(
            "find {root}/.rch-tmp -mindepth 1 -maxdepth 1 ! -name 'rch-cargo-cache-*' -mmin +{WORKER_TMP_PRUNE_MAX_AGE_MINS}"
        )));
        assert!(path_val.contains(&format!(
            "-name 'rch-cargo-cache-*' -mmin +{WORKER_DURABLE_CACHE_PRUNE_MAX_AGE_MINS}"
        )));
        // Issue #53: pooled stores use the CONFIGURED reaper retention (168 h
        // by default), never the 3-day durable-cache floor.
        let default_pooled_mins =
            u64::from(rch_common::remediation_config::DEFAULT_POOLED_REAPER_POOLED_IDLE_HOURS) * 60;
        assert!(path_val.contains(&format!(
            "-type d -name '.rch-target-*-pool-*' -mmin +{default_pooled_mins}"
        )));
        assert!(!path_val.contains(&format!(
            "-type d -name '.rch-target-*-pool-*' -mmin +{WORKER_DURABLE_CACHE_PRUNE_MAX_AGE_MINS}"
        )));

        // The prune must run before the mkdir/rsync pair it wraps.
        let mkdir_idx = path_val
            .find(&format!("mkdir -p {root} && rsync"))
            .expect("mkdir && rsync suffix");
        assert!(path_val.find("find ").expect("find prefix") < mkdir_idx);
    }

    #[test]
    fn pooled_target_prune_window_follows_configured_retention() {
        // Issue #53: one authoritative retention. The janitor window is the
        // configured pooled idle hours (floored at the reaper's 24 h defence),
        // and hours == 0 removes the pooled sweep entirely.
        let prefix = worker_cache_prune_rsync_path_prefix("/r/p", 240, None);
        assert!(prefix.contains("-name '.rch-target-*-pool-*' -mmin +14400 "));

        let floored = worker_cache_prune_rsync_path_prefix("/r/p", 1, None);
        assert!(floored.contains(&format!(
            "-name '.rch-target-*-pool-*' -mmin +{} ",
            rch_common::stale_target_reap::MIN_POOLED_IDLE_MINUTES
        )));

        let disabled = worker_cache_prune_rsync_path_prefix("/r/p", 0, None);
        assert!(!disabled.contains(".rch-target-*-pool-*"));
        // The scratch and durable-cache sweeps are unaffected by the pooled knob.
        assert!(disabled.contains(&format!(
            "! -name 'rch-cargo-cache-*' -mmin +{WORKER_TMP_PRUNE_MAX_AGE_MINS}"
        )));
        assert!(disabled.contains(&format!(
            "-name 'rch-cargo-cache-*' -mmin +{WORKER_DURABLE_CACHE_PRUNE_MAX_AGE_MINS}"
        )));
        assert_eq!(pooled_target_prune_max_age_mins(0), None);
        assert_eq!(pooled_target_prune_max_age_mins(168), Some(168 * 60));
    }

    #[test]
    fn test_rsync_commands_disable_owner_and_group_preservation() {
        let _guard = test_guard!();
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let sync = pipeline.build_sync_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );
        assert_portable_rsync_archive_args(&command_args(&sync));

        let sync_streaming = pipeline.build_sync_streaming_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );
        assert_portable_rsync_archive_args(&command_args(&sync_streaming));

        let retrieve =
            pipeline.build_retrieve_command(&worker, "/data/tmp/rch/test-project/abc123", &[]);
        assert_portable_rsync_archive_args(&command_args(&retrieve));

        let retrieve_streaming = pipeline.build_retrieve_streaming_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &[],
        );
        assert_portable_rsync_archive_args(&command_args(&retrieve_streaming));
    }

    #[test]
    fn test_build_retrieve_command_applies_rchignore_excludes_before_directory_include() {
        let _guard = test_guard!();
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        std::fs::write(
            temp_dir.path().join(".rchignore"),
            ".beads/\n.beads/recovery_*/\ncustom-cache/\ntarget/\n*.rlib\n",
        )
        .expect("write .rchignore");

        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_retrieve_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["target/debug/**".to_string()],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        let beads_exclude =
            arg_pair_position(&args, "--exclude", ".beads/").expect("missing .beads exclude");
        let recovery_exclude = arg_pair_position(&args, "--exclude", ".beads/recovery_*/")
            .expect("missing .beads recovery exclude");
        let custom_exclude = arg_pair_position(&args, "--exclude", "custom-cache/")
            .expect("missing custom cache exclude");
        let target_exclude = arg_pair_position(&args, "--exclude", "target/");
        let rlib_exclude = args
            .windows(2)
            .position(|window| window == ["--exclude", "*.rlib"]);
        let include_dirs = args
            .windows(2)
            .position(|window| window == ["--include", "*/"])
            .expect("missing directory include");
        // RCH bug d7xc3: artifact patterns are now anchored at the rsync
        // source root via `anchor_retrieval_pattern`, so `target/debug/**`
        // is emitted as `/target/debug/**` to prevent it from floating
        // and matching e.g. `<root>/anything/target/debug/...`.
        let target_include = args
            .windows(2)
            .position(|window| window == ["--include", "/target/debug/**"])
            .expect("missing artifact include");

        assert!(beads_exclude < include_dirs);
        assert!(recovery_exclude < include_dirs);
        assert!(custom_exclude < include_dirs);
        assert_eq!(target_exclude, None);
        assert_eq!(rlib_exclude, None);
        assert!(include_dirs < target_include);
        assert!(
            !args
                .windows(2)
                .any(|window| window == ["--exclude", "target/"]),
            "retrieve filters must not inherit upload-only target/ exclusion"
        );
        assert!(args.windows(2).any(|window| window == ["--exclude", "*"]));
    }

    #[test]
    fn test_build_retrieve_streaming_command_does_not_exclude_artifact_root_from_rchignore() {
        let _guard = test_guard!();
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        std::fs::write(temp_dir.path().join(".rchignore"), "build/\n.cache/\n")
            .expect("write .rchignore");

        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_retrieve_streaming_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["build/**".to_string()],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        assert!(
            !args
                .windows(2)
                .any(|window| window == ["--exclude", "build/"]),
            "streaming retrieval must not exclude the requested artifact root"
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["--exclude", ".cache/"]),
            "unrelated directory-only .rchignore entries should still prune traversal"
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["--include", "/build/**"]),
            "streaming retrieval should include the anchored form of requested artifact patterns (RCH bug d7xc3)"
        );
    }

    // =========================================================================
    // rsync flavour argv (issue #66)
    // =========================================================================

    fn pinned_rsync(flavor: RsyncFlavor) -> ResolvedRsync {
        ResolvedRsync {
            path: PathBuf::from("/pinned/rsync"),
            flavor,
            version_line: String::new(),
            source: RsyncSource::Config,
            shadowed: None,
        }
    }

    fn flavour_test_pipeline(flavor: RsyncFlavor, compression_level: u32) -> TransferPipeline {
        TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "flavour-project".to_string(),
            "abc123".to_string(),
            TransferConfig {
                compression_level,
                ..TransferConfig::default()
            },
        )
        .with_rsync(pinned_rsync(flavor))
    }

    fn flavour_test_worker() -> WorkerConfig {
        WorkerConfig {
            id: WorkerId::new("flavour-worker"),
            host: "worker.example".to_string(),
            user: "ubuntu".to_string(),
            identity_file: "~/.ssh/id_ed25519".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        }
    }

    /// Every rsync argv the pipeline builds for a flavour, keyed by builder.
    fn flavour_argvs(
        flavor: RsyncFlavor,
        compression_level: u32,
    ) -> Vec<(&'static str, Vec<String>)> {
        let pipeline = flavour_test_pipeline(flavor, compression_level);
        let worker = flavour_test_worker();
        let remote = "/data/tmp/rch/flavour-project/abc123";
        let destination = format!("ubuntu@worker.example:{remote}");
        let patterns = vec!["target/release/**".to_string()];
        let sync = pipeline.build_sync_command(&worker, &destination, remote, &[]);
        let sync_streaming =
            pipeline.build_sync_streaming_command(&worker, &destination, remote, &[]);
        let retrieve = pipeline.build_retrieve_command(&worker, remote, &patterns);
        let retrieve_streaming =
            pipeline.build_retrieve_streaming_command(&worker, remote, &patterns);
        let result_dir =
            pipeline.build_result_dir_retrieve_command(&worker, remote, Path::new("out"));
        for cmd in [
            &sync,
            &sync_streaming,
            &retrieve,
            &retrieve_streaming,
            &result_dir,
        ] {
            assert_eq!(
                cmd.as_std().get_program(),
                "/pinned/rsync",
                "builders must exec the resolved binary, not a bare `rsync`"
            );
            assert_eq!(
                cmd.as_std().get_envs().find(|(key, _)| *key == "LC_ALL"),
                Some((
                    std::ffi::OsStr::new("LC_ALL"),
                    Some(std::ffi::OsStr::new("C"))
                )),
                "rsync output parsing relies on the C locale"
            );
        }
        vec![
            ("sync", command_args(&sync)),
            ("sync_streaming", command_args(&sync_streaming)),
            ("retrieve", command_args(&retrieve)),
            ("retrieve_streaming", command_args(&retrieve_streaming)),
            ("result_dir", command_args(&result_dir)),
        ]
    }

    fn assert_has(args: &[String], flag: &str, builder: &str) {
        assert!(
            args.iter().any(|arg| arg == flag),
            "{builder}: expected {flag:?} in {args:?}"
        );
    }

    fn assert_lacks(args: &[String], flag: &str, builder: &str) {
        assert!(
            !args.iter().any(|arg| arg == flag),
            "{builder}: {flag:?} must not be passed, got {args:?}"
        );
    }

    #[test]
    fn test_rsync_argv_modern_flavour_uses_info_flags_and_zstd() {
        let _guard = test_guard!();
        let modern = RsyncFlavor::Rsync {
            major: 3,
            minor: 4,
            patch: 1,
        };
        for (builder, args) in flavour_argvs(modern, 7) {
            assert_has(&args, "--compress-choice=zstd", builder);
            assert_has(&args, "--compress-level=7", builder);
            assert_lacks(&args, "--progress", builder);
            assert_lacks(&args, "-vv", builder);
            match builder {
                "sync" | "result_dir" => {
                    assert_has(&args, "--stats", builder);
                    assert_lacks(&args, "--info=progress2", builder);
                }
                "sync_streaming" => {
                    assert_has(&args, "--info=progress2", builder);
                    assert_has(&args, "--info=stats2", builder);
                    assert_lacks(&args, "--stats", builder);
                }
                "retrieve" => {
                    assert_has(&args, "--stats", builder);
                    assert_has(&args, "--info=name2", builder);
                    assert_has(&args, "--out-format=%i %n", builder);
                }
                "retrieve_streaming" => {
                    assert_has(&args, "--info=progress2", builder);
                    assert_has(&args, "--info=stats2", builder);
                    assert_has(&args, "--info=name2", builder);
                    assert_has(&args, "--out-format=%i %n", builder);
                }
                other => panic!("unexpected builder {other}"), // ubs:ignore — exhaustive fixture-builder assertion
            }
        }
    }

    #[test]
    fn test_rsync_argv_openrsync_flavour_uses_compatible_flags() {
        let _guard = test_guard!();
        let openrsync = RsyncFlavor::OpenRsync { protocol: Some(29) };
        for (builder, args) in flavour_argvs(openrsync, 7) {
            // openrsync rejects every `--info=` value and `--compress-choice`.
            assert!(
                !args.iter().any(|arg| arg.starts_with("--info=")),
                "{builder}: openrsync rejects --info=*, got {args:?}"
            );
            assert_lacks(&args, "--compress-choice=zstd", builder);
            assert_has(&args, "--compress-level=7", builder);
            // `-z` (inside `-az`) still compresses, with zlib.
            assert_has(&args, "-az", builder);
            assert_has(&args, "--stats", builder);
            match builder {
                "sync" | "result_dir" => {
                    assert_lacks(&args, "--progress", builder);
                    assert_lacks(&args, "-vv", builder);
                }
                "sync_streaming" => {
                    assert_has(&args, "--progress", builder);
                    assert_lacks(&args, "-vv", builder);
                }
                "retrieve" => {
                    assert_has(&args, "-vv", builder);
                    assert_has(&args, "--out-format=%i %n", builder);
                    assert_lacks(&args, "--progress", builder);
                }
                "retrieve_streaming" => {
                    assert_has(&args, "--progress", builder);
                    assert_has(&args, "-vv", builder);
                    assert_has(&args, "--out-format=%i %n", builder);
                }
                other => panic!("unexpected builder {other}"), // ubs:ignore — exhaustive fixture-builder assertion
            }
        }
    }

    #[test]
    fn test_rsync_argv_legacy_flavour_clamps_compression_to_zlib_range() {
        let _guard = test_guard!();
        let apple_2_6_9 = RsyncFlavor::Rsync {
            major: 2,
            minor: 6,
            patch: 9,
        };
        for (builder, args) in flavour_argvs(apple_2_6_9, 19) {
            assert_lacks(&args, "--compress-choice=zstd", builder);
            assert_lacks(&args, "--compress-level=19", builder);
            assert_has(&args, "--compress-level=9", builder);
        }
        // A modern binary keeps the zstd level untouched.
        let modern = RsyncFlavor::Rsync {
            major: 3,
            minor: 2,
            patch: 7,
        };
        for (builder, args) in flavour_argvs(modern, 19) {
            assert_has(&args, "--compress-level=19", builder);
        }
    }

    #[test]
    fn test_rsync_argv_unknown_flavour_keeps_modern_argv() {
        let _guard = test_guard!();
        // Pre-#66 behaviour for a banner rch cannot classify: nothing changes.
        for (builder, args) in flavour_argvs(RsyncFlavor::Unknown, 3) {
            assert_has(&args, "--compress-choice=zstd", builder);
            if builder == "sync_streaming" || builder == "retrieve_streaming" {
                assert_has(&args, "--info=progress2", builder);
            }
        }
    }

    #[test]
    fn test_rsync_argv_zero_compression_level_emits_no_compression_flags() {
        let _guard = test_guard!();
        for flavor in [
            RsyncFlavor::Rsync {
                major: 3,
                minor: 4,
                patch: 1,
            },
            RsyncFlavor::OpenRsync { protocol: None },
        ] {
            for (builder, args) in flavour_argvs(flavor, 0) {
                assert!(
                    !args.iter().any(|arg| arg.starts_with("--compress-")),
                    "{builder}: level 0 must add no compression flags, got {args:?}"
                );
            }
        }
    }

    #[test]
    fn test_openrsync_output_feeds_existing_parsers() {
        // Captured from a real openrsync `--stats -vv --out-format='%i %n'`
        // retrieval over ssh (issue #66). The up-to-date `.f` line and the
        // pre-3.1 `Number of files transferred:` spelling must keep the
        // byte/file parsers and the zero-output manifest working; the
        // `(reg: N, ...)` breakdown is absent, so the completeness cross-check
        // yields `None` and the detector fails open.
        let output = concat!(
            "opening connection using: ssh worker rsync --server --sender -vv . /r/\n",
            "Delta transmission enabled for this transfer\n",
            "[sender] showing directory target because of pattern */\n",
            "[sender] showing file target/debug/mybin because of pattern /target/debug/**\n",
            "Transfer starting: 4 files\n",
            ".d        ./\n",
            ".d        target/\n",
            "cd+++++++ target/debug/\n",
            ">f+++++++ target/debug/mybin\n",
            ".f        target/debug/other\n",
            "total: matches=0  hash_hits=0  false_alarms=0 data=0\n",
            "\n",
            "sent 77 bytes  received 565 bytes  6420000 bytes/sec\n",
            "total size is 8  speedup is 0.01\n",
            "Number of files: 4\n",
            "Number of files transferred: 1\n",
            "Total file size: 8 B\n",
            "Total transferred file size: 8 B\n",
            "Total sent: 81 B\n",
            "Total received: 148 B\n",
        );
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Upload),
            77
        );
        assert_eq!(
            parse_rsync_bytes(output, RsyncTransferDirection::Download),
            565
        );
        assert_eq!(parse_rsync_files(output), 1);
        assert_eq!(
            parse_rsync_itemized_regular_files(output),
            vec![
                "target/debug/mybin".to_string(),
                "target/debug/other".to_string()
            ]
        );
        assert_eq!(parse_rsync_matched_regular_files(output), None);
    }

    #[test]
    fn test_build_retrieve_streaming_command_uses_safe_links() {
        let _guard = test_guard!();
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_retrieve_streaming_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["target/release/**".to_string()],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        assert!(
            args.iter().any(|arg| arg == "--safe-links"),
            "streaming artifact retrieval must keep symlink traversal protection"
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["--include", "/target/release/**"]),
            "streaming retrieval should still include requested artifact patterns (anchored per RCH bug d7xc3)"
        );
        assert!(args.windows(2).any(|window| window == ["--exclude", "*"]));
    }

    #[test]
    fn test_sync_result_struct() {
        let _guard = test_guard!();
        let result = SyncResult {
            bytes_transferred: 1024,
            files_transferred: 10,
            duration_ms: 500,
        };

        assert_eq!(result.bytes_transferred, 1024);
        assert_eq!(result.files_transferred, 10);
        assert_eq!(result.duration_ms, 500);

        // Test Clone
        let cloned = result.clone();
        assert_eq!(cloned.bytes_transferred, result.bytes_transferred);
    }

    /// A job already running on a worker must compile there, never offload
    /// onward. Without the loop-break the worker's own cargo shim re-offloads:
    /// a pure worker refuses local fallback (exit 103), a dispatcher bounces
    /// the build to a third host. See REMOTE_LOOP_BREAK_ENV.
    #[test]
    fn test_remote_env_plan_injects_loop_break() {
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        );

        let prefix = pipeline.build_env_prefix();
        assert!(
            prefix.prefix.contains("RCH_CARGO_WRAPPER_BYPASS=1"),
            "remote command must carry the loop-break, got: {}",
            prefix.prefix
        );
        assert!(
            prefix
                .applied
                .iter()
                .any(|k| k == "RCH_CARGO_WRAPPER_BYPASS"),
            "loop-break should be reported as applied"
        );
    }

    /// A caller that exports the bypass OFF locally must not be able to re-arm
    /// the worker's shim and reintroduce the bounce.
    #[test]
    fn test_remote_loop_break_overrides_forwarded_value() {
        let mut overrides = HashMap::new();
        overrides.insert("RCH_CARGO_WRAPPER_BYPASS".to_string(), "0".to_string());

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_allowlist(vec!["RCH_CARGO_WRAPPER_BYPASS".to_string()])
        .with_env_overrides(overrides);

        let prefix = pipeline.build_env_prefix();
        assert!(
            prefix.prefix.contains("RCH_CARGO_WRAPPER_BYPASS=1"),
            "loop-break must win over a forwarded value, got: {}",
            prefix.prefix
        );
        assert!(
            !prefix.prefix.contains("RCH_CARGO_WRAPPER_BYPASS=0"),
            "the disabling value must not survive, got: {}",
            prefix.prefix
        );
        assert_eq!(
            prefix
                .applied
                .iter()
                .filter(|k| *k == "RCH_CARGO_WRAPPER_BYPASS")
                .count(),
            1,
            "loop-break must be assigned exactly once"
        );
    }

    #[test]
    #[serial(mock_global)]
    fn test_execute_remote_applies_env_allowlist() {
        let _guard = test_guard!();
        mock::clear_global_invocations();
        mock::set_mock_enabled_override(Some(true));

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let mut overrides = HashMap::new();
        overrides.insert("RUSTFLAGS".to_string(), "-C target-cpu=native".to_string());
        overrides.insert("QUOTED".to_string(), "a'b".to_string());
        overrides.insert("BADVAL".to_string(), "line1\nline2".to_string());

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_color_mode(ColorMode::Auto)
        .with_env_allowlist(vec![
            "RUSTFLAGS".to_string(),
            "QUOTED".to_string(),
            "BADVAL".to_string(),
            "MISSING".to_string(),
            "BAD=KEY".to_string(),
        ])
        .with_env_overrides(overrides);

        let rt = tokio::runtime::Runtime::new().expect("runtime");
        rt.block_on(async {
            pipeline
                .execute_remote(&worker, "cargo build", None)
                .await
                .expect("execute_remote");
        });

        let invocations = mock::global_ssh_invocations_snapshot();
        let command = invocations
            .iter()
            .find(|inv| inv.phase == Phase::Execute)
            .and_then(|inv| inv.command.clone())
            .expect("execute invocation");

        let env_prefix = pipeline.build_env_prefix();
        assert!(env_prefix.applied.contains(&"RUSTFLAGS".to_string()));
        assert!(env_prefix.applied.contains(&"QUOTED".to_string()));
        assert!(env_prefix.rejected.contains(&"BADVAL".to_string()));
        assert!(env_prefix.rejected.contains(&"BAD=KEY".to_string()));

        assert!(command.contains("RUSTFLAGS="));
        assert!(command.contains("target-cpu=native"));
        // shell_escape uses '\'' style (end string, escaped quote, start string)
        assert!(command.contains("QUOTED='a'\\''b'"));
        assert!(!command.contains("BADVAL="));
        assert!(!command.contains("BAD=KEY"));
        assert!(command.contains("cargo build"));

        mock::clear_mock_overrides();
        mock::clear_global_invocations();
    }

    #[test]
    fn test_build_remote_command_rewrites_cargo_target_dir_and_tmpdir() {
        let _guard = test_guard!();
        let mut overrides = HashMap::new();
        overrides.insert(
            "CARGO_TARGET_DIR".to_string(),
            "/data/tmp/pi_agent_rust/pearleagle".to_string(),
        );
        overrides.insert(
            "TMPDIR".to_string(),
            "/data/tmp/pi_agent_rust/pearleagle/tmp".to_string(),
        );

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_allowlist(vec!["CARGO_TARGET_DIR".to_string(), "TMPDIR".to_string()])
        .with_env_overrides(overrides);

        let worker_scoped_root = pipeline.remote_path();
        let command = pipeline.build_remote_command("cargo test --no-run", None);
        assert!(command.contains(&managed_assignment(
            "CARGO_TARGET_DIR",
            &format!("{worker_scoped_root}/.rch-target")
        )));
        assert!(command.contains(&managed_assignment(
            "TMPDIR",
            &format!("{worker_scoped_root}/.rch-tmp")
        )));
        assert!(command.contains("mkdir -p"));
        assert!(command.contains(&format!("{}/.rch-target", worker_scoped_root)));
        assert!(command.contains(&format!("{}/.rch-tmp", worker_scoped_root)));
        assert!(command.contains("touch "));
        assert!(
            !command.contains("/data/tmp/pi_agent_rust/pearleagle"),
            "host-local tmpfs path should not be forwarded to worker"
        );
        assert!(command.contains(&worker_scoped_root));
    }

    /// Expected form of a managed directory assignment in the remote command:
    /// resolved to the physical path on the worker (see `format_env_assignment`).
    fn managed_assignment(key: &str, path: &str) -> String {
        format!("{key}=\"$(cd '{path}' 2>/dev/null && pwd -P || printf %s '{path}')\"")
    }

    #[test]
    fn test_managed_env_injected_when_nothing_forwarded() {
        let _guard = test_guard!();
        // FIX 2: even with an EMPTY allowlist and no forwarded env, the managed
        // build-artifact vars must be injected so wrapper/unclassified builds
        // still land in the managed remote zone.
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_overrides(HashMap::new()); // nothing forwarded

        let root = pipeline.remote_path();
        let command = pipeline.build_remote_command("cargo build", None);

        assert!(
            command.contains(&managed_assignment(
                "CARGO_TARGET_DIR",
                &format!("{root}/.rch-target")
            )),
            "CARGO_TARGET_DIR must be injected: {command}"
        );
        assert!(
            command.contains(&managed_assignment("TMPDIR", &format!("{root}/.rch-tmp"))),
            "TMPDIR must be injected: {command}"
        );
        assert!(
            command.contains(&managed_assignment(
                "GOCACHE",
                &format!("{root}/.rch-go/cache")
            )),
            "GOCACHE must be injected: {command}"
        );
        assert!(
            command.contains(&managed_assignment(
                "GOMODCACHE",
                &format!("{root}/.rch-go/mod")
            )),
            "GOMODCACHE must be injected: {command}"
        );
        assert!(
            command.contains(&managed_assignment(
                "GOPATH",
                &format!("{root}/.rch-go/path")
            )),
            "GOPATH must be injected: {command}"
        );
        // Each managed dir must be mkdir'd before the build.
        assert!(command.contains("mkdir -p"));
        assert!(command.contains(&format!("{}/.rch-go/cache", root)));
    }

    #[test]
    fn test_managed_temp_dir_is_chmodded_owner_only_with_sticky_bit() {
        let _guard = test_guard!();
        // A permissive worker umask (e.g. `002`) makes `mkdir -p` produce a
        // group-writable `.rch-tmp` without the sticky bit. Consumers that
        // validate ancestor permissions before atomic writes refuse such
        // parents, so rch must tighten every managed TEMP-family directory
        // right after creating it — and only those directories.
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_overrides(HashMap::new()); // nothing forwarded; forced keys inject TMPDIR

        let root = pipeline.remote_path();
        let command = pipeline.build_remote_command("cargo build", None);

        assert!(
            command.contains(&format!("chmod 1700 {}/.rch-tmp", root)),
            "managed .rch-tmp must be tightened to 1700: {command}"
        );
        assert!(
            !command.contains(&format!("chmod 1700 {}/.rch-target", root)),
            "target dir is not a TEMP-family dir and must not be chmodded: {command}"
        );
        assert!(
            !command.contains(&format!("chmod 1700 {}/.rch-go", root)),
            "go cache dirs are not TEMP-family dirs and must not be chmodded: {command}"
        );
    }

    #[test]
    fn test_managed_env_overrides_absolute_forwarded_target_dir() {
        let _guard = test_guard!();
        // FIX 2: a forwarded ABSOLUTE CARGO_TARGET_DIR (e.g. from a
        // .cargo/config.toml `target-dir` like /root/cass-ft-target) must be
        // rewritten to the managed worker-scoped path, never forwarded verbatim.
        let mut overrides = HashMap::new();
        overrides.insert(
            "CARGO_TARGET_DIR".to_string(),
            "/root/cass-ft-target".to_string(),
        );
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        // Note: NOT in the allowlist — the forced-injection pass must still
        // rewrite it because the value came through env_overrides. (And even a
        // fully-unforwarded build gets the managed value.)
        .with_env_overrides(overrides);

        let root = pipeline.remote_path();
        let command = pipeline.build_remote_command("cargo build", None);

        assert!(
            command.contains(&managed_assignment(
                "CARGO_TARGET_DIR",
                &format!("{root}/.rch-target")
            )),
            "absolute target-dir must be rewritten to managed path: {command}"
        );
        assert!(
            !command.contains("/root/cass-ft-target"),
            "host-absolute target-dir must NOT be forwarded: {command}"
        );
    }

    #[cfg(unix)]
    fn cargo_home_boundary_pipeline(root: &Path) -> TransferPipeline {
        TransferPipeline::new(
            root.to_path_buf(),
            "cache-boundary".to_owned(),
            "abcd1234".to_owned(),
            TransferConfig::default(),
        )
        .with_remote_path_override(root.to_str().unwrap())
        .with_compilation_config(rch_common::CompilationConfig {
            external_timeout_enabled: false,
            remote_build_jobs: RemoteBuildJobs::Off,
            ..Default::default()
        })
    }

    #[cfg(unix)]
    #[test]
    fn cargo_home_boundary_worker_base_survives_managed_env_and_cd() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir().unwrap().keep();
        let worker_base = retained.join("worker volume");
        std::fs::create_dir(&worker_base).unwrap();
        let alias = retained.join("worker alias");
        std::os::unix::fs::symlink(&worker_base, &alias).unwrap();
        let physical_base = worker_base.canonicalize().unwrap();
        let cdpath = retained.join("cdpath search");
        std::fs::create_dir_all(cdpath.join("worker volume")).unwrap();
        let dash_base = retained.join("-");
        std::fs::create_dir(&dash_base).unwrap();
        let temp_values = [
            (
                worker_base.as_os_str().to_os_string(),
                physical_base.clone(),
            ),
            (
                std::ffi::OsString::from("worker volume"),
                physical_base.clone(),
            ),
            (alias.as_os_str().to_os_string(), physical_base),
            (
                std::ffi::OsString::from("-"),
                dash_base.canonicalize().unwrap(),
            ),
        ];
        for (case, (temp_value, physical_base)) in temp_values.into_iter().enumerate() {
            let worker = WorkerId::new(format!("cache-worker-{case}"));
            let cache = physical_base.join(format!("rch-cargo-cache-{}", worker.as_str()));
            for round in 0..2 {
                let source = retained.join(format!("source-{case}-{round}"));
                std::fs::create_dir(&source).unwrap();
                let pipeline = cargo_home_boundary_pipeline(&source)
                    .with_env_allowlist(vec!["TMPDIR".to_owned(), "RCH_CH_BASE".to_owned()])
                    .with_env_overrides(HashMap::from([
                        (
                            "TMPDIR".to_owned(),
                            "/nonexistent/client-only-temp".to_owned(),
                        ),
                        (
                            "RCH_CH_BASE".to_owned(),
                            source.join("forwarded-poison").to_str().unwrap().to_owned(),
                        ),
                    ]));
                let workload = crate::hook::add_cargo_isolation(
                    "printf 'cargo-home=%s\\ntmpdir=%s\\n' \"$CARGO_HOME\" \"$TMPDIR\"; printf 'cache-probe-stderr' >&2",
                    &worker,
                );
                let command = pipeline.build_remote_command(&workload, None);
                let output = std::process::Command::new("sh")
                    .args(["-c", &command])
                    .current_dir(&retained)
                    .env("TMPDIR", &temp_value)
                    .env("CDPATH", &cdpath)
                    .env("OLDPWD", &cdpath)
                    .env("RCH_CH_BASE", retained.join("stale-inherited-base"))
                    .output()
                    .unwrap();
                assert!(output.status.success(), "{case}/{round}: {output:?}");
                assert_eq!(output.stderr, b"cache-probe-stderr");
                let physical_source = source.canonicalize().unwrap();
                assert_eq!(
                    output.stdout,
                    format!(
                        "cargo-home={}\ntmpdir={}\n",
                        cache.display(),
                        physical_source.join(".rch-tmp").display()
                    )
                    .into_bytes(),
                    "{case}/{round}"
                );
                assert!(!cache.starts_with(&physical_source));
                let marker = cache.join("cache-marker");
                if round == 0 {
                    std::fs::write(&marker, b"cache survives distinct source roots").unwrap();
                } else {
                    assert_eq!(
                        std::fs::read(&marker).unwrap(),
                        b"cache survives distinct source roots"
                    );
                }
            }
        }
    }

    #[cfg(unix)]
    #[test]
    fn cargo_home_boundary_preserves_explicit_home_streams_and_exit() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir().unwrap().keep();
        let worker_base = retained.join("worker volume");
        let source = retained.join("source");
        let explicit_home = retained.join("explicit home");
        for directory in [&worker_base, &source, &explicit_home] {
            std::fs::create_dir(directory).unwrap();
        }
        let inner = "printf '%s\\ncargo-out' \"$CARGO_HOME\"; printf 'cargo-err' >&2; exit 42";
        let requested = format!(
            "CARGO_HOME={} sh -c {}",
            shell_words::quote(explicit_home.to_str().unwrap()),
            shell_words::quote(inner)
        );
        let workload = crate::hook::add_cargo_isolation(&requested, &WorkerId::new("cache-worker"));
        let command = cargo_home_boundary_pipeline(&source).build_remote_command(&workload, None);
        let output = std::process::Command::new("sh")
            .args(["-c", &command])
            .current_dir(&retained)
            .env("TMPDIR", &worker_base)
            .output()
            .unwrap();
        assert_eq!(output.status.code(), Some(42), "{output:?}");
        assert_eq!(
            output.stdout,
            format!("{}\ncargo-out", explicit_home.display()).into_bytes()
        );
        assert_eq!(output.stderr, b"cargo-err");
    }

    #[cfg(unix)]
    #[test]
    fn cargo_home_boundary_worker_temp_fallbacks() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir().unwrap().keep();
        let source = retained.join("source");
        std::fs::create_dir(&source).unwrap();
        let fallback = if Path::new("/data/tmp").is_dir() {
            Path::new("/data/tmp")
        } else {
            Path::new("/tmp")
        }
        .canonicalize()
        .unwrap();
        let missing = retained.join("missing");
        for value in [
            None,
            Some(std::ffi::OsStr::new("")),
            Some(missing.as_os_str()),
        ] {
            let command = cargo_home_boundary_pipeline(&source)
                .build_remote_command("printf 'cargo-base=%s' \"$RCH_CH_BASE\"", None);
            let mut process = std::process::Command::new("sh");
            process
                .args(["-c", &command])
                .current_dir(&retained)
                .env("RCH_CH_BASE", retained.join("stale-inherited-base"));
            if let Some(value) = value {
                process.env("TMPDIR", value);
            } else {
                process.env_remove("TMPDIR");
            }
            let output = process.output().unwrap();
            assert!(output.status.success(), "{value:?}: {output:?}");
            assert_eq!(
                output.stdout,
                format!("cargo-base={}", fallback.display()).into_bytes()
            );
            assert!(output.stderr.is_empty(), "{output:?}");
        }
    }

    #[cfg(unix)]
    #[test]
    fn cargo_home_boundary_precedes_pgid_launch_and_skips_non_cargo() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir().unwrap().keep();
        let pipeline = cargo_home_boundary_pipeline(&retained).with_build_id(Some(1));
        let workload =
            crate::hook::add_cargo_isolation("cargo build", &WorkerId::new("cache-worker"));
        let command = pipeline.build_remote_command(&workload, None);
        let capture = command.find("export RCH_CH_BASE").unwrap();
        assert!(capture < command.find("touch ").unwrap());
        assert!(capture < command.find("&& cd ").unwrap());
        assert!(capture < command.find("setsid sh -c").unwrap());
        let plain = "printf plain";
        assert_eq!(
            crate::hook::add_cargo_isolation(plain, &WorkerId::new("cache-worker")),
            plain
        );
        assert!(
            !pipeline
                .build_remote_command(plain, None)
                .contains("RCH_CH_BASE")
        );
        let windows = cargo_home_boundary_pipeline(&retained)
            .with_worker_platform(WorkerPlatform::Windows)
            .build_remote_command(&workload, None);
        assert!(windows.contains("export RCH_CH_BASE"));
        assert!(!windows.contains("RCH_CH_BASE=\"$("));
    }

    // ---- issue #49: per-job CARGO_BUILD_JOBS cap on the worker ----

    fn jobs_pipeline(project_root: PathBuf, policy: RemoteBuildJobs) -> TransferPipeline {
        let compilation = rch_common::CompilationConfig {
            remote_build_jobs: policy,
            ..Default::default()
        };
        TransferPipeline::new(
            project_root,
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_config(compilation)
    }

    const JOBS_GUARD: &str = "if [ -z \"${CARGO_BUILD_JOBS:-}\" ]; then";

    #[test]
    fn test_remote_build_jobs_auto_is_default_and_exported_before_cd() {
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("tempdir");
        let pipeline = jobs_pipeline(temp.path().to_path_buf(), RemoteBuildJobs::default());
        let command = pipeline.build_remote_command("cargo test --workspace", None);
        assert!(
            command.contains(JOBS_GUARD),
            "auto policy must guard on an unset CARGO_BUILD_JOBS: {command}"
        );
        assert!(
            command.contains("CARGO_BUILD_JOBS=$__rch_j; export CARGO_BUILD_JOBS;"),
            "auto policy must export the derived count: {command}"
        );
        // The export must precede the `cd` into the project so the inner
        // `sh -lc` inherits it.
        let export_at = command.find("export CARGO_BUILD_JOBS").unwrap();
        let cd_at = command.find("&& cd ").unwrap();
        assert!(export_at < cd_at, "export must precede cd: {command}");
    }

    #[test]
    fn test_remote_build_jobs_off_injects_nothing() {
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("tempdir");
        let pipeline = jobs_pipeline(temp.path().to_path_buf(), RemoteBuildJobs::Off);
        let command = pipeline.build_remote_command("cargo build", None);
        assert!(
            !command.contains("CARGO_BUILD_JOBS"),
            "off policy must not mention CARGO_BUILD_JOBS: {command}"
        );
    }

    #[test]
    fn test_remote_build_jobs_fixed_exports_literal_but_stays_guarded() {
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("tempdir");
        let pipeline = jobs_pipeline(temp.path().to_path_buf(), RemoteBuildJobs::Fixed(3));
        let command = pipeline.build_remote_command("cargo build", None);
        assert!(
            command.contains(&format!(
                "{JOBS_GUARD} CARGO_BUILD_JOBS=3; export CARGO_BUILD_JOBS; fi; "
            )),
            "fixed policy must export the literal under the unset guard: {command}"
        );
        assert!(
            !command.contains("__rch_j"),
            "fixed policy must not probe: {command}"
        );
    }

    #[test]
    fn test_remote_build_jobs_skipped_on_windows_workers() {
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("tempdir");
        let pipeline = jobs_pipeline(temp.path().to_path_buf(), RemoteBuildJobs::Auto)
            .with_worker_platform(WorkerPlatform::Windows);
        let command = pipeline.build_remote_command("cargo build", None);
        assert!(
            !command.contains("CARGO_BUILD_JOBS"),
            "Windows workers have no nproc/meminfo/sysctl; must skip: {command}"
        );
    }

    #[test]
    fn test_remote_build_jobs_respects_project_cargo_config_jobs() {
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("tempdir");
        std::fs::create_dir_all(temp.path().join(".cargo")).unwrap();
        std::fs::write(
            temp.path().join(".cargo/config.toml"),
            "[build]\njobs = 2\ntarget-dir = \"target\"\n",
        )
        .unwrap();
        let pipeline = jobs_pipeline(temp.path().to_path_buf(), RemoteBuildJobs::Auto);
        let command = pipeline.build_remote_command("cargo build", None);
        assert!(
            !command.contains("CARGO_BUILD_JOBS"),
            "env CARGO_BUILD_JOBS outranks [build] jobs in cargo, so a project that pins jobs must be left alone: {command}"
        );

        // A config without `jobs` (target-dir only) does not suppress it.
        std::fs::write(
            temp.path().join(".cargo/config.toml"),
            "[build]\ntarget-dir = \"target\"\n",
        )
        .unwrap();
        let command = pipeline.build_remote_command("cargo build", None);
        assert!(command.contains("CARGO_BUILD_JOBS"), "{command}");
    }

    /// Run the auto fragment under a real POSIX `sh` with fake `nproc` and a
    /// fake meminfo, and return what `CARGO_BUILD_JOBS` ends up as
    /// (`unset` when nothing was exported). `preset` seeds the variable the
    /// way a worker's `/etc/environment` would.
    #[cfg(unix)]
    fn run_auto_fragment(
        nproc_out: &str,
        mem_total_kb: Option<u64>,
        preset: Option<&str>,
    ) -> String {
        use std::os::unix::fs::PermissionsExt;
        let temp = tempfile::tempdir().expect("tempdir");
        let bin = temp.path().join("bin");
        std::fs::create_dir_all(&bin).unwrap();
        // Fake nproc; fake sysctl that always fails so only the meminfo path
        // supplies memory (keeps the test identical on macOS and Linux).
        for (name, body) in [
            (
                "nproc",
                format!("#!/bin/sh\nprintf '%s\\n' '{nproc_out}'\n"),
            ),
            ("sysctl", "#!/bin/sh\nexit 1\n".to_string()),
        ] {
            let path = bin.join(name);
            std::fs::write(&path, body).unwrap();
            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap();
        }
        let meminfo = temp.path().join("meminfo");
        if let Some(kb) = mem_total_kb {
            std::fs::write(
                &meminfo,
                format!("MemTotal:       {kb} kB\nMemFree: 1 kB\n"),
            )
            .unwrap();
        }
        let fragment = remote_build_jobs_fragment(
            RemoteBuildJobs::Auto,
            meminfo.to_str().expect("utf8 tempdir"),
        );
        let script = format!("{fragment}printf '%s' \"${{CARGO_BUILD_JOBS:-unset}}\"");
        let mut cmd = std::process::Command::new("sh");
        cmd.arg("-c")
            .arg(&script)
            .env_clear()
            .env("PATH", format!("{}:/usr/bin:/bin", bin.display()));
        if let Some(value) = preset {
            cmd.env("CARGO_BUILD_JOBS", value);
        }
        let output = cmd.output().expect("run sh");
        assert!(
            output.status.success(),
            "fragment must never fail the session: {}",
            String::from_utf8_lossy(&output.stderr)
        );
        String::from_utf8(output.stdout).unwrap()
    }

    #[cfg(unix)]
    #[test]
    fn test_remote_build_jobs_auto_formula_under_real_sh() {
        let _guard = test_guard!();
        const GIB_KB: u64 = 1024 * 1024;
        // (nproc, MemTotal GiB, expected) — clamp(mem/8, 2, min(nproc, 8)).
        let cases = [
            ("16", 30, "3"),  // RAM-light 16-thread box: 30/8 = 3
            ("16", 58, "7"),  // 58/8 = 7
            ("6", 15, "2"),   // 15/8 = 1 -> floor 2
            ("64", 251, "8"), // 251/8 = 31 -> ceiling 8
            ("4", 251, "4"),  // ceiling also bounded by nproc
            ("1", 4, "1"),    // floor never exceeds nproc
        ];
        for (nproc, gib, expected) in cases {
            let got = run_auto_fragment(nproc, Some(gib * GIB_KB), None);
            assert_eq!(got, expected, "nproc={nproc} mem={gib}GiB");
        }
    }

    #[cfg(unix)]
    #[test]
    fn test_remote_build_jobs_auto_never_overrides_and_fails_open() {
        let _guard = test_guard!();
        const GIB_KB: u64 = 1024 * 1024;
        // Worker-side value (e.g. /etc/environment) wins untouched.
        assert_eq!(run_auto_fragment("16", Some(30 * GIB_KB), Some("5")), "5");
        // No readable memory fact -> export nothing, cargo keeps its default.
        assert_eq!(run_auto_fragment("16", None, None), "unset");
        // Garbage nproc -> export nothing.
        assert_eq!(
            run_auto_fragment("not-a-number", Some(30 * GIB_KB), None),
            "unset"
        );
    }

    #[test]
    fn test_managed_env_no_duplicate_when_allowlisted() {
        let _guard = test_guard!();
        // A managed key that IS forwarded + allowlisted must appear exactly once
        // (the allowlist pass and the forced-injection pass must not both emit it).
        let mut overrides = HashMap::new();
        overrides.insert(
            "CARGO_TARGET_DIR".to_string(),
            "/somewhere/else".to_string(),
        );
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_allowlist(vec!["CARGO_TARGET_DIR".to_string()])
        .with_env_overrides(overrides);

        let command = pipeline.build_remote_command("cargo build", None);
        let occurrences = command.matches("CARGO_TARGET_DIR=").count();
        assert_eq!(
            occurrences, 1,
            "CARGO_TARGET_DIR must be assigned exactly once: {command}"
        );
    }

    #[test]
    fn test_rch_go_dir_excluded_from_rsync_delete() {
        let _guard = test_guard!();
        // FIX 2: the injected Go cache dir must be protected from rsync --delete,
        // like .rch-target/.rch-tmp.
        assert!(
            REMOTE_RUNTIME_EXCLUDE_PATTERNS.contains(&".rch-go/"),
            "REMOTE_RUNTIME_EXCLUDE_PATTERNS must protect .rch-go/"
        );
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        );
        let excludes = pipeline.get_effective_excludes();
        assert!(
            excludes.iter().any(|e| e == ".rch-go/"),
            "effective excludes must include .rch-go/: {excludes:?}"
        );
    }

    #[test]
    fn test_build_remote_command_uses_custom_remote_cargo_target_dir_name() {
        let _guard = test_guard!();
        let mut overrides = HashMap::new();
        overrides.insert(
            "CARGO_TARGET_DIR".to_string(),
            "/data/tmp/pi_agent_rust/pearleagle".to_string(),
        );

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_allowlist(vec!["CARGO_TARGET_DIR".to_string()])
        .with_env_overrides(overrides)
        .with_remote_cargo_target_dir_name(".rch-target-worker-job-42");

        let worker_scoped_root = pipeline.remote_path();
        let command = pipeline.build_remote_command("cargo test --no-run", None);
        assert!(command.contains(&managed_assignment(
            "CARGO_TARGET_DIR",
            &format!("{worker_scoped_root}/.rch-target-worker-job-42")
        )));
        assert!(command.contains(&format!("{}/.rch-target-worker-job-42", worker_scoped_root)));
        assert!(!command.contains(&format!("{}/.rch-target'", worker_scoped_root)));
    }

    #[test]
    fn test_invalid_remote_cargo_target_dir_name_falls_back_to_default() {
        let _guard = test_guard!();
        let mut overrides = HashMap::new();
        overrides.insert(
            "CARGO_TARGET_DIR".to_string(),
            "/data/tmp/pi_agent_rust/pearleagle".to_string(),
        );

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_allowlist(vec!["CARGO_TARGET_DIR".to_string()])
        .with_env_overrides(overrides)
        .with_remote_cargo_target_dir_name("../bad");

        assert!(
            pipeline
                .build_remote_command("cargo test --no-run", None)
                .contains(".rch-target")
        );
        assert_eq!(
            pipeline.remote_cargo_target_dir(),
            format!("{}/.rch-target", pipeline.remote_path())
        );
    }

    /// Issue #60 regression: a pooled target-dir override must be STABLE
    /// across per-command (job-nonce-unique) remote roots, so consecutive
    /// clean-overlay gates reuse one warm cache instead of cold-building
    /// under each throwaway root.
    #[test]
    fn pooled_target_dir_override_is_stable_across_per_command_roots() {
        let _guard = test_guard!();
        let stable_pool = "/data/tmp/rch/project/.rch-target-w1-pool-0123456789abcdef";
        let make_pipeline = |per_command_root: &str| {
            let mut overrides = HashMap::new();
            overrides.insert(
                "CARGO_TARGET_DIR".to_string(),
                "/home/user/project/target".to_string(),
            );
            TransferPipeline::new(
                PathBuf::from("/tmp/project"),
                "project".to_string(),
                "hash".to_string(),
                TransferConfig::default(),
            )
            .with_env_allowlist(vec!["CARGO_TARGET_DIR".to_string()])
            .with_env_overrides(overrides)
            .with_remote_path_override(per_command_root.to_string())
            .with_remote_cargo_target_dir_name(".rch-target-w1-pool-0123456789abcdef")
            .with_remote_cargo_target_dir_override(stable_pool)
        };

        // Two different job nonces => two different per-command remote roots.
        let a = make_pipeline("/data/tmp/rch/project/aaaa1111aaaa1111");
        let b = make_pipeline("/data/tmp/rch/project/bbbb2222bbbb2222");

        assert_eq!(a.remote_cargo_target_dir(), stable_pool);
        assert_eq!(b.remote_cargo_target_dir(), stable_pool);
        // The pooled store must live OUTSIDE both throwaway roots (the
        // teardown reaper removes each root wholesale).
        assert!(!stable_pool.starts_with(&a.remote_path()));
        assert!(!stable_pool.starts_with(&b.remote_path()));

        // The remote CARGO_TARGET_DIR injection must use the stable path.
        let command = a.build_remote_command("cargo test --no-run", None);
        assert!(
            command.contains(&managed_assignment("CARGO_TARGET_DIR", stable_pool)),
            "override must drive the injected CARGO_TARGET_DIR: {command}"
        );
    }

    #[cfg(target_os = "linux")]
    fn source_pair_fixture(root: &Path, dependency: &Path, value: &str, inner: &str) {
        std::fs::create_dir_all(root.join("src")).unwrap();
        std::fs::create_dir_all(root.join("inner/src")).unwrap();
        let url = toml::Value::String(format!("file://{}", dependency.display()));
        std::fs::write(root.join("Cargo.toml"), format!(
            "[package]\nname='pair_fixture'\nversion='0.1.0'\nedition='2024'\n\
             [workspace]\nexclude=['inner']\n[dependencies]\ninner={{path='inner'}}\nexternal_fixture={{git={url}}}\n"
        )).unwrap();
        std::fs::write(
            root.join("inner/Cargo.toml"),
            "[package]\nname='inner'\nversion='0.1.0'\nedition='2024'\n",
        )
        .unwrap();
        std::fs::write(
            root.join("inner/src/lib.rs"),
            format!("pub fn value() -> &'static str {{ \"{inner}\" }}\n"),
        )
        .unwrap();
        std::fs::write(root.join("src/lib.rs"), format!(
            "pub fn value() -> &'static str {{ \"{value}\" }}\n\
             #[test] fn reads_current_fixture() {{\n\
             let root = std::path::Path::new(env!(\"CARGO_MANIFEST_DIR\"));\n\
             let expected = std::fs::read_to_string(root.join(\"fixture.txt\")).unwrap();\n\
             assert_eq!(expected, format!(\"{{}}|{{}}|{{}}\", value(), inner::value(), external_fixture::value()));\n\
             assert_eq!(std::fs::read_to_string(root.join(\"generation.txt\")).unwrap(), std::env::var(\"RCH_PAIR_GENERATION\").unwrap());\n\
             assert_eq!(root.canonicalize().unwrap(), std::env::current_dir().unwrap());\n}}\n"
        )).unwrap();
        std::fs::write(root.join("fixture.txt"), format!("{value}|{inner}|7")).unwrap();
        std::fs::write(root.join("generation.txt"), "initial").unwrap();
        let old = std::time::UNIX_EPOCH + Duration::from_secs(1_000_000);
        for file in [
            "Cargo.toml",
            "src/lib.rs",
            "inner/Cargo.toml",
            "inner/src/lib.rs",
            "fixture.txt",
            "generation.txt",
        ] {
            std::fs::File::options()
                .write(true)
                .open(root.join(file))
                .unwrap()
                .set_times(std::fs::FileTimes::new().set_modified(old))
                .unwrap();
        }
    }

    #[cfg(target_os = "linux")]
    fn source_pair_test_cargo(root: &Path, pool: &Path, checksum: bool) -> std::process::Output {
        let mut command = std::process::Command::new("cargo");
        command
            .current_dir(root)
            .args(["test", "--offline", "--jobs", "1", "--message-format=json"])
            .env("CARGO_TARGET_DIR", pool)
            .env(
                "RCH_PAIR_GENERATION",
                std::fs::read_to_string(root.join("generation.txt")).unwrap(),
            )
            .env_remove("CARGO_BUILD_BUILD_DIR")
            // Nested compiles execute on the worker running this RCH-offloaded
            // test, through its real Cargo rather than recursively offloading.
            .env("RCH_CARGO_WRAPPER_BYPASS", "1");
        if checksum {
            command.arg("-Zchecksum-freshness");
        }
        command.output().unwrap()
    }

    #[cfg(target_os = "linux")]
    fn source_pair_external_dependency_fresh(output: &std::process::Output) -> bool {
        String::from_utf8_lossy(&output.stdout)
            .lines()
            .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
            .any(|message| {
                message["reason"] == "compiler-artifact"
                    && message["target"]["name"] == "external_fixture"
                    && message["fresh"] == true
            })
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn source_pair_retirement_accepts_an_already_retired_root() {
        let evidence = tempfile::tempdir().unwrap().keep();
        let source = evidence.join("source with ' quotes");
        std::fs::create_dir(&source).unwrap();
        std::fs::write(source.join("fixture"), "retained source").unwrap();
        let retained = evidence.join("retired");
        std::fs::rename(&source, &retained).unwrap();
        // Only shell builtins are needed when the root is already absent.
        // An accidental removal invocation must fail instead of passing merely
        // because `rm -f` tolerates an absent path.
        let output = std::process::Command::new("/bin/sh")
            .env("PATH", "")
            .args([
                "-c",
                &TransferPipeline::remote_tree_retirement_command(source.to_str().unwrap()),
            ])
            .output()
            .unwrap();
        assert!(output.status.success());
        assert!(!source.exists());
        assert_eq!(
            std::fs::read_to_string(retained.join("fixture")).unwrap(),
            "retained source"
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn source_pair_real_cargo_preserves_unchanged_build_script_inputs_and_rollback() {
        use std::os::unix::fs::PermissionsExt;
        let retained = tempfile::tempdir().unwrap().keep();
        let root = retained.join("source with ' quotes");
        let pool = retained.join("pool");
        let counter = retained.join("build-script-count");
        let pipeline = TransferPipeline::new(
            root.clone(),
            "freshness".into(),
            "pair".into(),
            TransferConfig::default(),
        )
        .with_remote_path_override(root.to_string_lossy().into_owned())
        .with_remote_cargo_target_dir_override(pool.to_string_lossy().into_owned());
        let build_script = r#"use std::{env, fs, path::Path};
fn main() {
    println!("cargo:rerun-if-changed=watched.txt");
    println!("cargo:rerun-if-changed=watched");
    let counter = env::var("RCH_FRESHNESS_BUILD_COUNT").unwrap();
    let count = fs::read_to_string(&counter).unwrap_or_default().parse::<usize>().unwrap_or(0);
    fs::write(counter, (count + 1).to_string()).unwrap();
    let mut children: Vec<_> = fs::read_dir("watched").unwrap().map(|entry| entry.unwrap().path()).collect();
    children.sort();
    let mut value = fs::read_to_string("watched.txt").unwrap();
    for child in children {
        value.push('|');
        value.push_str(child.file_name().unwrap().to_str().unwrap());
        value.push('=');
        value.push_str(&fs::read_to_string(child).unwrap());
    }
    fs::write(Path::new(&env::var("OUT_DIR").unwrap()).join("value.rs"), format!("pub const VALUE: &str = {:?};", value)).unwrap();
}
"#;
        let mut watched_time = None;
        for (run, (code, watched, child, expected_count)) in [
            ("A", "x", Some("one"), 1),
            ("B", "x", Some("one"), 1), // unrelated Rust change
            ("A", "x", Some("one"), 1), // A -> B -> A must recompile Rust
            ("A", "y", Some("one"), 2),
            ("A", "x", Some("one"), 3), // watched-input rollback must rerun
            ("A", "x", Some("two"), 4), // delete/add with identical child bytes
            ("A", "x", Some("two"), 4), // new selected identity, identical bytes
            ("A", "x", None, 5),        // directory-watched deletion
            ("A", "x", None, 6),        // executable/mode identity is an input
        ]
        .into_iter()
        .enumerate()
        {
            std::fs::create_dir_all(root.join("src")).unwrap();
            std::fs::create_dir(root.join("watched")).unwrap();
            std::fs::write(root.join("Cargo.toml"), "[package]\nname='freshness_fixture'\nversion='0.1.0'\nedition='2024'\n[workspace]\n").unwrap();
            std::fs::write(root.join("build.rs"), build_script).unwrap();
            std::fs::write(root.join("watched.txt"), watched).unwrap();
            if run == 8 {
                std::fs::set_permissions(
                    root.join("watched.txt"),
                    std::fs::Permissions::from_mode(0o600),
                )
                .unwrap();
            }
            if let Some(child) = child {
                std::fs::write(root.join("watched").join(child), "d").unwrap();
            }
            std::fs::write(root.join("src/lib.rs"), format!(
                "include!(concat!(env!(\"OUT_DIR\"), \"/value.rs\"));\n#[test] fn actual_generated_value() {{ assert_eq!(format!(\"{code}:{{}}\", VALUE), std::env::var(\"RCH_EXPECTED_EFFECT\").unwrap()); }}\n"
            )).unwrap();
            // Simulate archives with timestamps older than every build output.
            for file in ["Cargo.toml", "build.rs", "watched.txt", "src/lib.rs"] {
                std::fs::File::options()
                    .write(true)
                    .open(root.join(file))
                    .unwrap()
                    .set_times(
                        std::fs::FileTimes::new()
                            .set_modified(std::time::UNIX_EPOCH + Duration::from_secs(1_000_000)),
                    )
                    .unwrap();
            }
            let identity = blake3::hash(format!("selected-commit-{run}").as_bytes())
                .to_hex()
                .to_string();
            let refresh = std::process::Command::new("sh")
                .args([
                    "-c",
                    &pipeline
                        .clean_overlay_source_refresh_command(&identity)
                        .unwrap(),
                ])
                .output()
                .unwrap();
            assert!(
                refresh.status.success(),
                "{}",
                String::from_utf8_lossy(&refresh.stderr)
            );
            assert!(String::from_utf8_lossy(&refresh.stdout).contains("RCH_SOURCE_FRESHNESS_V2"));
            let current_time = std::fs::metadata(root.join("watched.txt"))
                .unwrap()
                .modified()
                .unwrap();
            if run == 1 || run == 2 {
                assert_eq!(
                    Some(current_time),
                    watched_time,
                    "unchanged watched input was touched"
                );
            }
            if run == 0 {
                watched_time = Some(current_time);
            }
            let expected = format!(
                "{code}:{watched}{}",
                child.map_or_else(String::new, |name| format!("|{name}=d"))
            );
            let output = std::process::Command::new("cargo")
                .current_dir(&root)
                .args(["test", "--offline", "--jobs", "1", "--message-format=json"])
                .env("CARGO_TARGET_DIR", &pool)
                .env_remove("CARGO_BUILD_BUILD_DIR")
                .env("RCH_CARGO_WRAPPER_BYPASS", "1")
                .env("RCH_FRESHNESS_BUILD_COUNT", &counter)
                .env("RCH_EXPECTED_EFFECT", expected)
                .output()
                .unwrap();
            assert!(
                output.status.success(),
                "{}\n{}",
                String::from_utf8_lossy(&output.stdout),
                String::from_utf8_lossy(&output.stderr)
            );
            assert!(String::from_utf8_lossy(&output.stdout).contains("actual_generated_value"));
            assert_eq!(
                std::fs::read_to_string(&counter).unwrap(),
                expected_count.to_string(),
                "build-script run count at step {run}"
            );
            if run == 6 {
                let artifacts: Vec<_> = String::from_utf8_lossy(&output.stdout)
                    .lines()
                    .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
                    .filter(|message| message["reason"] == "compiler-artifact")
                    .collect();
                assert!(!artifacts.is_empty());
                assert!(artifacts.iter().all(|message| message["fresh"] == true));
            }
            std::fs::rename(&root, retained.join(format!("retired-{run}"))).unwrap();
        }
        eprintln!(
            "content freshness Cargo fixture retained at {}",
            retained.display()
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn source_pair_nested_root_ledgers_survive_container_retirement() {
        let retained = tempfile::tempdir().unwrap().keep();
        let container = retained.join("paired-source");
        let pool = retained.join("pool");
        let make_pipeline = |name: &str| {
            let root = container.join(name);
            TransferPipeline::new(
                root.clone(),
                name.into(),
                "pair".into(),
                TransferConfig::default(),
            )
            .with_remote_path_override(root.to_string_lossy().into_owned())
            .with_remote_cargo_target_dir_override(pool.to_string_lossy().into_owned())
        };
        let first = make_pipeline("first");
        let second = make_pipeline("second");
        let mut first_time = None;
        let mut second_time = None;
        for (run, contents) in ["before", "after", "before"].into_iter().enumerate() {
            for (name, contents, pipeline) in [
                ("first", "unchanged", &first),
                ("second", contents, &second),
            ] {
                let root = container.join(name);
                std::fs::create_dir_all(&root).unwrap();
                std::fs::write(root.join("input"), contents).unwrap();
                let identity = blake3::hash(format!("whole-closure-{run}").as_bytes())
                    .to_hex()
                    .to_string();
                let output = std::process::Command::new("sh")
                    .args([
                        "-c",
                        &pipeline
                            .clean_overlay_source_refresh_command_at(
                                &identity,
                                container.to_str().unwrap(),
                            )
                            .unwrap(),
                    ])
                    .output()
                    .unwrap();
                assert!(
                    output.status.success(),
                    "{}",
                    String::from_utf8_lossy(&output.stderr)
                );
                let modified = std::fs::metadata(root.join("input"))
                    .unwrap()
                    .modified()
                    .unwrap();
                if name == "first" {
                    if let Some(previous) = first_time {
                        assert_eq!(modified, previous);
                    }
                    first_time = Some(modified);
                } else {
                    if let Some(previous) = second_time {
                        assert!(modified > previous);
                    }
                    second_time = Some(modified);
                }
            }
            std::fs::rename(&container, retained.join(format!("retired-{run}"))).unwrap();
        }
        assert!(
            first
                .clean_overlay_source_refresh_command_at(&"a".repeat(64), "/unrelated")
                .is_err()
        );
        eprintln!(
            "nested freshness fixture retained at {}",
            retained.display()
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn source_pair_freshness_ledger_refuses_links_and_recovers_interrupted_metadata() {
        use std::os::unix::fs::symlink;
        let retained = tempfile::tempdir().unwrap().keep();
        let root = retained.join("source");
        let pool = retained.join("pool");
        let outside = retained.join("outside");
        let epoch = retained.join("source.freshness-epoch-v1");
        let ledger = retained.join("source.freshness-epoch-v1.files-v2");
        std::fs::write(&outside, "must not be read or touched through a link").unwrap();
        let outside_time = std::fs::metadata(&outside).unwrap().modified().unwrap();
        let pipeline = TransferPipeline::new(
            root.clone(),
            "fixture".into(),
            "pair".into(),
            TransferConfig::default(),
        )
        .with_remote_path_override(root.to_string_lossy().into_owned())
        .with_remote_cargo_target_dir_override(pool.to_string_lossy().into_owned());
        let materialize = || {
            std::fs::create_dir(&root).unwrap();
            std::fs::write(root.join("input\nwith ' quotes"), "identical").unwrap();
            symlink(&outside, root.join("outside-link")).unwrap();
        };
        let identity = "a".repeat(64);
        let run = || {
            std::process::Command::new("sh")
                .args([
                    "-c",
                    &pipeline
                        .clean_overlay_source_refresh_command(&identity)
                        .unwrap(),
                ])
                .output()
                .unwrap()
        };
        materialize();
        let result = run();
        assert!(
            result.status.success(),
            "{}",
            String::from_utf8_lossy(&result.stderr)
        );
        let first = std::fs::metadata(root.join("input\nwith ' quotes"))
            .unwrap()
            .modified()
            .unwrap();
        assert_eq!(
            std::fs::metadata(&outside).unwrap().modified().unwrap(),
            outside_time
        );
        // Deterministic I/O fault injection into the real helper, rather than a
        // chmod test that silently passes under a root worker. An unreadable
        // artifact subtree must not be omitted from the freshness upper bound.
        std::fs::create_dir(&pool).unwrap();
        let saved_epoch = std::fs::read(&epoch).unwrap();
        let saved_ledger = std::fs::read(&ledger).unwrap();
        std::fs::write(root.join("input\nwith ' quotes"), "changed").unwrap();
        let before_failure = std::fs::metadata(root.join("input\nwith ' quotes"))
            .unwrap()
            .modified()
            .unwrap();
        let fault_script = format!(
            "import os, sys\noriginal_scandir = os.scandir\ndef unreadable_pool(path):\n    if path == sys.argv[2]:\n        raise PermissionError('injected unreadable artifact pool')\n    return original_scandir(path)\nos.scandir = unreadable_pool\n{}",
            TransferPipeline::clean_overlay_freshness_ledger_script()
        );
        let failed = std::process::Command::new("python3")
            .args(["-c", &fault_script])
            .arg(&root)
            .arg(&pool)
            .arg(&epoch)
            .arg(&identity)
            .output()
            .unwrap();
        assert!(
            !failed.status.success(),
            "unreadable artifacts must fail closed"
        );
        assert!(
            String::from_utf8_lossy(&failed.stderr).contains("injected unreadable artifact pool")
        );
        assert_eq!(std::fs::read(&epoch).unwrap(), saved_epoch);
        assert_eq!(std::fs::read(&ledger).unwrap(), saved_ledger);
        assert_eq!(
            std::fs::metadata(root.join("input\nwith ' quotes"))
                .unwrap()
                .modified()
                .unwrap(),
            before_failure
        );
        std::fs::rename(&root, retained.join("retired-first")).unwrap();
        materialize();
        // A valid ledger with a different commit marker models interruption
        // between metadata publications; it must never grant timestamp reuse.
        std::fs::write(&epoch, format!("{}\n", "b".repeat(64))).unwrap();
        let result = run();
        assert!(
            result.status.success(),
            "{}",
            String::from_utf8_lossy(&result.stderr)
        );
        assert!(
            std::fs::metadata(root.join("input\nwith ' quotes"))
                .unwrap()
                .modified()
                .unwrap()
                > first
        );
        std::fs::rename(&root, retained.join("retired-second")).unwrap();
        materialize();
        std::fs::write(&ledger, "incomplete JSON").unwrap();
        let result = run();
        assert!(
            result.status.success(),
            "{}",
            String::from_utf8_lossy(&result.stderr)
        );
        let committed_epoch = std::fs::read(&epoch).unwrap();
        std::fs::rename(&ledger, retained.join("retained-ledger")).unwrap();
        symlink(&outside, &ledger).unwrap();
        let result = run();
        assert!(
            !result.status.success(),
            "symlink ledger must refuse before touching source"
        );
        assert_eq!(std::fs::read(&epoch).unwrap(), committed_epoch);
        assert_eq!(
            std::fs::metadata(&outside).unwrap().modified().unwrap(),
            outside_time
        );
        assert_eq!(
            std::fs::read_to_string(&outside).unwrap(),
            "must not be read or touched through a link"
        );
        eprintln!(
            "freshness metadata fixture retained at {}",
            retained.display()
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn source_pair_epoch_publication_requires_complete_refresh_and_valid_identity() {
        let retained = tempfile::tempdir().unwrap().keep();
        let root = retained.join("source");
        let pool = retained.join("pool");
        let epoch = retained.join("source.freshness-epoch-v1");
        let pipeline = TransferPipeline::new(
            root.clone(),
            "fixture".into(),
            "pair".into(),
            TransferConfig::default(),
        )
        .with_remote_path_override(root.to_string_lossy().into_owned())
        .with_remote_cargo_target_dir_override(pool.to_string_lossy().into_owned());
        let first = "a".repeat(64);
        let second = "b".repeat(64);
        let run = |identity: &str| {
            std::process::Command::new("sh")
                .args([
                    "-c",
                    &pipeline
                        .clean_overlay_source_refresh_command(identity)
                        .unwrap(),
                ])
                .output()
                .unwrap()
        };
        assert!(
            pipeline
                .clean_overlay_source_refresh_command("not-an-identity")
                .is_err()
        );
        // Refresh fails before publication when the complete root is absent.
        assert!(!run(&first).status.success());
        assert!(!epoch.exists());
        std::fs::create_dir(&root).unwrap();
        std::fs::write(root.join("input"), "first").unwrap();
        assert!(run(&first).status.success());
        let first_time = std::fs::metadata(&epoch).unwrap().modified().unwrap();
        let published = std::fs::read(&epoch).unwrap();
        std::fs::rename(&root, retained.join("retired")).unwrap();
        // A failed next epoch cannot replace the prior committed marker.
        assert!(!run(&second).status.success());
        assert_eq!(std::fs::read(&epoch).unwrap(), published);
        assert_eq!(
            std::fs::metadata(&epoch).unwrap().modified().unwrap(),
            first_time
        );
        std::fs::create_dir(&root).unwrap();
        std::fs::write(root.join("input"), "first").unwrap();
        assert!(run(&first).status.success());
        assert_eq!(
            std::fs::metadata(root.join("input"))
                .unwrap()
                .modified()
                .unwrap(),
            first_time
        );
        std::fs::write(&epoch, "malformed retained marker").unwrap();
        assert!(run(&first).status.success());
        assert_eq!(
            std::fs::read_to_string(&epoch).unwrap(),
            format!("{first}\n")
        );
        assert!(std::fs::metadata(&epoch).unwrap().modified().unwrap() > first_time);
        assert!(run(&second).status.success());
        assert_eq!(
            std::fs::read_to_string(&epoch).unwrap(),
            format!("{second}\n")
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn source_pair_real_cargo_reuses_dependencies_and_reads_current_source() {
        let dir = tempfile::tempdir().unwrap().keep();
        let dependency = dir.join("external");
        std::fs::create_dir_all(dependency.join("src")).unwrap();
        std::fs::write(
            dependency.join("Cargo.toml"),
            "[package]\nname='external_fixture'\nversion='0.1.0'\nedition='2024'\n",
        )
        .unwrap();
        std::fs::write(
            dependency.join("src/lib.rs"),
            "pub fn value() -> u32 { 7 }\n",
        )
        .unwrap();
        for args in [
            vec!["init", "-b", "main"],
            vec!["add", "."],
            vec![
                "-c",
                "user.name=RCH Test",
                "-c",
                "user.email=test@example.invalid",
                "commit",
                "-m",
                "fixture",
            ],
        ] {
            assert!(
                std::process::Command::new("git")
                    .current_dir(&dependency)
                    .args(args)
                    .status()
                    .unwrap()
                    .success()
            );
        }
        // Negative control: the previous nonce-root policy with unchanged,
        // old timestamps must reproduce a real runtime fixture-path failure.
        let legacy_pool = dir.join("legacy-pool");
        let legacy_a = dir.join("legacy-a");
        source_pair_fixture(&legacy_a, &dependency, "first", "alpha");
        // This fixture has only a local file:// Git dependency. Populate its
        // Cargo cache once before the deliberately offline regression runs.
        let fetch = std::process::Command::new("cargo")
            .current_dir(&legacy_a)
            .args(["fetch"])
            .output()
            .unwrap();
        assert!(
            fetch.status.success(),
            "{}",
            String::from_utf8_lossy(&fetch.stderr)
        );
        let first = source_pair_test_cargo(&legacy_a, &legacy_pool, false);
        assert!(
            first.status.success(),
            "{}",
            String::from_utf8_lossy(&first.stderr)
        );
        std::fs::rename(&legacy_a, dir.join("legacy-a-retired")).unwrap();
        let legacy_b = dir.join("legacy-b");
        source_pair_fixture(&legacy_b, &dependency, "first", "alpha");
        let stale = source_pair_test_cargo(&legacy_b, &legacy_pool, false);
        assert!(
            !stale.status.success(),
            "negative control did not reproduce the stale source path"
        );
        assert!(String::from_utf8_lossy(&stale.stdout).contains("reads_current_fixture"));

        for checksum in [false, true] {
            let root = dir.join(format!("source-{checksum}"));
            let pool = dir.join(format!("pool-{checksum}"));
            for (run, (value, inner, generation)) in [
                ("first", "alpha", 0),
                ("first", "alpha", 0),
                ("first", "alpha", 1),
                ("later", "omega", 2),
            ]
            .into_iter()
            .enumerate()
            {
                source_pair_fixture(&root, &dependency, value, inner);
                // Run three changes only a runtime fixture, so a fresh source
                // namespace must expose new data even when checksum Cargo
                // reuses the unchanged test executable.
                std::fs::write(
                    root.join("generation.txt"),
                    format!("generation-{generation}"),
                )
                .unwrap();
                let identity = blake3::hash(format!("{value}\0{inner}\0{generation}").as_bytes())
                    .to_hex()
                    .to_string();
                let pipeline = TransferPipeline::new(
                    root.clone(),
                    "fixture".into(),
                    "pair".into(),
                    TransferConfig::default(),
                )
                .with_remote_path_override(root.to_string_lossy().into_owned())
                .with_remote_cargo_target_dir_override(pool.to_string_lossy().into_owned());
                assert!(
                    std::process::Command::new("sh")
                        .args([
                            "-c",
                            &pipeline
                                .clean_overlay_source_refresh_command(&identity)
                                .unwrap()
                        ])
                        .status()
                        .unwrap()
                        .success()
                );
                let result = source_pair_test_cargo(&root, &pool, checksum);
                assert!(
                    result.status.success(),
                    "{}\n{}",
                    String::from_utf8_lossy(&result.stdout),
                    String::from_utf8_lossy(&result.stderr)
                );
                if run > 0 {
                    assert!(
                        source_pair_external_dependency_fresh(&result),
                        "external dependency was rebuilt"
                    );
                }
                if run == 1 || run == 3 {
                    for name in ["inner", "pair_fixture"] {
                        let artifacts = String::from_utf8_lossy(&result.stdout)
                            .lines()
                            .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
                            .filter(|message| {
                                message["reason"] == "compiler-artifact"
                                    && message["target"]["name"] == name
                            })
                            .collect::<Vec<_>>();
                        assert!(!artifacts.is_empty(), "missing actual artifact for {name}");
                        assert!(
                            artifacts
                                .iter()
                                .all(|artifact| artifact["fresh"] == (run == 1)),
                            "identical bytes must be Fresh; changed old-mtime bytes must rebuild: run={run} {name}: {artifacts:?}"
                        );
                    }
                }
                std::fs::rename(&root, dir.join(format!("retired-{checksum}-{run}"))).unwrap();
                assert!(
                    !root.exists(),
                    "source retirement must remove the old runtime path"
                );
            }
        }
        eprintln!(
            "source-pair Cargo regression artifacts retained at {}",
            dir.display()
        );
    }

    /// Issue #60: the transfer-start janitor must sweep the relocated pool's
    /// stable parent on the configured pooled retention, in addition to the
    /// per-command remote root.
    #[test]
    fn pooled_override_parent_joins_transfer_start_janitor_sweep() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_remote_path_override("/data/tmp/rch/project/aaaa1111aaaa1111")
        .with_remote_cargo_target_dir_override("/data/tmp/rch/project/.rch-target-w1-pool-abc123");

        let parent = pipeline
            .escaped_pooled_target_override_parent()
            .expect("override parent");
        let prefix = worker_cache_prune_rsync_path_prefix(
            "/data/tmp/rch/project/aaaa1111aaaa1111",
            168,
            Some(&parent),
        );
        assert!(
            prefix.contains(&format!(
                "find {parent} -mindepth 1 -maxdepth 1 -type d -name '.rch-target-*-pool-*'"
            )),
            "stable pool parent must be swept: {prefix}"
        );
        // Same parent as the remote path adds no duplicate sweep.
        let dup = worker_cache_prune_rsync_path_prefix("/r/p", 168, Some("/r/p"));
        assert_eq!(dup.matches(".rch-target-*-pool-*").count(), 1);
        // Disabled retention disables the parent sweep too.
        let disabled = worker_cache_prune_rsync_path_prefix("/r/p", 0, Some(&parent));
        assert!(!disabled.contains(".rch-target-*-pool-*"));
    }

    #[test]
    fn invalid_pooled_target_dir_override_is_ignored() {
        let _guard = test_guard!();
        let base = || {
            TransferPipeline::new(
                PathBuf::from("/tmp/project"),
                "project".to_string(),
                "hash".to_string(),
                TransferConfig::default(),
            )
        };
        for bad in [
            "",
            "relative/pool",
            "/data/tmp/../etc/pool",
            "/data/tmp/rch/.",
            "/data\ntmp/pool",
        ] {
            let pipeline = base().with_remote_cargo_target_dir_override(bad);
            assert_eq!(
                pipeline.remote_cargo_target_dir(),
                format!("{}/.rch-target", pipeline.remote_path()),
                "invalid override {bad:?} must fall back to the derived default"
            );
        }
        // Windows drive-letter absolute paths are accepted (Windows workers).
        let windows = base().with_remote_cargo_target_dir_override("C:/rch/project/.rch-target-p");
        assert_eq!(
            windows.remote_cargo_target_dir(),
            "C:/rch/project/.rch-target-p"
        );
    }

    /// Issue #62: the post-timeout kill script must target the recorded pgid
    /// with a dash-safe group SIGKILL (no `--`), verify death, and emit exactly
    /// one machine-readable verdict marker.
    #[test]
    fn remote_timeout_kill_script_shape_and_parse() {
        let script = remote_timeout_kill_script("/tmp/rch-run/proj-abc/42.pgid", 42);
        assert!(script.contains("/tmp/rch-run/proj-abc/42.pgid"));
        // Group kill with NO `--` (dash's kill builtin mishandles it).
        assert!(script.contains("kill -\"$1\" -\"$rch_pgid\""));
        assert!(!script.contains("kill -KILL -- "));
        // Boot/start identity and a process-table observation guard group signals.
        assert!(script.contains("rch_remote_leader_matches || return 43"));
        assert!(script.contains("ps -e -o pid= -o pgid= -o stat="));
        assert!(script.contains("rch_remote_cancel \"$f\" 42 kill"));
        assert!(script.contains("RCH_E104_KILL=verified_dead"));
        assert!(script.contains("RCH_E104_KILL=still_alive"));

        assert_eq!(
            parse_remote_timeout_kill_output("noise\nRCH_E104_KILL=verified_dead\n"),
            Some(RemoteTimeoutCleanup::Verified)
        );
        assert_eq!(
            parse_remote_timeout_kill_output("RCH_E104_KILL=no_pgid_file"),
            Some(RemoteTimeoutCleanup::Unverified)
        );
        assert_eq!(
            parse_remote_timeout_kill_output("RCH_E104_KILL=still_alive"),
            Some(RemoteTimeoutCleanup::Unverified)
        );
        assert_eq!(parse_remote_timeout_kill_output("garbage"), None);
    }

    /// Issue #62: the typed timeout error must keep the exact message prefix
    /// the hook's fail-closed classifier matches on.
    #[test]
    fn ssh_command_timed_out_error_keeps_classifier_prefix() {
        let err = SshCommandTimedOut {
            timeout: std::time::Duration::from_secs(1800),
            cleanup: RemoteTimeoutCleanup::Unverified,
            detail: "kill probe timed out".to_string(),
            evidence: None,
        };
        let text = err.to_string();
        assert!(text.starts_with("SSH command timed out after"));
        assert!(text.contains("NOT verified dead"));
        // Downcast through an anyhow chain (how the hook consumes it).
        let any: anyhow::Error = err.into();
        let found = any
            .chain()
            .find_map(|cause| cause.downcast_ref::<SshCommandTimedOut>())
            .expect("typed timeout error must be downcastable");
        assert_eq!(found.cleanup, RemoteTimeoutCleanup::Unverified);
    }

    #[cfg(target_os = "linux")]
    fn recorded_remote_pgid(record: &str, build_id: u64) -> u32 {
        let fields: Vec<_> = record.trim_end().split(':').collect();
        assert_eq!(fields.len(), 5, "versioned record: {record}");
        assert_eq!(fields[0], "RCH_REMOTE_PROCESS_V1");
        assert_eq!(fields[1], build_id.to_string());
        assert_eq!(
            fields[2],
            std::fs::read_to_string("/proc/sys/kernel/random/boot_id")
                .unwrap()
                .trim()
        );
        assert!(fields[4].parse::<u64>().unwrap() > 0);
        fields[3].parse::<u32>().unwrap()
    }

    /// The deadline must fire even when nobody drains the session's stderr.
    /// The workload fills the pipe (like cargo writing to a dispatcher that
    /// stopped reading), so a blocking marker write would wedge the timer.
    #[cfg(target_os = "linux")]
    #[test]
    fn remote_build_watchdog_kills_when_stderr_pipe_is_full() {
        use std::process::Command;
        use std::time::{Duration, Instant};

        let dir = tempfile::tempdir().unwrap().keep();
        let pgf = dir.join("job.pgid");
        let mut child = Command::new("/bin/sh")
            .arg("-c")
            .arg(r#"exec setsid sh -c "$1" rch-build "$2" 1 RCH_TEST_DEADLINE 42 sh -c "$3" 3>&2"#)
            .arg("rch-full-pipe")
            .arg(remote_build_watchdog_script())
            .arg(pgf.to_str().unwrap())
            .arg("head -c 1048576 /dev/zero >&2; sleep 60")
            .stdin(std::process::Stdio::null())
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::piped())
            .spawn()
            .unwrap();
        // Hold the read end open without reading so the pipe stays full.
        let _undrained = child.stderr.take();
        let start = Instant::now();
        let status = loop {
            if let Some(status) = child.try_wait().unwrap() {
                break Some(status);
            }
            if start.elapsed() > Duration::from_secs(20) {
                break None;
            }
            std::thread::sleep(Duration::from_millis(100));
        };
        if status.is_none() {
            let _ = Command::new("kill")
                .arg("-KILL")
                .arg(format!("-{}", child.id()))
                .status();
            let _ = child.wait();
        }
        let status = status.expect("deadline kill must not block on a full stderr pipe");
        assert_eq!(
            std::os::unix::process::ExitStatusExt::signal(&status),
            Some(9),
            "{status:?}"
        );
    }

    /// Exercise the production publisher and timeout consumer against a real
    /// group. Stale identity never signals it; its exact identity kills it.
    #[cfg(target_os = "linux")]
    #[test]
    fn remote_timeout_kill_script_reaps_process_group() {
        use std::process::Command;
        use std::time::{Duration, Instant};

        let dir = tempfile::tempdir().unwrap().keep();
        let pgf = dir.join("job.pgid");
        let mut child = Command::new("setsid")
            .arg("sh")
            .arg("-c")
            .arg(remote_build_watchdog_script())
            .arg("rch-e104-victim")
            .arg(pgf.to_str().unwrap())
            .arg("30")
            .arg("RCH_TEST_DEADLINE")
            .arg("42")
            .arg("sh")
            .arg("-c")
            .arg("trap '' TERM; sleep 30 & wait")
            .stdin(std::process::Stdio::null())
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null())
            .spawn()
            .unwrap();
        let start = Instant::now();
        while !pgf.exists() && start.elapsed() < Duration::from_secs(5) {
            std::thread::sleep(Duration::from_millis(50));
        }
        assert!(pgf.exists(), "victim should have recorded its pgid");
        let record = std::fs::read_to_string(&pgf).unwrap();
        let pgid = recorded_remote_pgid(&record, 42);
        assert_eq!(pgid, child.id(), "recorded the actual session leader");
        let stat = std::fs::read_to_string(format!("/proc/{pgid}/stat")).unwrap();
        let stat_fields: Vec<_> = stat
            .rsplit_once(") ")
            .unwrap()
            .1
            .split_whitespace()
            .collect();
        assert_eq!(stat_fields[2], pgid.to_string());
        let fields: Vec<_> = record.trim_end().split(':').collect();
        assert_eq!(
            fields[4], stat_fields[19],
            "recorded real kernel start ticks"
        );

        let wrong_boot = if fields[2] == "00000000-0000-0000-0000-000000000000" {
            "11111111-1111-1111-1111-111111111111"
        } else {
            "00000000-0000-0000-0000-000000000000"
        };
        for stale in [
            format!("{pgid}\n"),
            format!(
                "RCH_REMOTE_PROCESS_V1:41:{}:{pgid}:{}\n",
                fields[2], fields[4]
            ),
            format!(
                "RCH_REMOTE_PROCESS_V1:42:{wrong_boot}:{pgid}:{}\n",
                fields[4]
            ),
            format!(
                "RCH_REMOTE_PROCESS_V1:42:{}:{pgid}:{}\n",
                fields[2],
                fields[4].parse::<u64>().unwrap() + 1
            ),
            "malformed\n".to_string(),
            record.trim_end().to_string(),
            format!("{record}\n"),
            format!("{record}\0"),
        ] {
            std::fs::write(&pgf, &stale).unwrap();
            let output = Command::new("sh") // ubs:ignore — production verifier over this test's owned record
                .arg("-c")
                .arg(remote_timeout_kill_script(pgf.to_str().unwrap(), 42))
                .output()
                .unwrap();
            assert_eq!(
                parse_remote_timeout_kill_output(&String::from_utf8_lossy(&output.stdout)),
                Some(RemoteTimeoutCleanup::Unverified),
                "stale record must not grant signal authority: {stale}"
            );
            assert!(
                child.try_wait().unwrap().is_none(),
                "live leader was preserved"
            );
        }
        std::fs::write(&pgf, &record).unwrap();

        let script = remote_timeout_kill_script(pgf.to_str().unwrap(), 42);
        let output = Command::new("sh").arg("-c").arg(&script).output().unwrap(); // ubs:ignore — production kill script with this test's owned PGID path
        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
        // Safety net regardless of assertions.
        let _ = Command::new("sh") // ubs:ignore — test-owned process group; PGID parsed as positive integer above
            .arg("-c")
            .arg(format!("kill -KILL -{pgid} 2>/dev/null"))
            .status();
        let _ = child.kill();
        let _ = child.wait();

        assert_eq!(
            parse_remote_timeout_kill_output(&stdout),
            Some(RemoteTimeoutCleanup::Verified),
            "kill script must verify the group dead: {stdout}"
        );
        let retry = Command::new("sh") // ubs:ignore — production retry over the test's recorded identity
            .arg("-c")
            .arg(&script)
            .output()
            .unwrap();
        assert_eq!(
            parse_remote_timeout_kill_output(&String::from_utf8_lossy(&retry.stdout)),
            Some(RemoteTimeoutCleanup::Verified),
            "an already stopped group remains verifiably absent"
        );

        // Missing identity cannot prove the remote command never started.
        let missing = dir.join("never-started.pgid");
        let script = remote_timeout_kill_script(missing.to_str().unwrap(), 42);
        let output = Command::new("sh").arg("-c").arg(&script).output().unwrap(); // ubs:ignore — production missing-PGID script over this test's owned path
        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
        assert_eq!(
            parse_remote_timeout_kill_output(&stdout),
            Some(RemoteTimeoutCleanup::Unverified),
            "missing identity must preserve quarantine: {stdout}"
        );
    }

    #[test]
    fn test_build_remote_command_keeps_non_special_env_values() {
        let _guard = test_guard!();
        let mut overrides = HashMap::new();
        overrides.insert("RUSTFLAGS".to_string(), "-C target-cpu=native".to_string());

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_env_allowlist(vec!["RUSTFLAGS".to_string()])
        .with_env_overrides(overrides);

        let command = pipeline.build_remote_command("cargo build", None);
        assert!(command.contains("RUSTFLAGS='-C target-cpu=native'"));
        assert!(command.contains("cargo build"));
    }

    #[test]
    fn test_build_remote_command_preserves_inline_env_before_toolchain_runner() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_compilation_kind(Some(CompilationKind::CargoBuild));

        let command = pipeline.build_remote_command(
            "RUSTFLAGS='-C target-cpu=native' cargo build",
            Some(&ToolchainInfo::new("nightly", None, "")),
        );

        assert!(
            command.contains("RUSTFLAGS='-C target-cpu=native' rustup run nightly cargo build")
        );
        assert!(!command.contains("rustup run nightly RUSTFLAGS="));
    }

    #[test]
    fn test_build_remote_command_records_remote_pgid_for_cancellation() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_build_id(Some(42));

        let command = pipeline.build_remote_command("cargo test --no-run", None);
        let remote_pgid_file = pipeline
            .remote_pgid_file_path()
            .expect("build_id should enable remote pgid tracking");

        assert!(command.contains("/tmp/rch-run/"));
        assert!(!command.contains("/.rch-run/"));
        assert!(command.contains("RCH_REMOTE_PROCESS_V1:"));
        assert!(command.contains("rch_remote_record \"$1\" \"$4\""));
        assert!(command.contains("setsid sh -c"));
        assert!(command.contains(&remote_pgid_file));
    }

    #[test]
    fn test_build_id_path_uses_group_kill_watchdog_not_foreground_timeout() {
        // The pgid-tracked path must group-kill the whole session at the wall-clock
        // cap (so a livelocked test binary + fixtures are reaped together), NOT use
        // `timeout --foreground` (which only kills its direct child -> 20-45h orphans).
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_build_id(Some(7))
        .with_compilation_kind(Some(CompilationKind::CargoTest));

        let command = pipeline.build_remote_command("cargo test", None);

        // Watchdog group-kill of the recorded pgid (no `--`: dash mishandles it).
        assert!(
            command.contains("kill -KILL -\"$__p\""),
            "build_id path must SIGKILL the whole process group: {command}"
        );
        assert!(
            !command.contains("kill -KILL -- -"),
            "must not use the `--` form (broken in dash): {command}"
        );
        // The default cargo-test cap (1800s) is passed to the watchdog as an arg.
        assert!(
            command.contains(&format!("1800 {} 7 sh -lc", pipeline.deadline_marker)),
            "watchdog must receive the test timeout (1800s): {command}"
        );
        assert!(
            command.contains("wait \"$__c\""),
            "watchdog must wait on the job"
        );
        // The build_id path must NOT shell out to `timeout --foreground` (the bug).
        assert!(
            !command.contains("--foreground"),
            "build_id path must not use timeout --foreground: {command}"
        );
        // The timer subshell must detach stdio: its orphaned `sleep` otherwise
        // holds the SSH channel's pipe FDs open after a successful build, so
        // the session lasts the full cap and the client misreports a timeout (#20).
        assert!(
            command.contains(") >/dev/null 2>&1 </dev/null &"),
            "watchdog timer subshell must redirect stdio away from the channel: {command}"
        );
    }

    /// Functional proof: a SIGTERM-ignoring grandchild that the test harness forked
    /// is fully reaped by the watchdog at the cap. Under the old `timeout
    /// --foreground` behavior the grandchild would orphan and survive. Linux-only
    /// (needs setsid + process-group signaling); the build/CI workers are Linux.
    #[cfg(target_os = "linux")]
    #[test]
    fn test_watchdog_reaps_forking_orphan_at_cap() {
        use std::process::Command;
        use std::time::{Duration, Instant};

        if !Command::new("sh") // ubs:ignore — fixed test prerequisite probe, no interpolated input
            .arg("-c")
            .arg("command -v setsid")
            .output()
            .map(|o| o.status.success())
            .unwrap_or(false)
        {
            eprintln!("setsid unavailable; skipping functional watchdog test");
            return;
        }

        let dir = tempfile::tempdir().unwrap().keep();
        let pgf = dir.join("job.pgid");
        let marker = dir.join("grandchild-alive");

        // Job forks a grandchild that IGNORES SIGTERM and loops forever (a livelock
        // that only a group SIGKILL can stop), then waits on it.
        let inner = format!(
            "( trap '' TERM; touch '{}'; while true; do sleep 1; done ) & gc=$!; trap '' TERM; wait $gc",
            marker.display()
        );

        let mut child = Command::new("setsid")
            .arg("sh")
            .arg("-c")
            .arg(remote_build_watchdog_script())
            .arg("rch-build")
            .arg(pgf.to_str().unwrap())
            .arg("2") // 2-second cap
            .arg("RCH_TEST_DEADLINE")
            .arg("42")
            .arg("sh")
            .arg("-lc")
            .arg(&inner)
            // Detach all stdio: otherwise the forked grandchild inherits libtest's
            // captured stdout pipe and the harness blocks waiting for EOF.
            .stdin(std::process::Stdio::null())
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null())
            .spawn()
            .unwrap();

        // Grandchild must come up.
        let start = Instant::now();
        while !marker.exists() && start.elapsed() < Duration::from_secs(5) {
            std::thread::sleep(Duration::from_millis(50));
        }
        assert!(marker.exists(), "grandchild should have started");
        let pgid = recorded_remote_pgid(&std::fs::read_to_string(&pgf).unwrap(), 42);
        assert!(pgid > 1, "recorded a real pgid");

        // The session leader must die from the cap within a few seconds.
        let mut exited = false;
        let wstart = Instant::now();
        while wstart.elapsed() < Duration::from_secs(8) {
            if matches!(child.try_wait(), Ok(Some(_))) {
                exited = true;
                break;
            }
            std::thread::sleep(Duration::from_millis(100));
        }
        // If the deadline failed, cleanup still needs the exact live identity.
        if !exited {
            let _ = Command::new("sh") // ubs:ignore — production cleanup for the test-owned identity
                .arg("-c")
                .arg(remote_timeout_kill_script(pgf.to_str().unwrap(), 42))
                .status();
        }
        let _ = child.wait();

        assert!(
            exited,
            "watchdog should have killed the session at the ~2s cap"
        );

        // The whole process group (incl. the TERM-ignoring grandchild) must be gone.
        std::thread::sleep(Duration::from_millis(300));
        let probe = Command::new("sh") // ubs:ignore — read-only process-table probe for the test-owned group
            .arg("-c")
            .arg(format!(
                "{}\nrch_pgid={pgid}; rch_group_state",
                rch_common::REMOTE_PROCESS_IDENTITY_SCRIPT
            ))
            .status()
            .unwrap();
        assert!(
            probe.success(),
            "process group {pgid} (with the SIGTERM-ignoring grandchild) must be fully reaped"
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn test_watchdog_cancellation_keeps_identity_until_descendants_stop() {
        use std::process::{Command, Stdio};
        use std::time::{Duration, Instant};

        let dir = tempfile::tempdir().unwrap().keep();
        let pgf = dir.join("job.pgid");
        let descendant_file = dir.join("descendant.pid");
        let mut child = Command::new("setsid")
            .arg("sh")
            .arg("-c")
            .arg(remote_build_watchdog_script())
            .arg("rch-build")
            .arg(&pgf)
            .arg("15")
            .arg("RCH_TEST_DEADLINE")
            .arg("42")
            .arg("sh")
            .arg("-c")
            .arg("trap '' TERM; sleep 15 & printf '%s\\n' \"$!\" > \"$1\"; wait")
            .arg("rch-descendant")
            .arg(&descendant_file)
            .stdin(Stdio::null())
            .stdout(Stdio::null())
            .stderr(Stdio::null())
            .spawn()
            .unwrap();
        let started = Instant::now();
        while !descendant_file.exists() && started.elapsed() < Duration::from_secs(5) {
            std::thread::sleep(Duration::from_millis(20));
        }
        let descendant: u32 = std::fs::read_to_string(descendant_file)
            .unwrap()
            .trim()
            .parse()
            .unwrap();
        let output = Command::new("sh") // ubs:ignore — shared cancellation protocol and test-owned record
            .arg("-c")
            .arg(format!(
                "{}\nrch_remote_cancel \"$1\" 42 term",
                rch_common::REMOTE_PROCESS_IDENTITY_SCRIPT
            ))
            .arg("rch-cancel")
            .arg(&pgf)
            .output()
            .unwrap();
        let cancelled = output.status.success();
        if !cancelled {
            let _ = Command::new("sh") // ubs:ignore — bounded cleanup of this test's owned group
                .arg("-c")
                .arg(remote_timeout_kill_script(pgf.to_str().unwrap(), 42))
                .status();
        }
        let status = child.wait().unwrap();
        assert!(cancelled, "verified cancellation must complete: {output:?}");
        assert!(!status.success(), "watchdog must terminate with its group");
        match std::fs::read_to_string(format!("/proc/{descendant}/stat")) {
            Ok(stat) => assert!(
                matches!(
                    stat.rsplit_once(") ").unwrap().1.chars().next(),
                    Some('Z' | 'X')
                ),
                "TERM-ignoring descendant must no longer execute: {stat}"
            ),
            Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::NotFound),
        }
    }

    /// Functional proof of the #20 fix: after a SUCCESSFUL fast job, the
    /// watchdog session must release its stdout pipe (EOF) immediately — long
    /// before the timeout cap. Before the fix, the timer subshell's orphaned
    /// `sleep` inherited the pipe write-end, so EOF only arrived when the full
    /// cap expired and every remote build appeared to hang for build_timeout_sec.
    #[cfg(target_os = "linux")]
    #[test]
    fn test_watchdog_releases_output_pipe_immediately_on_success() {
        use std::io::Read;
        use std::process::Command;
        use std::time::{Duration, Instant};

        let dir = tempfile::tempdir().unwrap().keep();
        let pgf = dir.join("job.pgid");

        // Generous 20s cap; the job itself completes instantly. EOF must NOT
        // wait for the cap.
        let start = Instant::now();
        let mut child = Command::new("setsid")
            .arg("sh")
            .arg("-c")
            .arg(remote_build_watchdog_script())
            .arg("rch-build")
            .arg(pgf.to_str().unwrap())
            .arg("20")
            .arg("RCH_TEST_DEADLINE")
            .arg("42")
            .arg("sh")
            .arg("-c")
            .arg("echo job-done")
            .stdin(std::process::Stdio::null())
            .stdout(std::process::Stdio::piped())
            .stderr(std::process::Stdio::null())
            .spawn()
            .unwrap();

        let mut out = String::new();
        child
            .stdout
            .take()
            .unwrap()
            .read_to_string(&mut out)
            .unwrap();
        let eof_after = start.elapsed();
        let status = child.wait().unwrap();

        assert!(status.success(), "fast job must exit 0: {status:?}");
        assert!(out.contains("job-done"), "job output must arrive: {out:?}");
        assert!(
            eof_after < Duration::from_secs(5),
            "stdout EOF must arrive promptly after job success, not at the \
             timeout cap (took {eof_after:?})"
        );
        let pgid = recorded_remote_pgid(&std::fs::read_to_string(pgf).unwrap(), 42);
        let timer_state = Command::new("sh") // ubs:ignore — read-only observation of this test's own group
            .arg("-c")
            .arg(format!(
                "{}\nrch_pgid={pgid}; rch_group_state",
                rch_common::REMOTE_PROCESS_IDENTITY_SCRIPT
            ))
            .status()
            .unwrap();
        assert!(
            timer_state.success(),
            "successful jobs must reap the timer's sleep"
        );
    }

    #[test]
    fn test_remote_path_with_custom_remote_base() {
        let _guard = test_guard!();
        let config = TransferConfig {
            remote_base: "/var/rch-builds".to_string(),
            ..Default::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/home/user/project"),
            "myproject".to_string(),
            "abc123".to_string(),
            config,
        );

        assert_eq!(pipeline.remote_path(), "/var/rch-builds/myproject/abc123");
    }

    #[test]
    fn test_remote_path_with_home_directory_base() {
        let _guard = test_guard!();
        let config = TransferConfig {
            remote_base: "/home/builder/.rch".to_string(),
            ..Default::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/workspace/project"),
            "project".to_string(),
            "def456".to_string(),
            config,
        );

        assert_eq!(pipeline.remote_path(), "/home/builder/.rch/project/def456");
    }

    #[test]
    fn test_remote_path_override_absolute() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/workspace/project"),
            "project".to_string(),
            "def456".to_string(),
            TransferConfig::default(),
        )
        .with_remote_path_override("/data/projects/project");

        assert_eq!(pipeline.remote_path(), "/data/projects/project");
    }

    #[test]
    fn test_remote_path_override_rejects_relative() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/workspace/project"),
            "project".to_string(),
            "def456".to_string(),
            TransferConfig::default(),
        )
        .with_remote_path_override("relative/path");

        assert_eq!(pipeline.remote_path(), "/data/tmp/rch/project/def456");
    }

    #[test]
    fn test_is_windows_drive_abs_path_matches_drive_letter_forms() {
        assert!(is_windows_drive_abs_path("C:/rch"));
        assert!(is_windows_drive_abs_path("C:\\rch"));
        assert!(is_windows_drive_abs_path("d:/x/y"));
        assert!(!is_windows_drive_abs_path("/data/projects"));
        assert!(!is_windows_drive_abs_path("relative/path"));
        assert!(!is_windows_drive_abs_path("C:")); // no separator
        assert!(!is_windows_drive_abs_path("1:/rch")); // non-alpha drive
    }

    #[test]
    fn windows_artifact_filters_preserve_exact_outputs_and_exclude_cache_ancestors() {
        let includes = ["*/release/**", "bin/app[[]dev[]].exe", "*.obj"]
            .map(|pattern| WindowsArtifactFilter::new(pattern, true).unwrap());
        let excludes = ["*/release/incremental/", "*.d", "/src/"]
            .map(|pattern| WindowsArtifactFilter::new(pattern, false).unwrap());
        let inventory = b"4 ./x86_64-pc-windows-msvc/release/app.exe\0\
            8 ./x86_64-pc-windows-msvc/release/incremental/state\0\
            1 ./x86_64-pc-windows-msvc/release/app.d\0\
            2 ./bin/app[dev].exe\0\
            3 ./bin/appd.exe\0\
            5 ./root.obj\0\
            6 ./src/source.obj\0";
        let selected = windows_artifact_selection(inventory, &includes, &excludes).unwrap();
        assert_eq!(
            selected.into_keys().collect::<Vec<_>>(),
            [
                "bin/app[dev].exe",
                "root.obj",
                "x86_64-pc-windows-msvc/release/app.exe"
            ]
        );
        assert!(windows_artifact_selection(b"4 ./bin/app.exe", &includes, &excludes).is_err());
        for path in ["../outside", "/outside", "C:/outside", "a/../../b", "a\\b"] {
            let inventory = format!("1 {path}\0");
            assert!(
                windows_artifact_selection(inventory.as_bytes(), &includes, &excludes).is_err()
            );
        }
    }

    #[test]
    fn windows_artifact_staging_preserves_source_guard_and_recovery_phase_reference() {
        let _guard = test_guard!();
        let caller = tempfile::tempdir().unwrap();
        let stage = tempfile::tempdir().unwrap();
        std::fs::create_dir(caller.path().join("src")).unwrap();
        std::fs::create_dir(caller.path().join("bin")).unwrap();
        std::fs::write(caller.path().join("Cargo.toml"), b"local manifest").unwrap();
        std::fs::write(caller.path().join("main.c"), b"local source").unwrap();
        std::fs::write(caller.path().join(".rchignore"), b"ignored/\n").unwrap();
        let pipeline = TransferPipeline::new(
            caller.path().to_path_buf(),
            "project".into(),
            "abc".into(),
            TransferConfig::default(),
        )
        .with_local_root(stage.path().to_path_buf());
        let patterns = default_c_cpp_artifact_patterns();
        let excludes =
            pipeline.local_source_roots_to_exclude(&allowed_artifact_roots(&patterns), &patterns);
        assert!(excludes.contains(&"/Cargo.toml".to_string()));
        assert!(excludes.contains(&"/main.c".to_string()));
        assert!(excludes.contains(&"/src/".to_string()));
        assert!(!excludes.contains(&"/bin/".to_string()));
        assert!(
            pipeline
                .get_retrieval_excludes(&patterns)
                .contains(&"ignored/".to_string())
        );
        let includes = patterns
            .iter()
            .map(|pattern| WindowsArtifactFilter::new(pattern, true).unwrap())
            .collect::<Vec<_>>();
        let filters = excludes
            .iter()
            .map(|pattern| WindowsArtifactFilter::new(pattern, false).unwrap())
            .collect::<Vec<_>>();
        let selected = windows_artifact_selection(
            b"4 ./Cargo.toml\0\
            4 ./main.c\0\
            4 ./src/main.c\0\
            4 ./bin/app.exe\0\
            4 ./fresh.exe\0",
            &includes,
            &filters,
        )
        .unwrap();
        assert_eq!(
            selected.into_keys().collect::<Vec<_>>(),
            ["bin/app.exe", "fresh.exe"]
        );
        assert!(
            pipeline
                .validate_staged_artifact_paths(&[PathBuf::from("main.c")], &patterns)
                .is_err()
        );
        pipeline
            .validate_staged_artifact_paths(
                &[PathBuf::from("bin/app.exe"), PathBuf::from("fresh.exe")],
                &patterns,
            )
            .unwrap();
        let worker = worker_with_os(Some("linux"));
        for command in [
            pipeline.build_retrieve_command(&worker, "/remote", &patterns),
            pipeline.build_retrieve_streaming_command(&worker, "/remote", &patterns),
        ] {
            let args = command
                .as_std()
                .get_args()
                .map(|arg| arg.to_string_lossy().into_owned())
                .collect::<Vec<_>>();
            assert!(arg_pair_position(&args, "--exclude", "/main.c").is_some());
            assert!(arg_pair_position(&args, "--exclude", "/src/").is_some());
            assert_eq!(
                args.last().unwrap(),
                &format!("{}/", stage.path().display())
            );
        }

        // Recovery reconstructs one base pipeline, then replaces this reference
        // for each persisted phase (notably an external CARGO_TARGET_DIR).
        let target = tempfile::tempdir().unwrap();
        std::fs::write(target.path().join("private.txt"), b"caller-owned").unwrap();
        std::fs::write(target.path().join(".rchignore"), b"private-build/\n").unwrap();
        std::fs::create_dir(target.path().join("x86_64-pc-windows-msvc")).unwrap();
        let recovered = pipeline.with_retrieval_reference_root(target.path().to_path_buf());
        assert!(
            recovered
                .validate_staged_artifact_paths(
                    &[PathBuf::from("private.txt")],
                    &["*".to_string()],
                )
                .is_err(),
            "the recovered target reference must protect its own existing files"
        );
        let patterns = vec!["*/release/**".to_string()];
        let (includes, excludes) = recovered.artifact_retrieval_filters(&patterns).unwrap();
        // A wildcard first component keeps target directories traversable;
        // root-level files are refused by the complete include pattern, not
        // necessarily by an explicit source-root exclusion string.
        let selected = windows_artifact_selection(
            b"4 ./private.txt\0\
            4 ./Cargo.toml\0\
            4 ./x86_64-pc-windows-msvc/release/app.exe\0\
            4 ./private-build/release/secret.exe\0\
            4 ./ignored/release/kept.exe\0",
            &includes,
            &excludes,
        )
        .unwrap();
        assert_eq!(
            selected.into_keys().collect::<Vec<_>>(),
            [
                "ignored/release/kept.exe",
                "x86_64-pc-windows-msvc/release/app.exe"
            ]
        );
        for path in [
            "private.txt",
            "Cargo.toml",
            "private-build/release/secret.exe",
        ] {
            assert!(
                recovered
                    .validate_staged_artifact_paths(&[PathBuf::from(path)], &patterns)
                    .is_err(),
                "recovered publication must reject {path}"
            );
        }
        recovered
            .validate_staged_artifact_paths(
                &[
                    PathBuf::from("ignored/release/kept.exe"),
                    PathBuf::from("x86_64-pc-windows-msvc/release/app.exe"),
                ],
                &patterns,
            )
            .unwrap();
        assert_eq!(recovered.project_root, stage.path());
    }

    #[tokio::test]
    async fn windows_artifact_empty_policy_uses_tar_routing_in_both_entrypoints() {
        let _guard = test_guard!();
        let destination = tempfile::tempdir().unwrap();
        let pipeline = TransferPipeline::new(
            destination.path().to_path_buf(),
            "project".into(),
            "abc".into(),
            TransferConfig::default(),
        )
        .with_worker_platform(WorkerPlatform::Windows);
        let worker = worker_with_os(Some("windows"));
        for result in [
            pipeline.retrieve_artifacts(&worker, &[]).await,
            pipeline
                .retrieve_artifacts_streaming(&worker, &[], |_| {})
                .await,
        ] {
            let result = result.unwrap();
            assert_eq!(result.matched_regular_files, Some(0));
            assert_eq!(result.stats.files_transferred, 0);
        }
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn windows_artifact_real_tar_returns_selected_files_and_hardlinks_only() {
        let _guard = test_guard!();
        let source = tempfile::tempdir().unwrap();
        let destination = tempfile::tempdir().unwrap();
        for path in ["bin", "src", "target/debug", "peer/target/debug"] {
            std::fs::create_dir_all(source.path().join(path)).unwrap();
        }
        for (path, bytes) in [
            ("bin/app[dev] $cash.exe", b"actual executable".as_slice()),
            ("bin/app.pdb", b"debug symbols".as_slice()),
            ("target/debug/unrequested.exe", b"other target".as_slice()),
            ("peer/target/debug/foreign.exe", b"nested target".as_slice()),
            ("Cargo.toml", b"remote manifest".as_slice()),
            ("src/lib.rs", b"remote source".as_slice()),
        ] {
            std::fs::write(source.path().join(path), bytes).unwrap();
        }
        std::fs::hard_link(
            source.path().join("bin/app.pdb"),
            source.path().join("bin/app-link.pdb"),
        )
        .unwrap();
        std::fs::write(destination.path().join("Cargo.toml"), b"local manifest").unwrap();
        let mut inventory = Vec::new();
        let mut shell = Command::new("sh");
        shell.arg("-s");
        run_windows_artifact_process(
            shell,
            &windows_artifact_inventory_script(source.path().to_str().unwrap()),
            &mut inventory,
            1024 * 1024,
            TokioInstant::now() + Duration::from_secs(10),
            None,
            &mut |_| {},
        )
        .await
        .unwrap();
        let includes = ["bin/app[[]dev[]] $cash.exe", "bin/*.pdb"]
            .map(|pattern| WindowsArtifactFilter::new(pattern, true).unwrap());
        let selected = windows_artifact_selection(&inventory, &includes, &[]).unwrap();
        assert_eq!(selected.len(), 3);
        let archive = tempfile::NamedTempFile::new().unwrap();
        let mut output = tokio::fs::File::from_std(archive.reopen().unwrap());
        let mut shell = Command::new("sh");
        shell.arg("-s");
        let bytes = run_windows_artifact_process(
            shell,
            &windows_artifact_archive_script(source.path().to_str().unwrap(), &selected),
            &mut output,
            1024 * 1024,
            TokioInstant::now() + Duration::from_secs(10),
            None,
            &mut |_| {},
        )
        .await
        .unwrap();
        assert!(bytes > 0);
        drop(output);
        unpack_windows_artifact_archive(
            archive.path(),
            destination.path(),
            &selected,
            TokioInstant::now() + Duration::from_secs(10),
            None,
        )
        .unwrap();
        for path in selected.keys() {
            assert_eq!(
                std::fs::read(destination.path().join(path)).unwrap(),
                std::fs::read(source.path().join(path)).unwrap()
            );
        }
        assert_eq!(
            std::fs::read(destination.path().join("Cargo.toml")).unwrap(),
            b"local manifest"
        );
        assert!(!destination.path().join("src").exists());
        assert!(!destination.path().join("target").exists());
        assert!(!destination.path().join("peer").exists());
        let mut incomplete = selected;
        incomplete.insert("bin/missing.exe".to_string(), 4);
        assert!(
            unpack_windows_artifact_archive(
                archive.path(),
                destination.path(),
                &incomplete,
                TokioInstant::now() + Duration::from_secs(10),
                None
            )
            .is_err()
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn windows_artifact_transport_propagates_remote_failure_and_reaps_cancelled_child() {
        let _guard = test_guard!();
        let mut command = Command::new("sh");
        command.arg("-s");
        let mut output = Vec::new();
        let error = run_windows_artifact_process(
            command,
            "printf partial; printf 'missing requested file' >&2; exit 7\n",
            &mut output,
            1024,
            TokioInstant::now() + Duration::from_secs(5),
            None,
            &mut |_| {},
        )
        .await
        .unwrap_err();
        assert!(error.to_string().contains("Some(7)"), "{error}");
        assert!(
            error.to_string().contains("missing requested file"),
            "{error}"
        );

        let retained = tempfile::tempdir().unwrap();
        let pid_path = retained.path().join("child.pid");
        let script = format!(
            "printf '%s' \"$$\" > {}\nwhile :; do :; done\n",
            escape(Cow::from(pid_path.to_str().unwrap()))
        );
        let (cancel, receiver) = tokio::sync::watch::channel(false);
        let mut command = Command::new("sh");
        command.arg("-s");
        let mut output = Vec::new();
        let mut on_line = |_line: &str| {};
        let operation = run_windows_artifact_process(
            command,
            &script,
            &mut output,
            1024,
            TokioInstant::now() + Duration::from_secs(5),
            Some(receiver),
            &mut on_line,
        );
        let cancel_when_started = async {
            let deadline = TokioInstant::now() + Duration::from_secs(3);
            while !pid_path.exists() {
                assert!(TokioInstant::now() < deadline, "fixture did not start");
                tokio::time::sleep(Duration::from_millis(10)).await;
            }
            cancel.send(true).unwrap();
        };
        let (result, ()) = tokio::join!(operation, cancel_when_started);
        assert!(result.unwrap_err().is::<RetrievalCancelled>());
        let pid = std::fs::read_to_string(pid_path).unwrap();
        let status = std::process::Command::new("kill")
            .args(["-0", &pid])
            .status()
            .unwrap();
        assert!(!status.success(), "cancelled transport child remains alive");
    }

    #[cfg(unix)]
    #[test]
    fn windows_artifact_archive_refuses_unsafe_members_and_expansion_before_publication() {
        use std::io::Write;

        for case in [
            "unselected",
            "symlink",
            "outside-hardlink",
            "future-hardlink",
            "gzip-padding",
            "local-symlink",
        ] {
            let retained = tempfile::tempdir().unwrap();
            let destination = retained.path().join("destination");
            std::fs::create_dir_all(destination.join("bin")).unwrap();
            std::fs::write(destination.join("bin/app.exe"), b"old!").unwrap();
            let archive = tempfile::NamedTempFile::new().unwrap();
            let encoder = flate2::write::GzEncoder::new(
                archive.reopen().unwrap(),
                flate2::Compression::fast(),
            );
            let mut builder = tar::Builder::new(encoder);
            let mut header = tar::Header::new_gnu();
            header.set_mode(0o644);
            header.set_size(4);
            header.set_cksum();
            builder
                .append_data(&mut header, "bin/app.exe", b"new!".as_slice())
                .unwrap();
            let mut selected = std::collections::BTreeMap::from([("bin/app.exe".to_string(), 4)]);
            if matches!(case, "symlink" | "outside-hardlink" | "future-hardlink") {
                selected.insert("bin/linked.exe".to_string(), 4);
                let mut header = tar::Header::new_gnu();
                header.set_entry_type(if case == "symlink" {
                    tar::EntryType::Symlink
                } else {
                    tar::EntryType::Link
                });
                header.set_size(0);
                header.set_mode(0o644);
                header
                    .set_link_name(if case == "future-hardlink" {
                        "bin/future.exe"
                    } else {
                        "../outside"
                    })
                    .unwrap();
                header.set_cksum();
                builder
                    .append_data(&mut header, "bin/linked.exe", std::io::empty())
                    .unwrap();
                if case == "future-hardlink" {
                    selected.insert("bin/future.exe".to_string(), 4);
                }
            } else if case == "unselected" {
                builder
                    .append_data(&mut header, "Cargo.toml", b"evil".as_slice())
                    .unwrap();
            }
            let mut encoder = builder.into_inner().unwrap();
            if case == "gzip-padding" {
                std::io::copy(
                    &mut std::io::Read::take(std::io::repeat(0), 2 * 1024 * 1024),
                    &mut encoder,
                )
                .unwrap();
            }
            encoder.flush().unwrap();
            encoder.finish().unwrap();
            let publication_root = if case == "local-symlink" {
                let alias = retained.path().join("alias");
                std::fs::create_dir(&alias).unwrap();
                std::os::unix::fs::symlink(destination.join("bin"), alias.join("bin")).unwrap();
                alias
            } else {
                destination.clone()
            };
            let error = unpack_windows_artifact_archive(
                archive.path(),
                &publication_root,
                &selected,
                TokioInstant::now() + Duration::from_secs(5),
                None,
            )
            .unwrap_err();
            if case == "gzip-padding" {
                assert!(
                    error.to_string().contains("expanded size bound"),
                    "{error:#}"
                );
            }
            assert_eq!(
                std::fs::read(destination.join("bin/app.exe")).unwrap(),
                b"old!",
                "{case}: validation must precede every local write"
            );
            assert!(!destination.join("Cargo.toml").exists(), "{case}");
        }
    }

    #[test]
    fn test_remote_path_override_accepts_windows_drive_path() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/workspace/project"),
            "project".to_string(),
            "def456".to_string(),
            TransferConfig::default(),
        )
        .with_worker_platform(WorkerPlatform::Windows)
        .with_remote_path_override("C:/rch/project/def456");

        // The drive-letter override is accepted verbatim, not rejected as
        // "not absolute" (which would fall back to the default remote path and
        // send the target retrieval to the wrong directory).
        assert_eq!(pipeline.remote_path(), "C:/rch/project/def456");
    }

    #[test]
    fn test_build_remote_command_windows_skips_pgid_watchdog() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/workspace/project"),
            "project".to_string(),
            "def456".to_string(),
            TransferConfig::default(),
        )
        .with_worker_platform(WorkerPlatform::Windows)
        .with_build_id(Some(42));

        let cmd = pipeline.build_remote_command("cargo build", None);
        // Windows must NOT get the Unix pgid/setsid watchdog: its backgrounded
        // timer subshell keeps the SSH channel open so a successful build hangs
        // until timeout (the #20 failure mode re-triggered on Windows).
        assert!(
            !cmd.contains("setsid"),
            "windows cmd must not use setsid: {cmd}"
        );
        assert!(
            !cmd.contains("kill -KILL"),
            "windows cmd must not arm a group-kill watchdog: {cmd}"
        );
        assert!(
            cmd.contains("cargo build"),
            "windows cmd must still run the command: {cmd}"
        );
        assert!(
            cmd.contains("C:/rch/project/def456"),
            "windows cmd must cd into the C:/rch build root: {cmd}"
        );
    }

    #[test]
    fn test_build_remote_command_posix_keeps_pgid_watchdog() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/workspace/project"),
            "project".to_string(),
            "def456".to_string(),
            TransferConfig::default(),
        )
        .with_build_id(Some(42)); // default platform = Posix

        let cmd = pipeline.build_remote_command("cargo build", None);
        // Posix keeps the in-session pgid watchdog (setsid + group kill) so
        // this change is Windows-only and leaves the fleet path untouched.
        assert!(
            cmd.contains("setsid"),
            "posix cmd should still arm the pgid watchdog: {cmd}"
        );
    }

    // ==========================================================================
    // .rchignore Parser Tests
    // ==========================================================================

    #[test]
    fn test_parse_rchignore_content_basic() {
        let _guard = test_guard!();
        let content = "target/\n.git/\nnode_modules/";
        let patterns = parse_rchignore_content(content);
        assert_eq!(patterns, vec!["target/", ".git/", "node_modules/"]);
    }

    #[test]
    fn test_parse_rchignore_content_with_comments() {
        let _guard = test_guard!();
        let content = r#"# Build artifacts
target/
# Git metadata
.git/
# Node stuff
node_modules/"#;
        let patterns = parse_rchignore_content(content);
        assert_eq!(patterns, vec!["target/", ".git/", "node_modules/"]);
    }

    #[test]
    fn test_parse_rchignore_content_with_blank_lines() {
        let _guard = test_guard!();
        let content = r#"
target/

.git/


node_modules/
"#;
        let patterns = parse_rchignore_content(content);
        assert_eq!(patterns, vec!["target/", ".git/", "node_modules/"]);
    }

    #[test]
    fn test_parse_rchignore_content_trims_whitespace() {
        let _guard = test_guard!();
        let content = "  target/  \n\t.git/\t\n   node_modules/   ";
        let patterns = parse_rchignore_content(content);
        assert_eq!(patterns, vec!["target/", ".git/", "node_modules/"]);
    }

    #[test]
    fn test_parse_rchignore_content_empty() {
        let _guard = test_guard!();
        let content = "";
        let patterns = parse_rchignore_content(content);
        assert!(patterns.is_empty());
    }

    #[test]
    fn test_parse_rchignore_content_only_comments() {
        let _guard = test_guard!();
        let content = "# This is a comment\n# Another comment";
        let patterns = parse_rchignore_content(content);
        assert!(patterns.is_empty());
    }

    #[test]
    fn test_parse_rchignore_content_preserves_negation_literal() {
        let _guard = test_guard!();
        // Note: Unlike .gitignore, negation is not supported, ! is literal
        let content = "target/\n!important.txt\n.git/";
        let patterns = parse_rchignore_content(content);
        assert_eq!(patterns, vec!["target/", "!important.txt", ".git/"]);
    }

    #[test]
    fn test_parse_rchignore_file_not_found() {
        let _guard = test_guard!();
        let result = parse_rchignore(Path::new("/nonexistent/.rchignore"));
        assert!(result.is_err());
    }

    #[test]
    fn test_get_effective_excludes_without_rchignore() {
        let _guard = test_guard!();
        // When no .rchignore exists, should return config defaults + remote runtime guards.
        let config = TransferConfig::default();
        let default_excludes = config.exclude_patterns.clone();

        let pipeline = TransferPipeline::new(
            PathBuf::from("/nonexistent/project"),
            "project".to_string(),
            "hash".to_string(),
            config,
        );

        let effective = pipeline.get_effective_excludes();
        for pattern in &default_excludes {
            assert!(effective.contains(pattern));
        }
        assert!(effective.contains(&".git/".to_string()));
        assert!(
            !effective.contains(&".git/objects/".to_string()),
            "default upload excludes must avoid syncing a partial .git tree"
        );
        assert!(effective.contains(&".rch-target/".to_string()));
        assert!(effective.contains(&".rch-tmp/".to_string()));
        assert!(effective.contains(&".franken_whisper/tools/ffmpeg/".to_string()));
        assert!(effective.contains(&".venv/".to_string()));
        assert!(effective.contains(&".venv-*/".to_string()));
    }

    #[test]
    fn test_parse_worktree_gitdir_pointer_valid() {
        let _guard = test_guard!();
        let pointer =
            parse_worktree_gitdir_pointer("gitdir: /data/projects/rch/.git/worktrees/feature-x\n")
                .expect("linked worktree pointer");
        assert_eq!(
            pointer.gitdir,
            "/data/projects/rch/.git/worktrees/feature-x"
        );
    }

    #[test]
    fn test_parse_worktree_gitdir_pointer_tolerates_whitespace_and_crlf() {
        let _guard = test_guard!();
        // Leading blank line, CRLF, and surrounding spaces must all be tolerated.
        let pointer =
            parse_worktree_gitdir_pointer("\r\n  gitdir:   ../super/.git/worktrees/wt \r\n")
                .expect("linked worktree pointer");
        assert_eq!(pointer.gitdir, "../super/.git/worktrees/wt");
    }

    #[test]
    fn test_parse_worktree_gitdir_pointer_rejects_non_pointer() {
        let _guard = test_guard!();
        // A real `.git` dir never has file contents; a malformed / empty pointer
        // must not be treated as a worktree.
        assert!(parse_worktree_gitdir_pointer("").is_none());
        assert!(parse_worktree_gitdir_pointer("gitdir:").is_none());
        assert!(parse_worktree_gitdir_pointer("gitdir:   ").is_none());
        assert!(parse_worktree_gitdir_pointer("ref: refs/heads/main").is_none());
    }

    #[test]
    fn test_detect_linked_worktree_git_pointer_file_vs_dir_vs_absent() {
        let _guard = test_guard!();

        // 1) Linked worktree: `.git` is a FILE with a gitdir pointer -> detected.
        let wt = tempfile::tempdir().expect("create temp dir");
        std::fs::write(
            wt.path().join(".git"),
            "gitdir: /data/projects/rch/.git/worktrees/wt\n",
        )
        .expect("write .git file");
        let detected =
            detect_linked_worktree_git_pointer(wt.path()).expect("worktree should be detected");
        assert_eq!(detected.gitdir, "/data/projects/rch/.git/worktrees/wt");

        // 2) Normal repo: `.git` is a DIRECTORY -> not a linked worktree.
        let normal = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(normal.path().join(".git")).expect("create .git dir");
        assert!(detect_linked_worktree_git_pointer(normal.path()).is_none());

        // 3) No VCS at all -> None.
        let bare = tempfile::tempdir().expect("create temp dir");
        assert!(detect_linked_worktree_git_pointer(bare.path()).is_none());
    }

    #[test]
    fn test_get_effective_excludes_neutralizes_worktree_git_file() {
        let _guard = test_guard!();
        // A linked worktree's dangling `.git` FILE must be excluded from upload,
        // in ADDITION to the normal `.git/` directory exclude.
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        std::fs::write(
            temp_dir.path().join(".git"),
            "gitdir: /data/projects/super/.git/worktrees/wt\n",
        )
        .expect("write worktree .git file");

        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        );

        let effective = pipeline.get_effective_excludes();
        assert!(
            effective.contains(&"/.git".to_string()),
            "worktree `.git` file must be excluded from upload; got {effective:?}"
        );
        // The directory-form exclude is still present for the normal case.
        assert!(effective.contains(&".git/".to_string()));
    }

    #[test]
    fn test_get_effective_excludes_no_worktree_exclude_for_normal_repo() {
        let _guard = test_guard!();
        // A normal repo (`.git` directory) must NOT get the `/.git` file exclude.
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp_dir.path().join(".git")).expect("create .git dir");

        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        );

        let effective = pipeline.get_effective_excludes();
        assert!(
            !effective.contains(&"/.git".to_string()),
            "normal repo must not get the worktree file exclude; got {effective:?}"
        );
        assert!(effective.contains(&".git/".to_string()));
    }

    #[test]
    fn test_get_effective_excludes_with_rchignore() {
        let _guard = test_guard!();
        // Create a temp dir with .rchignore
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        let rchignore_path = temp_dir.path().join(".rchignore");
        std::fs::write(&rchignore_path, "large_data/\nsecrets/").expect("write .rchignore");

        let config = TransferConfig::default();
        let default_excludes = config.exclude_patterns.clone();

        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "project".to_string(),
            "hash".to_string(),
            config,
        );

        let effective = pipeline.get_effective_excludes();
        for pattern in &default_excludes {
            assert!(effective.contains(pattern));
        }
        assert!(effective.contains(&".rch-target/".to_string()));
        assert!(effective.contains(&".rch-tmp/".to_string()));
        assert!(effective.contains(&".franken_whisper/tools/ffmpeg/".to_string()));
        assert!(effective.contains(&"large_data/".to_string()));
        assert!(effective.contains(&"secrets/".to_string()));
    }

    #[test]
    fn test_get_effective_excludes_deduplicates() {
        let _guard = test_guard!();
        // Create a temp dir with .rchignore that overlaps with defaults
        let temp_dir = tempfile::tempdir().expect("create temp dir");
        let rchignore_path = temp_dir.path().join(".rchignore");
        // "target/" is already in defaults
        std::fs::write(&rchignore_path, "target/\ncustom/").expect("write .rchignore");

        let config = TransferConfig::default();
        let default_excludes = config.exclude_patterns.clone();

        let pipeline = TransferPipeline::new(
            temp_dir.path().to_path_buf(),
            "project".to_string(),
            "hash".to_string(),
            config,
        );

        let effective = pipeline.get_effective_excludes();
        for pattern in &default_excludes {
            assert!(effective.contains(pattern));
        }
        assert!(effective.contains(&".rch-target/".to_string()));
        assert!(effective.contains(&".rch-tmp/".to_string()));
        assert!(effective.contains(&".franken_whisper/tools/ffmpeg/".to_string()));
        assert!(effective.contains(&"custom/".to_string()));
        // target/ should appear only once
        let target_count = effective.iter().filter(|p| *p == "target/").count();
        assert_eq!(target_count, 1);
        // Runtime guards should appear only once too.
        let runtime_target_count = effective.iter().filter(|p| *p == ".rch-target/").count();
        assert_eq!(runtime_target_count, 1);
        let runtime_tmp_count = effective.iter().filter(|p| *p == ".rch-tmp/").count();
        assert_eq!(runtime_tmp_count, 1);
        let runtime_ffmpeg_count = effective
            .iter()
            .filter(|p| *p == ".franken_whisper/tools/ffmpeg/")
            .count();
        assert_eq!(runtime_ffmpeg_count, 1);
    }

    #[test]
    fn test_get_effective_excludes_rewrites_legacy_core_dump_globs() {
        let _guard = test_guard!();
        let config = TransferConfig {
            exclude_patterns: vec![
                "target/".to_string(),
                "core.*".to_string(),
                ".core.*".to_string(),
                "core.[0-9]*".to_string(),
            ],
            ..TransferConfig::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/nonexistent/project"),
            "project".to_string(),
            "hash".to_string(),
            config,
        );

        let effective = pipeline.get_effective_excludes();

        assert!(!effective.contains(&"core.*".to_string()));
        assert!(!effective.contains(&".core.*".to_string()));
        assert!(effective.contains(&"core.[0-9]*".to_string()));
        assert!(effective.contains(&".core.[0-9]*".to_string()));
        assert_eq!(effective.iter().filter(|p| *p == "core.[0-9]*").count(), 1);
    }

    #[test]
    fn test_get_effective_excludes_rewrites_legacy_git_objects_exclude() {
        let _guard = test_guard!();
        let config = TransferConfig {
            exclude_patterns: vec![
                "target/".to_string(),
                ".git/objects/".to_string(),
                "node_modules/".to_string(),
            ],
            ..TransferConfig::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/nonexistent/project"),
            "project".to_string(),
            "hash".to_string(),
            config,
        );

        let effective = pipeline.get_effective_excludes();

        assert!(effective.contains(&".git/".to_string()));
        assert!(!effective.contains(&".git/objects/".to_string()));
        assert_eq!(effective.iter().filter(|p| *p == ".git/").count(), 1);
    }

    // ==========================================================================
    // Transfer Optimization Tests (bd-3hho)
    // ==========================================================================

    #[test]
    fn test_parse_rsync_total_size_standard() {
        let _guard = test_guard!();
        let output = "Number of files: 1,234
Total file size: 56,789,012 bytes
Total transferred file size: 1,234,567 bytes";
        assert_eq!(parse_rsync_total_size(output), Some(56789012));
    }

    #[test]
    fn test_parse_rsync_total_size_no_commas() {
        let _guard = test_guard!();
        let output = "Total file size: 123456 bytes";
        assert_eq!(parse_rsync_total_size(output), Some(123456));
    }

    #[test]
    fn test_parse_rsync_total_size_transferred() {
        let _guard = test_guard!();
        let output = "Total transferred file size: 9,876,543 bytes";
        assert_eq!(parse_rsync_total_size(output), Some(9876543));
    }

    #[test]
    fn test_parse_rsync_total_size_missing() {
        let _guard = test_guard!();
        let output = "sent 100 bytes received 200 bytes";
        assert_eq!(parse_rsync_total_size(output), None);
    }

    #[test]
    fn test_parse_rsync_total_files_standard() {
        let _guard = test_guard!();
        let output = "Number of files: 1,234 (reg: 1,000, dir: 234)
Total file size: 56,789,012 bytes";
        assert_eq!(parse_rsync_total_files(output), Some(1234));
    }

    #[test]
    fn test_parse_rsync_total_files_no_commas() {
        let _guard = test_guard!();
        let output = "Number of files: 456
Total file size: 123 bytes";
        assert_eq!(parse_rsync_total_files(output), Some(456));
    }

    #[test]
    fn test_parse_rsync_total_files_transferred() {
        let _guard = test_guard!();
        let output = "Number of regular files transferred: 789";
        assert_eq!(parse_rsync_total_files(output), Some(789));
    }

    #[test]
    fn test_parse_rsync_total_files_missing() {
        let _guard = test_guard!();
        let output = "Total file size: 100 bytes";
        assert_eq!(parse_rsync_total_files(output), None);
    }

    #[test]
    fn test_transfer_config_optimization_defaults() {
        let _guard = test_guard!();
        let config = TransferConfig::default();
        assert!(config.max_transfer_mb.is_none());
        assert!(config.max_transfer_time_ms.is_none());
        assert!(config.bwlimit_kbps.is_none());
        assert!(config.estimated_bandwidth_bps.is_none());
    }

    #[test]
    fn test_transfer_config_with_optimization_options() {
        let _guard = test_guard!();
        let config = TransferConfig {
            max_transfer_mb: Some(500),
            max_transfer_time_ms: Some(5000),
            bwlimit_kbps: Some(10000),
            estimated_bandwidth_bps: Some(10 * 1024 * 1024),
            ..Default::default()
        };
        assert_eq!(config.max_transfer_mb, Some(500));
        assert_eq!(config.max_transfer_time_ms, Some(5000));
        assert_eq!(config.bwlimit_kbps, Some(10000));
        assert_eq!(config.estimated_bandwidth_bps, Some(10 * 1024 * 1024));
    }

    #[test]
    fn test_effective_rsync_retry_config_uses_transfer_time_override() {
        let _guard = test_guard!();
        let transfer_config = TransferConfig {
            max_transfer_time_ms: Some(5000),
            retry: RetryConfig {
                total_timeout_ms: 30000,
                ..RetryConfig::default()
            },
            ..TransferConfig::default()
        };
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            transfer_config,
        );

        assert_eq!(
            pipeline.effective_rsync_retry_config().total_timeout_ms,
            5000
        );
    }

    #[test]
    fn test_artifact_retry_config_for_size_boundaries() {
        let _guard = test_guard!();
        let mut pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/artifact-budget"),
            "artifact-budget".to_string(),
            "selected".to_string(),
            TransferConfig::default(),
        );
        pipeline.transfer_config.retry.total_timeout_ms = 30_000;
        assert_eq!(
            pipeline.artifact_retry_config_for_size(0).total_timeout_ms,
            30_000
        );
        assert_eq!(
            pipeline
                .artifact_retry_config_for_size(31 * 1024 * 1024)
                .total_timeout_ms,
            61_000
        );
        assert_eq!(
            pipeline.artifact_retry_config_for_size(1).total_timeout_ms,
            31_000
        );
        pipeline.transfer_config.bwlimit_kbps = Some(64);
        assert_eq!(
            pipeline
                .artifact_retry_config_for_size(1024 * 1024)
                .total_timeout_ms,
            46_000
        );
        pipeline.transfer_config.bwlimit_kbps = Some(0);
        assert_eq!(
            pipeline
                .artifact_retry_config_for_size(1024 * 1024)
                .total_timeout_ms,
            31_000
        );
        pipeline.transfer_config.bwlimit_kbps = Some(u64::MAX);
        assert_eq!(
            pipeline
                .artifact_retry_config_for_size(1024 * 1024)
                .total_timeout_ms,
            31_000
        );
        pipeline.transfer_config.bwlimit_kbps = Some(1);
        assert_eq!(
            pipeline
                .artifact_retry_config_for_size(u64::MAX)
                .total_timeout_ms,
            TransferConfig::MAX_SYNC_TIMEOUT_MS
        );
        pipeline.transfer_config.retry.total_timeout_ms = 90_000;
        assert_eq!(
            pipeline.artifact_retry_config_for_size(0).total_timeout_ms,
            90_000
        );
        pipeline.transfer_config.max_transfer_time_ms = Some(7);
        for bytes in [0, 31 * 1024 * 1024, u64::MAX] {
            assert_eq!(
                pipeline
                    .artifact_retry_config_for_size(bytes)
                    .total_timeout_ms,
                7
            );
        }
    }

    #[test]
    fn test_artifact_retry_size_requires_full_selected_stat() {
        assert_eq!(
            parse_rsync_selected_size(
                "Total transferred file size: 0 bytes\nTotal file size: 900,000,000 bytes"
            ),
            Some(900_000_000)
        );
        assert_eq!(
            parse_rsync_selected_size("Total transferred file size: 900 bytes"),
            None
        );
        assert_eq!(parse_rsync_selected_size("Total file size: invalid"), None);
        assert_eq!(
            parse_rsync_selected_size("Total file size: 0 bytes"),
            Some(0)
        );
    }

    #[cfg(target_os = "linux")]
    #[tokio::test]
    async fn test_artifact_retry_real_rsync_selected_payload_exceeds_old_budget() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir().expect("artifact fixture").keep();
        let source = retained.join("source");
        let destination = retained.join("destination");
        std::fs::create_dir_all(source.join("wrapper-release/incremental")).unwrap();
        std::fs::create_dir_all(&destination).unwrap();
        // Incompressible bytes keep the actual wire payload large even if the
        // peer negotiates compression. Repeated bytes did not exercise 30s.
        let mut payload = vec![0_u8; 2 * 1024 * 1024 + 128 * 1024];
        std::io::Read::read_exact(
            &mut std::fs::File::open("/dev/urandom").unwrap(),
            &mut payload,
        )
        .unwrap();
        std::fs::write(source.join("wrapper-release/rch"), &payload).unwrap();
        std::fs::File::create(source.join("wrapper-release/incremental/cache"))
            .unwrap()
            .set_len(64 * 1024 * 1024)
            .unwrap();
        std::fs::write(source.join("unselected.txt"), b"must not transfer").unwrap();
        let pipeline = TransferPipeline::new(
            destination.clone(),
            "artifact-local-peer".to_string(),
            "selected".to_string(),
            TransferConfig {
                bwlimit_kbps: Some(64),
                ..TransferConfig::default()
            },
        );
        let worker = worker_with_os(None);
        let patterns = vec![
            "- wrapper-release/incremental/***".to_string(),
            "wrapper-release/**".to_string(),
        ];
        let remote = source.to_str().unwrap();
        let command = || {
            let mut command = pipeline.build_retrieve_command(&worker, remote, &patterns);
            // Real rsync endpoints over local pipes; only SSH transport is
            // replaced. Retain production filter ordering and source selection.
            command.args([
                "--no-compress",
                "-e",
                r#"sh -c 'if [ "$1" = -l ]; then shift 2; fi; shift; exec "$@"' rch-local-peer"#,
            ]);
            command.env("LC_ALL", "C");
            command
        };
        let mut planning = command();
        planning.arg("--dry-run");
        let planned = tokio::time::timeout(
            std::time::Duration::from_secs(15),
            execute_rsync_attempt(planning),
        )
        .await
        .expect("bounded real rsync planning")
        .expect("real rsync planning");
        eprintln!(
            "retained fixture: {}\n{}",
            retained.display(),
            String::from_utf8_lossy(&planned.stderr)
        );
        assert!(
            planned.status.success(),
            "{}",
            String::from_utf8_lossy(&planned.stderr)
        );
        let selected = parse_rsync_selected_size(&String::from_utf8_lossy(&planned.stdout))
            .expect("selected file size");
        assert_eq!(selected, payload.len() as u64);
        assert!(
            !destination.join("wrapper-release/rch").exists(),
            "dry-run must not copy payload"
        );
        let retry = pipeline.artifact_retry_config_for_size(selected);
        let started = std::time::Instant::now();
        let copied = execute_rsync_with_retry(&retry, "real_artifact_retrieval", command)
            .await
            .expect("healthy transfer beyond old cap");
        eprintln!(
            "{}\n{}",
            String::from_utf8_lossy(&copied.stdout),
            String::from_utf8_lossy(&copied.stderr)
        );
        assert!(copied.status.success());
        assert!(
            started.elapsed() > std::time::Duration::from_secs(30),
            "fixture must exercise progress after the old deadline"
        );
        assert_eq!(
            blake3::hash(&std::fs::read(destination.join("wrapper-release/rch")).unwrap()),
            blake3::hash(&payload)
        );
        assert!(!destination.join("wrapper-release/incremental").exists());
        assert!(!destination.join("unselected.txt").exists());
        let mut noop = command();
        noop.arg("--dry-run");
        let noop = tokio::time::timeout(
            std::time::Duration::from_secs(15),
            execute_rsync_attempt(noop),
        )
        .await
        .unwrap()
        .unwrap();
        assert!(
            noop.status.success(),
            "{}",
            String::from_utf8_lossy(&noop.stderr)
        );
        assert_eq!(
            parse_rsync_selected_size(&String::from_utf8_lossy(&noop.stdout)),
            Some(selected),
            "planning counts the full selected payload even when already present"
        );
    }

    #[cfg(target_os = "linux")]
    #[tokio::test]
    async fn test_artifact_retry_explicit_cap_terminates_owned_child() {
        let _guard = test_guard!();
        let retained = tempfile::tempdir()
            .expect("artifact timeout fixture")
            .keep();
        let pid_file = retained.join("owned.pid");
        let pipeline = TransferPipeline::new(
            retained.clone(),
            "artifact-cap".to_string(),
            "selected".to_string(),
            TransferConfig {
                max_transfer_time_ms: Some(500),
                ..TransferConfig::default()
            },
        );
        let retry = pipeline.artifact_retry_config_for_size(u64::MAX);
        let result = execute_rsync_with_retry(&retry, "artifact_explicit_cap", || {
            let mut command = Command::new("/bin/sh");
            command
                .args([
                    "-c",
                    "printf '%s' \"$$\" > \"$1\"; exec sleep 10",
                    "artifact-cap",
                ])
                .arg(&pid_file)
                .stdout(Stdio::piped())
                .stderr(Stdio::piped());
            command
        })
        .await;
        assert!(
            result.is_err(),
            "explicit cap cannot be enlarged by payload planning"
        );
        let pid = std::fs::read_to_string(&pid_file).expect("owned child actually started");
        tokio::time::timeout(std::time::Duration::from_secs(3), async {
            while Path::new("/proc").join(pid.trim()).exists() {
                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
            }
        })
        .await
        .expect("deadline must terminate and reap its owned child");
    }

    #[tokio::test]
    async fn test_source_transfer_retry_unit_timeout_then_success_reuses_partial_base() {
        // Deterministic injected-runner unit proof only; this is not a live
        // network capture. The stable base string models the production rsync
        // destination retained across attempts.
        let _guard = test_guard!();
        let retry = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 0,
            max_delay_ms: 0,
            jitter_factor: 0.0,
            total_timeout_ms: 1,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let observed_bases = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
        let calls_in = std::sync::Arc::clone(&calls);
        let bases_in = std::sync::Arc::clone(&observed_bases);

        let (value, attempts) = run_source_transfer_attempts(
            &retry,
            std::time::Duration::from_millis(10),
            "cold_pinned_tree",
            move |_attempt| {
                let call = calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                bases_in
                    .lock()
                    .expect("record partial base")
                    .push("worker:/remote/job/.rch-clean-overlay-base.tar");
                async move {
                    if call == 0 {
                        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
                        Ok("completed too late")
                    } else {
                        Ok("continued")
                    }
                }
            },
        )
        .await
        .expect("second attempt should continue the partial base");

        assert_eq!(value, "continued");
        assert_eq!(attempts.len(), 2);
        assert_eq!(attempts[0].outcome, "retryable");
        assert_eq!(attempts[1].outcome, "succeeded");
        assert!(attempts[0].detail.contains("timed out after 10ms"));
        assert_eq!(
            observed_bases
                .lock()
                .expect("read partial bases")
                .as_slice(),
            [
                "worker:/remote/job/.rch-clean-overlay-base.tar",
                "worker:/remote/job/.rch-clean-overlay-base.tar"
            ]
        );
    }

    #[tokio::test]
    async fn test_source_transfer_retry_unit_exhausts_exact_attempt_count() {
        // Deterministic injected-runner unit proof only; no live network claim.
        let _guard = test_guard!();
        let retry = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 0,
            max_delay_ms: 0,
            jitter_factor: 0.0,
            total_timeout_ms: 1,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let error = run_source_transfer_attempts(
            &retry,
            std::time::Duration::from_secs(1),
            "cold_pinned_tree",
            move |_attempt| {
                calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                async { Err::<(), _>(anyhow::anyhow!("source sync timed out")) }
            },
        )
        .await
        .expect_err("all configured attempts should be exhausted");

        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 3);
        assert_eq!(error.attempts.len(), 3);
        assert_eq!(
            error
                .attempts
                .iter()
                .map(|attempt| attempt.attempt)
                .collect::<Vec<_>>(),
            vec![1, 2, 3]
        );
        assert!(
            error
                .to_string()
                .contains("before remote Cargo execution after 3 attempt(s)")
        );
    }

    #[tokio::test]
    async fn test_source_transfer_retry_unit_fatal_error_does_not_spin() {
        // Deterministic injected-runner unit proof only; no live network claim.
        let _guard = test_guard!();
        let retry = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 0,
            max_delay_ms: 0,
            jitter_factor: 0.0,
            total_timeout_ms: 1,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let error = run_source_transfer_attempts(
            &retry,
            std::time::Duration::from_secs(1),
            "cold_pinned_tree",
            move |_attempt| {
                calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                async { Err::<(), _>(anyhow::anyhow!("Permission denied")) }
            },
        )
        .await
        .expect_err("fatal error should fail immediately");

        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
        assert_eq!(error.attempts.len(), 1);
        assert_eq!(error.attempts[0].outcome, "fatal");
    }

    #[cfg(unix)]
    #[test]
    fn test_ordinary_warm_base_rsync_fast_path_is_unchanged() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/warm-project"),
            "warm-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = worker_with_os(Some("linux"));
        let remote_path = pipeline.remote_path();
        let escaped_remote_path = escape(Cow::from(remote_path.as_str()));
        let destination = format!("{}@{}:{}", worker.user, worker.host, escaped_remote_path);
        let command = pipeline.build_sync_command(
            &worker,
            &destination,
            &escaped_remote_path,
            &pipeline.get_effective_excludes(),
        );
        let args = command
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().into_owned())
            .collect::<Vec<_>>();

        assert!(args.iter().any(|arg| arg == "--delete"));
        assert!(!args.iter().any(|arg| arg == "--append-verify"));
        assert_eq!(args.iter().filter(|arg| *arg == "--partial").count(), 1);
        assert_eq!(
            args.iter()
                .filter(|arg| *arg == "--partial-dir=.rch-partial")
                .count(),
            1
        );
        assert_eq!(args.last(), Some(&destination));
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_ordinary_source_retry_rebuilds_same_resumable_rsync_command() {
        // Deterministic production-command spying boundary, not live-network
        // evidence: the real command builder is inspected on both attempts while
        // the injected operation makes attempt 1 exceed the real timeout future.
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/pinned-materialized-tree"),
            "pinned-tree".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = worker_with_os(Some("linux"));
        let excludes = pipeline.get_effective_excludes();
        let remote_path = pipeline.remote_path();
        let escaped_remote_path = escape(Cow::from(remote_path.as_str()));
        let destination = format!("{}@{}:{}", worker.user, worker.host, escaped_remote_path);
        let expected_destination = destination.clone();
        let observed = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
        let observed_in = std::sync::Arc::clone(&observed);
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let retry = RetryConfig {
            max_attempts: 2,
            base_delay_ms: 0,
            max_delay_ms: 0,
            jitter_factor: 0.0,
            // Deliberately smaller than one attempt. The source path must ignore
            // this old shared budget and still launch attempt 2 with a fresh cap.
            total_timeout_ms: 1,
        };

        let (_, attempts) = run_source_transfer_attempts(
            &retry,
            std::time::Duration::from_millis(200),
            "ordinary_source_sync",
            move |_attempt| {
                let command = pipeline.build_sync_command(
                    &worker,
                    &destination,
                    &escaped_remote_path,
                    &excludes,
                );
                observed_in
                    .lock()
                    .expect("record production rsync argv")
                    .push(
                        command
                            .as_std()
                            .get_args()
                            .map(|arg| arg.to_string_lossy().into_owned())
                            .collect::<Vec<_>>(),
                    );
                let call = calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                async move {
                    if call == 0 {
                        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
                    } else {
                        // This exceeds the obsolete 1ms shared budget, but fits
                        // comfortably inside the fresh 200ms attempt budget.
                        tokio::time::sleep(std::time::Duration::from_millis(25)).await;
                    }
                    Ok(())
                }
            },
        )
        .await
        .expect("attempt 2 should launch with a fresh source-sync budget");

        assert_eq!(attempts.len(), 2);
        assert_eq!(attempts[0].outcome, "retryable");
        assert_eq!(attempts[1].outcome, "succeeded");
        assert!(attempts[0].detail.contains("timed out after 200ms"));
        let observed = observed.lock().expect("read production rsync argv");
        assert_eq!(observed.len(), 2);
        assert_eq!(observed[0], observed[1]);
        assert_eq!(
            observed[0].iter().filter(|arg| *arg == "--partial").count(),
            1
        );
        assert_eq!(
            observed[0]
                .iter()
                .filter(|arg| *arg == "--partial-dir=.rch-partial")
                .count(),
            1
        );
        assert_eq!(observed[0].last(), Some(&expected_destination));
    }

    #[test]
    fn test_ordinary_source_sync_large_payload_gets_larger_per_attempt_budget() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/large-materialized-tree"),
            "large-tree".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_estimated_transfer_bytes(Some(512 * 1024 * 1024));

        assert_eq!(
            pipeline.source_sync_attempt_timeout(&[]),
            std::time::Duration::from_secs(30 + 512)
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_execute_rsync_with_retry_times_out_hanging_child() {
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 1,
            total_timeout_ms: 25,
            jitter_factor: 0.0,
            ..RetryConfig::default()
        };
        let start = std::time::Instant::now();

        let err = execute_rsync_with_retry(&retry_config, "test_hanging_rsync", || {
            let mut cmd = Command::new("sleep");
            cmd.arg("5");
            cmd.stdout(std::process::Stdio::piped());
            cmd.stderr(std::process::Stdio::piped());
            cmd
        })
        .await
        .expect_err("hanging child should time out");

        assert!(
            start.elapsed() < std::time::Duration::from_secs(2),
            "timeout should stop the child promptly"
        );
        assert!(
            err.to_string()
                .contains("test_hanging_rsync: timed out after 25ms")
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_execute_rsync_with_retry_retries_transient_transport_failure() {
        // Regression: a process that RAN but exited non-zero on a transient
        // transport error (rsync exit 12 / "connection unexpectedly closed")
        // must be RETRIED. Previously wait_with_output()'s Ok-on-any-exit made
        // retry_with_backoff return on attempt 0 — the retry subsystem was inert.
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 1,
            max_delay_ms: 2,
            jitter_factor: 0.0,
            total_timeout_ms: 5_000,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let err = execute_rsync_with_retry(&retry_config, "transient_rsync", move || {
            calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            let mut cmd = Command::new("sh");
            cmd.arg("-c").arg(
                "echo 'rsync: connection unexpectedly closed (0 bytes received)' >&2; exit 12",
            );
            cmd.stdout(std::process::Stdio::piped());
            cmd.stderr(std::process::Stdio::piped());
            cmd
        })
        .await
        .expect_err("transient transport failure should exhaust retries as Err");
        assert!(err.to_string().contains("transport error"));
        assert_eq!(
            calls.load(std::sync::atomic::Ordering::SeqCst),
            3,
            "must retry up to max_attempts on a retryable transport failure"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_execute_rsync_with_retry_does_not_retry_non_transport_failure() {
        // A non-zero exit that is NOT a transport error returns Ok(output) with
        // a non-success status (so the caller's existing failure handling runs)
        // and is NOT retried.
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 1,
            max_delay_ms: 2,
            jitter_factor: 0.0,
            total_timeout_ms: 5_000,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let output = execute_rsync_with_retry(&retry_config, "fatal_rsync", move || {
            calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            let mut cmd = Command::new("sh");
            cmd.arg("-c")
                .arg("echo 'rsync: mkstemp failed: Permission denied' >&2; exit 23");
            cmd.stdout(std::process::Stdio::piped());
            cmd.stderr(std::process::Stdio::piped());
            cmd
        })
        .await
        .expect("non-transport failure returns Ok(output) for caller handling");
        assert!(!output.status.success());
        assert_eq!(
            calls.load(std::sync::atomic::Ordering::SeqCst),
            1,
            "must NOT retry a non-transport failure"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_run_command_streaming_times_out_hanging_child() {
        let _guard = test_guard!();
        let mut cmd = Command::new("sleep");
        cmd.arg("5");
        cmd.stdout(std::process::Stdio::piped());
        cmd.stderr(std::process::Stdio::piped());
        let start = std::time::Instant::now();

        let err = run_command_streaming(
            cmd,
            "test_streaming_rsync",
            std::time::Duration::from_millis(25),
            None,
            |_| {},
        )
        .await
        .expect_err("streaming child should time out");

        assert!(
            start.elapsed() < std::time::Duration::from_secs(2),
            "timeout should stop the streaming child promptly"
        );
        assert!(
            err.to_string()
                .contains("test_streaming_rsync: timed out after 25ms")
        );
    }

    #[cfg(unix)]
    fn test_silence_policy(millis: u64) -> SyncSilencePolicy {
        SyncSilencePolicy {
            limit: std::time::Duration::from_millis(millis),
            worker_id: "stall-worker".to_string(),
            phase: "source_sync",
        }
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_streaming_silence_timeout_aborts_stalled_stream() {
        // Issue #59: a child that produces NO output for the silence window is
        // a dead channel, and must be aborted long before the wall-clock cap.
        let _guard = test_guard!();
        let mut cmd = Command::new("sleep");
        cmd.arg("5");
        cmd.stdout(std::process::Stdio::piped());
        cmd.stderr(std::process::Stdio::piped());
        let policy = test_silence_policy(100);
        let start = std::time::Instant::now();

        let err = run_command_streaming(
            cmd,
            "stalled_source_sync",
            std::time::Duration::from_secs(30),
            Some(&policy),
            |_| {},
        )
        .await
        .expect_err("a silent child must be aborted by the silence timeout");

        assert!(
            start.elapsed() < std::time::Duration::from_secs(3),
            "silence abort must not wait for the wall-clock cap"
        );
        let stall = find_source_sync_stall(&err)
            .expect("silence abort must surface the typed SourceSyncStalled");
        assert_eq!(stall.worker_id, "stall-worker");
        assert_eq!(stall.phase, "source_sync");
        assert_eq!(stall.silence, std::time::Duration::from_millis(100));
        // The stall must never be mistakable for the E104 fail-closed surface.
        let text = err.to_string();
        assert!(!text.contains("SSH command timed out after"));
        assert!(!text.contains("Command timed out after"));
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_streaming_silence_timeout_allows_slow_but_progressing_stream() {
        // Issue #59: a transfer that keeps producing output — however slowly
        // relative to its total duration — must NOT trip the silence timeout,
        // even when the whole run takes several silence windows.
        let _guard = test_guard!();
        let mut cmd = Command::new("sh");
        cmd.arg("-c")
            .arg("i=0; while [ $i -lt 6 ]; do echo tick$i; i=$((i+1)); sleep 0.15; done");
        cmd.stdout(std::process::Stdio::piped());
        cmd.stderr(std::process::Stdio::piped());
        let policy = test_silence_policy(600);

        let (output, _) = run_command_streaming(
            cmd,
            "progressing_source_sync",
            std::time::Duration::from_secs(30),
            Some(&policy),
            |_| {},
        )
        .await
        .expect("slow-but-progressing stream must complete (total > silence window)");
        assert!(output.contains("tick5"), "all output captured: {output}");
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_streaming_splits_progress_segments_on_carriage_returns() {
        // rsync --info=progress2 refreshes its progress line with bare `\r`
        // and no `\n` until a step completes; each refresh must count as a
        // forward-progress event for the silence detector and the heartbeat.
        let _guard = test_guard!();
        let mut cmd = Command::new("sh");
        cmd.arg("-c").arg("printf 'one\\rtwo\\rthree\\n'");
        cmd.stdout(std::process::Stdio::piped());
        cmd.stderr(std::process::Stdio::piped());
        let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
        let seen_in = std::sync::Arc::clone(&seen);

        let (output, _) = run_command_streaming(
            cmd,
            "cr_segment_split",
            std::time::Duration::from_secs(10),
            None,
            move |line| seen_in.lock().unwrap().push(line.to_string()),
        )
        .await
        .expect("segmented stream completes");

        assert_eq!(
            *seen.lock().unwrap(),
            vec!["one".to_string(), "two".to_string(), "three".to_string()],
            "each \\r-refresh must be delivered as its own progress event"
        );
        assert!(output.contains("three"));
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_streaming_with_retry_does_not_retry_source_sync_stall() {
        // Issue #59: a silence stall fails over to ANOTHER worker (hook side);
        // re-running the same dead channel would burn another full silence
        // window per configured attempt. The typed error must survive the
        // retry wrapper for the hook's downcast.
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 1,
            max_delay_ms: 2,
            jitter_factor: 0.0,
            total_timeout_ms: 60_000,
        };
        let policy = test_silence_policy(100);
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);

        let err = run_command_streaming_with_retry(
            &retry_config,
            "stalled_source_sync_retry",
            Some(std::time::Duration::from_secs(30)),
            Some(&policy),
            move || {
                calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                let mut cmd = Command::new("sleep");
                cmd.arg("5");
                cmd.stdout(std::process::Stdio::piped());
                cmd.stderr(std::process::Stdio::piped());
                cmd
            },
            |_| {},
        )
        .await
        .expect_err("stalled sync must fail without in-place retries");

        assert_eq!(
            calls.load(std::sync::atomic::Ordering::SeqCst),
            1,
            "a silence stall must NOT be retried on the same worker"
        );
        let stall = find_source_sync_stall(&err)
            .expect("typed stall must survive the retry wrapper for hook failover");
        assert_eq!(stall.worker_id, "stall-worker");
        assert_eq!(stall.phase, "source_sync");
    }

    #[test]
    fn test_streaming_error_is_retryable_reads_syncfailed_stderr() {
        // The whole point of the streaming classifier: TransferError::SyncFailed's
        // Display is only "Project sync failed: rsync failed", so the transport
        // signature must be read from the captured `stderr`, not the error chain.
        let _guard = test_guard!();
        let transient: anyhow::Error = TransferError::SyncFailed {
            reason: "rsync failed".to_string(),
            exit_code: Some(12),
            stderr: "rsync: connection unexpectedly closed (0 bytes received)".to_string(),
        }
        .into();
        assert!(
            streaming_error_is_retryable(&transient),
            "transport stderr inside SyncFailed must classify as retryable"
        );

        let fatal: anyhow::Error = TransferError::SyncFailed {
            reason: "rsync failed".to_string(),
            exit_code: Some(23),
            stderr: "rsync: mkstemp failed: Permission denied".to_string(),
        }
        .into();
        assert!(
            !streaming_error_is_retryable(&fatal),
            "permission-denied stderr inside SyncFailed must NOT be retryable"
        );

        // Non-SyncFailed errors fall back to the error-chain classifier.
        assert!(streaming_error_is_retryable(&anyhow::anyhow!(
            "op: timed out after 25ms"
        )));
        assert!(!streaming_error_is_retryable(&anyhow::anyhow!(
            "Failed to execute rsync"
        )));
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_run_command_streaming_with_retry_retries_transient_transport_failure() {
        // Regression (bd-review-transfer-streaming-noretry): streaming
        // upload/retrieve called run_command_streaming directly, so a transient
        // rsync transport drop failed the whole transfer with zero retries. The
        // wrapper must classify the SyncFailed stderr and retry to max_attempts.
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 1,
            max_delay_ms: 2,
            jitter_factor: 0.0,
            total_timeout_ms: 5_000,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let lines = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let lines_in = std::sync::Arc::clone(&lines);
        let err = run_command_streaming_with_retry(
            &retry_config,
            "transient_streaming_rsync",
            None,
            None,
            move || {
                calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                let mut cmd = Command::new("sh");
                cmd.arg("-c").arg(
                    "echo 'rsync: connection unexpectedly closed (0 bytes received)' >&2; exit 12",
                );
                cmd.stdout(std::process::Stdio::piped());
                cmd.stderr(std::process::Stdio::piped());
                cmd
            },
            |_line| {
                lines_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            },
        )
        .await
        .expect_err("transient transport failure should exhaust retries as Err");
        assert!(
            err.to_string().contains("Project sync failed"),
            "unexpected error: {err}"
        );
        assert_eq!(
            calls.load(std::sync::atomic::Ordering::SeqCst),
            3,
            "must retry up to max_attempts on a retryable transport failure"
        );
        assert!(
            lines.load(std::sync::atomic::Ordering::SeqCst) >= 3,
            "on_line must be re-invoked on every attempt"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_run_command_streaming_with_retry_does_not_retry_non_transport_failure() {
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 1,
            max_delay_ms: 2,
            jitter_factor: 0.0,
            total_timeout_ms: 5_000,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        run_command_streaming_with_retry(
            &retry_config,
            "fatal_streaming_rsync",
            None,
            None,
            move || {
                calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                let mut cmd = Command::new("sh");
                cmd.arg("-c")
                    .arg("echo 'rsync: mkstemp failed: Permission denied' >&2; exit 23");
                cmd.stdout(std::process::Stdio::piped());
                cmd.stderr(std::process::Stdio::piped());
                cmd
            },
            |_| {},
        )
        .await
        .expect_err("non-transport streaming failure should fail fast as Err");
        assert_eq!(
            calls.load(std::sync::atomic::Ordering::SeqCst),
            1,
            "must NOT retry a non-transport failure"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_run_command_streaming_with_retry_recovers_after_transient() {
        // A transient drop on the first attempt followed by success proves the
        // streaming path now reconnects instead of failing the whole transfer.
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 3,
            base_delay_ms: 1,
            max_delay_ms: 2,
            jitter_factor: 0.0,
            total_timeout_ms: 5_000,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let (out, _ms) = run_command_streaming_with_retry(
            &retry_config,
            "recovering_streaming_rsync",
            None,
            None,
            move || {
                let n = calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                let mut cmd = Command::new("sh");
                if n == 0 {
                    cmd.arg("-c")
                        .arg("echo 'rsync: connection unexpectedly closed' >&2; exit 12");
                } else {
                    cmd.arg("-c")
                        .arg("echo 'sent 100 bytes  received 50 bytes'; exit 0");
                }
                cmd.stdout(std::process::Stdio::piped());
                cmd.stderr(std::process::Stdio::piped());
                cmd
            },
            |_| {},
        )
        .await
        .expect("should recover on the second attempt");
        assert!(out.contains("received"), "stdout should be captured: {out}");
        assert_eq!(
            calls.load(std::sync::atomic::Ordering::SeqCst),
            2,
            "should succeed on the second attempt"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_source_streaming_timeout_still_launches_attempt_two_with_full_budget() {
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 2,
            base_delay_ms: 0,
            max_delay_ms: 0,
            jitter_factor: 0.0,
            total_timeout_ms: 1,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let (output, _) = run_command_streaming_with_retry(
            &retry_config,
            "ordinary_source_streaming",
            Some(std::time::Duration::from_millis(200)),
            None,
            move || {
                let call = calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                let mut command = Command::new("sh");
                if call == 0 {
                    command.arg("-c").arg("sleep 1");
                } else {
                    command.arg("-c").arg("sleep 0.025; echo attempt-two");
                }
                command.stdout(Stdio::piped()).stderr(Stdio::piped());
                command
            },
            |_| {},
        )
        .await
        .expect("attempt 2 should receive a fresh 200ms source-sync budget");

        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2);
        assert_eq!(output, "attempt-two\n");
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn test_source_streaming_exhaustion_retains_every_attempt() {
        let _guard = test_guard!();
        let retry_config = RetryConfig {
            max_attempts: 2,
            base_delay_ms: 0,
            max_delay_ms: 0,
            jitter_factor: 0.0,
            total_timeout_ms: 1,
        };
        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let calls_in = std::sync::Arc::clone(&calls);
        let error = run_command_streaming_with_retry(
            &retry_config,
            "ordinary_source_streaming",
            Some(std::time::Duration::from_millis(100)),
            None,
            move || {
                calls_in.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                let mut command = Command::new("sh");
                command
                    .arg("-c")
                    .arg("echo 'rsync: connection unexpectedly closed' >&2; exit 12")
                    .stdout(Stdio::piped())
                    .stderr(Stdio::piped());
                command
            },
            |_| {},
        )
        .await
        .expect_err("source streaming retries must exhaust exactly");

        let history = error
            .downcast_ref::<TransferAttemptsExhausted>()
            .expect("typed source streaming attempt history");
        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2);
        assert_eq!(history.attempts.len(), 2);
        assert_eq!(history.attempts[0].attempt, 1);
        assert_eq!(history.attempts[1].attempt, 2);
        assert_eq!(history.attempts[0].outcome, "retryable");
        assert_eq!(history.attempts[1].outcome, "retryable");
        assert!(
            history
                .attempts
                .iter()
                .all(|attempt| attempt.detail.contains("connection unexpectedly closed"))
        );
    }

    #[test]
    fn test_detect_partial_transfer_matches_indicators() {
        // Regression (bd-review-transfer-partial-warn-only): exit-0 rsync output
        // carrying a partial-transfer indicator must be detectable so the
        // upload/download paths fail instead of reporting trusted-but-wrong
        // success.
        let _guard = test_guard!();
        assert_eq!(detect_partial_transfer(""), None);
        assert_eq!(
            detect_partial_transfer("sent 100 bytes  received 50 bytes  total size 4096"),
            None
        );
        assert_eq!(
            detect_partial_transfer("rsync: connection unexpectedly closed (0 bytes received)"),
            Some("connection unexpectedly closed")
        );
        assert_eq!(
            detect_partial_transfer(
                "rsync warning: some files vanished\npartial transfer (code 23)"
            ),
            Some("partial transfer")
        );
        assert_eq!(
            detect_partial_transfer("rsync: read error: Connection reset by peer"),
            Some("read error")
        );
        // Case-insensitive.
        assert_eq!(
            detect_partial_transfer("WRITE ERROR: broken pipe (32)"),
            Some("write error")
        );
    }

    #[test]
    fn test_transfer_estimate_struct() {
        let _guard = test_guard!();
        let estimate = TransferEstimate {
            bytes: 1024 * 1024 * 50, // 50 MB
            files: 100,
            estimated_time_ms: 5000, // 5 seconds
            estimation_ms: 150,      // 150ms to estimate
        };
        assert_eq!(estimate.bytes, 52428800);
        assert_eq!(estimate.files, 100);
        assert_eq!(estimate.estimated_time_ms, 5000);
        assert_eq!(estimate.estimation_ms, 150);
    }

    #[test]
    fn test_build_sync_command_with_bwlimit() {
        let _guard = test_guard!();
        let config = TransferConfig {
            bwlimit_kbps: Some(5000),
            ..Default::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            config,
        );

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        assert!(args.contains(&"--bwlimit=5000".to_string()));
    }

    #[test]
    fn test_build_sync_command_without_bwlimit() {
        let _guard = test_guard!();
        let config = TransferConfig::default();

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            config,
        );

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        // Should not have any --bwlimit arg when not configured
        assert!(!args.iter().any(|arg| arg.starts_with("--bwlimit")));
    }

    #[test]
    fn test_build_sync_command_bwlimit_zero_disabled() {
        let _guard = test_guard!();
        let config = TransferConfig {
            bwlimit_kbps: Some(0), // Explicitly 0 = disabled
            ..Default::default()
        };

        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/test"),
            "test-project".to_string(),
            "abc123".to_string(),
            config,
        );

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        // bwlimit=0 should be treated as disabled (no flag)
        assert!(!args.iter().any(|arg| arg.starts_with("--bwlimit")));
    }

    #[test]
    fn test_build_sync_command_metadata_only_sync_omits_delete_and_uses_includes() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/workspace-root"),
            "workspace-root".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_sync_include_patterns(vec![
            "Cargo.toml".to_string(),
            "Cargo.lock".to_string(),
            ".cargo/".to_string(),
            ".cargo/**".to_string(),
        ])
        .with_sync_delete(false);

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        assert!(
            !args.iter().any(|arg| arg == "--delete"),
            "metadata-only syncs must not delete unrelated remote files"
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["--include", "Cargo.toml"]),
            "metadata-only syncs should include Cargo.toml"
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["--include", ".cargo/**"]),
            "metadata-only syncs should include workspace .cargo metadata"
        );
        assert!(
            args.windows(2).any(|window| window == ["--exclude", "*"]),
            "metadata-only syncs should exclude everything else"
        );
    }

    #[test]
    fn clean_overlay_include_patterns_are_anchored_literal_and_traversable() {
        let _guard = test_guard!();
        let root = tempfile::tempdir().expect("create include-pattern fixture");
        std::fs::create_dir_all(root.path().join("src/nested")).expect("create nested source");
        std::fs::write(
            root.path().join("src/nested/star*.rs"),
            "fn selected() {}\n",
        )
        .expect("write selected source");

        let patterns =
            clean_overlay_include_patterns(root.path(), &[PathBuf::from("src/nested/star*.rs")])
                .expect("build clean-overlay filters");

        assert_eq!(
            patterns,
            vec![
                "/src/".to_string(),
                "/src/nested/".to_string(),
                r"/src/nested/star\*.rs".to_string(),
            ]
        );
    }

    #[test]
    fn clean_overlay_directory_pattern_includes_only_selected_subtree() {
        let _guard = test_guard!();
        let root = tempfile::tempdir().expect("create directory-pattern fixture");
        std::fs::create_dir_all(root.path().join("src/selected")).expect("create selected dir");

        let patterns =
            clean_overlay_include_patterns(root.path(), &[PathBuf::from("src/selected")])
                .expect("build directory clean-overlay filters");

        assert_eq!(
            patterns,
            vec![
                "/src/".to_string(),
                "/src/selected/".to_string(),
                "/src/selected/***".to_string(),
            ]
        );
        assert!(!patterns.iter().any(|pattern| pattern.contains("peer")));
    }

    #[test]
    fn windows_clean_overlay_destination_quotes_spaces_and_refuses_shell_expansion() {
        for (input, expected) in [
            ("C:/rch/project/hash", "\"C:/rch/project/hash\""),
            (r"D:\rch\project name\hash", "\"D:/rch/project name/hash\""),
        ] {
            assert_eq!(
                TransferPipeline::windows_archive_destination(input).expect("literal drive path"),
                expected
            );
        }
        for input in [
            "relative/path",
            "/tmp/rch",
            "C:/rch/../peer",
            "C:/rch/./peer",
            "C:/rch/%TEMP%",
            "C:/rch/!PATH!",
            "C:/rch/\" & whoami",
            "C:/rch/$(whoami)",
            "C:/rch/a\nb",
            "C:/rch/a:b",
            "C:/rch/name.",
            "C:/rch/name /hash",
        ] {
            assert!(
                TransferPipeline::windows_archive_destination(input).is_err(),
                "must refuse unsafe remote archive destination {input:?}"
            );
        }
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn windows_clean_overlay_archives_preserve_base_and_only_selected_literal_paths() {
        let _guard = test_guard!();
        let root = tempfile::tempdir().expect("source fixture");
        let destination = tempfile::tempdir().expect("extraction fixture");
        for dir in ["src", "docs/owned", "target"] {
            std::fs::create_dir_all(root.path().join(dir)).expect("fixture directory");
        }
        for (path, bytes) in [
            ("src/a[1].rs", "base selected\n"),
            ("src/a1.rs", "base glob neighbor\n"),
            ("src/peer.rs", "base peer\n"),
            ("docs/owned/with space.md", "base document\n"),
            ("target/selected.rs", "base usually excluded\n"),
        ] {
            std::fs::write(root.path().join(path), bytes).expect("base source");
        }
        for args in [
            vec!["init", "-b", "main"],
            vec!["add", "."],
            vec![
                "-c",
                "user.name=Fixture",
                "-c",
                "user.email=fixture@example.invalid",
                "-c",
                "commit.gpgsign=false",
                "commit",
                "-m",
                "base fixture",
            ],
        ] {
            let mut git = Command::new("git");
            configure_clean_git_command(&mut git);
            let output = git
                .current_dir(root.path())
                .args(args)
                .output()
                .await
                .expect("fixture git");
            assert!(
                output.status.success(),
                "{}",
                String::from_utf8_lossy(&output.stderr)
            );
        }
        let mut git = Command::new("git");
        configure_clean_git_command(&mut git);
        let output = git
            .current_dir(root.path())
            .args(["rev-parse", "HEAD"])
            .output()
            .await
            .expect("base ID");
        assert!(output.status.success());
        let base = String::from_utf8(output.stdout).expect("base ID UTF-8");

        for (path, bytes) in [
            ("src/a[1].rs", "owned selected\n"),
            ("src/a1.rs", "unselected glob neighbor dirt\n"),
            ("src/peer.rs", "unselected peer dirt\n"),
            ("src/untracked.rs", "unselected new peer file\n"),
            ("docs/owned/with space.md", "owned document\n"),
            ("docs/owned/new.md", "owned descendant\n"),
            ("target/selected.rs", "owned usually excluded\n"),
            ("target/unselected.rs", "unselected target dirt\n"),
            (".rchignore", "target/\ndocs/\n"),
        ] {
            std::fs::write(root.path().join(path), bytes).expect("working-tree dirt");
        }
        let immutable = TransferPipeline::create_git_archive(root.path(), base.trim())
            .await
            .expect("actual immutable Git archive");
        tar::Archive::new(std::fs::File::open(immutable.path()).expect("open base archive"))
            .unpack(destination.path())
            .expect("extract base archive");
        assert_eq!(
            std::fs::read_to_string(destination.path().join("src/a[1].rs")).unwrap(),
            "base selected\n"
        );
        assert!(!destination.path().join("src/untracked.rs").exists());

        let selected = ["src/a[1].rs", "docs/owned", "target/selected.rs"].map(PathBuf::from);
        let pipeline = TransferPipeline::new(
            root.path().to_path_buf(),
            "fixture".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_worker_platform(WorkerPlatform::Windows)
        .with_sync_include_patterns(clean_overlay_include_patterns(root.path(), &selected).unwrap())
        .with_sync_delete(false);
        let (overlay, count) = pipeline
            .selected_source_archive()
            .await
            .expect("selected archive");
        assert_eq!(count, 4);
        let mut archive = tar::Archive::new(std::fs::File::open(overlay.path()).unwrap());
        let names = archive
            .entries()
            .unwrap()
            .map(|entry| {
                let entry = entry.unwrap();
                assert!(entry.header().entry_type().is_file());
                entry.path().unwrap().into_owned()
            })
            .collect::<Vec<_>>();
        assert_eq!(
            names,
            [
                "docs/owned/new.md",
                "docs/owned/with space.md",
                "src/a[1].rs",
                "target/selected.rs",
            ]
            .map(PathBuf::from)
        );
        tar::Archive::new(std::fs::File::open(overlay.path()).unwrap())
            .unpack(destination.path())
            .expect("extract selected overlay");
        for (path, expected) in [
            ("src/a[1].rs", "owned selected\n"),
            ("src/a1.rs", "base glob neighbor\n"),
            ("src/peer.rs", "base peer\n"),
            ("docs/owned/with space.md", "owned document\n"),
            ("docs/owned/new.md", "owned descendant\n"),
            ("target/selected.rs", "owned usually excluded\n"),
        ] {
            assert_eq!(
                std::fs::read_to_string(destination.path().join(path)).unwrap(),
                expected,
                "{path}"
            );
        }
        for absent in [
            "src/untracked.rs",
            "target/unselected.rs",
            ".rchignore",
            ".git",
        ] {
            assert!(
                !destination.path().join(absent).exists(),
                "unselected path {absent}"
            );
        }
        assert_eq!(
            pipeline
                .enumerate_source_content_files()
                .await
                .unwrap_err()
                .to_string(),
            "source-content receipts require the rsync transport"
        );
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn windows_clean_overlay_empty_selection_does_not_widen_and_symlinks_refuse() {
        let _guard = test_guard!();
        let root = tempfile::tempdir().expect("source fixture");
        std::fs::write(root.path().join("peer.rs"), "peer dirt").unwrap();
        let pipeline = TransferPipeline::new(
            root.path().to_path_buf(),
            "fixture".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_worker_platform(WorkerPlatform::Windows)
        .with_sync_include_patterns(Vec::new());
        let (archive, count) = pipeline
            .selected_source_archive()
            .await
            .expect("empty selection");
        assert_eq!(count, 0);
        assert_eq!(
            tar::Archive::new(std::fs::File::open(archive.path()).unwrap())
                .entries()
                .unwrap()
                .count(),
            0
        );

        std::os::unix::fs::symlink("peer.rs", root.path().join("selected.rs")).unwrap();
        let pipeline = pipeline.with_sync_include_patterns(vec!["/selected.rs".to_string()]);
        let error = pipeline.selected_source_archive().await.unwrap_err();
        assert!(
            error.to_string().contains("refuses non-regular rsync item"),
            "{error:#}"
        );
    }

    #[test]
    fn clean_overlay_sync_commands_force_content_checks() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/workspace-root"),
            "workspace-root".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_sync_include_patterns(vec!["/src/".to_string(), "/src/lib.rs".to_string()])
        .with_sync_delete(false)
        .with_sync_checksum(true);
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        for command in [
            pipeline.build_sync_command(
                &worker,
                "mockuser@mock://worker:/data/tmp/rch/workspace-root/abc123",
                "/data/tmp/rch/workspace-root/abc123",
                &[],
            ),
            pipeline.build_sync_streaming_command(
                &worker,
                "mockuser@mock://worker:/data/tmp/rch/workspace-root/abc123",
                "/data/tmp/rch/workspace-root/abc123",
                &[],
            ),
        ] {
            let args = command
                .as_std()
                .get_args()
                .map(|arg| arg.to_string_lossy().into_owned())
                .collect::<Vec<_>>();
            assert!(args.iter().any(|arg| arg == "--checksum"));
            assert!(!args.iter().any(|arg| arg == "--delete"));
            assert!(
                args.windows(2)
                    .any(|window| window == ["--include", "/src/lib.rs"])
            );
            assert!(args.windows(2).any(|window| window == ["--exclude", "*"]));
        }
    }

    #[cfg(unix)]
    #[test]
    fn stale_source_checksum_rsync_replaces_equal_metadata_divergent_bytes() {
        let _guard = test_guard!();
        let source = tempfile::tempdir().expect("create source fixture");
        let destination = tempfile::tempdir().expect("create destination fixture");
        let source_file = source.path().join("lib.rs");
        let destination_file = destination.path().join("lib.rs");
        let source_bytes = b"local-new\n";
        let stale_bytes = b"stale-old\n";
        assert_eq!(source_bytes.len(), stale_bytes.len());
        std::fs::write(&source_file, source_bytes).expect("write source bytes");
        std::fs::write(&destination_file, stale_bytes).expect("write stale destination bytes");

        let timestamp = std::time::SystemTime::UNIX_EPOCH
            .checked_add(std::time::Duration::from_secs(1_700_000_000))
            .expect("fixture timestamp");
        let times = std::fs::FileTimes::new()
            .set_accessed(timestamp)
            .set_modified(timestamp);
        std::fs::File::options()
            .write(true)
            .open(&source_file)
            .expect("open source fixture")
            .set_times(times)
            .expect("stamp source fixture");
        std::fs::File::options()
            .write(true)
            .open(&destination_file)
            .expect("open destination fixture")
            .set_times(times)
            .expect("stamp destination fixture");

        let plain = std::process::Command::new("rsync")
            .arg("-a")
            .arg(format!("{}/", source.path().display()))
            .arg(format!("{}/", destination.path().display()))
            .output()
            .expect("run ordinary rsync");
        assert!(
            plain.status.success(),
            "ordinary rsync failed: {}",
            String::from_utf8_lossy(&plain.stderr)
        );
        assert_eq!(
            std::fs::read(&destination_file).expect("read ordinary rsync destination"),
            stale_bytes,
            "size-and-mtime quick checks must reproduce the stale-byte failure"
        );

        let checksum = std::process::Command::new("rsync")
            .arg("-a")
            .arg("--checksum")
            .arg(format!("{}/", source.path().display()))
            .arg(format!("{}/", destination.path().display()))
            .output()
            .expect("run checksum rsync");
        assert!(
            checksum.status.success(),
            "checksum rsync failed: {}",
            String::from_utf8_lossy(&checksum.stderr)
        );
        assert_eq!(
            std::fs::read(&destination_file).expect("read checksum rsync destination"),
            source_bytes,
            "checksum-aware sync must replace stale bytes before remote Cargo"
        );
    }

    #[test]
    fn clean_overlay_git_command_clears_ambient_repository_selection() {
        let _guard = test_guard!();
        let mut command = Command::new("git");
        command
            .env("GIT_DIR", "/tmp/peer.git")
            .env("GIT_WORK_TREE", "/tmp/peer-worktree")
            .env("GIT_CONFIG_COUNT", "1");
        configure_clean_git_command(&mut command);

        let environment = command.as_std().get_envs().collect::<Vec<_>>();
        for key in ["GIT_DIR", "GIT_WORK_TREE", "GIT_CONFIG_COUNT"] {
            assert!(
                environment
                    .iter()
                    .any(|(candidate, value)| *candidate == key && value.is_none()),
                "{key} must be removed from clean Git commands: {environment:?}"
            );
        }
        assert!(environment.iter().any(|(key, value)| {
            *key == "GIT_NO_REPLACE_OBJECTS"
                && value.is_some_and(|value| value == std::ffi::OsStr::new("1"))
        }));
        let args = command
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().into_owned())
            .collect::<Vec<_>>();
        assert!(
            args.windows(2)
                .any(|window| { window == ["-c", "core.attributesFile=/dev/null"] })
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["-c", "tar.umask=0022"])
        );
    }

    #[test]
    fn test_build_sync_streaming_command_metadata_only_sync_omits_delete_and_uses_includes() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/workspace-root"),
            "workspace-root".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        )
        .with_sync_include_patterns(vec![
            "Cargo.toml".to_string(),
            "Cargo.lock".to_string(),
            ".cargo/".to_string(),
            ".cargo/**".to_string(),
        ])
        .with_sync_delete(false);

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_streaming_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        assert!(
            !args.iter().any(|arg| arg == "--delete"),
            "streaming metadata-only syncs must not delete unrelated remote files"
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["--include", "Cargo.toml"]),
            "streaming metadata-only syncs should include Cargo.toml"
        );
        assert!(
            args.windows(2)
                .any(|window| window == ["--include", ".cargo/**"]),
            "streaming metadata-only syncs should include workspace .cargo metadata"
        );
        assert!(
            args.windows(2).any(|window| window == ["--exclude", "*"]),
            "streaming metadata-only syncs should exclude everything else"
        );
    }

    #[test]
    fn test_build_sync_streaming_command_default_sync_includes_delete() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/workspace-root"),
            "workspace-root".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );

        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let cmd = pipeline.build_sync_streaming_command(
            &worker,
            "mockuser@mock://worker:/data/tmp/rch/test-project/abc123",
            "/data/tmp/rch/test-project/abc123",
            &[],
        );

        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        assert!(
            args.iter().any(|arg| arg == "--delete"),
            "normal streaming syncs should retain delete semantics"
        );
    }

    // =========================================================================
    // Source-Integrity Hardening Tests (RCH bug d7xc3)
    // =========================================================================
    //
    // These tests pin the contract that artifact retrieval cannot dirty the
    // local source checkout, regardless of the pattern shape or what other
    // agents may have left on the remote worker. The fix has three layers:
    //
    //   1. anchor_retrieval_pattern: every artifact pattern is anchored at
    //      the rsync source root with leading `/`.
    //   2. allowed_artifact_roots: derive the implied top-level allowed roots
    //      from anchored patterns; non-glob top-level components only.
    //   3. local_source_roots_to_exclude: emit explicit `--exclude /<entry>`
    //      rules for every top-level entry in the local project root that
    //      isn't an allowed artifact root.
    //
    // Together these mean: even if rsync's filter semantics had a subtle
    // bug, AND a malicious/stale remote tree contained source files at
    // unexpected paths, AND the artifact patterns were too permissive,
    // rsync STILL refuses to descend into local source roots.

    #[test]
    fn anchor_retrieval_pattern_prepends_slash_when_unanchored() {
        // TEST START: bare patterns get anchored
        assert_eq!(
            anchor_retrieval_pattern("target/debug/**"),
            "/target/debug/**"
        );
        assert_eq!(
            anchor_retrieval_pattern("target/release/**"),
            "/target/release/**"
        );
        assert_eq!(anchor_retrieval_pattern("coverage/**"), "/coverage/**");
        assert_eq!(
            anchor_retrieval_pattern("*.tsbuildinfo"),
            "/*.tsbuildinfo",
            "top-level glob files must still be anchored"
        );
        // TEST PASS: unanchored patterns get a leading `/`
    }

    #[test]
    fn anchor_retrieval_pattern_preserves_already_anchored() {
        // TEST START: patterns starting with `/` are passthrough
        assert_eq!(
            anchor_retrieval_pattern("/target/debug/**"),
            "/target/debug/**"
        );
        assert_eq!(anchor_retrieval_pattern("/coverage/**"), "/coverage/**");
        // TEST PASS: already-anchored patterns are returned unchanged
    }

    #[test]
    fn anchor_retrieval_pattern_preserves_recursive_globstar() {
        // TEST START: explicit recursion via `**/` is intentional, leave alone
        assert_eq!(
            anchor_retrieval_pattern("**/junit.xml"),
            "**/junit.xml",
            "explicit recursive globstar must be preserved"
        );
        // TEST PASS: explicit `**/` patterns pass through
    }

    #[test]
    fn anchor_retrieval_pattern_handles_edge_cases() {
        // TEST START: empty and whitespace patterns are returned unchanged
        assert_eq!(anchor_retrieval_pattern(""), "");
        assert_eq!(anchor_retrieval_pattern("   "), "   ");
        // TEST PASS: caller decides what to do with junk input
    }

    #[test]
    fn allowed_artifact_roots_derives_first_component() {
        // TEST START: implied allowed roots = first path component of each
        // anchored pattern, glob components excluded.
        let patterns = vec![
            "target/debug/**".to_string(),
            "target/release/**".to_string(),
            "coverage/**".to_string(),
            "*.tsbuildinfo".to_string(),
            "/build/output/**".to_string(),
        ];
        let roots = allowed_artifact_roots(&patterns);
        assert!(roots.contains("target"));
        assert!(roots.contains("coverage"));
        assert!(roots.contains("build"));
        // Top-level glob patterns cannot contribute a single allowed root.
        assert!(!roots.iter().any(|r| r.contains('*')));
        // TEST PASS: derived allowed roots
    }

    #[test]
    fn allowed_artifact_roots_handles_empty_input() {
        // TEST START: defensive — empty patterns yields empty set
        let roots = allowed_artifact_roots(&[]);
        assert!(roots.is_empty());
        // TEST PASS: empty input is safe
    }

    #[test]
    fn local_source_roots_to_exclude_emits_explicit_rules_for_each_top_level_source_entry() {
        // TEST START: scan local project root, emit `--exclude /<entry>/`
        // for every top-level dir that ISN'T an allowed artifact root.
        // This is the belt-and-suspenders defense — even if rsync filters
        // are subtly wrong, these explicit anchored excludes pin source
        // roots out of the retrieval set.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("rch")).expect("mkdir rch");
        std::fs::create_dir(temp.path().join("rch-common")).expect("mkdir rch-common");
        std::fs::create_dir(temp.path().join("rchd")).expect("mkdir rchd");
        std::fs::create_dir(temp.path().join("target")).expect("mkdir target");
        std::fs::write(temp.path().join("Cargo.toml"), b"[workspace]\n").expect("write toml");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let artifact_patterns = ["target/debug/**".to_string()];
        let allowed = allowed_artifact_roots(&artifact_patterns);
        let excludes = pipeline.local_source_roots_to_exclude(&allowed, &artifact_patterns);

        // target/ MUST NOT be excluded — it's the artifact root.
        assert!(
            !excludes.iter().any(|e| e == "/target/"),
            "target/ is an allowed artifact root and must NOT be in the exclude set"
        );
        // Every other source dir MUST be excluded (anchored with leading `/`).
        assert!(
            excludes.iter().any(|e| e == "/rch/"),
            "rch/ source dir must be excluded; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == "/rch-common/"),
            "rch-common/ source dir must be excluded; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == "/rchd/"),
            "rchd/ source dir must be excluded; got {excludes:?}"
        );
        // Top-level files also excluded, with no trailing slash.
        assert!(
            excludes.iter().any(|e| e == "/Cargo.toml"),
            "top-level files must be excluded (no trailing slash); got {excludes:?}"
        );
        // Excludes are sorted for stability.
        let mut sorted = excludes.clone();
        sorted.sort();
        assert_eq!(excludes, sorted, "excludes must be sorted for determinism");
        // TEST PASS: source-root exclusion contract
    }

    #[test]
    fn local_source_roots_to_exclude_does_not_hide_top_level_glob_artifacts() {
        // TEST START: Bun/typecheck retrieval includes top-level globs such
        // as `*.tsbuildinfo`. If the local file already exists, the
        // source-integrity guard must not emit a prior exclude that prevents
        // rsync from refreshing it.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("src")).expect("mkdir src");
        std::fs::write(temp.path().join("Cargo.toml"), b"[workspace]\n").expect("write toml");
        std::fs::write(temp.path().join("tsconfig.tsbuildinfo"), b"old").expect("write artifact");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let artifact_patterns = ["*.tsbuildinfo".to_string()];
        let allowed = allowed_artifact_roots(&artifact_patterns);
        let excludes = pipeline.local_source_roots_to_exclude(&allowed, &artifact_patterns);

        assert!(
            !excludes.iter().any(|e| e == "/tsconfig.tsbuildinfo"),
            "declared top-level glob artifact must not be source-excluded; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == "/Cargo.toml"),
            "unrelated top-level files must still be source-excluded; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == "/src/"),
            "source directories must still be source-excluded; got {excludes:?}"
        );
        // TEST PASS: top-level artifact glob stays retrievable without
        // opening unrelated source entries.
    }

    #[test]
    fn catch_all_pattern_does_not_prove_literal_star_entry_is_artifact() {
        // TEST START: a catch-all retrieval pattern may fetch newly-created
        // root outputs, but it must not prove that a local source entry named
        // `*` is safe to overwrite.
        assert!(!top_level_artifact_pattern_matches_entry("*", "*"));
        assert_eq!(escape_rsync_filter_literal_component("*").as_ref(), r"\*");
        assert_eq!(
            escape_rsync_filter_literal_component("question?mark.c").as_ref(),
            r"question\?mark.c"
        );
        assert_eq!(
            escape_rsync_filter_literal_component("array[0].c").as_ref(),
            r"array\[0].c"
        );
        assert_eq!(
            escape_rsync_filter_literal_component(r"back\slash.rs").as_ref(),
            r"back\slash.rs"
        );
        // TEST PASS: wildcard-looking local names stay literal in filter rules.
    }

    #[cfg(unix)]
    #[test]
    fn local_source_roots_to_exclude_keeps_catch_all_from_unprotecting_source_entries() {
        // TEST START: C/C++ retrieval includes a catch-all `*` for newly
        // created root-level outputs. That pattern must not make every
        // existing local source entry look safe to overwrite.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("src")).expect("mkdir src");
        std::fs::write(
            temp.path().join("main.c"),
            b"int main(void) { return 0; }\n",
        )
        .expect("write source file");
        std::fs::write(temp.path().join("*"), b"literal star source\n")
            .expect("write literal star source file");
        std::fs::write(temp.path().join("question?mark.c"), b"int q(void);\n")
            .expect("write literal question source file");
        std::fs::write(temp.path().join("array[0].c"), b"int a(void);\n")
            .expect("write literal bracket source file");
        std::fs::write(temp.path().join("Makefile"), b"all:\n\tcc main.c\n")
            .expect("write makefile");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let artifact_patterns = ["*".to_string()];
        let allowed = allowed_artifact_roots(&artifact_patterns);
        let excludes = pipeline.local_source_roots_to_exclude(&allowed, &artifact_patterns);

        assert!(
            excludes.iter().any(|e| e == "/src/"),
            "catch-all artifact pattern must not unprotect source directories; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == "/main.c"),
            "catch-all artifact pattern must not unprotect existing source files; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == "/Makefile"),
            "catch-all artifact pattern must not unprotect existing build files; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == r"/\*"),
            "literal local filenames with rsync glob syntax must be excluded literally; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == r"/question\?mark.c"),
            "literal question-mark filenames must not become rsync wildcards; got {excludes:?}"
        );
        assert!(
            excludes.iter().any(|e| e == r"/array\[0].c"),
            "literal bracket filenames must not become rsync character classes; got {excludes:?}"
        );
        // TEST PASS: catch-all retrieval stays subordinate to source protection.
    }

    #[test]
    fn local_source_roots_to_exclude_handles_unreadable_project_root() {
        // TEST START: defensive — if project_root is unreadable, we get an
        // empty exclude list (not a panic). Other retrieval guards still
        // apply, so retrieval remains safe.
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            std::path::PathBuf::from("/this/path/does/not/exist/anywhere/d7xc3-test"),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let allowed = std::collections::BTreeSet::new();
        let excludes = pipeline.local_source_roots_to_exclude(&allowed, &[]);
        assert!(
            excludes.is_empty(),
            "unreadable project_root must yield empty excludes (got {excludes:?})"
        );
        // TEST PASS: unreadable root is non-fatal
    }

    #[test]
    fn build_result_dir_retrieve_command_targets_declared_relative_dir() {
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir_all(temp.path().join("results/shard-a")).expect("mkdir results");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: rch_common::WorkerId::new("w1"),
            host: "203.0.113.9".to_string(),
            user: "ubuntu".to_string(),
            identity_file: "~/.ssh/id_rsa".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };
        let cmd = pipeline.build_result_dir_retrieve_command(
            &worker,
            "/tmp/rch/proj_abc123",
            Path::new("results/shard-a"),
        );
        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|a| a.to_string_lossy().into_owned())
            .collect();

        // Explicit SOURCE (not pattern machinery) so rsync itself fails when
        // the declared dir is missing on the worker — loud missing-output.
        let source = args
            .iter()
            .find(|a| a.contains("@") && a.contains(":/"))
            .expect("rsync remote source argument");
        assert!(
            source.ends_with("/tmp/rch/proj_abc123/results/shard-a/"),
            "source must be the declared dir under the remote root with trailing slash; got {source}"
        );
        // Destination mirrors the repository-relative location locally.
        let dest = args.last().expect("destination argument");
        assert!(
            dest.starts_with(&format!("{}/results/shard-a/", temp.path().display())),
            "dest must mirror the relative location in the project root; got {dest}"
        );
        // Symlink-traversal hardening stays on for job result pulls.
        assert!(args.iter().any(|a| a == "--safe-links"));
    }

    #[test]
    fn test_layer0_env_pairs_are_injected_verbatim() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        )
        .with_layer0_env(vec![
            ("CARGO_PROFILE_RELEASE_LTO".to_string(), "thin".to_string()),
            (
                "CARGO_PROFILE_RELEASE_CODEGEN_UNITS".to_string(),
                "1".to_string(),
            ),
        ]);
        let env_prefix = pipeline.build_env_prefix();
        assert!(
            env_prefix.prefix.contains("CARGO_PROFILE_RELEASE_LTO=thin"),
            "layer0 pairs must be emitted as assignments: {}",
            env_prefix.prefix
        );
        assert!(
            env_prefix
                .prefix
                .contains("CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1")
        );
        assert!(
            env_prefix
                .applied
                .contains(&"CARGO_PROFILE_RELEASE_LTO".to_string())
        );
        assert!(
            env_prefix
                .applied
                .contains(&"CARGO_PROFILE_RELEASE_CODEGEN_UNITS".to_string())
        );
    }

    #[test]
    fn test_layer0_env_absent_by_default() {
        let _guard = test_guard!();
        let pipeline = TransferPipeline::new(
            PathBuf::from("/tmp/project"),
            "project".to_string(),
            "hash".to_string(),
            TransferConfig::default(),
        );

        let env_prefix = pipeline.build_env_prefix();
        assert!(!env_prefix.prefix.contains("CARGO_PROFILE_"));
        assert!(
            !env_prefix
                .applied
                .iter()
                .any(|k| k.starts_with("CARGO_PROFILE_")),
            "no Layer 0 knob may inject unless resolved and attached"
        );
    }
    #[test]
    fn build_retrieve_command_excludes_local_source_dirs_at_anchored_paths() {
        // TEST START: integration test — build the retrieve command for a
        // workspace-shaped local project and verify both layers fire:
        //
        //   (a) The artifact pattern is anchored.
        //   (b) Every top-level source dir gets an explicit anchored exclude
        //       BEFORE the directory `--include "*/"`.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("rch")).expect("mkdir rch");
        std::fs::create_dir(temp.path().join("rch-common")).expect("mkdir rch-common");
        std::fs::create_dir(temp.path().join("rchd")).expect("mkdir rchd");
        std::fs::create_dir(temp.path().join("rch-wkr")).expect("mkdir rch-wkr");
        std::fs::create_dir(temp.path().join("target")).expect("mkdir target");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };
        let cmd = pipeline.build_retrieve_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["target/debug/**".to_string()],
        );
        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        // (a) Artifact pattern anchored.
        assert!(
            args.windows(2)
                .any(|w| w == ["--include", "/target/debug/**"]),
            "artifact pattern must be emitted in anchored form (RCH bug d7xc3); got args = {args:?}"
        );
        assert!(
            !args
                .windows(2)
                .any(|w| w == ["--include", "target/debug/**"]),
            "unanchored pattern form must NOT be emitted (RCH bug d7xc3)"
        );

        // (b) All four workspace source dirs explicitly excluded with anchored form.
        for src in ["/rch/", "/rch-common/", "/rchd/", "/rch-wkr/"] {
            assert!(
                args.windows(2).any(|w| w == ["--exclude", src]),
                "{src} must be in the exclude set (source-integrity guard); got args = {args:?}"
            );
        }

        // The artifact root MUST NOT be in the source-exclude set.
        assert!(
            !args.windows(2).any(|w| w == ["--exclude", "/target/"]),
            "target/ is the allowed artifact root and must NOT be source-excluded"
        );

        // Ordering: source-excludes are emitted BEFORE the `--include "*/"`
        // directive so rsync evaluates them first and refuses to descend.
        let include_dirs_pos = args
            .windows(2)
            .position(|w| w == ["--include", "*/"])
            .expect("missing directory include");
        let first_source_exclude_pos = args
            .windows(2)
            .position(|w| w == ["--exclude", "/rch/"])
            .expect("missing /rch/ exclude");
        assert!(
            first_source_exclude_pos < include_dirs_pos,
            "source-integrity excludes must be applied BEFORE --include */"
        );
        // TEST PASS: source-integrity guard wired through both helpers
    }

    #[test]
    fn build_retrieve_command_keeps_existing_top_level_glob_artifact_retrievable() {
        // TEST START: command-level proof for the Bun/default artifact case.
        // A local tsbuildinfo file must not be excluded before the artifact
        // include can match it.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("src")).expect("mkdir src");
        std::fs::write(temp.path().join("tsconfig.tsbuildinfo"), b"old").expect("write artifact");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };
        let cmd = pipeline.build_retrieve_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["*.tsbuildinfo".to_string()],
        );
        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();
        assert!(
            args.windows(2)
                .any(|w| w == ["--include", "/*.tsbuildinfo"]),
            "top-level glob artifact must be emitted in anchored form; got args = {args:?}"
        );
        assert!(
            !args
                .windows(2)
                .any(|w| w == ["--exclude", "/tsconfig.tsbuildinfo"]),
            "existing top-level artifact must not be excluded before its include; got args = {args:?}"
        );
        assert!(
            args.windows(2).any(|w| w == ["--exclude", "/src/"]),
            "source directories must remain protected; got args = {args:?}"
        );
        // TEST PASS: command preserves both source guard and top-level glob retrieval.
    }

    #[cfg(unix)]
    #[test]
    fn build_retrieve_command_with_catch_all_still_excludes_existing_source_entries() {
        // TEST START: command-level proof for the C/C++ catch-all artifact
        // pattern. The broad include may fetch new root outputs, but it must
        // not run before source excludes for existing entries.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("src")).expect("mkdir src");
        std::fs::write(
            temp.path().join("main.c"),
            b"int main(void) { return 0; }\n",
        )
        .expect("write source file");
        std::fs::write(temp.path().join("*"), b"literal star source\n")
            .expect("write literal star source file");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };
        let cmd = pipeline.build_retrieve_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["*".into()],
        );
        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();

        let include_dirs_pos = args
            .windows(2)
            .position(|w| w == ["--include", "*/"])
            .expect("missing directory include");
        let catch_all_include_pos = args
            .windows(2)
            .position(|w| w == ["--include", "/*"])
            .expect("missing anchored catch-all include");
        let src_exclude_pos = args
            .windows(2)
            .position(|w| w == ["--exclude", "/src/"])
            .expect("missing /src/ exclude");
        let main_exclude_pos = args
            .windows(2)
            .position(|w| w == ["--exclude", "/main.c"])
            .expect("missing /main.c exclude");
        let literal_star_exclude_pos = args
            .windows(2)
            .position(|w| w == ["--exclude", r"/\*"])
            .expect("missing escaped literal star exclude");

        assert!(src_exclude_pos < include_dirs_pos);
        assert!(main_exclude_pos < include_dirs_pos);
        assert!(literal_star_exclude_pos < include_dirs_pos);
        assert!(include_dirs_pos < catch_all_include_pos);
        assert!(
            !args.windows(2).any(|w| w == ["--exclude", "/*"]),
            "literal local star source must not become a broad /* exclude; got args = {args:?}"
        );
        // TEST PASS: broad include remains behind source-integrity excludes.
    }

    #[test]
    fn build_retrieve_streaming_command_also_applies_source_integrity_guard() {
        // TEST START: parity check — the streaming variant must apply the
        // same guard. If they drift, the bug returns under one code path.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("src")).expect("mkdir src");
        std::fs::create_dir(temp.path().join("target")).expect("mkdir target");
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };
        let cmd = pipeline.build_retrieve_streaming_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["target/release/**".to_string()],
        );
        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();
        assert!(
            args.windows(2)
                .any(|w| w == ["--include", "/target/release/**"]),
            "streaming variant must anchor patterns (RCH bug d7xc3)"
        );
        assert!(
            args.windows(2).any(|w| w == ["--exclude", "/src/"]),
            "streaming variant must apply source-integrity excludes (RCH bug d7xc3)"
        );
        // TEST PASS: streaming + non-streaming have identical safety contract
    }

    #[test]
    fn build_retrieve_command_with_absolute_cargo_target_dir_still_safe() {
        // TEST START: simulate the original d7xc3 scenario — operator set
        // CARGO_TARGET_DIR to an ABSOLUTE path outside the project (e.g.,
        // /tmp/rch-target-foo). The artifact pattern is still `target/...`
        // (relative to project), but no target/ exists locally. Our guard
        // must STILL prevent any local source dir from being a retrieval
        // candidate.
        let _guard = test_guard!();
        let temp = tempfile::tempdir().expect("create temp dir");
        std::fs::create_dir(temp.path().join("rch")).expect("mkdir rch");
        std::fs::create_dir(temp.path().join("rch-common")).expect("mkdir rch-common");
        // NO target/ dir locally — simulates absolute CARGO_TARGET_DIR.
        let pipeline = TransferPipeline::new(
            temp.path().to_path_buf(),
            "test-project".to_string(),
            "abc123".to_string(),
            TransferConfig::default(),
        );
        let worker = WorkerConfig {
            id: WorkerId::new("mock-worker"),
            host: "mock://worker".to_string(),
            user: "mockuser".to_string(),
            identity_file: "~/.ssh/mock".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };
        let cmd = pipeline.build_retrieve_command(
            &worker,
            "/data/tmp/rch/test-project/abc123",
            &["target/debug/**".to_string()],
        );
        let args: Vec<String> = cmd
            .as_std()
            .get_args()
            .map(|arg| arg.to_string_lossy().to_string())
            .collect();
        assert!(
            args.windows(2).any(|w| w == ["--exclude", "/rch/"]),
            "source-integrity excludes must fire even when local target/ is absent"
        );
        assert!(
            args.windows(2).any(|w| w == ["--exclude", "/rch-common/"]),
            "all local top-level source dirs must be excluded"
        );
        // TEST PASS: absolute CARGO_TARGET_DIR case
    }
}