mj_controller/hel_controller/

worker_binary.rs

//! Worker binary acquisition, profile staging, and worker installation.

use std::path::{Path, PathBuf};
use std::time::Instant;

use anyhow::{Context, Result, bail, ensure};
use rayon::prelude::*;
use sha2::{Digest, Sha256};

use crate::hel_session_manager::{
    ProjectMemorySyncTarget, RemoteWorkerBinaryRefresh, WorkerBinaryRefresh,
    WorkerBinaryRefreshPlan, WorkerLaunchRefreshPlan, WorkerRecoveryPlan,
};
use hel::hel_config::{
    ExecutionPolicy, HarnessKind, HarnessProfile, ProjectBundle, ProjectRepository, atomic_write,
    data_dir,
};
use hel::hel_project_memory::{ProjectMemoryIdentity, RepositoryMemoryIdentity};
use hel::hel_targets::{
    self, CommandExecutor, CommandPlan, CommandSpec, ProcessExecutor, ProvisionStage, SshTarget,
};
use hel::hel_worker_launch::{
    DISCOVER_LOGIN_PATH_ENV, ProjectMemoryLaunchConfig, ProjectMemoryMcpDelivery,
    WorkerLaunchConfig, WorkerOwnership,
};

use super::backend::backend_locator;
use super::readiness::WORKER_EXIT_RECORD_MARKER;
use super::{Controller, execute_checked, scp_command_spec, ssh_command_spec, target_profile_home};

impl Controller {
    /// Where this session's worker lives. This is decided from the session
    /// record and configuration alone, so a caller can name the worker root
    /// before anything is installed into it.
    pub(super) fn worker_placement(
        &self,
        session_id: &str,
    ) -> Result<(hel_targets::TargetLocator, String)> {
        let session = self
            .state
            .sessions
            .get(session_id)
            .with_context(|| format!("unknown session {session_id}"))?;
        let locator = session
            .target
            .as_ref()
            .context("session target is missing")?;
        let backend = backend_locator(locator, session, &self.config)?;
        let worker_root = hel_targets::worker_root(&backend, session_id)?;
        Ok((backend, worker_root))
    }

    pub(super) fn prepare_worker_files(
        &self,
        session_id: &str,
        backend: &hel_targets::TargetLocator,
        worker_root: &str,
        executor: &impl CommandExecutor,
    ) -> Result<()> {
        let session = self
            .state
            .sessions
            .get(session_id)
            .with_context(|| format!("unknown session {session_id}"))?;
        let profile = self
            .config
            .profiles
            .get(&session.last_profile)
            .context("session profile is missing")?;
        let bundle = session
            .project_directory
            .is_none()
            .then(|| self.config.bundles.get(&session.bundle_id))
            .flatten();
        let target = self
            .config
            .targets
            .get(&session.target_template_id)
            .context("session target template is missing")?;
        let (launch, project_memory, target_profile_home) = worker_launch_config(
            session,
            profile,
            bundle,
            backend,
            session_id,
            target.execution_policy(),
        )?;

        let staging = tempfile::tempdir().context("create worker staging directory")?;
        let launch_path = staging.path().join("launch.json");
        launch.write(&launch_path)?;
        let ownership_path = staging.path().join("ownership.json");
        WorkerOwnership {
            version: WorkerOwnership::VERSION,
            workspace_id: session.workspace_id.clone(),
            session_id: session_id.to_string(),
            profile_id: session.last_profile.clone(),
            bundle_id: session.bundle_id.clone(),
            target_template_id: session.target_template_id.clone(),
        }
        .write(&ownership_path)?;
        let profile_stage = staging.path().join("profile");
        if !matches!(backend, hel_targets::TargetLocator::LocalBare { .. }) {
            let started = Instant::now();
            let result = stage_profile(profile, &profile_stage);
            tracing::debug!(
                session_id,
                elapsed_ms = started.elapsed().as_millis(),
                "profile staging completed"
            );
            result?;
            append_hel_target_environment(profile.kind, &profile_stage, backend)?;
            stage_memory_replica(
                &project_memory,
                Path::new(&target_profile_home),
                &profile_stage,
            )?;
            if project_memory.mcp_delivery == ProjectMemoryMcpDelivery::HarnessProfile {
                configure_kimi_project_memory_mcp(&profile_stage, worker_root, &project_memory)?;
            }
        } else {
            seed_local_memory_replica(&project_memory)?;
        }
        let worker_binary = worker_binary_for(backend, executor)?;

        install_worker_files(
            executor,
            backend,
            session_id,
            worker_root,
            &target_profile_home,
            &worker_binary,
            &launch_path,
            &ownership_path,
            &profile_stage,
        )
    }

    /// Probe the installed binary and collect the dead worker's exit record
    /// and log tail after a session becomes unreachable. Best-effort; returns
    /// `None` when the target no longer exists or has no diagnostics.
    pub fn diagnose_worker(&self, session_id: &str) -> Option<String> {
        self.diagnose_worker_controlled(session_id, &ProcessExecutor)
    }

    pub fn diagnose_worker_controlled(
        &self,
        session_id: &str,
        executor: &impl CommandExecutor,
    ) -> Option<String> {
        let session = self.state.sessions.get(session_id)?;
        let locator = session.target.as_ref()?;
        let backend = match backend_locator(locator, session, &self.config) {
            Ok(backend) => backend,
            Err(error) => {
                tracing::debug!(
                    session_id,
                    error = format!("{error:#}"),
                    "could not construct a worker diagnostic probe"
                );
                return None;
            }
        };
        let worker_root = match hel_targets::worker_root(&backend, session_id) {
            Ok(root) => root,
            Err(error) => {
                tracing::debug!(
                    session_id,
                    error = format!("{error:#}"),
                    "could not derive the worker diagnostic root"
                );
                return None;
            }
        };
        let binary_failure = worker_binary_probe_failure(executor, &backend, &worker_root);
        let last_words = worker_last_words(executor, &backend, &worker_root);
        match (binary_failure, last_words) {
            (Some(binary_failure), Some(last_words)) => {
                Some(format!("{binary_failure}; {last_words}"))
            }
            (Some(binary_failure), None) => Some(binary_failure),
            (None, last_words) => last_words,
        }
    }

    /// A non-destructive liveness probe plus commands that replace a confirmed
    /// dead session worker without touching its durable relay files. The
    /// session manager runs both off its async actor.
    pub fn worker_recovery_plan(&self, session_id: &str) -> Result<WorkerRecoveryPlan> {
        let (backend, worker_root) = self.worker_placement(session_id)?;
        let session = self
            .state
            .sessions
            .get(session_id)
            .with_context(|| format!("unknown session {session_id}"))?;
        let profile = self
            .config
            .profiles
            .get(&session.last_profile)
            .context("session profile is missing")?;
        let bundle = session
            .project_directory
            .is_none()
            .then(|| self.config.bundles.get(&session.bundle_id))
            .flatten();
        let target = self
            .config
            .targets
            .get(&session.target_template_id)
            .context("session target template is missing")?;
        let (launch, _, _) = worker_launch_config(
            session,
            profile,
            bundle,
            &backend,
            session_id,
            target.execution_policy(),
        )?;
        Ok(WorkerRecoveryPlan {
            target: hel_targets::target_recovery_plan(&backend, session_id)?,
            liveness_probe: worker_liveness_command(&backend, &worker_root),
            binary_refresh: worker_binary_refresh_plan(&backend, session_id)?,
            launch_refresh: Some(worker_launch_refresh_plan(&backend, session_id, &launch)?),
            restart: CommandPlan {
                description: format!("restart Mjolnir worker for session {session_id}"),
                commands: vec![
                    stop_worker_command(&backend, &worker_root),
                    start_worker_command(&backend, &worker_root),
                ],
            },
        })
    }

    pub fn project_memory_sync_target(&self, session_id: &str) -> Result<ProjectMemorySyncTarget> {
        let session = self
            .state
            .sessions
            .get(session_id)
            .with_context(|| format!("unknown session {session_id}"))?;
        let locator = session
            .target
            .as_ref()
            .context("session target is missing")?;
        let backend = backend_locator(locator, session, &self.config)?;
        let profile = self
            .config
            .profiles
            .get(&session.last_profile)
            .context("session profile is missing")?;
        let bundle = session
            .project_directory
            .is_none()
            .then(|| self.config.bundles.get(&session.bundle_id))
            .flatten();
        let workspace = if let Some(project_directory) = &session.project_directory {
            (project_directory.to_string_lossy().into_owned(), Vec::new())
        } else {
            workspace_paths(
                &backend,
                bundle.context("session bundle is missing")?,
                session_id,
            )?
        };
        let target_home = target_profile_home(&backend, session_id, profile);
        let launch = project_memory_launch(session, bundle, &workspace, &target_home)?;
        Ok(ProjectMemorySyncTarget {
            canonical_root: canonical_memory_root(&launch.project_key),
        })
    }
}

fn worker_launch_config(
    session: &hel::hel_state::SessionRecord,
    profile: &hel::hel_config::HarnessProfile,
    bundle: Option<&ProjectBundle>,
    backend: &hel_targets::TargetLocator,
    session_id: &str,
    execution_policy: ExecutionPolicy,
) -> Result<(WorkerLaunchConfig, ProjectMemoryLaunchConfig, String)> {
    let target_profile_home = target_profile_home(backend, session_id, profile);
    let workspace = if let Some(project_directory) = &session.project_directory {
        (project_directory.to_string_lossy().into_owned(), Vec::new())
    } else {
        workspace_paths(
            backend,
            bundle.context("session bundle is missing")?,
            session_id,
        )?
    };
    let mut additional_directories = workspace.1.iter().map(PathBuf::from).collect::<Vec<_>>();
    additional_directories.extend(
        session
            .additional_mounts
            .iter()
            .map(|resource| resource.destination.clone()),
    );
    if profile.kind == hel::hel_config::HarnessKind::Deepseek && !additional_directories.is_empty()
    {
        bail!(
            "DeepSeek Harness ACP does not support multiple workspace roots; use a single-repository bundle"
        );
    }
    let (bridge_command, bridge_args) = bridge_launch(
        profile.kind,
        profile.executable.as_deref(),
        execution_policy,
    );
    let mut environment = profile.environment.clone();
    environment.insert(profile.home_env().into(), target_profile_home.clone());
    profile
        .kind
        .configure_execution_environment(execution_policy, &mut environment);
    configure_login_path_discovery(&mut environment, backend);
    let mut project_memory =
        project_memory_launch(session, bundle, &workspace, &target_profile_home)?;
    project_memory.mcp_delivery = project_memory_mcp_delivery(profile.kind, backend);
    if profile.kind == hel::hel_config::HarnessKind::Claude {
        environment.insert(
            "CLAUDE_CODE_PROJECT_DIR_NAME".into(),
            project_memory_replica_slug(&project_memory.project_key, session_id),
        );
    }
    apply_claude_setup_token(
        &mut environment,
        profile.kind,
        &hel::hel_credentials::claude_oauth_token_path(&session.last_profile),
    );
    Ok((
        WorkerLaunchConfig {
            session_id: session_id.to_string(),
            harness: profile.kind,
            bridge_command: PathBuf::from(bridge_command),
            bridge_args,
            environment,
            cwd: PathBuf::from(&workspace.0),
            additional_directories,
            native_session_id: session.native_session_id.clone(),
            project_memory: Some(project_memory.clone()),
            execution_policy,
        },
        project_memory,
        target_profile_home,
    ))
}

/// Hand a Claude worker the profile's long-lived setup token, when it has one.
///
/// Claude Code reads `CLAUDE_CODE_OAUTH_TOKEN` ahead of the `/login`
/// credentials file, and a setup token does not rotate, so a container copy
/// cannot lose the single-use refresh race with the host. A profile that sets
/// the variable itself stays authoritative.
fn apply_claude_setup_token(
    environment: &mut std::collections::BTreeMap<String, String>,
    kind: hel::hel_config::HarnessKind,
    token_path: &Path,
) {
    use hel::hel_credentials::CLAUDE_OAUTH_TOKEN_ENV;

    if kind != hel::hel_config::HarnessKind::Claude
        || environment.contains_key(CLAUDE_OAUTH_TOKEN_ENV)
    {
        return;
    }
    match hel::hel_credentials::read_claude_oauth_token(token_path) {
        Ok(Some(token)) => {
            environment.insert(CLAUDE_OAUTH_TOKEN_ENV.to_owned(), token);
        }
        Ok(None) => {}
        // A stored token Hel cannot read is worth reporting, but the session
        // still starts on the synced credentials file.
        Err(error) => tracing::warn!(
            path = %token_path.display(),
            %error,
            "ignoring an unreadable Claude setup token"
        ),
    }
}

fn configure_login_path_discovery(
    environment: &mut std::collections::BTreeMap<String, String>,
    backend: &hel_targets::TargetLocator,
) {
    environment.remove(DISCOVER_LOGIN_PATH_ENV);
    if !environment.contains_key("PATH")
        && matches!(
            backend,
            hel_targets::TargetLocator::LocalBare { .. }
                | hel_targets::TargetLocator::AwsEc2 { .. }
                | hel_targets::TargetLocator::SshBare { .. }
        )
    {
        environment.insert(DISCOVER_LOGIN_PATH_ENV.into(), "1".into());
    }
}

fn project_memory_launch(
    session: &hel::hel_state::SessionRecord,
    bundle: Option<&ProjectBundle>,
    workspace: &(String, Vec<String>),
    target_profile_home: &str,
) -> Result<ProjectMemoryLaunchConfig> {
    let identity = if let Some(worktree) = &session.managed_worktree {
        ProjectMemoryIdentity::Repository {
            repository: RepositoryMemoryIdentity::Local {
                canonical_root: std::fs::canonicalize(&worktree.source_repository)
                    .unwrap_or_else(|_| worktree.source_repository.clone()),
            },
        }
    } else if let Some(bundle) = bundle {
        let primary =
            configured_memory_identity(bundle.primary().context("bundle primary is missing")?)?;
        let members = bundle
            .repositories
            .iter()
            .map(configured_memory_identity)
            .collect::<Result<Vec<_>>>()?;
        ProjectMemoryIdentity::bundle(primary, members)
    } else {
        let project = session
            .project_directory
            .as_ref()
            .context("raw session project directory is missing")?;
        let repository = match session.target.as_ref() {
            Some(hel::hel_state::TargetLocator::LocalBare { .. }) => {
                RepositoryMemoryIdentity::Local {
                    canonical_root: std::fs::canonicalize(project)
                        .unwrap_or_else(|_| project.clone()),
                }
            }
            _ => RepositoryMemoryIdentity::Remote {
                target: session.target_template_id.clone(),
                canonical_root: project.clone(),
            },
        };
        ProjectMemoryIdentity::Repository { repository }
    };
    let project_key = identity.key()?;
    let replica_slug = project_memory_replica_slug(&project_key, &session.id);
    let project_root = PathBuf::from(target_profile_home)
        .join("projects")
        .join(replica_slug);
    let root = project_root.join("memory");
    let baseline_root = project_root.join(".hel-memory-baseline");
    let mut repository_roots = std::collections::BTreeMap::new();
    if let Some(bundle) = bundle {
        let target_roots =
            std::iter::once(workspace.0.as_str()).chain(workspace.1.iter().map(String::as_str));
        let repositories = std::iter::once(bundle.primary().context("bundle primary is missing")?)
            .chain(
                bundle
                    .repositories
                    .iter()
                    .filter(|repository| repository.id != bundle.primary_repo),
            );
        repository_roots.extend(
            repositories
                .zip(target_roots)
                .map(|(repository, root)| (repository.id.clone(), PathBuf::from(root))),
        );
    }
    Ok(ProjectMemoryLaunchConfig {
        project_key,
        root,
        baseline_root,
        repository_roots,
        mcp_delivery: ProjectMemoryMcpDelivery::Acp,
    })
}

fn project_memory_replica_slug(project_key: &str, session_id: &str) -> String {
    format!("hel-{}-{session_id}", &project_key[..16])
}

fn project_memory_mcp_delivery(
    harness: hel::hel_config::HarnessKind,
    target: &hel_targets::TargetLocator,
) -> ProjectMemoryMcpDelivery {
    if harness == hel::hel_config::HarnessKind::Kimi
        && !matches!(target, hel_targets::TargetLocator::LocalBare { .. })
    {
        ProjectMemoryMcpDelivery::HarnessProfile
    } else {
        ProjectMemoryMcpDelivery::Acp
    }
}

fn configured_memory_identity(repository: &ProjectRepository) -> Result<RepositoryMemoryIdentity> {
    if let Some(source) = repository.github.as_deref() {
        let github = crate::hel_setup::github_repository_from_origin(source)
            .with_context(|| format!("parse repository source {source:?} for project memory"))?;
        return Ok(RepositoryMemoryIdentity::Github {
            owner: github.owner.to_ascii_lowercase(),
            repository: github.repository.to_ascii_lowercase(),
        });
    }
    let root = repository
        .local
        .as_ref()
        .context("project repository has no source for memory identity")?;
    Ok(RepositoryMemoryIdentity::Local {
        canonical_root: hel::hel_local_git::main_worktree_root(root)
            .or_else(|_| std::fs::canonicalize(root).map_err(anyhow::Error::from))
            .unwrap_or_else(|_| root.clone()),
    })
}

fn canonical_memory_root(project_key: &str) -> PathBuf {
    data_dir().join("projects").join(project_key).join("memory")
}

fn stage_memory_replica(
    memory: &ProjectMemoryLaunchConfig,
    target_profile_home: &Path,
    profile_stage: &Path,
) -> Result<()> {
    let canonical = canonical_memory_root(&memory.project_key);
    std::fs::create_dir_all(&canonical)?;
    let replica = memory.root.strip_prefix(target_profile_home)?;
    let baseline = memory.baseline_root.strip_prefix(target_profile_home)?;
    copy_profile_entry(&canonical, &profile_stage.join(replica))?;
    copy_profile_entry(&canonical, &profile_stage.join(baseline))
}

fn seed_local_memory_replica(memory: &ProjectMemoryLaunchConfig) -> Result<()> {
    let canonical = canonical_memory_root(&memory.project_key);
    std::fs::create_dir_all(&canonical)?;
    let canonical_has_files = directory_has_files(&canonical)?;
    let replica_has_files = directory_has_files(&memory.root)?;
    match (canonical_has_files, replica_has_files) {
        (false, true) => copy_profile_entry(&memory.root, &canonical),
        (true, false) => copy_profile_entry(&canonical, &memory.root),
        _ => Ok(()),
    }?;
    copy_profile_entry(&canonical, &memory.baseline_root)
}

/// Kimi's runtime-aware engine cannot infer a runtime identity from an ACP
/// stdio server. Add Hel's server to the session-private profile instead,
/// where Kimi's native schema can bind it to the target's local runtime.
fn configure_kimi_project_memory_mcp(
    profile_stage: &Path,
    worker_root: &str,
    memory: &ProjectMemoryLaunchConfig,
) -> Result<()> {
    let path = profile_stage.join("mcp.json");
    let mut document = match std::fs::read(&path) {
        Ok(body) => serde_json::from_slice::<serde_json::Value>(&body)
            .with_context(|| format!("parse staged Kimi MCP configuration {}", path.display()))?,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
            serde_json::Value::Object(serde_json::Map::new())
        }
        Err(error) => {
            return Err(error)
                .with_context(|| format!("read staged Kimi MCP configuration {}", path.display()));
        }
    };
    let root = document.as_object_mut().with_context(|| {
        format!(
            "staged Kimi MCP configuration {} must contain a JSON object",
            path.display()
        )
    })?;
    let servers = root
        .entry("mcpServers")
        .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()))
        .as_object_mut()
        .with_context(|| {
            format!(
                "mcpServers in staged Kimi MCP configuration {} must be a JSON object",
                path.display()
            )
        })?;

    let worker = Path::new(worker_root).join("hel");
    let server = if worker.is_absolute() && memory.root.is_absolute() {
        serde_json::json!({
            "transport": "stdio",
            "command": worker,
            "args": ["worker", "memory-mcp", "--root", memory.root],
            "runtime_id": "local"
        })
    } else {
        let worker = worker.to_string_lossy();
        let memory_root = memory.root.to_string_lossy();
        serde_json::json!({
            "transport": "stdio",
            "command": "sh",
            "args": [
                "-c",
                "exec \"$HOME/$1\" worker memory-mcp --root \"$HOME/$2\"",
                "mj-project-memory",
                worker,
                memory_root
            ],
            "runtime_id": "local"
        })
    };
    servers.insert("mj-project-memory".into(), server);
    let mut body = serde_json::to_vec_pretty(&document)?;
    body.push(b'\n');
    atomic_write(&path, &body)
        .with_context(|| format!("write staged Kimi MCP configuration {}", path.display()))
}

fn directory_has_files(path: &Path) -> Result<bool> {
    let entries = match std::fs::read_dir(path) {
        Ok(entries) => entries,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
        Err(error) => return Err(error.into()),
    };
    for entry in entries {
        let entry = entry?;
        let metadata = entry.metadata()?;
        if metadata.is_file() || (metadata.is_dir() && directory_has_files(&entry.path())?) {
            return Ok(true);
        }
    }
    Ok(false)
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WorkerBinaryAvailability {
    Local {
        path: PathBuf,
        source: String,
    },
    Remote {
        url: String,
        sha256: String,
        triple: String,
    },
}

fn packaged_worker_binary_path(directory: &Path, triple: &str) -> PathBuf {
    directory.join(format!("mj-worker-{triple}"))
}

/// Linux exposes an unlinked running executable through `/proc` with a
/// ` (deleted)` suffix. `current_exe` preserves that suffix, but it is not
/// part of the executable's real file name and must not leak into sibling
/// lookup after `cargo` or a package upgrade replaces the controller.
fn running_executable_file_name(controller: &Path) -> Option<std::ffi::OsString> {
    let name = controller.file_name()?;
    #[cfg(target_os = "linux")]
    {
        use std::os::unix::ffi::{OsStrExt, OsStringExt};

        if let Some(name) = name.as_bytes().strip_suffix(b" (deleted)") {
            return Some(std::ffi::OsString::from_vec(name.to_vec()));
        }
    }
    Some(name.to_os_string())
}

/// File names a worker binary may carry when it sits beside the controller or
/// in a development sibling directory. The controller's own file name comes
/// first (after the 2.0 rename that is `mj`), then the legacy `hel` name that
/// older packages shipped, so both resolve without hardcoding one.
fn worker_sibling_names(controller: &Path) -> Vec<std::ffi::OsString> {
    use std::ffi::OsString;
    let mut names = Vec::new();
    if let Some(own) = running_executable_file_name(controller) {
        names.push(own);
    }
    let legacy = OsString::from("hel");
    if !names.contains(&legacy) {
        names.push(legacy);
    }
    names
}

/// Choose a worker binary that ships beside the controller or in a development
/// musl sibling directory. `is_file` probes the filesystem; tests pass a
/// hand-written probe. The static musl sibling is probed before the worker in
/// the controller's own directory, because in a development checkout that
/// same-directory candidate resolves to the controller itself, whose glibc may
/// be newer than the target's.
fn select_sibling_worker(
    controller: &Path,
    triple: &str,
    is_file: impl Fn(&Path) -> bool,
) -> Option<(PathBuf, &'static str)> {
    let directory = controller.parent()?;
    let names = worker_sibling_names(controller);
    let mut candidates: Vec<(PathBuf, &'static str)> = Vec::new();
    // Packaged worker beside the controller, named for the target triple.
    candidates.push((
        packaged_worker_binary_path(directory, triple),
        "beside the mj binary",
    ));
    // Development checkout: a controller at target/<profile>/<name> finds its
    // musl sibling at target/<triple>/<profile>/<name>. The static build is
    // preferred because the target's glibc may be older than the host's, so it
    // is probed before the same-directory worker (which is the controller
    // itself in a development checkout).
    if let (Some(profile), Some(target_dir)) = (directory.file_name(), directory.parent()) {
        for name in &names {
            candidates.push((
                target_dir.join(triple).join(profile).join(name),
                "development musl sibling",
            ));
        }
    }
    // A legacy package may put an `hel`-named worker beside an `mj`
    // controller. Never select the controller's own same-directory path: on
    // glibc Linux that is not a portable worker, and after an upgrade it is
    // the replacement controller rather than the still-running executable.
    let controller_name = running_executable_file_name(controller);
    for name in names
        .iter()
        .filter(|name| Some(name.as_os_str()) != controller_name.as_deref())
    {
        candidates.push((directory.join(name), "beside the running executable"));
    }
    candidates.into_iter().find(|(path, _)| is_file(path))
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum WorkerBinaryRequirement {
    PortableLinux,
    LocalHost,
}

/// Find a worker source without downloading it.
///
/// Container provisioning resolves this after discovering the target
/// architecture. Doctor uses the same lookup with the selected container's
/// expected architecture, so it can recommend a fix without creating a
/// container or making a network request.
pub fn worker_binary_prerequisite_for_arch(arch: &str) -> Result<WorkerBinaryAvailability> {
    worker_binary_for_arch(arch, WorkerBinaryRequirement::PortableLinux)
}

fn worker_binary_for_arch(
    arch: &str,
    requirement: WorkerBinaryRequirement,
) -> Result<WorkerBinaryAvailability> {
    let current = std::env::current_exe().context("resolve Mjolnir controller binary")?;
    worker_binary_prerequisite_for_current(arch, requirement, &current, &|path| path.is_file())
}

/// The lookup itself, with the controller's own path and the file probe passed
/// in so both can be exercised without the machine they describe.
fn worker_binary_prerequisite_for_current(
    arch: &str,
    requirement: WorkerBinaryRequirement,
    current: &Path,
    is_file: &dyn Fn(&Path) -> bool,
) -> Result<WorkerBinaryAvailability> {
    let triple = format!("{arch}-unknown-linux-musl");
    if let Some(path) = hel::hel_config::env_override_os("WORKER_BINARY").map(PathBuf::from) {
        if !is_file(&path) {
            bail!("MJ_WORKER_BINARY is not a file: {}", path.display());
        }
        return Ok(WorkerBinaryAvailability::Local {
            path,
            source: "MJ_WORKER_BINARY".into(),
        });
    }
    // A rebuilt or renamed checkout leaves a running controller pointing at a
    // path that no longer holds a binary. Every lookup derived from that path
    // is meaningless, so remember the fact and skip those lookups.
    let controller_replaced = !is_file(current);
    let mut candidates = Vec::new();
    if let Some(directory) = hel::hel_config::env_override_os("WORKER_DIR").map(PathBuf::from) {
        candidates.push((
            packaged_worker_binary_path(&directory, &triple),
            "MJ_WORKER_DIR",
        ));
        candidates.push((directory.join(&triple).join("hel"), "MJ_WORKER_DIR"));
    }
    if let Some((path, source)) = candidates.into_iter().find(|(path, _)| is_file(path)) {
        return Ok(WorkerBinaryAvailability::Local {
            path,
            source: source.into(),
        });
    }
    // The native branches survive a replaced controller: they copy
    // /proc/self/exe, which still names the running image.
    if cfg!(all(target_os = "linux", target_env = "musl"))
        && ((arch == "x86_64" && cfg!(target_arch = "x86_64"))
            || (arch == "aarch64" && cfg!(target_arch = "aarch64")))
    {
        return Ok(WorkerBinaryAvailability::Local {
            path: stable_running_executable(current)?,
            source: "native musl mj binary".into(),
        });
    }
    if !controller_replaced
        && let Some((path, source)) = select_sibling_worker(current, &triple, is_file)
    {
        return Ok(WorkerBinaryAvailability::Local {
            path,
            source: source.into(),
        });
    }
    if requirement == WorkerBinaryRequirement::LocalHost
        && cfg!(target_os = "linux")
        && ((arch == "x86_64" && cfg!(target_arch = "x86_64"))
            || (arch == "aarch64" && cfg!(target_arch = "aarch64")))
    {
        return Ok(WorkerBinaryAvailability::Local {
            path: stable_running_executable(current)?,
            source: "native Linux mj binary".into(),
        });
    }
    if let Some(template) = hel::hel_config::env_override("WORKER_URL") {
        let expected = hel::hel_config::env_override("WORKER_SHA256")
            .context("MJ_WORKER_URL requires MJ_WORKER_SHA256")?;
        validate_worker_sha256(&expected)?;
        return Ok(WorkerBinaryAvailability::Remote {
            url: template.replace("{target}", &triple),
            sha256: expected,
            triple,
        });
    }
    // Telling someone to install a worker beside a binary that is no longer
    // there sends them looking in the wrong place.
    ensure!(
        !controller_replaced,
        "the running mj binary was replaced or removed on disk ({}); restart the Mjolnir daemon so it runs the current build, then retry",
        display_path(current)
    );
    bail!(
        "no Linux worker for {triple}; install mj-worker-{triple} beside mj, set MJ_WORKER_DIR/MJ_WORKER_BINARY, or configure MJ_WORKER_URL and MJ_WORKER_SHA256"
    )
}

/// Linux appends " (deleted)" to `/proc/<pid>/exe` for a removed image. That
/// marker belongs in a message but never in a decision, which `is_file` makes.
fn display_path(path: &Path) -> String {
    let text = path.to_string_lossy();
    text.strip_suffix(" (deleted)").unwrap_or(&text).to_owned()
}

/// The architecture a configured template names outright, if it names one. A
/// container `platform` such as `linux/arm64` decides what the target runs
/// whatever the controller's own machine is, and it is the only architecture a
/// configured target template can state: the configured `AwsEc2` variant names
/// a launch template, whose instance type is only discoverable through the AWS
/// API.
fn template_architecture(template: &hel::hel_config::TargetTemplate) -> Option<&'static str> {
    use hel::hel_config::TargetTemplate as Template;
    let platform = match template {
        Template::LocalPodman { container }
        | Template::LocalDocker { container }
        | Template::AppleContainer { container }
        | Template::SshPodman { container, .. } => container.platform.as_deref()?,
        Template::LocalBare | Template::SshBare { .. } | Template::AwsEc2 { .. } => return None,
    };
    // Platform strings appear as "linux/arm64", "arm64", or "linux/arm64/v8".
    platform.split('/').find_map(|part| match part.trim() {
        "x86_64" | "amd64" => Some("x86_64"),
        "aarch64" | "arm64" => Some("aarch64"),
        _ => None,
    })
}

/// Architectures a resume must be able to serve, knowing only the configured
/// template. Provisioning learns the real answer by running `uname -m` on the
/// live target; a resume has no target yet, so this uses what is knowable
/// without one: an architecture the template names, else the controller's own
/// architecture for a target that runs on this machine, else either Linux
/// architecture for a remote target.
fn preflight_architectures(template: &hel::hel_config::TargetTemplate) -> Vec<&'static str> {
    use hel::hel_config::TargetTemplate as Template;
    if let Some(arch) = template_architecture(template) {
        return vec![arch];
    }
    match template {
        Template::LocalBare
        | Template::LocalPodman { .. }
        | Template::LocalDocker { .. }
        | Template::AppleContainer { .. } => vec![std::env::consts::ARCH],
        Template::SshBare { .. } | Template::SshPodman { .. } | Template::AwsEc2 { .. } => {
            vec!["x86_64", "aarch64"]
        }
    }
}

/// Whether this controller could produce a Linux worker binary for a target
/// that does not exist yet.
///
/// A resume compacts a cross-harness transcript before it provisions anything,
/// which costs minutes and paid model requests. Resolving the worker binary is
/// local and takes microseconds, so a resume that could never install a worker
/// must fail before spending any of that. This downloads nothing: a remote
/// source counts as available, because fetching it belongs to provisioning.
pub(super) fn preflight_worker_binary(template: &hel::hel_config::TargetTemplate) -> Result<()> {
    // Only a bare local target may run the controller's own host binary as
    // its worker; every other target needs a portable Linux worker.
    let requirement = if matches!(template, hel::hel_config::TargetTemplate::LocalBare) {
        WorkerBinaryRequirement::LocalHost
    } else {
        WorkerBinaryRequirement::PortableLinux
    };
    let mut failure = None;
    for arch in preflight_architectures(template) {
        match worker_binary_for_arch(arch, requirement) {
            Ok(_) => return Ok(()),
            Err(error) => failure = Some(error),
        }
    }
    match failure {
        // The message is the one provisioning would have printed later, so the
        // user reads the same fix, sooner.
        Some(error) => Err(error).context("preflight the worker binary before resuming"),
        None => Ok(()),
    }
}

fn stable_running_executable(current: &Path) -> Result<PathBuf> {
    if current.is_file() {
        return Ok(current.to_path_buf());
    }
    #[cfg(target_os = "linux")]
    {
        let proc_exe = PathBuf::from(format!("/proc/{}/exe", std::process::id()));
        let directory = data_dir().join("workers").join("running");
        let cached = directory.join(format!("hel-{}", std::process::id()));
        materialize_running_executable(current, &proc_exe, &cached)
    }
    #[cfg(not(target_os = "linux"))]
    bail!(
        "resolved Mjolnir controller executable is no longer readable: {}",
        current.display()
    )
}

#[cfg(target_os = "linux")]
fn materialize_running_executable(
    current: &Path,
    proc_exe: &Path,
    cached: &Path,
) -> Result<PathBuf> {
    if !proc_exe.is_file() {
        bail!(
            "resolved Mjolnir controller executable is no longer readable: {}",
            current.display()
        );
    }
    let parent = cached
        .parent()
        .context("worker executable cache has no parent")?;
    std::fs::create_dir_all(parent)
        .with_context(|| format!("create worker executable cache {}", parent.display()))?;
    std::fs::copy(proc_exe, cached).with_context(|| {
        format!(
            "copy running mj executable from {} after {} was replaced",
            proc_exe.display(),
            current.display()
        )
    })?;
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        std::fs::set_permissions(cached, std::fs::Permissions::from_mode(0o700))?;
    }
    Ok(cached.to_path_buf())
}

pub(super) fn worker_binary_for(
    locator: &hel_targets::TargetLocator,
    executor: &impl CommandExecutor,
) -> Result<PathBuf> {
    let arch = target_architecture(locator, executor)?;
    let requirement = if matches!(locator, hel_targets::TargetLocator::LocalBare { .. }) {
        WorkerBinaryRequirement::LocalHost
    } else {
        WorkerBinaryRequirement::PortableLinux
    };
    match worker_binary_for_arch(arch, requirement)? {
        WorkerBinaryAvailability::Local { path, .. } => Ok(path),
        WorkerBinaryAvailability::Remote {
            url,
            sha256,
            triple,
        } => download_worker(&url, &sha256, &triple),
    }
}

fn target_architecture(
    locator: &hel_targets::TargetLocator,
    executor: &impl CommandExecutor,
) -> Result<&'static str> {
    let command = match locator {
        hel_targets::TargetLocator::LocalBare { .. } => CommandSpec::new("uname", ["-m"]),
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => {
            CommandSpec::new("podman", ["exec", container_id, "uname", "-m"])
        }
        hel_targets::TargetLocator::LocalDocker { container_id } => {
            CommandSpec::new("docker", ["exec", container_id, "uname", "-m"])
        }
        hel_targets::TargetLocator::AppleContainer { container_id } => {
            CommandSpec::new("container", ["exec", container_id, "uname", "-m"])
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => ssh_command_spec(ssh, ["uname", "-m"]),
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(ssh, ["podman", "exec", container_id, "uname", "-m"]),
    }
    .purpose("detect target architecture");
    let output = execute_checked(executor, command)?;
    match String::from_utf8(output.stdout)?.trim() {
        "x86_64" | "amd64" => Ok("x86_64"),
        "aarch64" | "arm64" => Ok("aarch64"),
        architecture => bail!("unsupported target architecture {architecture:?}"),
    }
}

fn download_worker(url: &str, expected_sha256: &str, triple: &str) -> Result<PathBuf> {
    validate_worker_sha256(expected_sha256)?;
    let directory = data_dir()
        .join("workers")
        .join(env!("CARGO_PKG_VERSION"))
        .join(triple);
    std::fs::create_dir_all(&directory)?;
    let destination = directory.join("hel");
    if destination.is_file() {
        let bytes = std::fs::read(&destination)?;
        if format!("{:x}", Sha256::digest(&bytes)).eq_ignore_ascii_case(expected_sha256) {
            return Ok(destination);
        }
    }
    let bytes = reqwest::blocking::Client::builder()
        .timeout(std::time::Duration::from_secs(120))
        .build()?
        .get(url)
        .send()?
        .error_for_status()?
        .bytes()?;
    let actual = format!("{:x}", Sha256::digest(&bytes));
    if !actual.eq_ignore_ascii_case(expected_sha256) {
        bail!("downloaded worker checksum mismatch: expected {expected_sha256}, got {actual}");
    }
    let mut temporary = tempfile::NamedTempFile::new_in(&directory)?;
    std::io::Write::write_all(&mut temporary, &bytes)?;
    temporary.as_file_mut().sync_all()?;
    temporary
        .persist(&destination)
        .map_err(|error| error.error)?;
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        std::fs::set_permissions(&destination, std::fs::Permissions::from_mode(0o700))?;
    }
    Ok(destination)
}

fn validate_worker_sha256(expected_sha256: &str) -> Result<()> {
    if expected_sha256.len() != 64 || !expected_sha256.bytes().all(|byte| byte.is_ascii_hexdigit())
    {
        bail!("MJ_WORKER_SHA256 must be a 64-character hexadecimal digest");
    }
    Ok(())
}

fn workspace_paths(
    locator: &hel_targets::TargetLocator,
    bundle: &ProjectBundle,
    session_id: &str,
) -> Result<(String, Vec<String>)> {
    let root = match locator {
        hel_targets::TargetLocator::LocalBare { .. } => {
            bail!("local bare projects use their selected directory")
        }
        hel_targets::TargetLocator::LocalPodman { .. }
        | hel_targets::TargetLocator::LocalDocker { .. }
        | hel_targets::TargetLocator::AppleContainer { .. }
        | hel_targets::TargetLocator::SshPodman { .. } => "/workspace".to_string(),
        hel_targets::TargetLocator::AwsEc2 { workspace, .. }
        | hel_targets::TargetLocator::SshBare { workspace, .. } => workspace.clone(),
    };
    if matches!(locator, hel_targets::TargetLocator::AwsEc2 { .. }) {
        let expected = format!(".local/share/hel/workspaces/{session_id}");
        if root != expected {
            bail!("AWS workspace does not match session")
        }
    }
    let primary = bundle.primary().context("bundle primary is missing")?;
    let primary_path = format!("{root}/{}", primary.destination.to_string_lossy());
    let additional = bundle
        .repositories
        .iter()
        .filter(|repository| repository.id != bundle.primary_repo)
        .map(|repository| format!("{root}/{}", repository.destination.to_string_lossy()))
        .collect();
    Ok((primary_path, additional))
}

// Package versions for ACP bridges. Keep these in lockstep with the global npm installs in
// containers/Containerfile.agent-dev; bridge_pins_match_containerfile() below
// fails the build when they drift.
// Codex 0.148 reuses pending MCP startups during runtime reconciliation. Older
// releases could cancel the first project-memory startup while immediately
// replacing it with an equivalent connection, leaving a false failed-tool
// event at the beginning of every session.
const CODEX_ACP_FALLBACK_VERSION: &str = "1.8.0";

const CLAUDE_AGENT_ACP_FALLBACK_VERSION: &str = "0.73.0";

const DEEPSEEK_HARNESS_FALLBACK_VERSION: &str = "0.1.1-rc.2";

const DEEPSEEK_ACP_FALLBACK_VERSION: &str = "0.10.0";

/// Stage shown after the worker is reachable and while its ACP bridge becomes
/// ready. Default launchers that can fetch their own harness name that work;
/// explicit executables and DSH only have a process to start.
pub(super) fn bridge_readiness_stage(profile: &HarnessProfile) -> ProvisionStage {
    if profile.executable.is_none()
        && matches!(
            profile.kind,
            HarnessKind::Codex | HarnessKind::Claude | HarnessKind::Kimi | HarnessKind::Grok
        )
    {
        ProvisionStage::Installing(profile.kind)
    } else {
        ProvisionStage::Starting
    }
}

pub(super) fn bridge_launch(
    harness: hel::hel_config::HarnessKind,
    executable: Option<&Path>,
    policy: hel::hel_config::ExecutionPolicy,
) -> (String, Vec<String>) {
    if let Some(executable) = executable {
        let args = harness
            .bridge_override_args(policy)
            .into_iter()
            .map(str::to_owned)
            .collect();
        return (executable.to_string_lossy().into_owned(), args);
    }
    match harness {
        hel::hel_config::HarnessKind::Codex => (
            "sh".into(),
            vec![
                "-c".into(),
                format!("if command -v codex-acp >/dev/null 2>&1 && [ \"$(codex-acp --version 2>/dev/null)\" = \"@agentclientprotocol/codex-acp {CODEX_ACP_FALLBACK_VERSION}\" ]; then exec codex-acp; fi; {}; exec npx -y @agentclientprotocol/codex-acp@{CODEX_ACP_FALLBACK_VERSION}", ensure_node_script()),
            ],
        ),
        hel::hel_config::HarnessKind::Claude => (
            "sh".into(),
            vec![
                "-c".into(),
                format!("if command -v claude-agent-acp >/dev/null 2>&1; then exec claude-agent-acp; fi; {}; exec npx -y @agentclientprotocol/claude-agent-acp@{CLAUDE_AGENT_ACP_FALLBACK_VERSION}", ensure_node_script()),
            ],
        ),
        hel::hel_config::HarnessKind::Kimi => (
            "sh".into(),
            vec![
                "-c".into(),
                "if command -v kimi >/dev/null 2>&1; then exec kimi acp; elif [ -x \"$HOME/.kimi-code/bin/kimi\" ]; then exec \"$HOME/.kimi-code/bin/kimi\" acp; elif command -v curl >/dev/null 2>&1; then curl -fsSL https://code.kimi.com/kimi-code/install.sh | bash && exec \"$HOME/.kimi-code/bin/kimi\" acp; else echo 'Mjolnir needs compatible Kimi Code or curl for its official installer; configure the profile executable or environment PATH when the tool is installed elsewhere' >&2; exit 127; fi".into(),
            ],
        ),
        hel::hel_config::HarnessKind::Grok => {
            let acp = hel::hel_config::HarnessKind::Grok
                .bridge_override_args(policy)
                .join(" ");
            (
                "sh".into(),
                vec![
                    "-c".into(),
                    format!(
                        "if command -v grok >/dev/null 2>&1; then exec grok {acp}; elif [ -x \"$GROK_HOME/bin/grok\" ]; then exec \"$GROK_HOME/bin/grok\" {acp}; elif [ -x \"$HOME/.grok/bin/grok\" ]; then exec \"$HOME/.grok/bin/grok\" {acp}; elif command -v curl >/dev/null 2>&1; then curl -fsSL https://x.ai/cli/install.sh | bash && exec \"$HOME/.grok/bin/grok\" {acp}; else echo 'Mjolnir needs compatible Grok Build or curl for its official installer; configure the profile executable or environment PATH when the tool is installed elsewhere' >&2; exit 127; fi"
                    ),
                ],
            )
        }
        hel::hel_config::HarnessKind::Deepseek => (
            "sh".into(),
            vec![
                "-c".into(),
                format!(
                    "{}; if command -v dsh >/dev/null 2>&1 && command -v dsh-acp-server >/dev/null 2>&1; then exec dsh-acp-server; fi; echo 'Mjolnir needs @deepseek-ai/dsh@{DEEPSEEK_HARNESS_FALLBACK_VERSION} and dsh-acp-server@{DEEPSEEK_ACP_FALLBACK_VERSION} installed on PATH; configure the profile executable or environment PATH when they are installed elsewhere' >&2; exit 127",
                    ensure_node_22_script(),
                ),
            ],
        ),
    }
}

fn ensure_node_script() -> &'static str {
    "if ! command -v npx >/dev/null 2>&1; then if [ \"$(id -u)\" = 0 ]; then SUDO=''; elif command -v sudo >/dev/null 2>&1 && sudo -n true; then SUDO='sudo'; else echo 'Mjolnir needs Node/npx or passwordless sudo to install it; configure the profile executable or environment PATH when the tool is installed elsewhere' >&2; exit 127; fi; if command -v apt-get >/dev/null 2>&1; then $SUDO apt-get update && $SUDO apt-get install -y nodejs npm; elif command -v dnf >/dev/null 2>&1; then $SUDO dnf install -y nodejs npm; elif command -v yum >/dev/null 2>&1; then $SUDO yum install -y nodejs npm; elif command -v apk >/dev/null 2>&1; then $SUDO apk add --no-cache nodejs npm; else echo 'Mjolnir cannot install Node on this image; bake npx or a compatible ACP bridge into it, or configure the profile executable or environment PATH' >&2; exit 127; fi; fi"
}

fn ensure_node_22_script() -> String {
    format!(
        "{}; if ! node -e 'process.exit(Number(process.versions.node.split(\".\")[0]) >= 22 ? 0 : 1)'; then echo 'DeepSeek Harness requires Node.js 22 or newer' >&2; exit 127; fi",
        ensure_node_script()
    )
}

const MJ_CONTAINER_ENVIRONMENT: &str = "## Mjolnir disposable environment\n\nThis session runs in a disposable Mjolnir container. When the session closes, Mjolnir checkpoints everything in project workspace directories under `/workspace`, including committed work, staged and unstaged changes, and untracked files. Mjolnir then removes the container.\n\nEverything outside `/workspace`, including installed packages, `$HOME`, and `/tmp`, is ephemeral and will be lost. Keep durable results in the workspace or push them to a remote.\n";

pub(super) fn stage_profile(
    profile: &hel::hel_config::HarnessProfile,
    destination: &Path,
) -> Result<()> {
    let harness = profile.kind;
    let source = profile.home.as_path();
    std::fs::create_dir_all(destination)?;
    let allowlist: &[&str] = match harness {
        hel::hel_config::HarnessKind::Codex => &[
            "auth.json",
            "config.toml",
            "AGENTS.md",
            "instructions.md",
            "rules",
            "skills",
        ],
        hel::hel_config::HarnessKind::Claude => &[
            ".claude.json",
            ".credentials.json",
            "settings.json",
            "CLAUDE.md",
            "skills",
            "plugins",
        ],
        hel::hel_config::HarnessKind::Kimi => &[
            "credentials",
            "config.toml",
            "device_id",
            "AGENTS.md",
            "SYSTEM.md",
            "mcp.json",
            "skills",
            "agents",
            "plugins",
        ],
        hel::hel_config::HarnessKind::Grok => &[
            "auth.json",
            "config.toml",
            "AGENTS.md",
            "agent_id",
            "skills",
            "plugins",
        ],
        hel::hel_config::HarnessKind::Deepseek => &[
            ".credentials.yaml",
            "settings.yaml",
            "AGENTS.md",
            "skills",
            ".agent-presets",
        ],
    };
    // Allowlist entries (and, within each, a copied directory's children) are
    // independent of one another, so copying them concurrently shortens the
    // stage step for profiles with large skills/plugins trees.
    allowlist.par_iter().try_for_each(|name| -> Result<()> {
        let from = source.join(name);
        if from.exists() {
            copy_profile_entry(&from, &destination.join(name))?;
        }
        Ok(())
    })?;
    Ok(())
}

/// Add lifecycle guidance only for targets that Hel destroys as a whole.
fn append_hel_target_environment(
    harness: hel::hel_config::HarnessKind,
    destination: &Path,
    target: &hel_targets::TargetLocator,
) -> Result<()> {
    let environment = match target {
        hel_targets::TargetLocator::LocalPodman { .. }
        | hel_targets::TargetLocator::LocalDocker { .. }
        | hel_targets::TargetLocator::AppleContainer { .. }
        | hel_targets::TargetLocator::SshPodman { .. } => MJ_CONTAINER_ENVIRONMENT.to_owned(),
        hel_targets::TargetLocator::AwsEc2 { workspace, .. } => format!(
            "## Mjolnir disposable environment\n\nThis session runs on a disposable Mjolnir EC2 instance. When the session closes, Mjolnir checkpoints everything in project workspace directories under `$HOME/{workspace}`, including committed work, staged and unstaged changes, and untracked files. Mjolnir then terminates the instance.\n\nEverything outside `$HOME/{workspace}`, including installed packages, the rest of `$HOME`, and `/tmp`, is ephemeral and will be lost. Keep durable results in the workspace or push them to a remote.\n"
        ),
        hel_targets::TargetLocator::LocalBare { .. }
        | hel_targets::TargetLocator::SshBare { .. } => return Ok(()),
    };
    let instructions = match harness {
        hel::hel_config::HarnessKind::Codex => "AGENTS.md",
        hel::hel_config::HarnessKind::Claude => "CLAUDE.md",
        hel::hel_config::HarnessKind::Kimi => "AGENTS.md",
        hel::hel_config::HarnessKind::Grok => "AGENTS.md",
        hel::hel_config::HarnessKind::Deepseek => "AGENTS.md",
    };
    let path = destination.join(instructions);
    let separator = match std::fs::read_to_string(&path) {
        Ok(contents) if !contents.is_empty() && !contents.ends_with('\n') => "\n\n",
        Ok(contents) if !contents.is_empty() => "\n",
        Ok(_) => "",
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => "",
        Err(error) => return Err(error.into()),
    };
    use std::io::Write;

    let mut file = std::fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(&path)
        .with_context(|| format!("open staged harness instructions {}", path.display()))?;
    file.write_all(separator.as_bytes())?;
    file.write_all(environment.as_bytes())?;
    Ok(())
}

fn copy_profile_entry(source: &Path, destination: &Path) -> Result<()> {
    let metadata = std::fs::symlink_metadata(source)
        .with_context(|| format!("read staged profile entry metadata {}", source.display()))?;
    if metadata.file_type().is_symlink() {
        return Ok(());
    }
    if metadata.is_file() {
        if let Some(parent) = destination.parent() {
            std::fs::create_dir_all(parent)
                .with_context(|| format!("create staged profile directory {}", parent.display()))?;
        }
        std::fs::copy(source, destination).with_context(|| {
            format!(
                "copy staged profile file {} to {}",
                source.display(),
                destination.display()
            )
        })?;
        return Ok(());
    }
    if metadata.is_dir() {
        std::fs::create_dir_all(destination).with_context(|| {
            format!("create staged profile directory {}", destination.display())
        })?;
        let entries = std::fs::read_dir(source)
            .with_context(|| format!("list staged profile directory {}", source.display()))?
            .collect::<std::io::Result<Vec<_>>>()
            .with_context(|| {
                format!(
                    "read staged profile directory entries in {}",
                    source.display()
                )
            })?;
        // Sibling entries in one directory are independent, so recurse in
        // parallel; this is the level most likely to hold many files (e.g. a
        // skills or plugins tree).
        entries.par_iter().try_for_each(|entry| {
            copy_profile_entry(&entry.path(), &destination.join(entry.file_name()))
        })?;
        std::fs::set_permissions(destination, metadata.permissions()).with_context(|| {
            format!(
                "set permissions for staged profile directory {}",
                destination.display()
            )
        })?;
    }
    Ok(())
}

#[allow(clippy::too_many_arguments)]
fn install_worker_files(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    session_id: &str,
    worker_root: &str,
    profile_home: &str,
    worker_binary: &Path,
    launch_config: &Path,
    ownership: &Path,
    profile_stage: &Path,
) -> Result<()> {
    match locator {
        hel_targets::TargetLocator::LocalBare { .. } => {
            for command in [
                CommandSpec::new("mkdir", ["-p", worker_root])
                    .purpose("create local bare worker directory"),
                CommandSpec::new(
                    "cp",
                    [
                        worker_binary.to_string_lossy().into_owned(),
                        format!("{worker_root}/hel"),
                    ],
                )
                .purpose("install local Mjolnir worker"),
                CommandSpec::new(
                    "cp",
                    [
                        launch_config.to_string_lossy().into_owned(),
                        format!("{worker_root}/launch.json"),
                    ],
                )
                .purpose("install local worker launch configuration"),
                CommandSpec::new(
                    "cp",
                    [
                        ownership.to_string_lossy().into_owned(),
                        format!("{worker_root}/ownership.json"),
                    ],
                )
                .purpose("install local worker ownership marker"),
                CommandSpec::new("chmod", ["700", &format!("{worker_root}/hel")])
                    .purpose("make local Mjolnir worker executable"),
            ] {
                execute_checked(executor, command)?;
            }
        }
        hel_targets::TargetLocator::LocalPodman { container_id, .. }
        | hel_targets::TargetLocator::LocalDocker { container_id }
        | hel_targets::TargetLocator::AppleContainer { container_id } => {
            let engine = match locator {
                hel_targets::TargetLocator::LocalPodman { .. } => "podman",
                hel_targets::TargetLocator::LocalDocker { .. } => "docker",
                hel_targets::TargetLocator::AppleContainer { .. } => "container",
                _ => unreachable!("matched local container target"),
            };
            for command in [
                CommandSpec::new(
                    engine,
                    [
                        "exec".into(),
                        container_id.clone(),
                        "mkdir".into(),
                        "-p".into(),
                        worker_root.into(),
                        profile_home.into(),
                    ],
                )
                .purpose("create target worker directories"),
                CommandSpec::new(
                    engine,
                    [
                        "cp".into(),
                        worker_binary.to_string_lossy().into_owned(),
                        format!("{container_id}:{worker_root}/hel"),
                    ],
                )
                .purpose("upload Mjolnir worker"),
                CommandSpec::new(
                    engine,
                    [
                        "cp".into(),
                        launch_config.to_string_lossy().into_owned(),
                        format!("{container_id}:{worker_root}/launch.json"),
                    ],
                )
                .purpose("upload worker launch configuration"),
                CommandSpec::new(
                    engine,
                    [
                        "cp".into(),
                        ownership.to_string_lossy().into_owned(),
                        format!("{container_id}:{worker_root}/ownership.json"),
                    ],
                )
                .purpose("upload worker ownership marker"),
                CommandSpec::new(
                    engine,
                    [
                        "cp".into(),
                        format!("{}/.", profile_stage.display()),
                        format!("{container_id}:{profile_home}"),
                    ],
                )
                .purpose("upload harness profile allowlist"),
                CommandSpec::new(
                    engine,
                    [
                        "exec".into(),
                        container_id.clone(),
                        "chmod".into(),
                        "700".into(),
                        format!("{worker_root}/hel"),
                    ],
                )
                .purpose("make Mjolnir worker executable"),
                CommandSpec::new(
                    engine,
                    [
                        "exec".into(),
                        container_id.clone(),
                        "chmod".into(),
                        "-R".into(),
                        "go-rwx".into(),
                        profile_home.into(),
                    ],
                )
                .purpose("restrict harness profile permissions"),
            ] {
                execute_checked(executor, command)?;
            }
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            install_worker_over_ssh(
                executor,
                ssh,
                worker_root,
                profile_home,
                worker_binary,
                launch_config,
                ownership,
                profile_stage,
            )?;
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => {
            // The worker binary is 10-30 MB and identical across sessions, so
            // keep it in a content-addressed cache on the remote host and copy
            // it over the wire only once per unique binary.
            let digest = hel::hel_worker_launch::worker_executable_digest(worker_binary)?;
            // Home-relative, not "~/": ssh_command_spec single-quotes every
            // argument, so a tilde would stay literal in the remote shell
            // while scp expands it, and the two sides would disagree. Both
            // ssh commands (cwd is the login home) and scp resolve a relative
            // path against the remote home.
            let cache_dir = format!(".cache/mjolnir/workers/{digest}");
            let cached_worker = format!("{cache_dir}/hel");
            let cached = matches!(
                executor.execute(
                    &ssh_command_spec(ssh, ["test", "-f", &cached_worker])
                        .purpose("probe cached remote Mjolnir worker"),
                ),
                Ok(output) if output.status == 0
            );
            if !cached {
                execute_checked(
                    executor,
                    ssh_command_spec(ssh, ["mkdir", "-p", &cache_dir])
                        .purpose("create remote worker cache"),
                )?;
                let partial = format!("{cache_dir}/hel.partial-{session_id}");
                execute_checked(
                    executor,
                    scp_command_spec(ssh, worker_binary, &partial, false)
                        .purpose("upload remote Podman worker binary"),
                )?;
                // Rename within the cache directory so the final path only
                // ever names a complete upload.
                execute_checked(
                    executor,
                    ssh_command_spec(ssh, ["mv", &partial, &cached_worker])
                        .purpose("publish cached remote Mjolnir worker"),
                )?;
            }
            let upload = format!(".cache/mjolnir/uploads/{session_id}");
            execute_checked(
                executor,
                ssh_command_spec(ssh, ["mkdir", "-p", &upload])
                    .purpose("create remote upload staging"),
            )?;
            for (source, name) in [
                (launch_config, "launch.json"),
                (ownership, "ownership.json"),
            ] {
                execute_checked(
                    executor,
                    scp_command_spec(ssh, source, &format!("{upload}/{name}"), false)
                        .purpose("upload remote Podman worker file"),
                )?;
            }
            execute_checked(
                executor,
                scp_command_spec(ssh, profile_stage, &format!("{upload}/profile"), true)
                    .purpose("upload remote Podman profile allowlist"),
            )?;
            let remote = [
                vec![
                    "podman".into(),
                    "exec".into(),
                    container_id.clone(),
                    "mkdir".into(),
                    "-p".into(),
                    worker_root.into(),
                    profile_home.into(),
                ],
                vec![
                    "podman".into(),
                    "cp".into(),
                    cached_worker.clone(),
                    format!("{container_id}:{worker_root}/hel"),
                ],
                vec![
                    "podman".into(),
                    "cp".into(),
                    format!("{upload}/launch.json"),
                    format!("{container_id}:{worker_root}/launch.json"),
                ],
                vec![
                    "podman".into(),
                    "cp".into(),
                    format!("{upload}/ownership.json"),
                    format!("{container_id}:{worker_root}/ownership.json"),
                ],
                vec![
                    "podman".into(),
                    "cp".into(),
                    format!("{upload}/profile/."),
                    format!("{container_id}:{profile_home}"),
                ],
                vec![
                    "podman".into(),
                    "exec".into(),
                    container_id.clone(),
                    "chmod".into(),
                    "700".into(),
                    format!("{worker_root}/hel"),
                ],
                vec![
                    "podman".into(),
                    "exec".into(),
                    container_id.clone(),
                    "chmod".into(),
                    "-R".into(),
                    "go-rwx".into(),
                    profile_home.into(),
                ],
                vec!["rm".into(), "-rf".into(), "--".into(), upload.clone()],
            ];
            for args in remote {
                execute_checked(
                    executor,
                    ssh_command_spec(ssh, args).purpose("install remote Podman worker"),
                )?;
            }
        }
    }
    Ok(())
}

#[allow(clippy::too_many_arguments)]
fn install_worker_over_ssh(
    executor: &impl CommandExecutor,
    ssh: &SshTarget,
    worker_root: &str,
    profile_home: &str,
    worker_binary: &Path,
    launch_config: &Path,
    ownership: &Path,
    profile_stage: &Path,
) -> Result<()> {
    execute_checked(
        executor,
        ssh_command_spec(ssh, ["mkdir", "-p", worker_root, profile_home])
            .purpose("create SSH worker directories"),
    )?;
    for (source, remote, recursive) in [
        (worker_binary, format!("{worker_root}/hel"), false),
        (launch_config, format!("{worker_root}/launch.json"), false),
        (ownership, format!("{worker_root}/ownership.json"), false),
    ] {
        execute_checked(
            executor,
            scp_command_spec(ssh, source, &remote, recursive).purpose("upload SSH worker file"),
        )?;
    }
    let incoming_profile = format!("{profile_home}.incoming");
    execute_checked(
        executor,
        scp_command_spec(ssh, profile_stage, &incoming_profile, true)
            .purpose("upload SSH harness profile allowlist"),
    )?;
    execute_checked(
        executor,
        ssh_command_spec(
            ssh,
            ["cp", "-R", &format!("{incoming_profile}/."), profile_home],
        )
        .purpose("install SSH harness profile allowlist"),
    )?;
    execute_checked(
        executor,
        ssh_command_spec(ssh, ["rm", "-rf", "--", &incoming_profile])
            .purpose("remove SSH profile staging"),
    )?;
    execute_checked(
        executor,
        ssh_command_spec(ssh, ["chmod", "700", &format!("{worker_root}/hel")])
            .purpose("make SSH worker executable"),
    )?;
    execute_checked(
        executor,
        ssh_command_spec(ssh, ["chmod", "-R", "go-rwx", profile_home])
            .purpose("restrict SSH harness profile permissions"),
    )?;
    Ok(())
}

/// Replace `{worker_root}/hel` with the controller's current worker binary.
///
/// Checkpoint export starts that path as a new process. A live daemon already
/// has the previous inode mapped, so this does not restart it. Writing through
/// `hel.next` and renaming avoids `ETXTBSY` on a running image.
pub(super) fn replace_installed_worker_binary(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    session_id: &str,
    worker_binary: &Path,
) -> Result<()> {
    let plan = installed_worker_binary_replacement_plan(locator, session_id, worker_binary)?;
    for command in plan.commands {
        execute_checked(executor, command)?;
    }
    Ok(())
}

fn installed_worker_binary_replacement_plan(
    locator: &hel_targets::TargetLocator,
    session_id: &str,
    worker_binary: &Path,
) -> Result<CommandPlan> {
    let worker_root = hel_targets::worker_root(locator, session_id)?;
    let installed = format!("{worker_root}/hel");
    let staged = format!("{worker_root}/hel.next");
    let commands = match locator {
        hel_targets::TargetLocator::LocalBare { .. } => vec![
            CommandSpec::new(
                "cp",
                [worker_binary.to_string_lossy().into_owned(), staged.clone()],
            )
            .purpose("stage replacement Mjolnir worker"),
            CommandSpec::new("mv", ["-f", &staged, &installed])
                .purpose("replace installed Mjolnir worker"),
            CommandSpec::new("chmod", ["700", &installed])
                .purpose("make replaced Mjolnir worker executable"),
        ],
        hel_targets::TargetLocator::LocalPodman { container_id, .. }
        | hel_targets::TargetLocator::LocalDocker { container_id }
        | hel_targets::TargetLocator::AppleContainer { container_id } => {
            let engine = match locator {
                hel_targets::TargetLocator::LocalPodman { .. } => "podman",
                hel_targets::TargetLocator::LocalDocker { .. } => "docker",
                hel_targets::TargetLocator::AppleContainer { .. } => "container",
                _ => unreachable!("matched local container target"),
            };
            vec![
                CommandSpec::new(
                    engine,
                    [
                        "cp".into(),
                        worker_binary.to_string_lossy().into_owned(),
                        format!("{container_id}:{staged}"),
                    ],
                )
                .purpose("stage replacement Mjolnir worker"),
                CommandSpec::new(
                    engine,
                    [
                        "exec".into(),
                        container_id.clone(),
                        "mv".into(),
                        "-f".into(),
                        staged,
                        installed.clone(),
                    ],
                )
                .purpose("replace installed Mjolnir worker"),
                CommandSpec::new(
                    engine,
                    [
                        "exec".into(),
                        container_id.clone(),
                        "chmod".into(),
                        "700".into(),
                        installed,
                    ],
                )
                .purpose("make replaced Mjolnir worker executable"),
            ]
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => vec![
            scp_command_spec(ssh, worker_binary, &staged, false)
                .purpose("stage replacement Mjolnir worker"),
            ssh_command_spec(ssh, ["mv", "-f", "--", &staged, &installed])
                .purpose("replace installed Mjolnir worker"),
            ssh_command_spec(ssh, ["chmod", "700", &installed])
                .purpose("make replaced Mjolnir worker executable"),
        ],
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => {
            let upload = format!(".cache/mjolnir/uploads/{session_id}-hel.next");
            vec![
                ssh_command_spec(ssh, ["mkdir", "-p", ".cache/mjolnir/uploads"])
                    .purpose("create remote replacement worker staging"),
                scp_command_spec(ssh, worker_binary, &upload, false)
                    .purpose("stage replacement Mjolnir worker"),
                ssh_command_spec(
                    ssh,
                    ["podman", "cp", &upload, &format!("{container_id}:{staged}")],
                )
                .purpose("stage replacement Mjolnir worker"),
                ssh_command_spec(
                    ssh,
                    [
                        "podman",
                        "exec",
                        container_id,
                        "mv",
                        "-f",
                        "--",
                        &staged,
                        &installed,
                    ],
                )
                .purpose("replace installed Mjolnir worker"),
                ssh_command_spec(
                    ssh,
                    ["podman", "exec", container_id, "chmod", "700", &installed],
                )
                .purpose("make replaced Mjolnir worker executable"),
                ssh_command_spec(ssh, ["rm", "-f", "--", &upload])
                    .purpose("remove remote replacement worker staging"),
            ]
        }
    };
    Ok(CommandPlan {
        description: format!("replace stale Mjolnir worker for session {session_id}"),
        commands,
    })
}

fn installed_file_digest_command(
    locator: &hel_targets::TargetLocator,
    path: &str,
    purpose: &str,
) -> CommandSpec {
    match locator {
        hel_targets::TargetLocator::LocalBare { .. } => CommandSpec::new("sha256sum", [path]),
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => {
            CommandSpec::new("podman", ["exec", container_id, "sha256sum", path])
        }
        hel_targets::TargetLocator::LocalDocker { container_id } => {
            CommandSpec::new("docker", ["exec", container_id, "sha256sum", path])
        }
        hel_targets::TargetLocator::AppleContainer { container_id } => {
            CommandSpec::new("container", ["exec", container_id, "sha256sum", path])
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            ssh_command_spec(ssh, ["sha256sum", path])
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(ssh, ["podman", "exec", container_id, "sha256sum", path]),
    }
    .purpose(purpose)
}

fn worker_launch_refresh_plan(
    locator: &hel_targets::TargetLocator,
    session_id: &str,
    launch: &WorkerLaunchConfig,
) -> Result<WorkerLaunchRefreshPlan> {
    let worker_root = hel_targets::worker_root(locator, session_id)?;
    let installed = format!("{worker_root}/launch.json");
    let staged = format!("{installed}.next");
    let staged_arg = hel_targets::join_remote_command(std::slice::from_ref(&staged));
    let installed_arg = hel_targets::join_remote_command(std::slice::from_ref(&installed));
    let script = format!("umask 077; cat > {staged_arg} && mv -f -- {staged_arg} {installed_arg}");
    let body = serde_json::to_vec_pretty(launch).context("serialize worker launch config")?;
    let expected_sha256 = format!("{:x}", Sha256::digest(&body));
    let replace = match locator {
        hel_targets::TargetLocator::LocalBare { .. } => CommandSpec::new("sh", ["-c", &script]),
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => {
            CommandSpec::new("podman", ["exec", "-i", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::LocalDocker { container_id } => {
            CommandSpec::new("docker", ["exec", "-i", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::AppleContainer { container_id } => CommandSpec::new(
            "container",
            ["exec", "-i", container_id, "sh", "-c", &script],
        ),
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            ssh_command_spec(ssh, ["sh", "-c", &script])
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(
            ssh,
            ["podman", "exec", "-i", container_id, "sh", "-c", &script],
        ),
    }
    .purpose("replace stale Mjolnir worker launch config")
    .with_sensitive_stdin(body);
    Ok(WorkerLaunchRefreshPlan {
        expected_sha256,
        installed_digest: installed_file_digest_command(
            locator,
            &installed,
            "identify installed Mjolnir worker launch config",
        ),
        replace: CommandPlan {
            description: format!("replace stale Mjolnir launch config for session {session_id}"),
            commands: vec![replace],
        },
    })
}

/// Prepare a local refresh without hashing the controller binary. Digesting
/// happens only after recovery has proved that the worker needs a restart.
fn worker_binary_refresh_plan(
    locator: &hel_targets::TargetLocator,
    session_id: &str,
) -> Result<Option<WorkerBinaryRefresh>> {
    let worker_root = hel_targets::worker_root(locator, session_id)?;
    let installed = format!("{worker_root}/hel");
    // Remote targets defer source selection to the recovery task: choosing the
    // binary needs the target's architecture, and probing it (plus hashing the
    // remote binary) is blocking ssh work that must not run on this UI/event
    // path. Building the refresh here stays cheap.
    if matches!(
        locator,
        hel_targets::TargetLocator::AwsEc2 { .. }
            | hel_targets::TargetLocator::SshBare { .. }
            | hel_targets::TargetLocator::SshPodman { .. }
    ) {
        return Ok(Some(WorkerBinaryRefresh::Remote(
            RemoteWorkerBinaryRefresh {
                locator: locator.clone(),
                session_id: session_id.to_owned(),
                installed_digest: installed_file_digest_command(
                    locator,
                    &installed,
                    "identify installed Mjolnir worker binary",
                ),
            },
        )));
    }
    // Local: resolve the source now. Resolving a deleted running executable
    // materializes /proc/self/exe and can copy hundreds of megabytes; target
    // lists are assembled on UI/event loops, so leave refresh disabled until
    // the next controller start rather than doing that work here.
    if !std::env::current_exe().is_ok_and(|path| path.is_file()) {
        return Ok(None);
    }
    let requirement = if matches!(locator, hel_targets::TargetLocator::LocalBare { .. }) {
        WorkerBinaryRequirement::LocalHost
    } else {
        WorkerBinaryRequirement::PortableLinux
    };
    let source = match worker_binary_for_arch(std::env::consts::ARCH, requirement) {
        Ok(WorkerBinaryAvailability::Local { path, .. }) => path,
        Ok(WorkerBinaryAvailability::Remote { .. }) | Err(_) => return Ok(None),
    };
    Ok(Some(WorkerBinaryRefresh::Prepared(
        WorkerBinaryRefreshPlan {
            replace: installed_worker_binary_replacement_plan(locator, session_id, &source)?,
            source,
            installed_digest: installed_file_digest_command(
                locator,
                &installed,
                "identify installed Mjolnir worker binary",
            ),
        },
    )))
}

/// Refresh a remote worker binary during recovery: pick the worker binary for
/// the target's own architecture, and copy it over the installed one only when
/// their digests differ. This runs inside the recovery task, where blocking
/// ssh work is allowed; it must never be called from a UI/event loop.
///
/// The digest gate is what stops a redeploy loop: once the right binary is
/// installed, its digest matches the source and nothing is copied again, even
/// though recovery may still restart the worker.
pub(crate) fn refresh_remote_worker_binary_if_stale(
    executor: &impl CommandExecutor,
    refresh: &RemoteWorkerBinaryRefresh,
) -> Result<()> {
    let source = worker_binary_for(&refresh.locator, executor)
        .context("resolve the worker binary for the recovering target")?;
    replace_remote_worker_binary_if_stale(
        executor,
        &refresh.locator,
        &refresh.session_id,
        &refresh.installed_digest,
        &source,
    )
    .map(|_| ())
}

/// Copy `source` over the installed remote worker only when the installed
/// digest differs from `source`'s. Returns whether a copy ran. Split from the
/// resolver above so the digest gate is testable without resolving a real
/// worker binary for a target architecture.
fn replace_remote_worker_binary_if_stale(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    session_id: &str,
    installed_digest: &CommandSpec,
    source: &Path,
) -> Result<bool> {
    let expected = hel::hel_worker_launch::worker_executable_digest(source)?;
    let installed = executor
        .execute(installed_digest)
        .context("read the installed remote worker digest")?;
    let matches = installed.status == 0
        && String::from_utf8_lossy(&installed.stdout)
            .split_whitespace()
            .next()
            .is_some_and(|digest| digest.eq_ignore_ascii_case(&expected));
    if matches {
        return Ok(false);
    }
    installed_worker_binary_replacement_plan(locator, session_id, source)?
        .execute(executor)
        .context("replace stale remote relay worker binary")?;
    Ok(true)
}

/// Stop the detached worker daemon at `worker_root` without deleting its files.
///
/// The script signals the worker's process group so a wedged ACP child dies
/// with it. Checkpoint then restarts the daemon against the same relay root.
pub(super) fn stop_worker(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    worker_root: &str,
) -> Result<()> {
    execute_checked(executor, stop_worker_command(locator, worker_root))?;
    Ok(())
}

/// Restore a stopped Podman target before signaling its worker. Checkpoint
/// recovery uses this instead of assuming every persisted target is running.
pub(super) fn stop_worker_after_target_recovery(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    session_id: &str,
    worker_root: &str,
) -> Result<()> {
    let target = hel_targets::target_recovery_plan(locator, session_id)?;
    hel_targets::ensure_recovery_target_running(executor, target.as_ref())
        .context("restore Mjolnir worker target")?;
    stop_worker(executor, locator, worker_root)
}

fn stop_worker_command(locator: &hel_targets::TargetLocator, worker_root: &str) -> CommandSpec {
    let script = hel_targets::stop_worker_daemon_script(worker_root);
    match locator {
        hel_targets::TargetLocator::LocalBare { .. } => CommandSpec::new("sh", ["-c", &script]),
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => {
            CommandSpec::new("podman", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::LocalDocker { container_id } => {
            CommandSpec::new("docker", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::AppleContainer { container_id } => {
            CommandSpec::new("container", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            ssh_command_spec(ssh, ["sh", "-c", &script])
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(ssh, ["podman", "exec", container_id, "sh", "-c", &script]),
    }
    .purpose("stop Mjolnir worker daemon")
}

fn worker_liveness_command(locator: &hel_targets::TargetLocator, worker_root: &str) -> CommandSpec {
    let script = hel_targets::worker_daemon_liveness_script(worker_root);
    match locator {
        hel_targets::TargetLocator::LocalBare { .. } => CommandSpec::new("sh", ["-c", &script]),
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => {
            CommandSpec::new("podman", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::LocalDocker { container_id } => {
            CommandSpec::new("docker", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::AppleContainer { container_id } => {
            CommandSpec::new("container", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            ssh_command_spec(ssh, ["sh", "-c", &script])
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(ssh, ["podman", "exec", container_id, "sh", "-c", &script]),
    }
    .purpose("probe Mjolnir worker daemon liveness")
}

pub(super) fn start_worker(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    worker_root: &str,
) -> Result<()> {
    execute_checked(executor, start_worker_command(locator, worker_root))?;
    Ok(())
}

fn start_worker_command(locator: &hel_targets::TargetLocator, worker_root: &str) -> CommandSpec {
    let binary = format!("{worker_root}/hel");
    let config = format!("{worker_root}/launch.json");
    // These files describe the worker's previous life. Clear them as part of
    // the launch, before the new daemon can be probed: a stale exit record
    // aborts startup, while a stale socket makes a recovering daemon look
    // ready and invites the reconnect actor to kill it as unresponsive.
    let clear_stale_runtime = format!(
        "rm -f {} {}; ",
        hel_targets::join_remote_command(&[format!("{worker_root}/worker-exit.json")]),
        hel_targets::join_remote_command(&[format!("{worker_root}/control.sock")]),
    );
    let detached_script = format!(
        "{clear_stale_runtime}nohup {} >{} 2>&1 </dev/null &",
        hel_targets::join_remote_command(&[
            binary.clone(),
            "worker".into(),
            "run".into(),
            "--root".into(),
            worker_root.into(),
            "--config".into(),
            config.clone(),
        ]),
        hel_targets::join_remote_command(&[format!("{worker_root}/worker.log")]),
    );
    // Redirect daemon output to worker.log in every launch mode; an
    // unexplained dead worker is undebuggable without it.
    let exec_script = format!(
        "{clear_stale_runtime}exec {} >{} 2>&1",
        hel_targets::join_remote_command(&[
            binary.clone(),
            "worker".into(),
            "run".into(),
            "--root".into(),
            worker_root.into(),
            "--config".into(),
            config.clone(),
        ]),
        hel_targets::join_remote_command(&[format!("{worker_root}/worker.log")]),
    );
    match locator {
        hel_targets::TargetLocator::LocalBare { .. } => {
            CommandSpec::new("sh", ["-c", &detached_script])
        }
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => CommandSpec::new(
            "podman",
            ["exec", "--detach", container_id, "sh", "-c", &exec_script],
        ),
        hel_targets::TargetLocator::LocalDocker { container_id } => CommandSpec::new(
            "docker",
            ["exec", "--detach", container_id, "sh", "-c", &exec_script],
        ),
        hel_targets::TargetLocator::AppleContainer { container_id } => CommandSpec::new(
            "container",
            ["exec", "--detach", container_id, "sh", "-c", &exec_script],
        ),
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            ssh_command_spec(ssh, ["sh", "-c", &detached_script])
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(
            ssh,
            [
                "podman",
                "exec",
                "--detach",
                container_id,
                "sh",
                "-c",
                &exec_script,
            ],
        ),
    }
    .purpose("start detached Mjolnir worker")
    // Everything before this moves data into the target and reports as Sync.
    // Start begins here, with the daemon launch.
    .stage(ProvisionStage::Starting)
}

/// Enrich an opaque handshake failure by running the installed worker binary
/// directly in the target. This surfaces loader errors (for example a
/// glibc-linked worker inside an older-glibc container) that a detached start
/// swallows.
pub(super) fn worker_probe_diagnosis(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    worker_root: &str,
    error: anyhow::Error,
) -> anyhow::Error {
    let error = match worker_binary_probe_failure(executor, locator, worker_root) {
        Some(failure) => error.context(failure),
        None => error,
    };
    match worker_last_words(executor, locator, worker_root) {
        Some(last_words) => error.context(last_words),
        None => error,
    }
}

fn worker_binary_probe_failure(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    worker_root: &str,
) -> Option<String> {
    let binary = format!("{worker_root}/hel");
    let command = match locator {
        hel_targets::TargetLocator::LocalBare { .. } => {
            CommandSpec::new(binary.clone(), ["--version"])
        }
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => {
            CommandSpec::new("podman", ["exec", container_id, &binary, "--version"])
        }
        hel_targets::TargetLocator::LocalDocker { container_id } => {
            CommandSpec::new("docker", ["exec", container_id, &binary, "--version"])
        }
        hel_targets::TargetLocator::AppleContainer { container_id } => {
            CommandSpec::new("container", ["exec", container_id, &binary, "--version"])
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            ssh_command_spec(ssh, [binary.as_str(), "--version"])
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(
            ssh,
            ["podman", "exec", container_id, binary.as_str(), "--version"],
        ),
    }
    .purpose("probe installed worker binary");
    match executor.execute(&command) {
        Ok(output) if output.status == 0 => None,
        Ok(output) => {
            let stderr = String::from_utf8_lossy(&output.stderr);
            let stdout = String::from_utf8_lossy(&output.stdout);
            let detail = if !stderr.trim().is_empty() {
                stderr.trim()
            } else if !stdout.trim().is_empty() {
                stdout.trim()
            } else {
                "the process exited unsuccessfully without output"
            };
            Some(format!(
                "worker binary {binary} fails to run in the target: {detail}; \
                 if this is a loader/glibc error, provide a musl worker \
                 (cargo build --release --target <arch>-unknown-linux-musl, \
                 or set MJ_WORKER_BINARY/MJ_WORKER_DIR)"
            ))
        }
        Err(probe_error) => Some(format!("worker probe failed: {probe_error:#}")),
    }
}

/// Fetch the dead worker's structured exit record and log tail from the
/// target, so unreachable-worker errors carry the root cause.
pub(super) fn worker_last_words(
    executor: &impl CommandExecutor,
    locator: &hel_targets::TargetLocator,
    worker_root: &str,
) -> Option<String> {
    let script = format!(
        "if [ -f {root}/worker-exit.json ]; then echo '{marker}'; cat {root}/worker-exit.json; fi; if [ -f {root}/worker.log ]; then echo '--- worker.log (tail) ---'; tail -n 20 {root}/worker.log; fi",
        root = worker_root,
        marker = WORKER_EXIT_RECORD_MARKER
    );
    let command = match locator {
        hel_targets::TargetLocator::LocalBare { .. } => CommandSpec::new("sh", ["-c", &script]),
        hel_targets::TargetLocator::LocalPodman { container_id, .. } => {
            CommandSpec::new("podman", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::LocalDocker { container_id } => {
            CommandSpec::new("docker", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::AppleContainer { container_id } => {
            CommandSpec::new("container", ["exec", container_id, "sh", "-c", &script])
        }
        hel_targets::TargetLocator::AwsEc2 { ssh, .. }
        | hel_targets::TargetLocator::SshBare { ssh, .. } => {
            ssh_command_spec(ssh, ["sh", "-c", &script])
        }
        hel_targets::TargetLocator::SshPodman {
            ssh, container_id, ..
        } => ssh_command_spec(ssh, ["podman", "exec", container_id, "sh", "-c", &script]),
    }
    .purpose("collect worker last words");
    let output = match executor.execute(&command) {
        Ok(output) => output,
        Err(error) => {
            tracing::debug!(
                worker_root,
                %error,
                "could not collect worker diagnostics"
            );
            return None;
        }
    };
    if output.status != 0 {
        tracing::debug!(
            worker_root,
            status = output.status,
            "worker diagnostic probe returned a failure"
        );
        return None;
    }
    let text = String::from_utf8_lossy(&output.stdout).trim().to_string();
    (!text.is_empty()).then(|| format!("worker diagnostics:\n{text}"))
}

#[cfg(test)]
mod tests {
    use super::*;

    use anyhow::Result;

    use hel::hel_config::ExecutionPolicy;
    use hel::hel_targets::{self, CommandExecutor, CommandOutput, CommandSpec, SshTarget};

    use sha2::{Digest, Sha256};
    use std::cell::RefCell;
    use std::collections::BTreeMap;

    use std::path::{Path, PathBuf};

    #[test]
    fn a_stored_setup_token_reaches_only_claude_workers_that_do_not_set_their_own() {
        use hel::hel_config::HarnessKind;
        use hel::hel_credentials::{CLAUDE_OAUTH_TOKEN_ENV, write_claude_oauth_token};

        let directory = tempfile::tempdir().unwrap();
        let token_path = directory.path().join("profiles/claude/claude-oauth-token");
        let missing = directory.path().join("profiles/absent/claude-oauth-token");
        write_claude_oauth_token(&token_path, b"sk-ant-oat01-stored").unwrap();

        let mut claude = BTreeMap::new();
        apply_claude_setup_token(&mut claude, HarnessKind::Claude, &token_path);
        assert_eq!(
            claude.get(CLAUDE_OAUTH_TOKEN_ENV).map(String::as_str),
            Some("sk-ant-oat01-stored")
        );

        // Every other harness ignores the variable, so it must not appear.
        for kind in HarnessKind::ALL
            .into_iter()
            .filter(|kind| *kind != HarnessKind::Claude)
        {
            let mut environment = BTreeMap::new();
            apply_claude_setup_token(&mut environment, kind, &token_path);
            assert!(environment.is_empty(), "{kind:?} must not read the token");
        }

        // A profile that sets the variable itself stays authoritative.
        let mut overridden = BTreeMap::from([(
            CLAUDE_OAUTH_TOKEN_ENV.to_owned(),
            "profile-token".to_owned(),
        )]);
        apply_claude_setup_token(&mut overridden, HarnessKind::Claude, &token_path);
        assert_eq!(
            overridden.get(CLAUDE_OAUTH_TOKEN_ENV).map(String::as_str),
            Some("profile-token")
        );

        // A profile with no stored token launches exactly as before.
        let mut without = BTreeMap::new();
        apply_claude_setup_token(&mut without, HarnessKind::Claude, &missing);
        assert!(without.is_empty());
    }

    #[test]
    fn packaged_worker_names_match_release_archives() {
        let directory = Path::new("/opt/hel/bin");
        assert_eq!(
            packaged_worker_binary_path(directory, "x86_64-unknown-linux-musl"),
            directory.join("mj-worker-x86_64-unknown-linux-musl")
        );
        assert_eq!(
            packaged_worker_binary_path(directory, "aarch64-unknown-linux-musl"),
            directory.join("mj-worker-aarch64-unknown-linux-musl")
        );
    }

    #[test]
    fn dev_checkout_prefers_the_musl_sibling_over_the_glibc_controller() {
        let controller = PathBuf::from("target/debug/mj");
        let musl = PathBuf::from("target/x86_64-unknown-linux-musl/debug/mj");
        // Both the controller (glibc) and its musl sibling exist on disk.
        let present = [controller.clone(), musl.clone()];
        let selected = select_sibling_worker(&controller, "x86_64-unknown-linux-musl", |path| {
            present.iter().any(|p| p == path)
        });
        assert_eq!(
            selected,
            Some((musl, "development musl sibling")),
            "the static musl sibling must win over the glibc controller itself"
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn replaced_dev_controller_still_finds_its_musl_sibling() {
        let controller = PathBuf::from("target/debug/mj (deleted)");
        let musl = PathBuf::from("target/x86_64-unknown-linux-musl/debug/mj");
        let selected = select_sibling_worker(&controller, "x86_64-unknown-linux-musl", |path| {
            path == musl
        });

        assert_eq!(selected, Some((musl, "development musl sibling")));
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn replaced_dev_controller_never_selects_the_new_glibc_controller_as_its_worker() {
        let controller = PathBuf::from("target/debug/mj (deleted)");
        let replacement = PathBuf::from("target/debug/mj");
        let selected = select_sibling_worker(&controller, "x86_64-unknown-linux-musl", |path| {
            path == replacement
        });

        assert_eq!(selected, None);
    }

    /// A configured container template for the preflight tests. Only the
    /// platform matters here; the rest is the smallest valid template.
    fn container_template(platform: Option<&str>) -> hel::hel_config::ContainerTemplate {
        hel::hel_config::ContainerTemplate {
            image: "example.invalid/mj-test:latest".into(),
            pull_policy: Default::default(),
            platform: platform.map(str::to_owned),
            cpus: None,
            memory: None,
            environment: BTreeMap::new(),
            workspace_storage: Default::default(),
        }
    }

    fn ssh_connection() -> hel::hel_config::SshConnection {
        hel::hel_config::SshConnection {
            host: "builder".into(),
            user: Some("dev".into()),
            identity_file: None,
            extra_args: Vec::new(),
        }
    }

    #[test]
    fn preflight_reads_the_architecture_a_template_names() {
        use hel::hel_config::TargetTemplate;

        for (platform, expected) in [
            ("linux/arm64", "aarch64"),
            ("linux/arm64/v8", "aarch64"),
            ("linux/amd64", "x86_64"),
            ("aarch64", "aarch64"),
        ] {
            assert_eq!(
                preflight_architectures(&TargetTemplate::LocalPodman {
                    container: container_template(Some(platform)),
                }),
                vec![expected],
                "platform {platform}"
            );
        }
        // A named platform decides a remote container target too, so a resume
        // onto an arm64 container never asks about the host's architecture.
        assert_eq!(
            preflight_architectures(&TargetTemplate::SshPodman {
                ssh: ssh_connection(),
                container: container_template(Some("linux/arm64")),
            }),
            vec!["aarch64"]
        );
    }

    #[test]
    fn preflight_uses_the_host_architecture_for_a_local_target() {
        use hel::hel_config::TargetTemplate;

        for template in [
            TargetTemplate::LocalBare,
            TargetTemplate::LocalPodman {
                container: container_template(None),
            },
            TargetTemplate::LocalDocker {
                container: container_template(None),
            },
            TargetTemplate::AppleContainer {
                container: container_template(None),
            },
        ] {
            assert_eq!(
                preflight_architectures(&template),
                vec![std::env::consts::ARCH],
                "{template:?}"
            );
        }
    }

    #[test]
    fn preflight_accepts_either_linux_architecture_for_a_remote_target() {
        use hel::hel_config::TargetTemplate;

        // Nothing in the configuration says what a remote machine runs, so the
        // preflight passes as long as one architecture could be served; the
        // real architecture is read from the live target during provisioning.
        for template in [
            TargetTemplate::SshBare {
                ssh: ssh_connection(),
                permissions: hel::hel_config::PermissionMode::Yolo,
                workspace_prefix: PathBuf::from(".local/share/hel/workspaces"),
            },
            TargetTemplate::SshPodman {
                ssh: ssh_connection(),
                container: container_template(None),
            },
            TargetTemplate::AwsEc2 {
                aws_profile: None,
                region: "us-east-1".into(),
                launch_template: "lt-mj".into(),
                launch_template_version: None,
                ssh_user: "dev".into(),
                address_source: Default::default(),
                identity_file: None,
                ssh_args: Vec::new(),
            },
        ] {
            assert_eq!(
                preflight_architectures(&template),
                vec!["x86_64", "aarch64"],
                "{template:?}"
            );
        }
    }

    #[test]
    fn dev_checkout_still_finds_a_hel_named_sibling() {
        let controller = PathBuf::from("target/debug/hel");
        let musl = PathBuf::from("target/x86_64-unknown-linux-musl/debug/hel");
        let present = [controller.clone(), musl.clone()];
        let selected = select_sibling_worker(&controller, "x86_64-unknown-linux-musl", |path| {
            present.iter().any(|p| p == path)
        });
        assert_eq!(selected, Some((musl, "development musl sibling")));
    }

    /// An architecture no host builds for, so the lookup cannot take one of
    /// the "native mj binary" shortcuts and reaches the end on any machine.
    const FOREIGN_ARCH: &str = "riscv64";

    /// A rebuilt or renamed checkout leaves a running daemon pointing at a
    /// path that holds nothing. Searching beside that path finds nothing and
    /// blames the user for a worker that may well be installed correctly.
    #[test]
    fn a_replaced_controller_is_reported_instead_of_a_missing_worker() {
        let stale = PathBuf::from("/src/.backup-vHXvCs/target/debug/mj (deleted)");
        let probed = RefCell::new(Vec::new());

        let error = worker_binary_prerequisite_for_current(
            FOREIGN_ARCH,
            WorkerBinaryRequirement::PortableLinux,
            &stale,
            &|path| {
                probed.borrow_mut().push(path.to_path_buf());
                false
            },
        )
        .unwrap_err();

        let detail = format!("{error:#}");
        assert!(
            detail.contains("was replaced or removed on disk"),
            "{detail}"
        );
        assert!(detail.contains("restart the Mjolnir daemon"), "{detail}");
        // The path is named without the kernel's deletion marker.
        assert!(
            detail.contains("/src/.backup-vHXvCs/target/debug/mj)"),
            "{detail}"
        );
        assert!(!detail.contains("(deleted)"), "{detail}");
        assert_eq!(
            probed.into_inner(),
            vec![stale],
            "nothing beside a path that no longer exists is worth probing"
        );
    }

    /// The guard is about a controller path that no longer exists and nothing
    /// else: a controller still on disk keeps its whole sibling lookup, and
    /// keeps the plain "no Linux worker" answer when that lookup comes up
    /// empty. A present controller is never its own portable worker, so with
    /// nothing installed beside it the lookup ends in that plain answer.
    #[test]
    fn a_present_controller_still_looks_beside_itself() {
        let controller = PathBuf::from("/opt/brokk/mj");
        let probed = RefCell::new(Vec::new());

        let error = worker_binary_prerequisite_for_current(
            FOREIGN_ARCH,
            WorkerBinaryRequirement::PortableLinux,
            &controller,
            &|path| {
                probed.borrow_mut().push(path.to_path_buf());
                path == controller
            },
        )
        .unwrap_err();

        let probed = probed.into_inner();
        assert!(
            probed
                .iter()
                .any(|path| path.ends_with("mj-worker-riscv64-unknown-linux-musl")),
            "the packaged worker name must still be probed: {probed:?}"
        );
        let detail = format!("{error:#}");
        assert!(
            detail.contains("no Linux worker for riscv64-unknown-linux-musl"),
            "{detail}"
        );
        assert!(!detail.contains("restart the Mjolnir daemon"), "{detail}");

        // With nothing beside it either, a present controller still gets the
        // generic message; only a replaced one is told to restart.
        let root = PathBuf::from("/");
        let error = worker_binary_prerequisite_for_current(
            FOREIGN_ARCH,
            WorkerBinaryRequirement::PortableLinux,
            &root,
            &|path| path == root,
        )
        .unwrap_err();
        let detail = format!("{error:#}");
        assert!(
            detail.contains("no Linux worker for riscv64-unknown-linux-musl"),
            "{detail}"
        );
        assert!(!detail.contains("restart the Mjolnir daemon"), "{detail}");
    }

    const WORKER_BINARY_OVERRIDE_CHILD: &str = "MJ_WORKER_BINARY_OVERRIDE_CHILD";

    /// The override names a worker outright, so it does not care where the
    /// controller lives or whether that path still exists.
    #[test]
    fn a_replaced_controller_still_honors_the_worker_binary_override() {
        // MJ_WORKER_BINARY is process-global and other tests resolve worker
        // binaries, so set it only in an exact child test.
        if std::env::var_os(WORKER_BINARY_OVERRIDE_CHILD).is_none() {
            let directory = tempfile::tempdir().unwrap();
            let worker = directory.path().join("mj-worker");
            std::fs::write(&worker, b"worker").unwrap();
            let test_name = format!(
                "{}::a_replaced_controller_still_honors_the_worker_binary_override",
                module_path!()
                    .strip_prefix("mj_controller::")
                    .unwrap_or(module_path!())
            );
            let output = std::process::Command::new(std::env::current_exe().unwrap())
                .args(["--exact", &test_name, "--nocapture"])
                .env(WORKER_BINARY_OVERRIDE_CHILD, "1")
                .env("MJ_WORKER_BINARY", &worker)
                .output()
                .unwrap();
            assert!(
                output.status.success(),
                "isolated worker override test failed\nstdout:\n{}\nstderr:\n{}",
                String::from_utf8_lossy(&output.stdout),
                String::from_utf8_lossy(&output.stderr)
            );
            return;
        }

        let stale = PathBuf::from("/src/.backup-vHXvCs/target/debug/mj (deleted)");
        let availability = worker_binary_prerequisite_for_current(
            FOREIGN_ARCH,
            WorkerBinaryRequirement::PortableLinux,
            &stale,
            &|path| path.is_file(),
        )
        .unwrap();

        match availability {
            WorkerBinaryAvailability::Local { source, .. } => {
                assert_eq!(source, "MJ_WORKER_BINARY");
            }
            other => panic!("expected the override to resolve, got {other:?}"),
        }
    }

    #[test]
    fn sibling_lookup_falls_back_to_the_legacy_hel_name_beside_an_mj_controller() {
        let controller = PathBuf::from("/opt/brokk/mj");
        let legacy = PathBuf::from("/opt/brokk/hel");
        let selected = select_sibling_worker(&controller, "x86_64-unknown-linux-musl", |path| {
            path == legacy
        });
        assert_eq!(selected, Some((legacy, "beside the running executable")));
    }

    #[test]
    fn worker_diagnosis_surfaces_a_loader_failure_from_the_installed_binary() {
        struct FailedProbe;

        impl CommandExecutor for FailedProbe {
            fn execute(&self, _command: &CommandSpec) -> Result<CommandOutput> {
                Ok(CommandOutput {
                    status: 1,
                    stdout: Vec::new(),
                    stderr: b"libc.so.6: version `GLIBC_2.39' not found\n".to_vec(),
                })
            }
        }

        let failure = worker_binary_probe_failure(
            &FailedProbe,
            &hel_targets::TargetLocator::LocalBare {
                worker_root: "/worker/root".into(),
            },
            "/worker/root",
        )
        .expect("an unsuccessful --version probe should explain the dead worker");

        assert!(failure.contains("GLIBC_2.39"), "{failure}");
        assert!(failure.contains("provide a musl worker"), "{failure}");
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn replaced_running_executable_is_materialized_for_worker_upload() {
        use std::os::unix::fs::PermissionsExt;

        let directory = tempfile::tempdir().unwrap();
        let replaced = directory.path().join("hel (deleted)");
        let proc_exe = directory.path().join("proc-exe");
        let cached = directory.path().join("workers/running/hel-1");
        std::fs::write(&proc_exe, b"running executable").unwrap();

        assert_eq!(
            materialize_running_executable(&replaced, &proc_exe, &cached).unwrap(),
            cached
        );
        assert_eq!(std::fs::read(&cached).unwrap(), b"running executable");
        assert_eq!(
            std::fs::metadata(&cached).unwrap().permissions().mode() & 0o777,
            0o700
        );
    }
    /// A worker that died leaves an exit record behind. Starting a new worker
    /// must clear it first, or the startup connect loop reads the previous
    /// death as this worker's and gives up on a healthy daemon.
    #[test]
    fn starting_a_worker_clears_stale_runtime_files_before_launching() {
        struct RecordingExecutor {
            commands: RefCell<Vec<CommandSpec>>,
        }

        impl CommandExecutor for RecordingExecutor {
            fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
                self.commands.borrow_mut().push(command.clone());
                Ok(CommandOutput {
                    status: 0,
                    stdout: Vec::new(),
                    stderr: Vec::new(),
                })
            }
        }

        for locator in [
            hel_targets::TargetLocator::LocalBare {
                worker_root: "/worker/root".into(),
            },
            hel_targets::TargetLocator::LocalPodman {
                container_id: "container-1".into(),
                workspace_storage: Default::default(),
            },
        ] {
            let executor = RecordingExecutor {
                commands: RefCell::new(Vec::new()),
            };
            start_worker(&executor, &locator, "/worker/root").unwrap();

            let commands = executor.commands.borrow();
            let script = commands
                .iter()
                .flat_map(|command| command.args.iter())
                .find(|argument| argument.contains("worker-exit.json"))
                .unwrap_or_else(|| {
                    panic!("no launch script cleared the exit record: {commands:?}")
                });
            let cleared = script.find("rm -f").expect("the exit record is removed");
            let launched = script.find("worker").expect("the daemon is launched");
            assert!(
                script.contains("control.sock"),
                "the stale relay endpoint must be cleared before startup: {script}"
            );
            assert!(
                cleared < launched,
                "stale runtime files must be cleared before the daemon starts: {script}"
            );
        }
    }
    #[test]
    fn stopping_a_worker_runs_the_daemon_stop_script() {
        struct RecordingExecutor {
            commands: RefCell<Vec<CommandSpec>>,
        }

        impl CommandExecutor for RecordingExecutor {
            fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
                self.commands.borrow_mut().push(command.clone());
                Ok(CommandOutput {
                    status: 0,
                    stdout: Vec::new(),
                    stderr: Vec::new(),
                })
            }
        }

        let locator = hel_targets::TargetLocator::SshBare {
            ssh: SshTarget {
                destination: "user@example.test".into(),
                ssh_args: Vec::new(),
            },
            workspace: "/workspace".into(),
        };
        let executor = RecordingExecutor {
            commands: RefCell::new(Vec::new()),
        };
        stop_worker(&executor, &locator, "/worker/root").unwrap();

        let commands = executor.commands.borrow();
        assert_eq!(commands.len(), 1);
        assert_eq!(commands[0].purpose, "stop Mjolnir worker daemon");
        assert!(
            commands[0]
                .args
                .last()
                .is_some_and(|remote| remote.starts_with("'sh' '-c' ")),
            "raw SSH worker management must not source login profiles: {commands:?}"
        );
        let script = commands[0]
            .args
            .iter()
            .find(|argument| argument.contains("worker run --root"))
            .unwrap_or_else(|| panic!("stop script missing from {commands:?}"));
        assert!(
            script.contains("hel_match=\"hel worker run --root $hel_root\""),
            "stop must match only this session's worker: {script}"
        );
        assert!(
            script.contains("hel_match_home=\"hel worker run --root $HOME/$hel_root\""),
            "stop must also match a login-home-absolute --root: {script}"
        );
        assert!(
            !script.contains("grep -F"),
            "leftover detection must not grep the match string: {script}"
        );
    }
    #[test]
    fn checkpoint_worker_stop_restores_a_stopped_podman_target_first() {
        struct RecordingExecutor {
            commands: RefCell<Vec<CommandSpec>>,
            outputs: RefCell<Vec<CommandOutput>>,
        }

        impl CommandExecutor for RecordingExecutor {
            fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
                self.commands.borrow_mut().push(command.clone());
                Ok(self.outputs.borrow_mut().remove(0))
            }
        }

        let session = "0123456789abcdef0123456789abcdef";
        let container_id = hel_targets::resource_name(session).unwrap();
        let inspection = |status: &str| CommandOutput {
            status: 0,
            stdout: serde_json::to_vec(&serde_json::json!([{
                "Config": { "Labels": {
                    (hel_targets::MANAGED_LABEL): "true",
                    (hel_targets::SESSION_LABEL): session,
                }},
                "State": { "Status": status },
            }]))
            .unwrap(),
            stderr: Vec::new(),
        };
        let executor = RecordingExecutor {
            commands: RefCell::new(Vec::new()),
            outputs: RefCell::new(vec![
                CommandOutput {
                    status: 0,
                    stdout: Vec::new(),
                    stderr: Vec::new(),
                },
                inspection("exited"),
                CommandOutput {
                    status: 0,
                    stdout: Vec::new(),
                    stderr: Vec::new(),
                },
                inspection("running"),
                CommandOutput {
                    status: 0,
                    stdout: Vec::new(),
                    stderr: Vec::new(),
                },
            ]),
        };
        let locator = hel_targets::TargetLocator::LocalPodman {
            container_id,
            workspace_storage: Default::default(),
        };

        stop_worker_after_target_recovery(&executor, &locator, session, "/worker/root").unwrap();

        let commands = executor.commands.borrow();
        let purposes = commands
            .iter()
            .map(|command| command.purpose.as_str())
            .collect::<Vec<_>>();
        assert_eq!(
            purposes,
            [
                "check for Mjolnir session container",
                "inspect Mjolnir session container",
                "start stopped Mjolnir session container",
                "inspect Mjolnir session container",
                "stop Mjolnir worker daemon",
            ]
        );
    }

    struct PodmanInstallExecutor {
        commands: RefCell<Vec<CommandSpec>>,
        worker_cached: bool,
    }
    impl CommandExecutor for PodmanInstallExecutor {
        fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
            self.commands.borrow_mut().push(command.clone());
            let probing_cache = command
                .args
                .iter()
                .any(|argument| argument.contains("'test' '-f'"));
            let status = if probing_cache && !self.worker_cached {
                1
            } else {
                0
            };
            Ok(CommandOutput {
                status,
                stdout: Vec::new(),
                stderr: Vec::new(),
            })
        }
    }
    struct PodmanInstallFixture {
        _root: tempfile::TempDir,
        worker_binary: PathBuf,
        launch_config: PathBuf,
        ownership: PathBuf,
        profile_stage: PathBuf,
        locator: hel_targets::TargetLocator,
        digest: String,
    }
    fn podman_install_fixture() -> PodmanInstallFixture {
        let root = tempfile::tempdir().unwrap();
        let worker_binary = root.path().join("hel");
        std::fs::write(&worker_binary, b"worker-binary-bytes").unwrap();
        let launch_config = root.path().join("launch.json");
        std::fs::write(&launch_config, b"{}").unwrap();
        let ownership = root.path().join("ownership.json");
        std::fs::write(&ownership, b"{}").unwrap();
        let profile_stage = root.path().join("profile");
        std::fs::create_dir_all(&profile_stage).unwrap();
        let digest = format!("{:x}", Sha256::digest(b"worker-binary-bytes"));
        PodmanInstallFixture {
            _root: root,
            worker_binary,
            launch_config,
            ownership,
            profile_stage,
            locator: hel_targets::TargetLocator::SshPodman {
                ssh: SshTarget {
                    destination: "user@example.test".into(),
                    ssh_args: Vec::new(),
                },
                container_id: "container-1".into(),
                workspace_storage: Default::default(),
            },
            digest,
        }
    }
    fn run_podman_install(worker_cached: bool) -> (Vec<CommandSpec>, PodmanInstallFixture) {
        let fixture = podman_install_fixture();
        let executor = PodmanInstallExecutor {
            commands: RefCell::new(Vec::new()),
            worker_cached,
        };
        install_worker_files(
            &executor,
            &fixture.locator,
            "0123456789abcdef0123456789abcdef",
            "/workspace/.hel/worker",
            "/workspace/.hel/profile",
            &fixture.worker_binary,
            &fixture.launch_config,
            &fixture.ownership,
            &fixture.profile_stage,
        )
        .unwrap();
        let commands = executor.commands.borrow().clone();
        (commands, fixture)
    }
    fn rendered(commands: &[CommandSpec]) -> Vec<String> {
        commands
            .iter()
            .map(|command| format!("{} {}", command.program, command.args.join(" ")))
            .collect()
    }
    #[test]
    fn ssh_podman_install_caches_the_worker_binary_on_a_cache_miss() {
        let (commands, fixture) = run_podman_install(false);
        let lines = rendered(&commands);
        let digest = &fixture.digest;
        let cache_dir = format!(".cache/mjolnir/workers/{digest}");
        let session = "0123456789abcdef0123456789abcdef";

        assert!(
            lines
                .iter()
                .any(|line| line.starts_with("ssh") && line.contains("'test' '-f'")),
            "expected a cache probe, got {lines:#?}"
        );
        assert!(
            !lines.iter().any(|line| line.contains('~')),
            "remote staging paths must be home-relative: ssh arguments are \
                 single-quoted so a tilde stays literal in the remote shell while \
                 scp expands it, got {lines:#?}"
        );
        assert!(
            lines.iter().any(|line| line.starts_with("ssh")
                && line.contains(&format!("'mkdir' '-p' '{cache_dir}'"))),
            "expected the cache directory to be created, got {lines:#?}"
        );
        let partial = format!("{cache_dir}/hel.partial-{session}");
        assert!(
            lines.iter().any(|line| line
                == &format!(
                    "scp {} user@example.test:{partial}",
                    fixture.worker_binary.display()
                )),
            "expected the worker to be uploaded to the partial cache path, got {lines:#?}"
        );
        assert!(
            lines.iter().any(|line| line.starts_with("ssh")
                && line.contains(&format!("'mv' '{partial}' '{cache_dir}/hel'"))),
            "expected an atomic rename into the cache, got {lines:#?}"
        );
        assert!(
            lines.iter().any(|line| line.contains("'podman' 'cp'")
                && line.contains(&format!("'{cache_dir}/hel'"))),
            "expected podman cp to read the cached worker, got {lines:#?}"
        );
        assert!(
            !lines.iter().any(|line| line.starts_with("scp")
                && line.ends_with(&format!(
                    "user@example.test:.cache/mjolnir/uploads/{session}/hel"
                ))),
            "the worker must not be staged in the per-session upload directory, got {lines:#?}"
        );
    }
    #[test]
    fn ssh_podman_install_skips_the_worker_upload_on_a_cache_hit() {
        let (commands, fixture) = run_podman_install(true);
        let lines = rendered(&commands);
        let digest = &fixture.digest;
        let cache_dir = format!(".cache/mjolnir/workers/{digest}");
        let session = "0123456789abcdef0123456789abcdef";

        assert!(
            !lines.iter().any(|line| line.starts_with("scp")
                && line.contains(&fixture.worker_binary.display().to_string())),
            "a cached worker must not be re-uploaded, got {lines:#?}"
        );
        assert!(
            !lines.iter().any(|line| line.contains("'mv'")),
            "a cache hit must not rename anything, got {lines:#?}"
        );
        assert!(
            lines.iter().any(|line| line.contains("'podman' 'cp'")
                && line.contains(&format!("'{cache_dir}/hel'"))),
            "expected podman cp to read the cached worker, got {lines:#?}"
        );
        for name in ["launch.json", "ownership.json"] {
            assert!(
                lines.iter().any(|line| line.starts_with("scp")
                    && line.ends_with(&format!(
                        "user@example.test:.cache/mjolnir/uploads/{session}/{name}"
                    ))),
                "expected {name} to still be uploaded per session, got {lines:#?}"
            );
        }
    }
    #[test]
    fn replacing_an_installed_podman_worker_writes_through_a_next_path() {
        struct RecordingExecutor {
            commands: RefCell<Vec<CommandSpec>>,
        }
        impl CommandExecutor for RecordingExecutor {
            fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
                self.commands.borrow_mut().push(command.clone());
                Ok(CommandOutput {
                    status: 0,
                    stdout: Vec::new(),
                    stderr: Vec::new(),
                })
            }
        }

        let session = "0123456789abcdef0123456789abcdef";
        let container_id = hel_targets::resource_name(session).unwrap();
        let locator = hel_targets::TargetLocator::LocalPodman {
            container_id: container_id.clone(),
            workspace_storage: Default::default(),
        };
        let executor = RecordingExecutor {
            commands: RefCell::new(Vec::new()),
        };
        replace_installed_worker_binary(&executor, &locator, session, Path::new("/controller/hel"))
            .unwrap();

        let lines = rendered(&executor.commands.borrow());
        assert_eq!(
            lines,
            vec![
                format!(
                    "podman cp /controller/hel {container_id}:/var/lib/hel/workers/{session}/hel.next"
                ),
                format!(
                    "podman exec {container_id} mv -f /var/lib/hel/workers/{session}/hel.next /var/lib/hel/workers/{session}/hel"
                ),
                format!("podman exec {container_id} chmod 700 /var/lib/hel/workers/{session}/hel"),
            ]
        );
    }
    #[test]
    fn default_bridges_pin_command_capable_adapter_versions() {
        let (codex_command, codex_arguments) = bridge_launch(
            hel::hel_config::HarnessKind::Codex,
            None,
            ExecutionPolicy::Unconstrained,
        );
        assert_eq!(codex_command, "sh");
        assert_eq!(codex_arguments[0], "-c");
        assert!(codex_arguments[1].contains("@agentclientprotocol/codex-acp@1.8.0"));
        assert!(codex_arguments[1].contains("codex-acp --version"));

        let (claude_command, claude_arguments) = bridge_launch(
            hel::hel_config::HarnessKind::Claude,
            None,
            ExecutionPolicy::Unconstrained,
        );
        assert_eq!(claude_command, "sh");
        assert_eq!(claude_arguments[0], "-c");
        assert!(claude_arguments[1].contains("@agentclientprotocol/claude-agent-acp@0.73.0"));

        let (deepseek_command, deepseek_arguments) = bridge_launch(
            hel::hel_config::HarnessKind::Deepseek,
            None,
            ExecutionPolicy::Unconstrained,
        );
        assert_eq!(deepseek_command, "sh");
        assert_eq!(deepseek_arguments[0], "-c");
        assert!(deepseek_arguments[1].contains("@deepseek-ai/dsh@0.1.1-rc.2"));
        assert!(deepseek_arguments[1].contains("dsh-acp-server@0.10.0"));
        assert!(!deepseek_arguments[1].contains("npx -y -p @deepseek-ai/dsh"));
        assert!(deepseek_arguments[1].contains("exec dsh-acp-server"));
        assert!(deepseek_arguments[1].contains("Mjolnir needs @deepseek-ai/dsh"));
        assert!(!deepseek_arguments[1].contains("Hel"));
    }

    #[test]
    fn readiness_stage_names_only_install_capable_default_harnesses() {
        let profile = |kind, executable| hel::hel_config::HarnessProfile {
            kind,
            home: PathBuf::from("/profiles/test"),
            executable,
            environment: BTreeMap::new(),
            context_window_bytes: None,
        };

        for harness in [
            HarnessKind::Codex,
            HarnessKind::Claude,
            HarnessKind::Kimi,
            HarnessKind::Grok,
        ] {
            assert_eq!(
                bridge_readiness_stage(&profile(harness, None)),
                ProvisionStage::Installing(harness)
            );
        }
        assert_eq!(
            bridge_readiness_stage(&profile(HarnessKind::Deepseek, None)),
            ProvisionStage::Starting
        );
        assert_eq!(
            bridge_readiness_stage(&profile(
                HarnessKind::Codex,
                Some(PathBuf::from("/opt/bin/codex-acp")),
            )),
            ProvisionStage::Starting
        );
    }
    #[test]
    fn codex_execution_environment_follows_the_target_policy() {
        let mut podman_environment =
            BTreeMap::from([("INITIAL_AGENT_MODE".to_owned(), "read-only".to_owned())]);
        hel::hel_config::HarnessKind::Codex.configure_execution_environment(
            ExecutionPolicy::Unconstrained,
            &mut podman_environment,
        );
        assert_eq!(
            podman_environment
                .get("INITIAL_AGENT_MODE")
                .map(String::as_str),
            Some("agent-full-access")
        );

        let mut bare_environment =
            BTreeMap::from([("INITIAL_AGENT_MODE".to_owned(), "read-only".to_owned())]);
        hel::hel_config::HarnessKind::Codex.configure_execution_environment(
            ExecutionPolicy::ConfiguredApprovals,
            &mut bare_environment,
        );
        assert_eq!(
            bare_environment
                .get("INITIAL_AGENT_MODE")
                .map(String::as_str),
            Some("read-only"),
            "raw localhost must preserve the profile's configured mode"
        );
    }
    #[test]
    fn only_raw_targets_without_an_explicit_path_request_login_path_discovery() {
        let raw = hel_targets::TargetLocator::LocalBare {
            worker_root: "/worker".into(),
        };
        let managed = hel_targets::TargetLocator::LocalPodman {
            container_id: "container".into(),
            workspace_storage: Default::default(),
        };

        let mut environment = BTreeMap::new();
        configure_login_path_discovery(&mut environment, &raw);
        assert_eq!(
            environment.get(DISCOVER_LOGIN_PATH_ENV).map(String::as_str),
            Some("1")
        );

        let mut explicit = BTreeMap::from([
            ("PATH".into(), "/configured/bin".into()),
            (DISCOVER_LOGIN_PATH_ENV.into(), "stale".into()),
        ]);
        configure_login_path_discovery(&mut explicit, &raw);
        assert_eq!(
            explicit.get("PATH").map(String::as_str),
            Some("/configured/bin")
        );
        assert!(!explicit.contains_key(DISCOVER_LOGIN_PATH_ENV));

        let mut managed_environment =
            BTreeMap::from([(DISCOVER_LOGIN_PATH_ENV.into(), "stale".into())]);
        configure_login_path_discovery(&mut managed_environment, &managed);
        assert!(!managed_environment.contains_key(DISCOVER_LOGIN_PATH_ENV));
    }
    #[test]
    fn grok_sandbox_environment_follows_the_target_policy() {
        let mut isolated = BTreeMap::from([("GROK_SANDBOX".to_owned(), "strict".to_owned())]);
        hel::hel_config::HarnessKind::Grok
            .configure_execution_environment(ExecutionPolicy::Unconstrained, &mut isolated);
        assert_eq!(
            isolated.get("GROK_SANDBOX").map(String::as_str),
            Some("off")
        );

        let mut local = BTreeMap::from([("GROK_SANDBOX".to_owned(), "strict".to_owned())]);
        hel::hel_config::HarnessKind::Grok
            .configure_execution_environment(ExecutionPolicy::ConfiguredApprovals, &mut local);
        assert_eq!(
            local.get("GROK_SANDBOX").map(String::as_str),
            Some("strict"),
            "raw localhost must preserve the profile's configured sandbox"
        );
    }
    #[test]
    fn bridge_fallback_pins_match_the_agent_dev_containerfile() {
        const CONTAINERFILE: &str = include_str!("../../../containers/Containerfile.agent-dev");

        let codex = format!("codex-acp@{CODEX_ACP_FALLBACK_VERSION}");
        assert!(
            CONTAINERFILE.contains(&codex),
            "containers/Containerfile.agent-dev must install {codex}. The image and the \
                 bridge_launch() npx fallbacks have to stay in lockstep, otherwise a container \
                 session and an npx session run different adapter versions."
        );

        let claude = format!("claude-agent-acp@{CLAUDE_AGENT_ACP_FALLBACK_VERSION}");
        assert!(
            CONTAINERFILE.contains(&claude),
            "containers/Containerfile.agent-dev must install {claude}. The image and the \
                 bridge_launch() npx fallbacks have to stay in lockstep, otherwise a container \
                 session and an npx session run different adapter versions."
        );

        for package in [
            format!("@deepseek-ai/dsh@{DEEPSEEK_HARNESS_FALLBACK_VERSION}"),
            format!("dsh-acp-server@{DEEPSEEK_ACP_FALLBACK_VERSION}"),
        ] {
            assert!(
                CONTAINERFILE.contains(&package),
                "containers/Containerfile.agent-dev must install {package}"
            );
        }
    }
    #[test]
    fn kimi_default_bridge_is_non_login_and_uses_bash_for_the_official_installer() {
        let (command, arguments) = bridge_launch(
            hel::hel_config::HarnessKind::Kimi,
            None,
            ExecutionPolicy::Unconstrained,
        );
        assert_eq!(command, "sh");
        assert_eq!(arguments[0], "-c");
        assert!(arguments[1].contains("install.sh | bash &&"));
        assert!(arguments[1].contains("$HOME/.kimi-code/bin/kimi"));
        assert!(arguments[1].contains("Mjolnir needs compatible Kimi Code"));
        assert!(!arguments[1].contains("Hel"));
    }
    #[test]
    fn grok_default_bridge_is_non_login_and_uses_bash_for_the_official_installer() {
        let (command, arguments) = bridge_launch(
            hel::hel_config::HarnessKind::Grok,
            None,
            ExecutionPolicy::ConfiguredApprovals,
        );
        assert_eq!(command, "sh");
        assert_eq!(arguments[0], "-c");
        let script = &arguments[1];
        assert!(script.contains("https://x.ai/cli/install.sh | bash &&"));
        assert!(script.contains("command -v grok"));
        assert!(script.contains("[ -x \"$GROK_HOME/bin/grok\" ]"));
        assert!(script.contains("[ -x \"$HOME/.grok/bin/grok\" ]"));
        assert!(script.contains("exit 127"));
        assert!(script.contains("exec grok agent stdio"));
        assert!(!script.contains("--always-approve"));
        assert!(script.contains("Mjolnir needs compatible Grok Build"));
        assert!(!script.contains("Hel"));
    }
    #[test]
    fn node_bootstrap_errors_name_mjolnir() {
        let script = ensure_node_script();
        assert!(script.contains("Mjolnir needs Node/npx or passwordless sudo"));
        assert!(script.contains("Mjolnir cannot install Node on this image"));
        assert!(!script.contains("Hel"));
    }
    #[test]
    fn grok_default_bridge_adds_the_always_approve_flag_when_unrestricted() {
        let (_, arguments) = bridge_launch(
            hel::hel_config::HarnessKind::Grok,
            None,
            ExecutionPolicy::Unconstrained,
        );
        let script = &arguments[1];
        assert!(script.contains("exec grok agent --always-approve stdio"));
        assert!(script.contains("exec \"$GROK_HOME/bin/grok\" agent --always-approve stdio"));
        assert!(script.contains("exec \"$HOME/.grok/bin/grok\" agent --always-approve stdio"));
    }
    #[test]
    fn bridge_executable_override_carries_the_acp_subcommand_per_harness() {
        let executable = std::path::PathBuf::from("/opt/harness");
        for policy in [
            ExecutionPolicy::ConfiguredApprovals,
            ExecutionPolicy::Unconstrained,
        ] {
            for (kind, expected) in [
                (hel::hel_config::HarnessKind::Codex, Vec::new()),
                (hel::hel_config::HarnessKind::Claude, Vec::new()),
                (hel::hel_config::HarnessKind::Kimi, vec!["acp"]),
                (
                    hel::hel_config::HarnessKind::Grok,
                    if policy.is_unconstrained() {
                        vec!["agent", "--always-approve", "stdio"]
                    } else {
                        vec!["agent", "stdio"]
                    },
                ),
                (hel::hel_config::HarnessKind::Deepseek, Vec::new()),
            ] {
                let (command, arguments) = bridge_launch(kind, Some(&executable), policy);
                assert_eq!(command, "/opt/harness");
                assert_eq!(arguments, expected, "{kind:?} policy: {policy:?}");
            }
        }
    }
    #[test]
    fn kimi_uses_runtime_aware_memory_delivery_only_on_staged_targets() {
        let local = hel_targets::TargetLocator::LocalBare {
            worker_root: "/worker".into(),
        };
        let podman = hel_targets::TargetLocator::LocalPodman {
            container_id: "container".into(),
            workspace_storage: Default::default(),
        };

        assert_eq!(
            project_memory_mcp_delivery(hel::hel_config::HarnessKind::Kimi, &local),
            ProjectMemoryMcpDelivery::Acp
        );
        assert_eq!(
            project_memory_mcp_delivery(hel::hel_config::HarnessKind::Kimi, &podman),
            ProjectMemoryMcpDelivery::HarnessProfile
        );
        assert_eq!(
            project_memory_mcp_delivery(hel::hel_config::HarnessKind::Codex, &podman),
            ProjectMemoryMcpDelivery::Acp
        );
    }
    #[test]
    fn stage_grok_profile_copies_authentication_and_agent_identity() {
        let home = tempfile::tempdir().unwrap();
        std::fs::write(
            home.path().join("auth.json"),
            "{\"https://auth.x.ai::1\":{}}",
        )
        .unwrap();
        std::fs::write(home.path().join("agent_id"), "stable-agent-id").unwrap();
        std::fs::write(home.path().join("config.toml"), "model = \"grok-4.6\"\n").unwrap();
        // Native session storage is checkpointed, never staged.
        std::fs::create_dir(home.path().join("sessions")).unwrap();
        std::fs::write(home.path().join("sessions/session_search.sqlite"), "x").unwrap();
        let staged = tempfile::tempdir().unwrap();
        let profile = hel::hel_config::HarnessProfile {
            kind: hel::hel_config::HarnessKind::Grok,
            home: home.path().to_path_buf(),
            executable: None,
            environment: BTreeMap::new(),
            context_window_bytes: None,
        };

        stage_profile(&profile, staged.path()).unwrap();

        assert_eq!(
            std::fs::read_to_string(staged.path().join("agent_id")).unwrap(),
            "stable-agent-id"
        );
        assert!(staged.path().join("auth.json").is_file());
        assert!(staged.path().join("config.toml").is_file());
        assert!(!staged.path().join("sessions").exists());
    }
    #[test]
    fn stage_claude_profile_preserves_rollout_identity() {
        let home = tempfile::tempdir().unwrap();
        let identity = r#"{
                "machineID": "stable-machine",
                "userID": "stable-user",
                "cachedGrowthBookFeatures": {
                    "tengu_velvet_mallet_fable_5": true
                }
            }"#;
        std::fs::write(home.path().join(".claude.json"), identity).unwrap();
        let staged = tempfile::tempdir().unwrap();
        let profile = hel::hel_config::HarnessProfile {
            kind: hel::hel_config::HarnessKind::Claude,
            home: home.path().to_path_buf(),
            executable: None,
            environment: BTreeMap::new(),
            context_window_bytes: None,
        };

        stage_profile(&profile, staged.path()).unwrap();

        assert_eq!(
            std::fs::read_to_string(staged.path().join(".claude.json")).unwrap(),
            identity
        );
    }
    #[test]
    fn stage_kimi_profile_preserves_device_identity() {
        let home = tempfile::tempdir().unwrap();
        std::fs::write(home.path().join("config.toml"), "default_model = \"k3\"\n").unwrap();
        std::fs::write(home.path().join("device_id"), "stable-device-id").unwrap();
        std::fs::create_dir(home.path().join("credentials")).unwrap();
        std::fs::write(
            home.path().join("credentials/kimi-code.json"),
            "{\"access_token\":\"secret\"}",
        )
        .unwrap();
        let staged = tempfile::tempdir().unwrap();
        let profile = hel::hel_config::HarnessProfile {
            kind: hel::hel_config::HarnessKind::Kimi,
            home: home.path().to_path_buf(),
            executable: None,
            environment: BTreeMap::new(),
            context_window_bytes: None,
        };

        stage_profile(&profile, staged.path()).unwrap();

        assert_eq!(
            std::fs::read_to_string(staged.path().join("device_id")).unwrap(),
            "stable-device-id"
        );
        assert!(staged.path().join("credentials/kimi-code.json").is_file());
    }
    #[test]
    fn staged_kimi_profile_binds_project_memory_to_the_target_runtime() {
        let home = tempfile::tempdir().unwrap();
        let original = serde_json::json!({
            "mcpServers": {
                "user-server": {
                    "command": "user-mcp",
                    "args": ["serve"]
                }
            },
            "userSetting": true
        });
        let original_body = serde_json::to_vec_pretty(&original).unwrap();
        std::fs::write(home.path().join("mcp.json"), &original_body).unwrap();
        let staged = tempfile::tempdir().unwrap();
        let profile = hel::hel_config::HarnessProfile {
            kind: hel::hel_config::HarnessKind::Kimi,
            home: home.path().to_path_buf(),
            executable: None,
            environment: BTreeMap::new(),
            context_window_bytes: None,
        };
        stage_profile(&profile, staged.path()).unwrap();
        let memory = ProjectMemoryLaunchConfig {
            project_key: "project".into(),
            root: "/var/lib/hel/profiles/session/projects/project/memory".into(),
            baseline_root: PathBuf::new(),
            repository_roots: BTreeMap::new(),
            mcp_delivery: ProjectMemoryMcpDelivery::HarnessProfile,
        };

        configure_kimi_project_memory_mcp(staged.path(), "/var/lib/hel/workers/session", &memory)
            .unwrap();

        let configured: serde_json::Value =
            serde_json::from_slice(&std::fs::read(staged.path().join("mcp.json")).unwrap())
                .unwrap();
        assert_eq!(configured["userSetting"], true);
        assert_eq!(
            configured["mcpServers"]["user-server"]["command"],
            "user-mcp"
        );
        assert_eq!(
            configured["mcpServers"]["mj-project-memory"],
            serde_json::json!({
                "transport": "stdio",
                "command": "/var/lib/hel/workers/session/hel",
                "args": [
                    "worker",
                    "memory-mcp",
                    "--root",
                    "/var/lib/hel/profiles/session/projects/project/memory"
                ],
                "runtime_id": "local"
            })
        );
        assert_eq!(
            std::fs::read(home.path().join("mcp.json")).unwrap(),
            original_body,
            "the controller-side Kimi profile must remain unchanged"
        );
    }

    #[test]
    fn staged_kimi_project_memory_resolves_ssh_paths_from_target_home() {
        let staged = tempfile::tempdir().unwrap();
        let memory = ProjectMemoryLaunchConfig {
            project_key: "project".into(),
            root: ".local/share/hel/profiles/session/projects/project/memory".into(),
            baseline_root: PathBuf::new(),
            repository_roots: BTreeMap::new(),
            mcp_delivery: ProjectMemoryMcpDelivery::HarnessProfile,
        };

        configure_kimi_project_memory_mcp(
            staged.path(),
            ".local/share/hel/workers/session",
            &memory,
        )
        .unwrap();

        let configured: serde_json::Value =
            serde_json::from_slice(&std::fs::read(staged.path().join("mcp.json")).unwrap())
                .unwrap();
        let server = &configured["mcpServers"]["mj-project-memory"];
        assert_eq!(server["command"], "sh");
        assert_eq!(server["runtime_id"], "local");
        assert_eq!(
            server["args"],
            serde_json::json!([
                "-c",
                "exec \"$HOME/$1\" worker memory-mcp --root \"$HOME/$2\"",
                "mj-project-memory",
                ".local/share/hel/workers/session/hel",
                ".local/share/hel/profiles/session/projects/project/memory"
            ])
        );
    }
    #[test]
    fn stage_deepseek_profile_copies_only_portable_configuration() {
        let home = tempfile::tempdir().unwrap();
        std::fs::write(
            home.path().join(".credentials.yaml"),
            "version: 1\nrefs: {}\n",
        )
        .unwrap();
        std::fs::write(home.path().join("settings.yaml"), "models: {}\n").unwrap();
        std::fs::create_dir(home.path().join("sessions")).unwrap();
        std::fs::write(home.path().join("sessions/native-session"), "private state").unwrap();
        std::fs::create_dir(home.path().join("profiles")).unwrap();
        let staged = tempfile::tempdir().unwrap();
        let profile = hel::hel_config::HarnessProfile {
            kind: hel::hel_config::HarnessKind::Deepseek,
            home: home.path().to_path_buf(),
            executable: None,
            environment: BTreeMap::new(),
            context_window_bytes: None,
        };

        stage_profile(&profile, staged.path()).unwrap();

        assert!(staged.path().join(".credentials.yaml").is_file());
        assert!(staged.path().join("settings.yaml").is_file());
        assert!(!staged.path().join("sessions").exists());
        assert!(!staged.path().join("profiles").exists());
    }
    #[test]
    fn disposable_container_guidance_reaches_each_harness_without_touching_home() {
        let target = hel_targets::TargetLocator::LocalPodman {
            container_id: "container".into(),
            workspace_storage: Default::default(),
        };
        for (kind, instructions) in [
            (hel::hel_config::HarnessKind::Codex, "AGENTS.md"),
            (hel::hel_config::HarnessKind::Claude, "CLAUDE.md"),
            (hel::hel_config::HarnessKind::Kimi, "AGENTS.md"),
            (hel::hel_config::HarnessKind::Grok, "AGENTS.md"),
            (hel::hel_config::HarnessKind::Deepseek, "AGENTS.md"),
        ] {
            let home = tempfile::tempdir().unwrap();
            let original = "# Controller instructions\n\nKeep this source unchanged.\n";
            let source_instructions = home.path().join(instructions);
            std::fs::write(&source_instructions, original).unwrap();
            let staged = tempfile::tempdir().unwrap();
            let profile = hel::hel_config::HarnessProfile {
                kind,
                home: home.path().to_path_buf(),
                executable: None,
                environment: std::collections::BTreeMap::new(),
                context_window_bytes: None,
            };

            stage_profile(&profile, staged.path()).unwrap();
            append_hel_target_environment(kind, staged.path(), &target).unwrap();

            let guidance = std::fs::read_to_string(staged.path().join(instructions)).unwrap();
            assert_eq!(
                guidance,
                format!("{original}\n{MJ_CONTAINER_ENVIRONMENT}"),
                "{instructions} receives the section in the staged profile"
            );
            assert!(guidance.contains("## Mjolnir disposable environment"));
            assert!(!guidance.contains("## Hel disposable environment"));
            assert_eq!(
                std::fs::read_to_string(source_instructions).unwrap(),
                original,
                "{instructions} in the controller-side home stays untouched"
            );
        }
    }
    #[test]
    fn kimi_guidance_uses_agents_md_without_mutating_the_system_override() {
        let home = tempfile::tempdir().unwrap();
        let system_override = "# Custom Kimi system prompt\n";
        std::fs::write(home.path().join("SYSTEM.md"), system_override).unwrap();
        let staged = tempfile::tempdir().unwrap();
        let profile = hel::hel_config::HarnessProfile {
            kind: hel::hel_config::HarnessKind::Kimi,
            home: home.path().to_path_buf(),
            executable: None,
            environment: std::collections::BTreeMap::new(),
            context_window_bytes: None,
        };

        stage_profile(&profile, staged.path()).unwrap();
        append_hel_target_environment(
            profile.kind,
            staged.path(),
            &hel_targets::TargetLocator::LocalPodman {
                container_id: "container".into(),
                workspace_storage: Default::default(),
            },
        )
        .unwrap();

        assert_eq!(
            std::fs::read_to_string(staged.path().join("AGENTS.md")).unwrap(),
            MJ_CONTAINER_ENVIRONMENT
        );
        assert_eq!(
            std::fs::read_to_string(staged.path().join("SYSTEM.md")).unwrap(),
            system_override
        );
        assert!(!home.path().join("AGENTS.md").exists());
        assert_eq!(
            std::fs::read_to_string(home.path().join("SYSTEM.md")).unwrap(),
            system_override
        );
    }

    #[test]
    fn ec2_guidance_names_its_real_workspace_and_ssh_bare_gets_none() {
        let ec2 = tempfile::tempdir().unwrap();
        append_hel_target_environment(
            hel::hel_config::HarnessKind::Codex,
            ec2.path(),
            &hel_targets::TargetLocator::AwsEc2 {
                profile: "profile".into(),
                region: "region".into(),
                instance_id: "instance".into(),
                ssh: hel_targets::SshTarget {
                    destination: "host".into(),
                    ssh_args: Vec::new(),
                },
                workspace: ".local/share/hel/workspaces/session".into(),
            },
        )
        .unwrap();
        let guidance = std::fs::read_to_string(ec2.path().join("AGENTS.md")).unwrap();
        assert_eq!(
            guidance,
            "## Mjolnir disposable environment\n\nThis session runs on a disposable Mjolnir EC2 instance. When the session closes, Mjolnir checkpoints everything in project workspace directories under `$HOME/.local/share/hel/workspaces/session`, including committed work, staged and unstaged changes, and untracked files. Mjolnir then terminates the instance.\n\nEverything outside `$HOME/.local/share/hel/workspaces/session`, including installed packages, the rest of `$HOME`, and `/tmp`, is ephemeral and will be lost. Keep durable results in the workspace or push them to a remote.\n"
        );
        assert!(!guidance.contains("## Hel disposable environment"));

        let ssh_bare = tempfile::tempdir().unwrap();
        append_hel_target_environment(
            hel::hel_config::HarnessKind::Codex,
            ssh_bare.path(),
            &hel_targets::TargetLocator::SshBare {
                ssh: hel_targets::SshTarget {
                    destination: "host".into(),
                    ssh_args: Vec::new(),
                },
                workspace: ".local/share/hel/workspaces/session".into(),
            },
        )
        .unwrap();
        assert!(!ssh_bare.path().join("AGENTS.md").exists());
    }

    #[test]
    fn project_memory_replicas_are_session_private() {
        let key = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
        assert_eq!(
            project_memory_replica_slug(key, "session-a"),
            "hel-0123456789abcdef-session-a"
        );
        assert_ne!(
            project_memory_replica_slug(key, "session-a"),
            project_memory_replica_slug(key, "session-b")
        );
    }

    /// Returns a fixed digest line for every command and records what it ran,
    /// so a remote refresh can be driven without a real ssh host.
    struct DigestExecutor {
        installed_line: String,
        commands: RefCell<Vec<CommandSpec>>,
    }

    impl CommandExecutor for DigestExecutor {
        fn execute(&self, command: &CommandSpec) -> Result<CommandOutput> {
            self.commands.borrow_mut().push(command.clone());
            Ok(CommandOutput {
                status: 0,
                stdout: self.installed_line.clone().into_bytes(),
                stderr: Vec::new(),
            })
        }
    }

    // The SshBare worker_root guard requires the workspace to end in the exact
    // session ID, so build the locator around the session under test.
    fn ssh_bare_locator(session_id: &str) -> hel_targets::TargetLocator {
        hel_targets::TargetLocator::SshBare {
            ssh: SshTarget {
                destination: "user@host.test".into(),
                ssh_args: Vec::new(),
            },
            workspace: format!("/srv/mj/{session_id}"),
        }
    }

    #[test]
    fn a_remote_worker_with_a_mismatched_binary_is_replaced_before_restart() {
        let directory = tempfile::tempdir().unwrap();
        let source = directory.path().join("worker");
        std::fs::write(&source, b"fresh musl worker").unwrap();
        let executor = DigestExecutor {
            installed_line: format!("{}  /root/hel\n", "0".repeat(64)),
            commands: RefCell::new(Vec::new()),
        };
        let replaced = replace_remote_worker_binary_if_stale(
            &executor,
            &ssh_bare_locator("session-remote"),
            "session-remote",
            &CommandSpec::new("true", Vec::<String>::new()),
            &source,
        )
        .unwrap();
        assert!(replaced, "a stale remote binary must be replaced");
        assert!(
            executor.commands.borrow().len() > 1,
            "the digest probe must be followed by replacement commands"
        );
    }

    #[test]
    fn a_remote_worker_already_current_is_restarted_without_recopying() {
        let directory = tempfile::tempdir().unwrap();
        let source = directory.path().join("worker");
        std::fs::write(&source, b"fresh musl worker").unwrap();
        let current = hel::hel_worker_launch::worker_executable_digest(&source).unwrap();
        let executor = DigestExecutor {
            installed_line: format!("{current}  /root/hel\n"),
            commands: RefCell::new(Vec::new()),
        };
        let replaced = replace_remote_worker_binary_if_stale(
            &executor,
            &ssh_bare_locator("session-remote"),
            "session-remote",
            &CommandSpec::new("true", Vec::<String>::new()),
            &source,
        )
        .unwrap();
        assert!(!replaced, "a current remote binary must not be recopied");
        assert_eq!(
            executor.commands.borrow().len(),
            1,
            "only the digest probe runs when the binary is already current"
        );
    }

    #[test]
    fn a_remote_recovery_plan_defers_binary_refresh_to_the_recovery_task() {
        let locator = ssh_bare_locator("session-remote");
        let refresh = worker_binary_refresh_plan(&locator, "session-remote")
            .unwrap()
            .expect("a remote target now gets a binary refresh");
        match refresh {
            WorkerBinaryRefresh::Remote(remote) => {
                assert_eq!(remote.session_id, "session-remote");
                assert_eq!(remote.locator, locator);
            }
            WorkerBinaryRefresh::Prepared(_) => {
                panic!("a remote target must defer, not prepare, its binary refresh")
            }
        }
    }
}