brokk-mj-controller 2.23.2

//! Actionable host and configuration prerequisite checks.

use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use anyhow::Result;
use serde::Serialize;

use crate::controller::{WorkerBinaryAvailability, worker_binary_prerequisite_for_arch};
use crate::setup::{
    DiscoveredHome, discover_harness_homes_with_executor, harness_is_authenticated_with_executor,
};
use crate::targets::{
    BoundedProcessExecutor, CommandExecutor, CommandSpec, CommandTimedOut,
    ContainerTemplate as RuntimeContainerTemplate, DockerUnavailable, PODMAN_DOCUMENTATION_URL,
    PodmanPostcondition, ProcessExecutor, SshTarget as RuntimeSshTarget,
    TargetTemplate as RuntimeTargetTemplate, failed_podman_postcondition, podman_probe_observation,
    run_setup_smoke_test, ssh_command, ssh_connectivity_probe, ssh_validation_command,
    verify_local_docker, verify_local_podman, verify_ssh_docker, verify_ssh_podman,
};
use mj_core::config::{
    Config, ContainerTemplate, HarnessKind, HarnessProfile, TargetTemplate, config_path,
};
use mj_core::credentials::login_command;

// Only the image for the Apple container smoke test when the config has no
// apple-container target. This intentionally stays a small stock image rather
// than setup::DEFAULT_IMAGE: the check just proves the runtime can start a
// container, and pulling the multi-gigabyte agent-dev image to do that would be
// a poor trade.
const DEFAULT_CONTAINER_IMAGE: &str = "ubuntu:24.04";
const APPLE_CONTAINER_INSTALL_URL: &str = "https://github.com/apple/container#initial-install";

/// How long a single prerequisite probe may take before doctor reports it as a
/// fixable check instead of waiting for it.
///
/// Every probe outside the opt-in smoke tests is a local or short network call,
/// so this only ever fires for a wedged runtime socket, a blackholed network,
/// or a credential helper waiting on something that will never arrive.
pub const PROBE_TIMEOUT: Duration = Duration::from_secs(15);

/// The executor `mj doctor` and `mj setup` run their prerequisite probes
/// through: one deadline per probe, so a wedged runtime cannot hang the run.
pub const fn probe_executor() -> BoundedProcessExecutor {
    BoundedProcessExecutor::new(PROBE_TIMEOUT)
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum CheckStatus {
    Ready,
    Warning,
    Fixable,
    Unsupported,
}

impl CheckStatus {
    pub const fn label(self) -> &'static str {
        match self {
            Self::Ready => "ready",
            Self::Warning => "warning",
            Self::Fixable => "fixable",
            Self::Unsupported => "unsupported",
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DoctorCheck {
    pub id: String,
    pub title: String,
    pub status: CheckStatus,
    pub detail: String,
    pub remediation: Option<String>,
}

impl DoctorCheck {
    fn ready(id: impl Into<String>, title: impl Into<String>, detail: impl Into<String>) -> Self {
        Self {
            id: id.into(),
            title: title.into(),
            status: CheckStatus::Ready,
            detail: detail.into(),
            remediation: None,
        }
    }

    fn warning(
        id: impl Into<String>,
        title: impl Into<String>,
        detail: impl Into<String>,
        remediation: impl Into<String>,
    ) -> Self {
        Self {
            id: id.into(),
            title: title.into(),
            status: CheckStatus::Warning,
            detail: detail.into(),
            remediation: Some(remediation.into()),
        }
    }

    pub(crate) fn fixable(
        id: impl Into<String>,
        title: impl Into<String>,
        detail: impl Into<String>,
        remediation: impl Into<String>,
    ) -> Self {
        Self {
            id: id.into(),
            title: title.into(),
            status: CheckStatus::Fixable,
            detail: detail.into(),
            remediation: Some(remediation.into()),
        }
    }

    fn unsupported(
        id: impl Into<String>,
        title: impl Into<String>,
        detail: impl Into<String>,
    ) -> Self {
        Self {
            id: id.into(),
            title: title.into(),
            status: CheckStatus::Unsupported,
            detail: detail.into(),
            remediation: None,
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DoctorOptions {
    pub smoke: bool,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ApplePlatform {
    Linux,
    Macos {
        architecture: String,
        major_version: u32,
    },
    Other(String),
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum InstructionsPlatform {
    Linux,
    Macos,
}

pub fn run_current(options: DoctorOptions) -> Vec<DoctorCheck> {
    if options.smoke {
        // A smoke test may legitimately pull a multi-gigabyte image, which no
        // probe deadline could tell apart from a hung runtime, so an opt-in
        // `--smoke` run keeps waiting for its commands.
        return run_with(
            &ProcessExecutor,
            current_apple_platform(&ProcessExecutor),
            options,
        );
    }
    let executor = probe_executor();
    run_with(&executor, current_apple_platform(&executor), options)
}

pub fn run_with(
    executor: &impl CommandExecutor,
    apple_platform: ApplePlatform,
    options: DoctorOptions,
) -> Vec<DoctorCheck> {
    run_with_config_path(&config_path(), executor, apple_platform, options)
}

/// The same checks as [`run_with`], against an explicit configuration file.
///
/// `mj setup` uses this to report on the configuration it just wrote, so a
/// first run ends with exactly the summary and remediations `mj doctor`
/// would print.
pub fn run_with_config_path(
    config_path: &Path,
    executor: &impl CommandExecutor,
    apple_platform: ApplePlatform,
    options: DoctorOptions,
) -> Vec<DoctorCheck> {
    let (loaded, mut checks) = configuration_checks(config_path);
    let config: ConfigStatus<'_> = loaded.as_ref().map_err(|gap| *gap);
    checks.push(harness_discovery_check(config, executor));
    checks.extend(harness_checks(config, executor));
    checks.extend(subagent_eligibility_checks(config));
    let podman = podman_checks(config, executor, options.smoke, &apple_platform);
    let docker = docker_checks(config, executor, options.smoke);
    let apple_container = apple_container_check(
        &apple_platform,
        executor,
        options.smoke,
        apple_container_image(config),
    );
    let offered = config.map(|config| {
        offered_targets(
            config,
            podman.iter().chain(&docker).chain([&apple_container]),
        )
    });
    let offered = offered.as_ref().map_err(|gap| *gap);
    checks.extend(podman);
    checks.extend(docker);
    checks.extend(ssh_bare_checks(config, executor));
    checks.extend(ssh_podman_checks(config, executor, options.smoke));
    checks.extend(ssh_docker_checks(config, executor, options.smoke));
    checks.extend(build_cache_checks(offered, executor));
    checks.extend(aws_checks(config, executor));
    checks.extend(worker_binary_checks(offered));
    checks.push(daemon_build_check());
    checks.extend(worker_freshness_checks(offered));
    checks.extend(review_residue_checks(config));
    // Reported last, where it has always been.
    checks.push(apple_container);
    checks
}

fn build_cache_checks(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
) -> Vec<DoctorCheck> {
    let Ok(config) = config else {
        return Vec::new();
    };
    crate::controller::doctor_host_mbx(config, executor)
        .into_iter()
        .map(|host| {
            let id = format!("build-cache.{}", host.host);
            let title = format!("Build cache on {}", host.host);
            let targets = host.targets.join(", ");
            match host.status {
                crate::controller::DoctorHostMbxStatus::Absent => DoctorCheck::ready(
                    id,
                    title,
                    format!(
                        "No native mbx is installed; targets {targets} can use Mjolnir's mbx {}.",
                        crate::controller::MBX_VERSION
                    ),
                ),
                crate::controller::DoctorHostMbxStatus::Compatible(version) => DoctorCheck::ready(
                    id,
                    title,
                    format!(
                        "Host mbx {version} is compatible with Mjolnir's mbx {} for targets {targets}.",
                        crate::controller::MBX_VERSION
                    ),
                ),
                crate::controller::DoctorHostMbxStatus::TooOld(version) => DoctorCheck::warning(
                    id,
                    title,
                    format!(
                        "Host mbx {version} is older than Mjolnir's mbx {}; sessions on targets {targets} run without the shared build cache.",
                        crate::controller::MBX_VERSION
                    ),
                    format!(
                        "Upgrade mbx on {} to {} or newer, then rerun `mj doctor`.",
                        host.host,
                        crate::controller::MBX_VERSION
                    ),
                ),
                crate::controller::DoctorHostMbxStatus::Unknown(error) => DoctorCheck::warning(
                    id,
                    title,
                    format!("Could not check host mbx for targets {targets}: {error}"),
                    format!(
                        "Check access to {} and run `mbx --version` there, then rerun `mj doctor`.",
                        host.host
                    ),
                ),
            }
        })
        .collect()
}

fn harness_discovery_check(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
) -> DoctorCheck {
    let home = dirs::home_dir();
    let overrides = HarnessKind::ALL.into_iter().filter_map(|kind| {
        std::env::var_os(kind.home_env()).map(|path| (kind, kind.home_from_environment(path)))
    });
    let discovered = discover_harness_homes_with_executor(home.as_deref(), overrides, executor);
    harness_discovery_check_from(
        &discovered,
        config.is_ok_and(|config| !config.profiles.is_empty()),
        &settings_key(config.ok()),
    )
}

fn harness_discovery_check_from(
    discovered: &[DiscoveredHome],
    has_configured_profiles: bool,
    settings_key: &str,
) -> DoctorCheck {
    if discovered.is_empty() {
        return if has_configured_profiles {
            DoctorCheck::ready(
                "harness.discovery",
                "Harness home discovery",
                "No default or environment-overridden harness homes were found; configured profile homes are checked below.",
            )
        } else {
            DoctorCheck::fixable(
                "harness.discovery",
                "Harness home discovery",
                format!(
                    "No {} home was found in the default or environment-overridden locations.",
                    HarnessKind::every_display_name_or()
                ),
                format!(
                    "Install and sign in to a supported harness, then open Mjolnir, press {settings_key} for Settings, and choose Agent Profiles."
                ),
            )
        };
    }

    let homes = discovered
        .iter()
        .map(|home| {
            let authentication = if home.authenticated {
                "authenticated"
            } else {
                "not authenticated"
            };
            format!(
                "{} at {} ({authentication})",
                home.kind.display_name(),
                home.path.display()
            )
        })
        .collect::<Vec<_>>()
        .join("; ");
    DoctorCheck::ready(
        "harness.discovery",
        "Harness home discovery",
        format!("Discovered {homes}. Configured profile authentication is checked below."),
    )
}

/// The key that opens Settings, as the help overlay labels it (`ctrl+b s`
/// by default). Without a readable configuration the default bindings apply.
fn settings_key(config: Option<&Config>) -> String {
    let keybinds = config.map_or_else(mj_core::config::Keybinds::default, Config::keybinds);
    keybinds
        .labels(mj_core::config::KeyAction::OpenSettings)
        .into_iter()
        .next()
        .unwrap_or_else(|| "the Settings command in the command palette".to_owned())
}

pub fn all_ready(checks: &[DoctorCheck]) -> bool {
    checks
        .iter()
        .all(|check| check.status != CheckStatus::Fixable)
}

pub fn render_human(checks: &[DoctorCheck], output: &mut impl Write) -> Result<()> {
    for check in checks {
        writeln!(
            output,
            "{} {}: {}",
            check.status.label(),
            check.title,
            check.detail
        )?;
        if let Some(remediation) = &check.remediation {
            writeln!(output, "  remediation: {remediation}")?;
        }
    }
    Ok(())
}

pub fn setup_instructions(platform: InstructionsPlatform) -> String {
    match platform {
        InstructionsPlatform::Linux => format!(
            "# Mjolnir setup instructions for Linux\n\n\
This page is self-contained. Follow this exact loop as the user who will run `mj`:\n\n\
1. Run `mj doctor --json`.\n\
2. Follow every `fixable` remediation from its JSON output.\n\
3. Run `mj doctor --json` again. Repeat until no check is `fixable`.\n\
4. Finish with `mj doctor --json --smoke` to verify every configured container\n\
   image end to end, and resolve anything it reports as `fixable`.\n\n\
For a coding-agent handoff, provide this entire instructions page together with\n\
the latest `mj doctor --json` output.\n\n\
## Local bare runtime\n\n\
A local bare runtime runs the agent directly on this machine. It needs the\n\
native `mj-worker` installed beside `mj`; the release installer and the npm\n\
package include it. The worker installs the pinned harness version itself.\n\
Codex and Claude need Node.js 22 or newer and npm on `PATH`. Kimi and Grok\n\
need curl and Bash. Muse needs curl and tar. Mjolnir does not install these\n\
prerequisites.\n\n\
## Linux container-runtime postconditions\n\n{}\n\n{}",
            crate::targets::PODMAN_DOCUMENTATION,
            crate::targets::DOCKER_DOCUMENTATION
        ),
        InstructionsPlatform::Macos => format!(
            "# Mjolnir setup instructions for macOS\n\n\
Run these commands as the user who will run `mj`:\n\n\
1. Install and sign in to a supported coding agent, then run `mj setup`.\n\
   Setup discovers agent homes and local runtimes, offers AWS when the AWS CLI\n\
   has credentials, and offers SSH hosts from `~/.ssh/config`. Review its\n\
   proposed configuration and confirm writing `config.toml`. It runs smoke\n\
   tests for newly added container targets and ends with a doctor report.\n\
2. Run `mj doctor --json` and follow every `fixable` remediation. Repeat until\n\
   no check is `fixable`. Review warnings for any target you intend to use;\n\
   `unsupported` runtimes are unavailable, not ready.\n\
3. If you intend to use containers, start the runtime and run\n\
   `mj doctor --json --smoke`. Do this for Docker Desktop as well as Apple\n\
   container. Resolve failures for the target you intend to use.\n\
4. Run `mj`, then press n in the Sessions pane to start your first session.\n\
   If no profile was discovered, open Settings (Ctrl-B then s by default)\n\
   to add one first.\n\n\
You can also configure profiles and targets in Settings instead of `mj setup`.\n\
For a named instance, use the same `--instance <name>` on every command above.\n\n\
For a coding-agent handoff, provide this entire instructions page together with\n\
the latest `mj doctor --json` output.\n\n\
## Local bare runtime\n\n\
A local bare runtime runs the agent directly on this machine. It needs the\n\
native `mj-worker` installed beside `mj`; the release installer, npm package,\n\
and Homebrew formula include it (Homebrew keeps both binaries in libexec).\n\
The worker installs the pinned harness version itself. Codex and Claude need\n\
Node.js 22 or newer and npm on `PATH`. Kimi and Grok need curl and Bash.\n\
Muse needs curl and tar. Mjolnir does not install these prerequisites.\n\n\
## Docker Desktop\n\n\
Start Docker Desktop and wait for its Linux daemon to be ready before setup\n\
or the smoke test. `mj doctor --json --smoke` checks the container image and\n\
the OverlayFS copy-on-write attachment, including disposable container cleanup.\n\
A connected daemon alone does not establish that these operations work.\n\n\
## Apple container runtime\n\n\
Mjolnir's Apple container target requires Apple silicon and macOS 26 or newer.\n\
On an Intel Mac or an older macOS release, the target is unsupported; use the\n\
local bare runtime, an SSH target, or an AWS target instead.\n\n\
On a supported Mac, if `container` is absent, install only the official signed package:\n\n\
<https://github.com/apple/container#initial-install>\n\n\
Mjolnir never downloads or installs that package. If doctor reports a stopped\n\
daemon, run exactly:\n\n```console\ncontainer system start\n```\n\n\
Finish with the opt-in disposable runtime test in JSON mode:\n\n```console\nmj doctor --json --smoke\n```\n\n\
Apple container is ready only when that smoke test creates a disposable\n\
container, executes `true` in it, and removes it successfully. Use the image\n\
configured by an `apple-container` target; without one, doctor uses\n\
`{DEFAULT_CONTAINER_IMAGE}` for the smoke test.\n\n\
## Shared Mjolnir prerequisites\n\n\
`mj doctor --json` also checks the configuration, each configured harness home\n\
and authentication marker, selected container worker binaries, and any relevant\n\
Podman prerequisites. Resolve every `fixable` status before starting a session.\n"
        ),
    }
}

/// Why `mj doctor` has no configuration for the checks that need one.
///
/// The cases call for different advice, so every dependent check is told
/// which one it is: a file this build cannot read because a newer Mjolnir
/// wrote it is not broken, and telling the user to fix or replace it would
/// destroy that build's settings; a file that does not exist yet has nothing
/// to fix (R14-3).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ConfigGap {
    /// A newer Mjolnir wrote the file; the value is the version it carries.
    NewerVersion(u32),
    /// There is no file yet. The first `mj` that adds an agent profile, or
    /// `mj setup`, writes it.
    Missing,
    /// The file is not valid Mjolnir TOML.
    Unreadable,
}

impl ConfigGap {
    /// What a dependent check waits for, completing "cannot be checked
    /// until ...".
    fn awaited(self) -> &'static str {
        match self {
            ConfigGap::NewerVersion(_) => "this Mjolnir can read config.toml",
            ConfigGap::Missing => "config.toml exists",
            ConfigGap::Unreadable => "config.toml is valid",
        }
    }

    /// The fix for a check that waits for a configuration it can read.
    ///
    /// It names plain `mj doctor`: the checks are the same in the human and
    /// the JSON report, and a person reading the human one has no use for
    /// `--json` (R14-3).
    fn remediation(self) -> &'static str {
        match self {
            ConfigGap::NewerVersion(_) => {
                "Update Mjolnir to the build that wrote config.toml or newer, then rerun `mj doctor`."
            }
            ConfigGap::Missing => {
                "Run `mj` and add an agent profile, or run `mj setup`; either one writes config.toml. Then rerun `mj doctor`."
            }
            ConfigGap::Unreadable => "Fix config.toml, then rerun `mj doctor`.",
        }
    }
}

/// What a dependent check works from: the loaded configuration, or why there
/// is none.
type ConfigStatus<'a> = std::result::Result<&'a Config, ConfigGap>;

/// What a check reports when the configuration came from a newer Mjolnir.
///
/// Nothing about the check can be evaluated, and nothing the user does to
/// `config.toml` would help, so the check skips and names the one real fix.
fn newer_config_skip(id: &str, title: &str, version: u32) -> DoctorCheck {
    DoctorCheck::unsupported(
        id,
        title,
        format!(
            "Skipped: config.toml was written by a newer Mjolnir (config version {version}; this build supports {}). Update Mjolnir to that build or newer.",
            mj_core::config::CONFIG_VERSION
        ),
    )
}

fn configuration_checks(path: &Path) -> (std::result::Result<Config, ConfigGap>, Vec<DoctorCheck>) {
    if !path.exists() {
        return (
            Err(ConfigGap::Missing),
            vec![DoctorCheck::fixable(
                "config",
                "Mjolnir configuration",
                format!("{} does not exist", path.display()),
                format!(
                    "Open Mjolnir and press {} for Settings to add an agent profile.",
                    settings_key(None)
                ),
            )],
        );
    }
    // A config a newer build wrote is not broken TOML: replacing it with
    // `mj setup` would discard that build's settings. Say what is actually
    // wrong before the load below reports it as invalid.
    if let Some(found) = mj_core::config::newer_version_on_disk(path) {
        return (
            Err(ConfigGap::NewerVersion(found)),
            vec![DoctorCheck::fixable(
                "config",
                "Mjolnir configuration",
                format!(
                    "{} was written by a newer Mjolnir (config version {found}; this build supports {})",
                    path.display(),
                    mj_core::config::CONFIG_VERSION
                ),
                "Update Mjolnir to that build or newer. Do not lower the version value by hand or replace the file.",
            )],
        );
    }
    match Config::load_from(path) {
        Ok(config) => {
            let mut checks = vec![DoctorCheck::ready(
                "config",
                "Mjolnir configuration",
                format!("{} is valid", path.display()),
            )];
            // A bundle only names a set of repositories to start from; a
            // session can start from any project directory without one, so
            // only the missing profile keeps sessions from starting.
            if config.enabled_profiles().next().is_none() {
                checks.push(DoctorCheck::fixable(
                    "config.session-prerequisites",
                    "Session configuration",
                    "No agent profile is enabled, so no session can start. Local targets are supplied automatically.",
                    format!(
                        "Open Mjolnir and press {} for Settings to add or enable an agent profile.",
                        settings_key(Some(&config))
                    ),
                ));
            } else if config.bundles.is_empty() {
                checks.push(DoctorCheck::ready(
                    "config.session-prerequisites",
                    "Session configuration",
                    "An agent profile is enabled. No project bundle is configured; sessions start from a project directory, and a bundle is only needed to start from a saved set of repositories.",
                ));
            } else {
                checks.push(DoctorCheck::ready(
                    "config.session-prerequisites",
                    "Session configuration",
                    "At least one profile, bundle, and target are configured.",
                ));
            }
            (Ok(config), checks)
        }
        Err(error) => (
            Err(ConfigGap::Unreadable),
            vec![DoctorCheck::fixable(
                "config",
                "Mjolnir configuration",
                format!("{} is invalid: {error:#}", path.display()),
                "Fix the reported TOML error in config.toml, or run `mj setup` to replace it.",
            )],
        ),
    }
}

fn harness_checks(config: ConfigStatus<'_>, executor: &impl CommandExecutor) -> Vec<DoctorCheck> {
    let config = match config {
        Ok(config) => config,
        Err(ConfigGap::NewerVersion(version)) => {
            return vec![newer_config_skip(
                "harness.profiles",
                "Harness profiles",
                version,
            )];
        }
        Err(gap @ (ConfigGap::Missing | ConfigGap::Unreadable)) => {
            return vec![DoctorCheck::fixable(
                "harness.profiles",
                "Harness profiles",
                format!("Harness homes cannot be checked until {}.", gap.awaited()),
                gap.remediation(),
            )];
        }
    };
    if config.profiles.is_empty() {
        return vec![DoctorCheck::fixable(
            "harness.profiles",
            "Harness profiles",
            "No harness profiles are configured.",
            format!(
                "Open Mjolnir, press {} for Settings, and choose Agent Profiles to detect accounts or add a profile.",
                settings_key(Some(config))
            ),
        )];
    }
    config
        .profiles
        .iter()
        .map(|(id, profile)| {
            let mut check = harness_profile_check(id, profile, executor);
            check.detail = format!("{} {}", profile_summary(config, id, profile), check.detail);
            check
        })
        .collect()
}

/// The readiness of one profile: its home, and whether it can authenticate.
fn harness_profile_check(
    id: &str,
    profile: &HarnessProfile,
    executor: &impl CommandExecutor,
) -> DoctorCheck {
    let title = format!("Harness profile {id}");
    if !profile.enabled {
        return DoctorCheck::ready(
            format!("harness.{id}"),
            title,
            "Profile is disabled; home and authentication checks were skipped.",
        );
    }
    if !profile.home.is_dir() {
        return DoctorCheck::fixable(
            format!("harness.{id}"),
            title,
            format!("{} does not exist", profile.home.display()),
            format!(
                "{} If this profile should use an existing installation, select its home in Setup.",
                harness_login_remediation(id, profile)
            ),
        );
    }
    if !harness_is_authenticated_with_executor(profile, executor) {
        return DoctorCheck::fixable(
            format!("harness.{id}"),
            title,
            format!(
                "No usable authentication was detected for {}",
                profile.home.display()
            ),
            harness_login_remediation(id, profile),
        );
    }
    DoctorCheck::ready(
        format!("harness.{id}"),
        title,
        format!(
            "{} is present and authentication is available",
            profile.home.display()
        ),
    )
}

/// One sentence saying what a profile is: its harness, where its quota comes
/// from, and whether other sessions' sub-agents may use it. Quota ranking and
/// delegation both depend on these, and none of them shows in the profile's
/// own table in config.toml.
fn profile_summary(config: &Config, id: &str, profile: &HarnessProfile) -> String {
    let delegation = if config
        .subagents
        .eligible_profiles
        .get(id)
        .copied()
        .unwrap_or(false)
    {
        "any session's sub-agents may use it"
    } else {
        "only its own sessions' sub-agents may use it"
    };
    format!(
        "{}; {}; {delegation}.",
        profile.kind.display_name(),
        profile_quota_source(profile)
    )
}

/// Where a profile's quota report comes from, in the terms the quota refresh
/// uses: a Codex profile is read through its custom provider only when that
/// provider's API key is in the profile's environment.
fn profile_quota_source(profile: &HarnessProfile) -> String {
    match profile.kind {
        HarnessKind::Claude => "Claude subscription quota".to_owned(),
        HarnessKind::Codex => match crate::quota::provider_credential(profile) {
            Some(provider) if crate::zai_usage::serves_quota(&provider.host) => {
                format!("quota from {}", provider.host)
            }
            Some(provider) => format!(
                "pay-per-use through {}, counted as 100% left when choosing a sub-agent's profile",
                provider.host
            ),
            None => match profile.codex_provider() {
                Ok(None) => "ChatGPT subscription quota".to_owned(),
                Ok(Some(provider)) => format!(
                    "no quota report, because custom provider {:?} has no API key in this profile's environment",
                    provider.id
                ),
                Err(error) => format!("its Codex config.toml could not be read ({error:#})"),
            },
        },
        kind => format!("quota as {} reports it", kind.display_name()),
    }
}

/// The sub-agent policy in one line, then a warning for each profile that is
/// both listed for sub-agent use and disabled.
///
/// The daemon keeps running with such a profile and simply does not offer it
/// to a parent, because the delegation candidates and the spawn gate both
/// require an enabled profile. This surfaces the contradiction so the eligible
/// list and the profile's `enabled` flag can be reconciled. An eligible id
/// that names no profile never reaches here: the configuration fails to load,
/// and the configuration check reports it.
fn subagent_eligibility_checks(config: ConfigStatus<'_>) -> Vec<DoctorCheck> {
    let Ok(config) = config else {
        return Vec::new();
    };
    let mut checks = vec![subagent_policy_check(config)];
    checks.extend(
        config
            .subagents
            .eligible_profiles
            .iter()
            .filter(|(_, eligible)| **eligible)
            .filter_map(|(id, _)| match config.profiles.get(id) {
                Some(profile) if !profile.enabled => Some(DoctorCheck::warning(
                    format!("subagents.{id}"),
                    format!("Sub-agent profile {id}"),
                    format!(
                        "Profile {id:?} is listed in [subagents.eligible_profiles] but is disabled, so it is not offered for sub-agent use."
                    ),
                    format!(
                        "Re-enable profile {id:?}, or remove it from [subagents.eligible_profiles]."
                    ),
                )),
                _ => None,
            }),
    );
    checks
}

/// How many sub-agents a session may start, and on which profiles. Whether a
/// given session uses Mjolnir sub-agents at all is a per-session choice, not
/// a global policy, so this check only describes the shared limits.
fn subagent_policy_check(config: &Config) -> DoctorCheck {
    let subagents = &config.subagents;
    let eligible = subagents
        .eligible_profiles
        .iter()
        .filter(|(_, eligible)| **eligible)
        .map(|(id, _)| id.as_str())
        .collect::<Vec<_>>();
    let others = if eligible.is_empty() {
        "no other profile".to_owned()
    } else {
        eligible.join(", ")
    };
    let detail = format!(
        "Claude and Codex sessions may opt in, up to {} sub-agents at once per session. A session's sub-agents may use its own profile and: {others}.",
        subagents.max_concurrent
    );
    DoctorCheck::ready("subagents.policy", "Sub-agent policy", detail)
}

/// Point an unauthenticated profile at `mj login`, which already knows how to
/// sign each harness in.
///
/// The underlying command is named only for the reader's benefit; it comes from
/// [`login_command`], the one place that tracks what each harness CLI actually
/// accepts, so this text cannot drift away from what `mj login` runs.
fn harness_login_remediation(id: &str, profile: &HarnessProfile) -> String {
    let (program, arguments) = match login_command(profile) {
        Ok(command) => command,
        // An API-key profile has no login to recommend; say what is missing
        // instead. The authentication gate normally passes such a profile, so
        // this text appears only when its configuration file is absent.
        Err(error) => return format!("{error} Check {}.", profile.home.display()),
    };
    format!(
        "Run `mj login --profile {id}`; it runs `{program} {}` against {}.",
        arguments.join(" "),
        profile.home.display()
    )
}

/// Host Podman prerequisites, then one image check per `local-podman` target.
///
/// The image checks run only after the host preflight passes, because a broken
/// Podman installation already reports its own actionable check.
fn podman_checks(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
    smoke: bool,
    platform: &ApplePlatform,
) -> Vec<DoctorCheck> {
    let effective = config.map(|config| config.clone().with_local_targets());
    let effective = effective.as_ref().map_err(|gap| *gap);
    let explicit = config.is_ok_and(|config| {
        local_podman_targets(config)
            .iter()
            .any(|(id, _)| config.configures_target(id))
    });
    let preflight = builtin_target_availability(
        podman_check(effective, executor, platform),
        explicit,
        "Podman",
        "podman",
    );
    let preflight_passed = preflight.status == CheckStatus::Ready;
    let mut checks = vec![preflight];
    if preflight_passed {
        checks.extend(
            podman_image_checks(effective, executor, smoke)
                .into_iter()
                .map(|(id, check)| builtin_image_check(config, &id, check)),
        );
    }
    checks
}

/// An image check for a standard local target the user never configured.
/// The dashboard downloads that image itself when it starts, so a missing
/// image is a warning rather than a fault.
fn builtin_image_check(
    config: ConfigStatus<'_>,
    target_id: &str,
    check: DoctorCheck,
) -> DoctorCheck {
    let explicit = config.is_ok_and(|config| config.configures_target(target_id));
    if explicit || check.status != CheckStatus::Fixable {
        return check;
    }
    DoctorCheck::warning(
        check.id,
        check.title,
        format!(
            "{} (built-in `{target_id}` target; the dashboard downloads its image when it starts)",
            check.detail
        ),
        check.remediation.unwrap_or_default(),
    )
}

/// Doctor checks the same target set the dashboard lists: the configured
/// targets plus the standard local ones [`Config::with_local_targets`]
/// supplies whether or not their engine is installed. A standard target whose
/// engine is missing or not running is reported as unavailable, as the
/// dashboard's Targets pane marks it, rather than as a fault to fix: nobody
/// asked for it. A target the user configured keeps the fixable result; a
/// block that only repeats a standard target does not count as configured
/// ([`Config::configures_target`]).
fn builtin_target_availability(
    check: DoctorCheck,
    explicit: bool,
    engine: &str,
    target_id: &str,
) -> DoctorCheck {
    if explicit || check.status != CheckStatus::Fixable {
        return check;
    }
    DoctorCheck::unsupported(
        check.id,
        check.title,
        format!(
            "{engine} is not available, so the built-in `{target_id}` target is marked unavailable: {}",
            check.detail
        ),
    )
}

fn podman_check(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
    platform: &ApplePlatform,
) -> DoctorCheck {
    let config = match config {
        Ok(config) => config,
        Err(ConfigGap::NewerVersion(version)) => {
            return newer_config_skip("runtime.podman", "Rootless Podman", version);
        }
        Err(gap @ (ConfigGap::Missing | ConfigGap::Unreadable)) => {
            return DoctorCheck::unsupported(
                "runtime.podman",
                "Rootless Podman",
                format!(
                    "Podman prerequisites cannot be evaluated until {}.",
                    gap.awaited()
                ),
            );
        }
    };
    if local_podman_targets(config).is_empty() {
        return DoctorCheck::unsupported(
            "runtime.podman",
            "Rootless Podman",
            "No local-podman target is configured.",
        );
    }
    local_podman_runtime_check(executor, platform)
}

/// Probe the local rootless Podman prerequisites and phrase the result as a
/// doctor check.
///
/// This is the single source of truth for Podman availability wording and
/// remediation. `mj setup` calls it directly so its runtime list reports the
/// same detail and fix that `mj doctor` would.
pub fn local_podman_runtime_check(
    executor: &impl CommandExecutor,
    platform: &ApplePlatform,
) -> DoctorCheck {
    if !matches!(platform, ApplePlatform::Linux) {
        return DoctorCheck::unsupported(
            "runtime.podman",
            "Rootless Podman",
            "Mjolnir's local Podman target requires a Linux host; Podman machine is not supported. Use localhost or an SSH target on a Linux host.",
        );
    }
    match verify_local_podman(executor) {
        Ok(preflight) => DoctorCheck::ready(
            "runtime.podman",
            "Rootless Podman",
            format!("Podman {} has a valid rootless UID map.", preflight.version),
        ),
        Err(error) => DoctorCheck::fixable(
            "runtime.podman",
            "Rootless Podman",
            podman_failure_detail(&error),
            podman_remediation(&error),
        ),
    }
}

fn local_podman_targets(config: &Config) -> Vec<(&String, &ContainerTemplate)> {
    config
        .targets
        .iter()
        .filter_map(|(id, target)| match target {
            TargetTemplate::LocalPodman { container } => Some((id, container)),
            _ => None,
        })
        .collect()
}

fn podman_image_checks(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> Vec<(String, DoctorCheck)> {
    let Ok(config) = config else {
        return Vec::new();
    };
    local_podman_targets(config)
        .into_iter()
        .map(|(id, container)| {
            (
                id.clone(),
                podman_image_check(id, &container.image, executor, smoke),
            )
        })
        .collect()
}

fn podman_image_check(
    id: &str,
    image: &str,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> DoctorCheck {
    let check_id = format!("runtime.podman.image.{id}");
    let title = format!("Podman image for target {id}");
    if smoke {
        let target = RuntimeTargetTemplate::LocalPodman(RuntimeContainerTemplate {
            build_cache: None,
            image: image.to_owned(),
            pull_policy: Default::default(),
            extra_run_args: vec![],
            workspace_storage: Default::default(),
        });
        return match run_setup_smoke_test(&target, &doctor_smoke_id(), executor) {
            Ok(()) => DoctorCheck::ready(
                check_id,
                title,
                format!("Disposable run/exec/remove smoke test passed for image {image}."),
            ),
            Err(error) => DoctorCheck::fixable(
                check_id,
                title,
                format!(
                    "Disposable run/exec/remove smoke test failed for image {image}: {error:#}"
                ),
                "Fix the configured image or Podman runtime, then run `mj doctor --json --smoke` again.",
            ),
        };
    }

    let command = CommandSpec::new("podman", ["image", "exists", image])
        .purpose("check Podman image presence");
    match executor.execute(&command) {
        Ok(output) if output.status == 0 => DoctorCheck::ready(
            check_id,
            title,
            format!("Image {image} is present in local Podman storage."),
        ),
        Ok(_) => DoctorCheck::fixable(
            check_id,
            title,
            format!("Image {image} is not present in local Podman storage."),
            missing_image_remediation(image),
        ),
        Err(error) => DoctorCheck::fixable(
            check_id,
            title,
            format!(
                "Could not check whether image {image} is present in local Podman storage: {error}"
            ),
            missing_image_remediation(image),
        ),
    }
}

fn missing_image_remediation(image: &str) -> String {
    format!(
        "Pull it with `podman pull {image}`, build it from containers/Containerfile.agent-dev, or run `mj doctor --json --smoke` to verify the full pull-and-run path."
    )
}

/// Host Docker prerequisites, then one image check per `local-docker` target.
fn docker_checks(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> Vec<DoctorCheck> {
    let config = match config {
        Ok(config) => config,
        Err(ConfigGap::NewerVersion(version)) => {
            return vec![newer_config_skip("runtime.docker", "Docker", version)];
        }
        Err(gap @ (ConfigGap::Missing | ConfigGap::Unreadable)) => {
            return vec![DoctorCheck::unsupported(
                "runtime.docker",
                "Docker",
                format!(
                    "Docker prerequisites cannot be evaluated until {}.",
                    gap.awaited()
                ),
            )];
        }
    };
    let explicit = local_docker_targets(config)
        .iter()
        .any(|(id, _)| config.configures_target(id));
    let effective = config.clone().with_local_targets();
    let targets = local_docker_targets(&effective);
    if targets.is_empty() {
        return vec![DoctorCheck::unsupported(
            "runtime.docker",
            "Docker",
            "No local-docker target is configured.",
        )];
    }
    let preflight = builtin_target_availability(
        local_docker_runtime_check(executor),
        explicit,
        "Docker",
        "docker",
    );
    if preflight.status != CheckStatus::Ready {
        return vec![preflight];
    }
    let mut checks = vec![preflight];
    checks.extend(targets.into_iter().map(|(id, container)| {
        builtin_image_check(
            Ok(config),
            id,
            docker_image_check(id, &container.image, executor, smoke),
        )
    }));
    checks
}

pub fn local_docker_runtime_check(executor: &impl CommandExecutor) -> DoctorCheck {
    match verify_local_docker(executor) {
        Ok(preflight) => DoctorCheck::ready(
            "runtime.docker",
            "Docker",
            format!(
                "Docker {} is connected to a Linux daemon.",
                preflight.version
            ),
        ),
        Err(error) => match error.downcast_ref::<DockerUnavailable>() {
            Some(problem) => DoctorCheck::fixable(
                "runtime.docker",
                "Docker",
                problem.to_string(),
                problem.remediation(),
            ),
            None => DoctorCheck::fixable(
                "runtime.docker",
                "Docker",
                format!("{error:#}"),
                "Install and start Docker, then make sure `docker info` succeeds as the user running mj.",
            ),
        },
    }
}

fn local_docker_targets(config: &Config) -> Vec<(&String, &ContainerTemplate)> {
    config
        .targets
        .iter()
        .filter_map(|(id, target)| match target {
            TargetTemplate::LocalDocker { container } => Some((id, container)),
            _ => None,
        })
        .collect()
}

fn docker_image_check(
    id: &str,
    image: &str,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> DoctorCheck {
    let check_id = format!("runtime.docker.image.{id}");
    let title = format!("Docker image for target {id}");
    if smoke {
        let target = RuntimeTargetTemplate::LocalDocker(RuntimeContainerTemplate {
            build_cache: None,
            image: image.to_owned(),
            pull_policy: Default::default(),
            extra_run_args: vec![],
            workspace_storage: Default::default(),
        });
        return match run_setup_smoke_test(&target, &doctor_smoke_id(), executor) {
            Ok(()) => DoctorCheck::ready(
                check_id,
                title,
                format!(
                    "Disposable run/exec/remove and OverlayFS attachment smoke test passed for image {image}."
                ),
            ),
            Err(error) => DoctorCheck::fixable(
                check_id,
                title,
                format!(
                    "Disposable run/exec/remove smoke test failed for image {image}: {error:#}"
                ),
                "Fix the configured image or Docker runtime, then run `mj doctor --json --smoke` again.",
            ),
        };
    }
    let command = CommandSpec::new("docker", ["image", "inspect", image])
        .purpose("check Docker image presence");
    match executor.execute(&command) {
        Ok(output) if output.status == 0 => DoctorCheck::ready(
            check_id,
            title,
            format!("Image {image} is present in Docker storage."),
        ),
        Ok(_) => DoctorCheck::fixable(
            check_id,
            title,
            format!("Image {image} is not present in Docker storage."),
            format!("Pull it with `docker pull {image}`, or run `mj doctor --json --smoke`."),
        ),
        Err(error) => DoctorCheck::fixable(
            check_id,
            title,
            format!("Could not inspect Docker image {image}: {error}"),
            format!("Make sure `docker info` succeeds, then run `docker pull {image}`."),
        ),
    }
}

/// The outcome of the shared SSH connectivity probe.
///
/// Both SSH-backed checks run this first: an unreachable host makes every
/// later probe fail with a misleading message.
enum SshConnectivity {
    Reachable,
    Failed { detail: String, remediation: String },
}

/// Probe `ssh <destination> true` and map any failure to a copy-paste fix.
///
/// Hel never generates keys, runs `ssh-copy-id`, or accepts a host key on the
/// user's behalf; it only says exactly which command would fix the failure.
fn ssh_connectivity(ssh: &RuntimeSshTarget, executor: &impl CommandExecutor) -> SshConnectivity {
    let destination = &ssh.destination;
    let command = ssh_connectivity_probe(ssh);
    match executor.execute(&command) {
        Err(error) => SshConnectivity::Failed {
            detail: format!("Could not run `ssh {destination} true`: {error:#}"),
            remediation: ssh_launch_failure_remediation(&error, ssh),
        },
        Ok(output) if output.status != 0 => {
            let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
            SshConnectivity::Failed {
                detail: format!("`ssh {destination} true` failed: {stderr}"),
                remediation: ssh_failure_remediation(&stderr, ssh),
            }
        }
        Ok(_) => SshConnectivity::Reachable,
    }
}

const SSH_MISSING_REMEDIATION: &str = "Install an OpenSSH client and put `ssh` on PATH: `sudo apt update && sudo apt install -y openssh-client` (Debian/Ubuntu) or `sudo dnf install -y openssh-clients` (Fedora).";

/// What OpenSSH reported, as far as doctor needs to tell the cases apart.
///
/// OpenSSH is an external tool, so its wording is the only signal available.
/// This is the one place in doctor that reads it; everything downstream works
/// from the classification rather than the text.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SshFailure {
    UntrustedHostKey,
    Unauthenticated,
    ClientMissing,
    /// The host answered nothing at all.
    Unreachable,
    Unrecognized,
}

fn classify_ssh_stderr(stderr: &str) -> SshFailure {
    const UNTRUSTED_HOST_KEY: [&str; 3] = [
        "Host key verification failed",
        "No ECDSA host key is known",
        "REMOTE HOST IDENTIFICATION HAS CHANGED",
    ];
    const UNAUTHENTICATED: [&str; 4] = [
        "Permission denied",
        "Too many authentication failures",
        "no matching host key",
        "Authentication failed",
    ];
    const CLIENT_MISSING: [&str; 2] = ["ssh: command not found", "No such file or directory"];
    const UNREACHABLE: [&str; 3] = [
        "Connection timed out",
        "No route to host",
        "Network is unreachable",
    ];

    let reported = |signatures: &[&str]| signatures.iter().any(|text| stderr.contains(text));
    if reported(&UNTRUSTED_HOST_KEY) {
        SshFailure::UntrustedHostKey
    } else if reported(&UNAUTHENTICATED) {
        SshFailure::Unauthenticated
    } else if reported(&CLIENT_MISSING) {
        SshFailure::ClientMissing
    } else if reported(&UNREACHABLE) {
        SshFailure::Unreachable
    } else {
        SshFailure::Unrecognized
    }
}

/// Map a failure to run `ssh` at all (as opposed to `ssh` exiting nonzero)
/// to the command that fixes it.
fn ssh_launch_failure_remediation(error: &anyhow::Error, ssh: &RuntimeSshTarget) -> String {
    if error.downcast_ref::<CommandTimedOut>().is_some() {
        return ssh_unreachable_remediation(ssh);
    }
    let missing_binary = error.chain().any(|cause| {
        cause
            .downcast_ref::<std::io::Error>()
            .is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound)
    });
    if missing_binary {
        return SSH_MISSING_REMEDIATION.to_owned();
    }
    format!(
        "Run `ssh {} true` by hand and resolve the error it reports: {error:#}",
        ssh.destination
    )
}

/// The host answered nothing: it is asleep, behind a down VPN, or the cloud
/// session that exposes it has expired.
fn ssh_unreachable_remediation(ssh: &RuntimeSshTarget) -> String {
    let host = ssh_host_only(&ssh.destination);
    format!(
        "Check that {host} is up and reachable from this machine: wake it, bring up the VPN, or refresh the cloud session that exposes it, then run `ssh {} true` by hand.",
        ssh.destination
    )
}

/// Map `ssh -o BatchMode=yes` stderr to the command that fixes it.
fn ssh_failure_remediation(stderr: &str, ssh: &RuntimeSshTarget) -> String {
    let destination = &ssh.destination;
    match classify_ssh_stderr(stderr) {
        SshFailure::UntrustedHostKey => {
            let host = ssh_host_only(destination);
            format!(
                "Add the host key with `ssh-keyscan -H {host} >> ~/.ssh/known_hosts`. Verify the fingerprint out of band before trusting it; if the key changed, remove the stale entry with `ssh-keygen -R {host}` first."
            )
        }
        SshFailure::Unauthenticated => match ssh_identity_file(ssh) {
            Some(identity) => format!(
                "Install your public key on the host with `ssh-copy-id -i {identity}.pub {destination}`."
            ),
            None => {
                format!("Install your public key on the host with `ssh-copy-id {destination}`.")
            }
        },
        SshFailure::ClientMissing => SSH_MISSING_REMEDIATION.to_owned(),
        SshFailure::Unreachable => ssh_unreachable_remediation(ssh),
        SshFailure::Unrecognized => {
            format!(
                "Run `ssh {destination} true` by hand and resolve the error it reports: {stderr}"
            )
        }
    }
}

/// The host part of an OpenSSH destination, without any `user@` prefix.
fn ssh_host_only(destination: &str) -> &str {
    destination
        .rsplit_once('@')
        .map_or(destination, |(_, host)| host)
}

/// The identity file provisioning passes, recovered from the built ssh args.
fn ssh_identity_file(ssh: &RuntimeSshTarget) -> Option<&str> {
    let position = ssh.ssh_args.iter().position(|arg| arg == "-i")?;
    ssh.ssh_args.get(position + 1).map(String::as_str)
}

/// One check per `ssh-bare` target: can Hel reach the host noninteractively?
fn ssh_bare_checks(config: ConfigStatus<'_>, executor: &impl CommandExecutor) -> Vec<DoctorCheck> {
    let Ok(config) = config else {
        return Vec::new();
    };
    config
        .targets
        .iter()
        .filter_map(|(id, target)| match target {
            TargetTemplate::SshBare { ssh, .. } => {
                Some(ssh_bare_check(id, &RuntimeSshTarget::from(ssh), executor))
            }
            _ => None,
        })
        .collect()
}

fn ssh_bare_check(
    id: &str,
    ssh: &RuntimeSshTarget,
    executor: &impl CommandExecutor,
) -> DoctorCheck {
    let check_id = format!("runtime.ssh-bare.{id}");
    let title = format!("SSH access for target {id}");
    match ssh_connectivity(ssh, executor) {
        SshConnectivity::Reachable => DoctorCheck::ready(
            check_id,
            title,
            format!(
                "`ssh {} true` succeeds noninteractively from this host.",
                ssh.destination
            ),
        ),
        SshConnectivity::Failed {
            detail,
            remediation,
        } => DoctorCheck::fixable(check_id, title, detail, remediation),
    }
}

/// Two checks per `ssh-podman` target: the same Podman probes run over SSH,
/// then the host limits that only bite under provisioning load.
fn ssh_podman_checks(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> Vec<DoctorCheck> {
    let Ok(config) = config else {
        return Vec::new();
    };
    config
        .targets
        .iter()
        .flat_map(|(id, target)| match target {
            TargetTemplate::SshPodman { ssh, container, .. } => {
                let ssh = RuntimeSshTarget::from(ssh);
                let (check, reachable) =
                    ssh_podman_check(id, &ssh, &container.image, executor, smoke);
                let mut checks = vec![check];
                // An unreachable host has one problem, not two.
                if reachable {
                    checks.push(ssh_podman_limits_check(id, &ssh, executor));
                }
                checks
            }
            _ => Vec::new(),
        })
        .collect()
}

/// The Podman check for one target, paired with whether the host answered SSH
/// at all: the caller skips its follow-up probes when it did not.
fn ssh_podman_check(
    id: &str,
    ssh: &RuntimeSshTarget,
    image: &str,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> (DoctorCheck, bool) {
    let check_id = format!("runtime.ssh-podman.{id}");
    let title = format!("Remote Podman for target {id}");
    // Connectivity first: a remote Podman probe on an unreachable host reports
    // a Podman problem the user does not have.
    if let SshConnectivity::Failed {
        detail,
        remediation,
    } = ssh_connectivity(ssh, executor)
    {
        return (
            DoctorCheck::fixable(check_id, title, detail, remediation),
            false,
        );
    }
    (
        ssh_podman_runtime_check(check_id, title, ssh, image, executor, smoke),
        true,
    )
}

/// The Podman half of the target's checks, on a host already known reachable.
fn ssh_podman_runtime_check(
    check_id: String,
    title: String,
    ssh: &RuntimeSshTarget,
    image: &str,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> DoctorCheck {
    let destination = &ssh.destination;
    let preflight = match verify_ssh_podman(ssh, executor) {
        Ok(preflight) => preflight,
        Err(error) => {
            let detail = podman_failure_detail(&error);
            let remediation = match podman_remediation_match(&error) {
                Some(remediation) => {
                    format!("On {destination}: {remediation} See {PODMAN_DOCUMENTATION_URL}.")
                }
                None => format!(
                    "Verify `ssh {destination}` succeeds noninteractively from this host, then install rootless Podman 4.3 or newer there. See {PODMAN_DOCUMENTATION_URL}."
                ),
            };
            return DoctorCheck::fixable(check_id, title, detail, remediation);
        }
    };
    let linger_warning = preflight.warnings.first();
    if !smoke && let Some(warning) = linger_warning {
        return DoctorCheck::warning(
            check_id,
            title,
            format!(
                "Remote rootless Podman {} is available via {destination}, but {}",
                preflight.version, warning.detail
            ),
            &warning.remediation,
        );
    }
    if !smoke {
        return DoctorCheck::ready(
            check_id,
            title,
            format!(
                "Remote rootless Podman {} is available via {destination}. Run `mj doctor --json --smoke` to verify the image end to end.",
                preflight.version
            ),
        );
    }

    let target = RuntimeTargetTemplate::SshPodman {
        ssh: ssh.clone(),
        container: RuntimeContainerTemplate {
            build_cache: None,
            image: image.to_owned(),
            pull_policy: Default::default(),
            extra_run_args: vec![],
            workspace_storage: Default::default(),
        },
    };
    match run_setup_smoke_test(&target, &doctor_smoke_id(), executor) {
        Ok(()) => match linger_warning {
            Some(warning) => DoctorCheck::warning(
                check_id,
                title,
                format!(
                    "Disposable run/exec/remove smoke test passed for image {image} on {destination}, but {}",
                    warning.detail
                ),
                &warning.remediation,
            ),
            None => DoctorCheck::ready(
                check_id,
                title,
                format!(
                    "Disposable run/exec/remove smoke test passed for image {image} on {destination}."
                ),
            ),
        },
        Err(error) => DoctorCheck::fixable(
            check_id,
            title,
            format!(
                "Disposable run/exec/remove smoke test failed for image {image} on {destination}: {error:#}"
            ),
            format!(
                "Fix the configured image or Podman runtime on {destination}, then run `mj doctor --json --smoke` again."
            ),
        ),
    }
}

/// Host limits that cause provisioning failures under load, read on their own SSH
/// round trip so the provisioning preflight never pays for them.
///
/// Every crun container takes a session keyring, so `podman run` fails with
/// `crun: create keyring` once the login user's keyring quota is exhausted, and
/// sshd refuses new connections past `MaxStartups`. `sshd -T` needs root, so the
/// directive is read from the config files instead; drop-ins may be unreadable,
/// which the script reports rather than guessing.
const SSH_PODMAN_HOST_LIMITS_SCRIPT: &str = r#"
if [ -r /proc/sys/kernel/keys/maxkeys ]; then
    printf 'keys.max=%s\n' "$(cat /proc/sys/kernel/keys/maxkeys)"
fi
if [ -r /proc/key-users ]; then
    awk -v uid="$(id -u)" '
        { user = $1; sub(/:$/, "", user) }
        user == uid {
            split($4, quota, "/")
            printf "keys.used=%s\nkeys.quota=%s\n", quota[1], quota[2]
        }
    ' /proc/key-users
fi
unreadable=0
maxstartups=
# A drop-in directory that cannot be listed hides any override it holds.
if [ -d /etc/ssh/sshd_config.d ] && ! [ -r /etc/ssh/sshd_config.d ]; then
    unreadable=1
fi
for file in /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf; do
    [ -e "$file" ] || continue
    if [ -r "$file" ]; then
        match=$(grep -i '^[[:space:]]*maxstartups[[:space:]]' "$file" 2>/dev/null | tail -n 1)
        [ -n "$match" ] && maxstartups=$(printf '%s\n' "$match" | awk '{ print $2 }')
    else
        unreadable=1
    fi
done
[ -n "$maxstartups" ] && printf 'maxstartups=%s\n' "$maxstartups"
[ "$unreadable" = 1 ] && printf 'maxstartups.unreadable=1\n'
exit 0
"#;

/// Keyring use at or above this share of the quota is reported as a warning:
/// the remaining headroom is a few concurrent containers, not a comfortable
/// margin.
const KEYRING_PRESSURE_PERCENT: u64 = 80;

/// What `SSH_PODMAN_HOST_LIMITS_SCRIPT` managed to read. Every field is
/// optional: an unreadable file is reported, never guessed at.
#[derive(Debug, Default, PartialEq, Eq)]
struct HostLimits {
    keys_used: Option<u64>,
    keys_quota: Option<u64>,
    keys_max: Option<u64>,
    max_startups: Option<String>,
    max_startups_unreadable: bool,
}

fn parse_host_limits(stdout: &[u8]) -> HostLimits {
    let text = String::from_utf8_lossy(stdout);
    let mut limits = HostLimits::default();
    for line in text.lines() {
        let Some((name, value)) = line.split_once('=') else {
            continue;
        };
        let value = value.trim();
        match name.trim() {
            "keys.used" => limits.keys_used = value.parse().ok(),
            "keys.quota" => limits.keys_quota = value.parse().ok(),
            "keys.max" => limits.keys_max = value.parse().ok(),
            "maxstartups" if !value.is_empty() => limits.max_startups = Some(value.to_owned()),
            "maxstartups.unreadable" => limits.max_startups_unreadable = value == "1",
            _ => {}
        }
    }
    limits
}

impl HostLimits {
    /// True when the script produced nothing a reader could act on.
    fn is_empty(&self) -> bool {
        self.keys_used.is_none()
            && self.keys_quota.is_none()
            && self.keys_max.is_none()
            && self.max_startups.is_none()
            && !self.max_startups_unreadable
    }

    fn keyring_is_under_pressure(&self) -> bool {
        match (self.keys_used, self.keys_quota) {
            (Some(used), Some(quota)) if quota > 0 => {
                used.saturating_mul(100) >= quota.saturating_mul(KEYRING_PRESSURE_PERCENT)
            }
            _ => false,
        }
    }

    fn keyring_sentence(&self, destination: &str) -> String {
        match (self.keys_used, self.keys_quota) {
            (Some(used), Some(quota)) => {
                let system = match self.keys_max {
                    Some(max) => format!(", and `kernel.keys.maxkeys` is {max}"),
                    None => String::new(),
                };
                format!(
                    "The login user on {destination} holds {used} of its {quota} kernel keyring quota{system}."
                )
            }
            _ => format!(
                "The kernel keyring quota for the login user on {destination} could not be read."
            ),
        }
    }

    fn max_startups_sentence(&self) -> String {
        match (&self.max_startups, self.max_startups_unreadable) {
            (Some(value), _) => format!("sshd MaxStartups is {value}."),
            (None, true) => "sshd MaxStartups is not set in a readable sshd_config file, so sshd's default applies unless an unreadable drop-in overrides it.".to_owned(),
            (None, false) => {
                "sshd MaxStartups is not set in sshd_config, so sshd's default applies.".to_owned()
            }
        }
    }
}

/// Report the two host limits that made provisioning fail under load. The
/// target still works when they cannot be read, so an unreadable host is a
/// warning with a manual command, never a `fixable` runtime failure.
fn ssh_podman_limits_check(
    id: &str,
    ssh: &RuntimeSshTarget,
    executor: &impl CommandExecutor,
) -> DoctorCheck {
    let check_id = format!("runtime.ssh-podman.{id}.limits");
    let title = format!("Host limits for target {id}");
    let destination = &ssh.destination;
    let manual = || {
        format!(
            "Read them by hand on {destination}: `cat /proc/key-users /proc/sys/kernel/keys/maxkeys` and `grep -ri maxstartups /etc/ssh/sshd_config /etc/ssh/sshd_config.d`."
        )
    };
    let command = ssh_validation_command(
        ssh,
        vec![
            "sh".to_owned(),
            "-c".to_owned(),
            SSH_PODMAN_HOST_LIMITS_SCRIPT.to_owned(),
        ],
        "read ssh-podman host limits",
    );
    let limits = match executor.execute(&command) {
        Ok(output) if output.status == 0 => parse_host_limits(&output.stdout),
        Ok(output) => {
            let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
            return DoctorCheck::warning(
                check_id,
                title,
                format!(
                    "Could not read the kernel keyring quota or sshd MaxStartups from {destination}: {stderr}"
                ),
                manual(),
            );
        }
        Err(error) => {
            return DoctorCheck::warning(
                check_id,
                title,
                format!(
                    "Could not read the kernel keyring quota or sshd MaxStartups from {destination}: {error}"
                ),
                manual(),
            );
        }
    };
    if limits.is_empty() {
        return DoctorCheck::warning(
            check_id,
            title,
            format!("{destination} reported no readable kernel keyring or sshd limits."),
            manual(),
        );
    }
    let detail = format!(
        "{} {}",
        limits.keyring_sentence(destination),
        limits.max_startups_sentence()
    );
    if limits.keyring_is_under_pressure() {
        return DoctorCheck::warning(
            check_id,
            title,
            format!(
                "{detail} Every container takes a session keyring, so `podman run` fails with `crun: create keyring` once the quota is gone."
            ),
            format!(
                "Raise `kernel.keys.maxkeys` and `kernel.keys.maxbytes` with sysctl on {destination}, and close finished sessions promptly."
            ),
        );
    }
    DoctorCheck::ready(check_id, title, detail)
}

/// One check per `ssh-docker` target: Docker daemon, image, and optional
/// remote OverlayFS smoke test, all executed on the SSH host.
fn ssh_docker_checks(
    config: ConfigStatus<'_>,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> Vec<DoctorCheck> {
    let Ok(config) = config else {
        return Vec::new();
    };
    config
        .targets
        .iter()
        .filter_map(|(id, target)| match target {
            TargetTemplate::SshDocker { ssh, container } => Some(ssh_docker_check(
                id,
                &RuntimeSshTarget::from(ssh),
                &container.image,
                executor,
                smoke,
            )),
            _ => None,
        })
        .collect()
}

fn ssh_docker_check(
    id: &str,
    ssh: &RuntimeSshTarget,
    image: &str,
    executor: &impl CommandExecutor,
    smoke: bool,
) -> DoctorCheck {
    let check_id = format!("runtime.ssh-docker.{id}");
    let title = format!("Remote Docker for target {id}");
    let destination = &ssh.destination;
    if let SshConnectivity::Failed {
        detail,
        remediation,
    } = ssh_connectivity(ssh, executor)
    {
        return DoctorCheck::fixable(check_id, title, detail, remediation);
    }

    let preflight = match verify_ssh_docker(ssh, executor) {
        Ok(preflight) => preflight,
        Err(error) => {
            let detail = format!("{error:#}");
            return DoctorCheck::fixable(
                check_id,
                title,
                detail,
                format!(
                    "Verify `ssh {destination}` succeeds noninteractively from this host, then install and start Docker Engine there; make sure `docker info` succeeds for the configured SSH user."
                ),
            );
        }
    };

    if smoke {
        let target = RuntimeTargetTemplate::SshDocker {
            ssh: ssh.clone(),
            container: RuntimeContainerTemplate {
                build_cache: None,
                image: image.to_owned(),
                pull_policy: Default::default(),
                extra_run_args: vec![],
                workspace_storage: Default::default(),
            },
        };
        return match run_setup_smoke_test(&target, &doctor_smoke_id(), executor) {
            Ok(()) => DoctorCheck::ready(
                check_id,
                title,
                format!(
                    "Remote Docker {} is available via {destination}; disposable run/exec/remove and remote OverlayFS attachment smoke test passed for image {image}.",
                    preflight.version
                ),
            ),
            Err(error) => DoctorCheck::fixable(
                check_id,
                title,
                format!(
                    "Disposable run/exec/remove smoke test failed for image {image} on {destination}: {error:#}"
                ),
                format!(
                    "Fix the configured image or Docker runtime on {destination}, then run `mj doctor --json --smoke` again."
                ),
            ),
        };
    }

    let image_command = ssh_command(
        ssh,
        [
            "docker".to_owned(),
            "image".to_owned(),
            "inspect".to_owned(),
            image.to_owned(),
        ]
        .to_vec(),
    )
    .purpose("check remote Docker image presence");
    match executor.execute(&image_command) {
        Ok(output) if output.status == 0 => DoctorCheck::ready(
            check_id,
            title,
            format!(
                "Remote Docker {} is available via {destination}; image {image} is present. Run `mj doctor --json --smoke` to verify remote OverlayFS attachments.",
                preflight.version
            ),
        ),
        Ok(output) => DoctorCheck::fixable(
            check_id,
            title,
            format!(
                "Image {image} is not present in remote Docker storage on {destination}: {}",
                String::from_utf8_lossy(&output.stderr).trim()
            ),
            format!(
                "Pull it on {destination} with `ssh {destination} docker pull {image}`, or run `mj doctor --json --smoke`."
            ),
        ),
        Err(error) => DoctorCheck::fixable(
            check_id,
            title,
            format!("Could not inspect remote Docker image {image} on {destination}: {error}"),
            format!(
                "Verify `ssh {destination} docker info` succeeds, then pull {image} on that host."
            ),
        ),
    }
}

/// Shared disposable-container identity for every doctor smoke test.
fn doctor_smoke_id() -> String {
    format!(
        "doctor-{}-{:x}",
        std::process::id(),
        SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap_or_default()
            .as_nanos()
    )
}

fn podman_remediation(error: &anyhow::Error) -> String {
    let fix = podman_remediation_match(error).unwrap_or(
        "Install Podman with `sudo apt update && sudo apt install -y podman uidmap` (Debian/Ubuntu) or `sudo dnf install -y podman shadow-utils` (Fedora).",
    );
    format!("{fix} See {PODMAN_DOCUMENTATION_URL}.")
}

/// A Podman failure without its fix, which the check reports separately.
fn podman_failure_detail(error: &anyhow::Error) -> String {
    podman_probe_observation(error).map_or_else(|| format!("{error:#}"), str::to_owned)
}

/// Map a Podman preflight failure to its specific remediation, if one applies.
///
/// The preflight reports which postcondition failed on the error itself, so
/// the fix is chosen from that probe rather than by matching the message text
/// this repository just produced. A failure that is not a probe result, such
/// as an unreachable SSH host, has no specific fix here.
fn podman_remediation_match(error: &anyhow::Error) -> Option<&'static str> {
    failed_podman_postcondition(error).map(PodmanPostcondition::remediation)
}

const AWS_CLI_INSTALL_URL: &str =
    "https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html";

/// One check per `aws-ec2` target: the AWS CLI, its credentials, and the
/// configured launch template.
fn aws_checks(config: ConfigStatus<'_>, executor: &impl CommandExecutor) -> Vec<DoctorCheck> {
    let Ok(config) = config else {
        return Vec::new();
    };
    config
        .targets
        .iter()
        .filter_map(|(id, target)| match target {
            TargetTemplate::AwsEc2 {
                aws_profile,
                region,
                launch_template,
                ..
            } => Some(aws_target_check(
                id,
                aws_profile.as_deref(),
                region,
                launch_template,
                executor,
            )),
            _ => None,
        })
        .collect()
}

/// The profile and region every AWS probe carries, applied exactly the way
/// provisioning applies them in `targets`.
fn aws_global_args<'a>(profile: Option<&'a str>, region: &'a str) -> Vec<String> {
    vec![
        "--profile".to_owned(),
        profile.unwrap_or("default").to_owned(),
        "--region".to_owned(),
        region.to_owned(),
    ]
}

fn aws_target_check(
    id: &str,
    profile: Option<&str>,
    region: &str,
    launch_template: &str,
    executor: &impl CommandExecutor,
) -> DoctorCheck {
    let check_id = format!("runtime.aws-ec2.{id}");
    let title = format!("AWS EC2 target {id}");
    let profile_label = profile.unwrap_or("default");

    let version = CommandSpec::new("aws", ["--version"]).purpose("check AWS CLI installation");
    match executor.execute(&version) {
        Err(error) => {
            return DoctorCheck::fixable(
                check_id,
                title,
                format!("The `aws` command is not available: {error}"),
                format!("Install the AWS CLI and put `aws` on PATH: {AWS_CLI_INSTALL_URL}"),
            );
        }
        Ok(output) if output.status != 0 => {
            return DoctorCheck::fixable(
                check_id,
                title,
                format!(
                    "`aws --version` failed: {}",
                    String::from_utf8_lossy(&output.stderr).trim()
                ),
                format!("Reinstall the AWS CLI: {AWS_CLI_INSTALL_URL}"),
            );
        }
        Ok(_) => {}
    }

    let mut identity_args = aws_global_args(profile, region);
    identity_args.extend(["sts".to_owned(), "get-caller-identity".to_owned()]);
    identity_args.extend(["--output".to_owned(), "json".to_owned()]);
    let identity =
        CommandSpec::new("aws", identity_args).purpose("check AWS credentials for a doctor target");
    match executor.execute(&identity) {
        Err(error) => {
            return DoctorCheck::fixable(
                check_id,
                title,
                format!("Could not run `aws sts get-caller-identity`: {error}"),
                format!(
                    "Configure credentials with `aws configure --profile {profile_label}`, or sign in with `aws sso login --profile {profile_label}`."
                ),
            );
        }
        Ok(output) if output.status != 0 => {
            return DoctorCheck::fixable(
                check_id,
                title,
                format!(
                    "AWS credentials for profile {profile_label} are not usable: {}",
                    String::from_utf8_lossy(&output.stderr).trim()
                ),
                format!(
                    "Configure credentials with `aws configure --profile {profile_label}`, or sign in with `aws sso login --profile {profile_label}`."
                ),
            );
        }
        Ok(_) => {}
    }

    // Launch templates are addressed by id when they carry the `lt-` prefix
    // and by name otherwise, the same split provisioning uses.
    let by_id = launch_template.starts_with("lt-");
    let mut template_args = aws_global_args(profile, region);
    template_args.extend(["ec2".to_owned(), "describe-launch-templates".to_owned()]);
    template_args.extend([
        if by_id {
            "--launch-template-ids".to_owned()
        } else {
            "--launch-template-names".to_owned()
        },
        launch_template.to_owned(),
    ]);
    template_args.extend(["--output".to_owned(), "json".to_owned()]);
    let template =
        CommandSpec::new("aws", template_args).purpose("check the configured AWS launch template");
    let template_remediation = format!(
        "Create the launch template in {region}, or point this target at an existing one; `aws --profile {profile_label} --region {region} ec2 describe-launch-templates` lists them."
    );
    match executor.execute(&template) {
        Err(error) => DoctorCheck::fixable(
            check_id,
            title,
            format!("Could not query launch template {launch_template}: {error}"),
            template_remediation,
        ),
        Ok(output) if output.status != 0 => DoctorCheck::fixable(
            check_id,
            title,
            format!(
                "Launch template {launch_template} was not found in {region}: {}",
                String::from_utf8_lossy(&output.stderr).trim()
            ),
            template_remediation,
        ),
        Ok(_) => DoctorCheck::ready(
            check_id,
            title,
            format!(
                "The AWS CLI is installed, profile {profile_label} has valid credentials, and launch template {launch_template} exists in {region}."
            ),
        ),
    }
}

/// Whether the running daemon is this build.
///
/// Two Mjolnir builds carry the same version string, so the version line in
/// `mj daemon status` cannot answer it. A daemon left over from before a
/// rebuild keeps serving the old code, and, when its executable was unlinked
/// by the rebuild, also loses every portable worker source it would have
/// pinned. Both are invisible without this check.
fn daemon_build_check() -> DoctorCheck {
    const ID: &str = "daemon.build";
    const TITLE: &str = "Daemon build";
    let Ok(metadata) = mj_client::daemon::read_metadata_any() else {
        return DoctorCheck::ready(
            ID,
            TITLE,
            "No Mjolnir daemon is running; the next command starts one from this build.",
        );
    };
    let pid = metadata.pid;
    match mj_client::executable::process_runs_this_executable(pid) {
        Ok(Some(true)) => DoctorCheck::ready(
            ID,
            TITLE,
            format!(
                "Daemon {pid} runs this build (version {}).",
                metadata.build_version
            ),
        ),
        Ok(Some(false)) => DoctorCheck::warning(
            ID,
            TITLE,
            format!(
                "{}. Two builds can report the same version, so the files are what tell them apart. Code rebuilt since that daemon started is not running.",
                mj_client::executable::describe_running_daemon_and_client_builds(
                    pid,
                    &metadata.build_version,
                ),
            ),
            "Run `mj daemon restart` from this build. It now fails rather than reporting success if another client's build wins.",
        ),
        Ok(None) => DoctorCheck::ready(
            ID,
            TITLE,
            format!(
                "Daemon {pid} is recorded but not running; the next command starts one from this build."
            ),
        ),
        Err(error) => DoctorCheck::warning(
            ID,
            TITLE,
            format!("Could not tell which build daemon {pid} runs: {error:#}"),
            "Run `mj daemon restart` from this build if rebuilt code is not taking effect.",
        ),
    }
}

/// Whether a new session would run the worker binary as it is on disk now.
///
/// The daemon copies each worker it can find into a content-addressed cache
/// when it starts and serves that copy for the rest of its life, so rebuilding
/// `mj-worker` does not reach a running daemon. Nothing else reports this, and
/// the digests are what make it checkable at all: two worker builds differ by
/// content, not by name or version.
fn worker_freshness_checks(config: ConfigStatus<'_>) -> Vec<DoctorCheck> {
    let mut checks = Vec::new();
    let daemon = mj_client::daemon::read_metadata_any()
        .ok()
        .filter(|metadata| mj_client::daemon::process_is_alive(metadata.pid));
    let pinned = pinned_worker_digests();
    let mut sources: Vec<(String, Result<WorkerBinaryAvailability>)> = vec![(
        "this host".to_owned(),
        crate::controller::native_worker_binary_prerequisite(),
    )];
    for arch in container_worker_architectures(config) {
        sources.push((
            format!("{arch} Linux targets"),
            worker_binary_prerequisite_for_arch(&arch),
        ));
    }
    for (label, availability) in sources {
        let id = format!("worker.freshness.{}", label.replace(' ', "-"));
        let title = format!("Worker binary for {label}");
        let path = match availability {
            Ok(WorkerBinaryAvailability::Local { path, .. }) => path,
            // A remote worker is fetched by digest when a target is
            // provisioned, so it cannot go stale behind a running daemon.
            Ok(WorkerBinaryAvailability::Remote { .. }) => continue,
            Err(error) => {
                checks.push(DoctorCheck::unsupported(
                    id,
                    title,
                    format!("No worker binary resolves for {label}: {error:#}"),
                ));
                continue;
            }
        };
        let digest = mj_core::worker_launch::worker_executable_digest(&path)
            .unwrap_or_else(|error| format!("unreadable ({error:#})"));
        let pinned_note = if pinned.is_empty() {
            "the daemon has pinned no worker".to_owned()
        } else if pinned.contains(&digest) {
            "this content is in the daemon's pinned worker cache".to_owned()
        } else {
            format!(
                "the pinned worker cache holds {} instead",
                pinned.join(", ")
            )
        };
        let detail = format!("{} has digest {digest}; {pinned_note}.", path.display());
        let Some(metadata) = daemon.as_ref() else {
            checks.push(DoctorCheck::ready(
                id,
                title,
                format!("{detail} No daemon is running, so the next session uses this file."),
            ));
            continue;
        };
        match worker_changed_since_daemon_start(&path, &metadata.started_at) {
            Ok(true) => checks.push(DoctorCheck::warning(
                id,
                title,
                format!("{detail} It was rebuilt after daemon {} started, which froze the copy it serves.", metadata.pid),
                "Run `mj daemon restart` so new sessions use the rebuilt worker. Sessions already running keep their worker until they are quiet enough to be upgraded.",
            )),
            Ok(false) => checks.push(DoctorCheck::ready(id, title, detail)),
            Err(error) => checks.push(DoctorCheck::warning(
                id,
                title,
                format!("{detail} Could not compare it with the daemon's start time: {error:#}"),
                "Run `mj daemon restart` if rebuilt worker code is not taking effect.",
            )),
        }
    }
    checks
}

/// The architectures this configuration needs a portable Linux worker for.
fn container_worker_architectures(config: ConfigStatus<'_>) -> Vec<String> {
    let Ok(config) = config else {
        return Vec::new();
    };
    let mut architectures = Vec::new();
    for target in config.targets.values() {
        let container = match target {
            TargetTemplate::LocalPodman { container }
            | TargetTemplate::LocalDocker { container }
            | TargetTemplate::AppleContainer { container }
            | TargetTemplate::SshPodman { container, .. }
            | TargetTemplate::SshDocker { container, .. } => container,
            _ => continue,
        };
        let arch = container
            .platform
            .as_deref()
            .and_then(|platform| platform.rsplit('/').next())
            .map_or_else(
                || std::env::consts::ARCH.to_owned(),
                normalized_worker_architecture,
            );
        if !architectures.contains(&arch) {
            architectures.push(arch);
        }
    }
    architectures
}

/// Container platforms name architectures the way Docker does; worker files
/// are named the way Rust target triples do.
fn normalized_worker_architecture(platform_arch: &str) -> String {
    crate::targets::normalize_architecture(platform_arch)
        .unwrap_or(platform_arch)
        .to_owned()
}

/// The digests the daemon's immutable worker cache holds.
///
/// The cache is never pruned, so this is what any daemon on this machine has
/// pinned at some point, which is why it is reported rather than judged.
fn pinned_worker_digests() -> Vec<String> {
    let root = mj_core::config::data_dir().join("workers").join("pinned");
    let Ok(entries) = std::fs::read_dir(root) else {
        return Vec::new();
    };
    let mut digests: Vec<String> = entries
        .flatten()
        .filter(|entry| entry.path().is_dir())
        .filter_map(|entry| entry.file_name().into_string().ok())
        .collect();
    digests.sort();
    digests
}

/// Whether a worker file was written after the daemon started.
///
/// The daemon copies the file it finds at startup, so a later modification is
/// exactly the case where the running daemon serves older content.
fn worker_changed_since_daemon_start(path: &Path, started_at: &str) -> Result<bool> {
    let started: SystemTime = chrono::DateTime::parse_from_rfc3339(started_at)
        .map_err(|error| anyhow::anyhow!("parse daemon start time {started_at:?}: {error}"))?
        .into();
    let modified = std::fs::metadata(path)?.modified()?;
    Ok(modified > started)
}

/// The targets the build-cache and worker-binary checks cover: every target
/// the user configured, plus each standard local target
/// ([`Config::with_local_targets`]) whose engine check in `engine_checks`
/// passed. That is the set a new session can use. With no target blocks in
/// config.toml, the built-in `podman` target on a host with working Podman
/// still needs a worker (launch finding R5-2). A standard target whose
/// engine is unavailable is shown as unavailable and needs none.
fn offered_targets<'a>(
    config: &Config,
    engine_checks: impl IntoIterator<Item = &'a DoctorCheck>,
) -> Config {
    let ready = engine_checks
        .into_iter()
        .filter(|check| check.status == CheckStatus::Ready)
        .map(|check| check.id.as_str())
        .collect::<std::collections::BTreeSet<_>>();
    let engine_ready = |target_id: &str| match target_id {
        "podman" => ready.contains("runtime.podman"),
        "docker" => ready.contains("runtime.docker"),
        "apple-container" => ready.contains("runtime.apple-container"),
        _ => true,
    };
    let mut offered = config.clone().with_local_targets();
    offered
        .targets
        .retain(|id, _| config.configures_target(id) || engine_ready(id));
    offered
}

fn worker_binary_checks(config: ConfigStatus<'_>) -> Vec<DoctorCheck> {
    let config = match config {
        Ok(config) => config,
        Err(ConfigGap::NewerVersion(version)) => {
            return vec![newer_config_skip(
                "worker.containers",
                "Container worker binary",
                version,
            )];
        }
        Err(gap @ (ConfigGap::Missing | ConfigGap::Unreadable)) => {
            return vec![DoctorCheck::fixable(
                "worker.containers",
                "Container worker binary",
                format!(
                    "Worker availability cannot be checked until {}.",
                    gap.awaited()
                ),
                gap.remediation(),
            )];
        }
    };
    let containers = config
        .targets
        .iter()
        .filter_map(|(id, target)| match target {
            TargetTemplate::LocalPodman { container }
            | TargetTemplate::LocalDocker { container }
            | TargetTemplate::AppleContainer { container } => Some((id, container, None)),
            TargetTemplate::SshPodman { container, .. } => {
                Some((id, container, Some("ssh-podman")))
            }
            TargetTemplate::SshDocker { container, .. } => {
                Some((id, container, Some("ssh-docker")))
            }
            _ => None,
        })
        .collect::<Vec<_>>();
    if containers.is_empty() {
        return vec![DoctorCheck::unsupported(
            "worker.containers",
            "Container worker binary",
            "No container target is configured.",
        )];
    }
    containers
        .into_iter()
        .map(|(id, container, remote_kind)| {
            if let Some(remote_kind) = remote_kind
                && container.platform.is_none()
            {
                // The remote CPU architecture is only observable once the host
                // is reachable, so an explicit `platform` is required here.
                return DoctorCheck::unsupported(
                    format!("worker.{id}"),
                    format!("Container worker binary for target {id}"),
                    format!(
                        "Set `platform` on this {remote_kind} target to check its worker binary; the remote architecture is unknown until provisioning."
                    ),
                );
            }
            worker_binary_check(id, container)
        })
        .collect()
}

fn worker_binary_check(id: &str, container: &ContainerTemplate) -> DoctorCheck {
    let title = format!("Container worker binary for target {id}");
    let arch = match container_architecture(container.platform.as_deref()) {
        Ok(arch) => arch,
        Err(reason) => {
            return DoctorCheck::unsupported(format!("worker.{id}"), title, reason);
        }
    };
    let triple = format!("{arch}-unknown-linux-musl");
    match worker_binary_prerequisite_for_arch(arch) {
        Ok(WorkerBinaryAvailability::Local { path, source }) => DoctorCheck::ready(
            format!("worker.{id}"),
            title,
            format!(
                "{triple} worker is available from {source}: {}",
                path.display()
            ),
        ),
        Ok(WorkerBinaryAvailability::Remote { url, .. }) => DoctorCheck::ready(
            format!("worker.{id}"),
            title,
            format!("{triple} worker will be verified and downloaded from {url} when needed."),
        ),
        Err(error) => DoctorCheck::fixable(
            format!("worker.{id}"),
            title,
            format!("No usable {triple} worker source: {error:#}"),
            format!(
                "Build it with `cargo build --release --target {triple} -p brokk-mj-worker --bin mj-worker`, install `mj-worker-{triple}` beside `mj`, or set MJ_WORKER_BINARY, MJ_WORKER_DIR, or MJ_WORKER_URL with MJ_WORKER_SHA256."
            ),
        ),
    }
}

fn container_architecture(platform: Option<&str>) -> std::result::Result<&'static str, String> {
    let candidate = platform.unwrap_or(std::env::consts::ARCH);
    let candidate = candidate
        .split('/')
        .rev()
        .find(|part| matches!(*part, "x86_64" | "amd64" | "aarch64" | "arm64"))
        .unwrap_or(candidate);
    crate::targets::normalize_architecture(candidate).map_err(|_| format!(
        "Container architecture {candidate:?} is unsupported; Mjolnir supports x86_64 and aarch64 Linux workers."
    ))
}

fn apple_container_image(config: ConfigStatus<'_>) -> String {
    config
        .ok()
        .and_then(|config| {
            config.targets.values().find_map(|target| match target {
                TargetTemplate::AppleContainer { container } => Some(container.image.clone()),
                _ => None,
            })
        })
        .unwrap_or_else(|| DEFAULT_CONTAINER_IMAGE.into())
}

/// Platform support and daemon readiness shared by setup and doctor.
pub fn apple_container_runtime_check(
    platform: &ApplePlatform,
    executor: &impl CommandExecutor,
) -> DoctorCheck {
    match platform {
        ApplePlatform::Linux => {
            return DoctorCheck::unsupported(
                "runtime.apple-container",
                "Apple container runtime",
                "macOS only",
            );
        }
        ApplePlatform::Other(current) => {
            return DoctorCheck::unsupported(
                "runtime.apple-container",
                "Apple container runtime",
                format!("macOS only (current platform: {current})"),
            );
        }
        ApplePlatform::Macos {
            architecture,
            major_version,
        } if architecture != "aarch64" && architecture != "arm64" => {
            return DoctorCheck::unsupported(
                "runtime.apple-container",
                "Apple container runtime",
                "Apple container requires Apple silicon; Intel Macs are unsupported.",
            );
        }
        ApplePlatform::Macos { major_version, .. } if *major_version < 26 => {
            return DoctorCheck::unsupported(
                "runtime.apple-container",
                "Apple container runtime",
                format!("Apple container requires macOS 26 or newer (found {major_version})."),
            );
        }
        ApplePlatform::Macos { .. } => {}
    }

    apple_container_daemon_check(executor)
}

pub fn apple_container_check(
    platform: &ApplePlatform,
    executor: &impl CommandExecutor,
    smoke: bool,
    image: String,
) -> DoctorCheck {
    let daemon = apple_container_runtime_check(platform, executor);
    if daemon.status != CheckStatus::Ready {
        return daemon;
    }

    if !smoke {
        return DoctorCheck::fixable(
            "runtime.apple-container",
            "Apple container runtime",
            "The daemon is running, but the required disposable smoke test was not requested.",
            "Run `mj doctor --json --smoke`.",
        );
    }

    let target = RuntimeTargetTemplate::AppleContainer(RuntimeContainerTemplate {
        build_cache: None,
        image,
        pull_policy: Default::default(),
        extra_run_args: vec![],
        workspace_storage: Default::default(),
    });
    match run_setup_smoke_test(&target, &doctor_smoke_id(), executor) {
        Ok(()) => DoctorCheck::ready(
            "runtime.apple-container",
            "Apple container runtime",
            "Installed, daemon running, and disposable run/exec/remove smoke test passed.",
        ),
        Err(error) => DoctorCheck::fixable(
            "runtime.apple-container",
            "Apple container runtime",
            format!("Disposable run/exec/remove smoke test failed: {error:#}"),
            "Fix the configured image or container runtime, then run `mj doctor --json --smoke` again.",
        ),
    }
}

/// Probe that the Apple `container` command is installed and its daemon is
/// running, phrased as a doctor check.
///
/// Called only after [`apple_container_runtime_check`] checks platform support.
fn apple_container_daemon_check(executor: &impl CommandExecutor) -> DoctorCheck {
    let installed =
        CommandSpec::new("container", ["--version"]).purpose("check Apple container installation");
    match executor.execute(&installed) {
        Err(error) => {
            return DoctorCheck::fixable(
                "runtime.apple-container",
                "Apple container runtime",
                format!(
                    "The `container` command is not available: {}",
                    error.root_cause()
                ),
                format!("Install the official signed package: {APPLE_CONTAINER_INSTALL_URL}"),
            );
        }
        Ok(output) if output.status != 0 => {
            return DoctorCheck::fixable(
                "runtime.apple-container",
                "Apple container runtime",
                format!(
                    "The installed `container --version` command failed: {}",
                    String::from_utf8_lossy(&output.stderr).trim()
                ),
                format!("Reinstall the official signed package: {APPLE_CONTAINER_INSTALL_URL}"),
            );
        }
        Ok(_) => {}
    }

    let status =
        CommandSpec::new("container", ["system", "status"]).purpose("check Apple container daemon");
    match executor.execute(&status) {
        Ok(output) if output.status == 0 => DoctorCheck::ready(
            "runtime.apple-container",
            "Apple container runtime",
            "Installed, and the Apple container daemon is running.",
        ),
        Ok(output) => DoctorCheck::fixable(
            "runtime.apple-container",
            "Apple container runtime",
            format!(
                "The Apple container daemon is stopped: {}",
                String::from_utf8_lossy(&output.stderr).trim()
            ),
            "Run `container system start`.",
        ),
        Err(error) => DoctorCheck::fixable(
            "runtime.apple-container",
            "Apple container runtime",
            format!("Could not query the Apple container daemon: {error}"),
            "Run `container system start`.",
        ),
    }
}

pub fn current_apple_platform(executor: &impl CommandExecutor) -> ApplePlatform {
    if cfg!(target_os = "linux") {
        return ApplePlatform::Linux;
    }
    if !cfg!(target_os = "macos") {
        return ApplePlatform::Other(std::env::consts::OS.into());
    }
    let major_version = executor
        .execute(&CommandSpec::new("sw_vers", ["-productVersion"]).purpose("detect macOS version"))
        .ok()
        .filter(|output| output.status == 0)
        .and_then(|output| {
            String::from_utf8(output.stdout)
                .ok()
                .and_then(|value| value.trim().split('.').next()?.parse().ok())
        })
        .unwrap_or(0);
    ApplePlatform::Macos {
        architecture: std::env::consts::ARCH.into(),
        major_version,
    }
}

#[cfg(test)]
mod tests;

/// What one repository still holds from a Mjolnir review capture.
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct ReviewResidue {
    /// `refs/hel/*` refs in the repository.
    pub refs: Vec<String>,
    /// Scratch index files left in the Git directory by an interrupted capture.
    pub scratch_indexes: Vec<PathBuf>,
}

impl ReviewResidue {
    fn is_empty(&self) -> bool {
        self.refs.is_empty() && self.scratch_indexes.is_empty()
    }
}

/// Read what a repository still holds from Mjolnir's review captures.
///
/// Releases before this one staged the whole working tree into the user's own
/// object store and pinned it with two refs, and a capture that was killed
/// partway left its scratch index behind. Both are the user's to remove, so
/// this only reads.
pub(crate) fn review_residue(repository: &Path) -> ReviewResidue {
    let mut residue = ReviewResidue::default();
    let git_dir = repository.join(".git");
    if !git_dir.exists() {
        return residue;
    }
    for reference in ["review-baseline", "review-capture"] {
        if git_dir.join("refs/hel").join(reference).is_file() {
            residue.refs.push(format!("refs/hel/{reference}"));
        }
    }
    // A packed ref survives `git pack-refs`, which a `git gc` runs.
    if let Ok(packed) = std::fs::read_to_string(git_dir.join("packed-refs")) {
        for line in packed.lines() {
            if let Some((_, reference)) = line.split_once(' ')
                && reference.starts_with("refs/hel/")
                && !residue.refs.iter().any(|known| known == reference)
            {
                residue.refs.push(reference.to_owned());
            }
        }
    }
    if let Ok(entries) = std::fs::read_dir(&git_dir) {
        for entry in entries.filter_map(Result::ok) {
            if entry
                .file_name()
                .to_str()
                .is_some_and(|name| name.starts_with("hel-review-index-"))
            {
                residue.scratch_indexes.push(entry.path());
            }
        }
    }
    residue.refs.sort();
    residue.scratch_indexes.sort();
    residue
}

/// The repositories a person works in by hand, where review leftovers are
/// worth reporting.
///
/// These are the configured bundle repositories and the directories sessions
/// started with `--project-directory` use. A session's own managed checkout
/// (`<repository>/.mj/clones/<id>` or `<repository>/.mj/worktrees/<id>`) is
/// Mjolnir's working state, not a leftover, so it is never reported. Neither is
/// a repository a live session is working in, because its refs are in use; a
/// linked worktree shares its refs with the repository it came from, so a live
/// worktree session keeps that repository out too.
pub(crate) fn review_residue_repositories(
    configured: impl IntoIterator<Item = PathBuf>,
    sessions: &[&mj_core::state::SessionRecord],
) -> Vec<PathBuf> {
    use mj_core::state::ManagedCheckoutKind;

    let managed_roots = sessions
        .iter()
        .filter_map(|session| session.managed_worktree.as_ref())
        .map(|worktree| worktree.worktree_root.as_path())
        .collect::<Vec<_>>();
    let mut in_use = Vec::new();
    for session in sessions.iter().filter(|session| session.state.is_active()) {
        if let Some(directory) = &session.project_directory {
            in_use.push(directory.as_path());
        }
        if let Some(worktree) = &session.managed_worktree
            && worktree.kind == ManagedCheckoutKind::Worktree
        {
            in_use.push(worktree.source_repository.as_path());
        }
    }
    let mut repositories = configured
        .into_iter()
        .chain(
            sessions
                .iter()
                .filter_map(|session| session.project_directory.clone()),
        )
        .filter(|repository| {
            !is_inside_managed_checkout(repository)
                && !managed_roots
                    .iter()
                    .any(|root| repository.starts_with(root))
                && !in_use.iter().any(|directory| repository == directory)
        })
        .collect::<Vec<_>>();
    repositories.sort();
    repositories.dedup();
    repositories
}

/// Whether `path` is at or under `<repository>/.mj/clones/<id>` or
/// `<repository>/.mj/worktrees/<id>`.
fn is_inside_managed_checkout(path: &Path) -> bool {
    let components = path
        .components()
        .map(|component| component.as_os_str())
        .collect::<Vec<_>>();
    components
        .windows(3)
        .any(|window| window[0] == ".mj" && (window[1] == "clones" || window[1] == "worktrees"))
}

/// Report Mjolnir's own leftovers in the repositories the configuration names.
///
/// This deletes nothing. Removing refs and running `git gc` in someone else's
/// repository without asking is the same mistake as writing to it without
/// asking, which is what left this residue in the first place.
fn review_residue_checks(config: ConfigStatus<'_>) -> Vec<DoctorCheck> {
    let Ok(config) = config else {
        return Vec::new();
    };
    let configured = config
        .bundles
        .values()
        .flat_map(|bundle| bundle.repositories.iter())
        .filter_map(|repository| repository.local.clone());
    // A daemon-less machine has no session database, which is not a reason to
    // skip the configured repositories.
    let state = crate::database::load_state().ok();
    let sessions = state
        .as_ref()
        .map(|state| state.sessions.values().collect::<Vec<_>>())
        .unwrap_or_default();
    let repositories = review_residue_repositories(configured, &sessions);
    if repositories.is_empty() {
        return Vec::new();
    }
    let found = repositories
        .into_iter()
        .map(|repository| {
            let residue = review_residue(&repository);
            (repository, residue)
        })
        .filter(|(_, residue)| !residue.is_empty())
        .collect::<Vec<_>>();
    if found.is_empty() {
        return vec![DoctorCheck::ready(
            "review.residue",
            "Review leftovers in your repositories",
            "No Mjolnir refs or scratch index files were found in the configured repositories.",
        )];
    }
    let detail = found
        .iter()
        .map(|(repository, residue)| {
            let mut parts = Vec::new();
            if !residue.refs.is_empty() {
                parts.push(residue.refs.join(", "));
            }
            if !residue.scratch_indexes.is_empty() {
                parts.push(mj_core::text::counted(
                    residue.scratch_indexes.len(),
                    "leftover scratch index file",
                    "leftover scratch index files",
                ));
            }
            format!("{}: {}", repository.display(), parts.join("; "))
        })
        .collect::<Vec<_>>()
        .join(". ");
    let commands = found
        .iter()
        .flat_map(|(repository, residue)| {
            let repository = repository.display().to_string();
            let mut commands = residue
                .refs
                .iter()
                .map(|reference| format!("git -C {repository} update-ref -d {reference}"))
                .collect::<Vec<_>>();
            commands.extend(
                residue
                    .scratch_indexes
                    .iter()
                    .map(|index| format!("rm -f {}", index.display())),
            );
            commands.push(format!("git -C {repository} gc --prune=now"));
            commands
        })
        .collect::<Vec<_>>()
        .join("\n");
    vec![DoctorCheck::fixable(
        "review.residue",
        "Review leftovers in your repositories",
        format!("Mjolnir left these in repositories it does not own: {detail}."),
        format!("Remove them yourself when you are ready:\n{commands}"),
    )]
}