xbp_cli/commands/

runners_runtime.rs

use crate::cli::commands::{
    ApiTargetOptions, RunnersAgentServeCmd, RunnersGroupsAccessCmd, RunnersHostsPreflightCmd,
    RunnersHostsStatusCmd, RunnersLogsCmd, RunnersStatusCmd,
};
use crate::commands::api_request::resolve_request_url;
use crate::config::{
    global_runner_root_dir, resolve_github_oauth2_key, resolve_xbp_api_token, SshConfig,
};
use crate::strategies::{GitHubRunnersProjectConfig, XbpConfig};
use crate::utils::{find_xbp_config_upwards, parse_config_with_auto_heal};
use chrono::Utc;
use colored::Colorize;
use flate2::read::GzDecoder;
use reqwest::{Client, Method, StatusCode};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::env;
use std::fs;
use std::io::{self, Cursor};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::Duration;
use tar::Archive;
use tokio::time::sleep;
use xbp_core::{
    RunnerExecutionRuntime, RunnerHostPreflightRecord, RunnerHostRecord, RunnerInstallPhase,
    RunnerJobRecord, RunnerPlatform, RunnerRegistrationState, RunnerRuntimeState,
    RunnerServiceManager, RunnerServiceStatus, RunnerSyncState,
};
use zip::ZipArchive;

#[derive(Debug, Deserialize)]
struct RunnerHostListResponse {
    runner_hosts: Vec<RunnerHostRecord>,
}

#[derive(Debug, Deserialize)]
struct RunnerHostPreflightListResponse {
    preflights: Vec<RunnerHostPreflightRecord>,
}

#[derive(Debug, Deserialize)]
struct RunnerJobListResponse {
    jobs: Vec<RunnerJobRecord>,
}

#[derive(Debug, Deserialize)]
struct RunnerGroupRepositoryListResponse {
    repositories: Vec<Value>,
}

#[derive(Debug, Clone, Deserialize)]
struct RunnerApiReleaseResponse {
    version: String,
    file_name: String,
    download_url: String,
}

#[derive(Debug, Deserialize)]
struct RunnerApiTokenResponse {
    token: String,
    #[serde(rename = "expires_at")]
    _expires_at: Option<String>,
}

#[derive(Debug, Deserialize)]
struct RunnerGitHubListResponse {
    runners: Vec<RunnerGitHubRunner>,
}

#[derive(Debug, Deserialize)]
struct RunnerGitHubRunner {
    id: i64,
    name: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct LocalRunnerRuntimeMetadata {
    #[serde(default)]
    runner_host_id: Option<String>,
    #[serde(default)]
    hostname: Option<String>,
    #[serde(default)]
    runner_name_prefix: Option<String>,
    #[serde(default)]
    runner_name: Option<String>,
    #[serde(default)]
    runner_group: Option<String>,
    #[serde(default)]
    organization_login: Option<String>,
    #[serde(default)]
    github_installation_id: Option<String>,
    #[serde(default)]
    github_runner_id: Option<i64>,
    #[serde(default)]
    removed_runner_names: Vec<String>,
    #[serde(default)]
    removed_runner_ids: Vec<i64>,
}

#[derive(Debug, Clone, Default)]
struct RunnerCleanupSummary {
    removed_runner_names: Vec<String>,
    removed_runner_ids: Vec<i64>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LocalRunnerJobPayload {
    pub runtime: RunnerExecutionRuntime,
    #[serde(default)]
    pub organization_login: Option<String>,
    #[serde(default)]
    pub github_installation_id: Option<String>,
    #[serde(default)]
    pub runner_group: Option<String>,
    #[serde(default)]
    pub labels: Vec<String>,
    #[serde(default)]
    pub runner_name: Option<String>,
    #[serde(default)]
    pub runner_version: Option<String>,
    #[serde(default)]
    pub ephemeral: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LocalRunnerJobResult {
    pub runtime_root: String,
    pub service_manager: RunnerServiceManager,
    pub service_status: RunnerServiceStatus,
    pub install_phase: RunnerInstallPhase,
    pub registration_state: RunnerRegistrationState,
    pub installed_version: Option<String>,
    pub generated_files: Vec<String>,
}

pub async fn run_runner_host_preflight(cmd: &RunnersHostsPreflightCmd) -> Result<(), String> {
    let host = fetch_runner_host_by_id(&cmd.runner_host_id, &cmd.target).await?;
    let preflight = build_local_preflight(&host)?;
    print_preflight(&host, &preflight);

    if cmd.write_api {
        let body = json!({
            "runner_host_id": host.id,
            "platform": preflight.platform,
            "status": preflight.status,
            "docker_available": preflight.docker_available,
            "service_manager": preflight.service_manager,
            "checks": preflight.checks,
            "checked_at": preflight.checked_at,
        });
        let _: RunnerHostPreflightRecord = api_json(
            Method::POST,
            "/runner-host-preflights",
            &cmd.target,
            Some(body),
        )
        .await?;
        println!(
            "{}",
            "Persisted preflight result to the control plane.".green()
        );
    }

    Ok(())
}

pub async fn run_runner_host_status(cmd: &RunnersHostsStatusCmd) -> Result<(), String> {
    let host = fetch_runner_host_by_id(&cmd.runner_host_id, &cmd.target).await?;
    let response: RunnerHostPreflightListResponse = api_json(
        Method::GET,
        &format!("/runner-host-preflights?runner_host_id={}", host.id),
        &cmd.target,
        None,
    )
    .await?;

    println!("{}", "Runner Host".bold());
    println!("  id: {}", host.id);
    println!("  hostname: {}", host.hostname);
    println!("  platform: {}", host.platform);
    println!("  status: {}", color_status(host.status.as_str()));
    println!("  host kind: {}", host.host_kind);
    println!("  prefix: {}", host.runner_name_prefix);
    if let Some(preflight) = response.preflights.first() {
        println!("{}", "Latest preflight".bold());
        println!("  status: {}", color_status(preflight.status.as_str()));
        println!(
            "  service manager: {}",
            preflight
                .service_manager
                .map(|value| value.to_string())
                .unwrap_or_else(|| "unknown".to_string())
        );
        println!(
            "  docker available: {}",
            preflight
                .docker_available
                .map(|value| if value { "yes" } else { "no" })
                .unwrap_or("unknown")
        );
    }
    Ok(())
}

pub async fn run_runner_group_access(cmd: &RunnersGroupsAccessCmd) -> Result<(), String> {
    let response: RunnerGroupRepositoryListResponse = api_json(
        Method::GET,
        &format!("/runner-groups/{}/repositories", cmd.runner_group_id),
        &cmd.target,
        None,
    )
    .await?;

    println!("{}", "Runner Group Access".bold());
    if response.repositories.is_empty() {
        println!("No explicit repositories are stored for this runner group.");
        return Ok(());
    }

    let rows = response
        .repositories
        .iter()
        .map(|repository| {
            vec![
                read_json_string(repository, "repository_full_name"),
                if repository
                    .get("allowed")
                    .and_then(Value::as_bool)
                    .unwrap_or(true)
                {
                    "allowed".green().to_string()
                } else {
                    "blocked".red().to_string()
                },
            ]
        })
        .collect::<Vec<_>>();
    print_table(&["Repository", "Access"], &rows);
    Ok(())
}

pub async fn run_runner_status(cmd: &RunnersStatusCmd) -> Result<(), String> {
    let hosts_path = format!("/runner-hosts?organization_id={}", cmd.organization_id);
    let jobs_path = {
        let mut params = vec![("organization_id", Some(cmd.organization_id.as_str()))];
        params.push(("runner_host_id", cmd.runner_host_id.as_deref()));
        params.push(("runner_id", cmd.runner_id.as_deref()));
        params.push(("daemon_id", cmd.daemon_id.as_deref()));
        params.push(("status", cmd.status.as_deref()));
        params.push(("phase", cmd.phase.as_deref()));
        let limit = cmd.limit.to_string();
        params.push(("limit", Some(limit.as_str())));
        with_query("/runner-jobs", &params)
    };
    let hosts: RunnerHostListResponse =
        api_json(Method::GET, &hosts_path, &cmd.target, None).await?;
    let jobs: RunnerJobListResponse = api_json(Method::GET, &jobs_path, &cmd.target, None).await?;

    println!("{}", "Runner Hosts".bold());
    let host_rows = hosts
        .runner_hosts
        .iter()
        .filter(|host| {
            cmd.runner_host_id
                .as_deref()
                .map(|runner_host_id| runner_host_id == host.id)
                .unwrap_or(true)
        })
        .map(|host| {
            vec![
                host.id.clone(),
                host.hostname.clone(),
                host.platform.to_string(),
                color_status(host.status.as_str()),
            ]
        })
        .collect::<Vec<_>>();
    print_table(&["Host ID", "Hostname", "Platform", "Status"], &host_rows);

    println!();
    println!("{}", "Runner Jobs".bold());
    let job_rows = jobs
        .jobs
        .iter()
        .filter(|job| {
            cmd.runner_host_id
                .as_deref()
                .map(|runner_host_id| job.runner_host_id.as_deref() == Some(runner_host_id))
                .unwrap_or(true)
        })
        .map(|job| {
            vec![
                job.id.clone(),
                job.job_kind.to_string(),
                color_status(job.status.as_str()),
                job.phase
                    .map(|phase| phase.to_string())
                    .unwrap_or_else(|| "-".to_string()),
                job.runner_host_id
                    .clone()
                    .unwrap_or_else(|| "-".to_string()),
            ]
        })
        .collect::<Vec<_>>();
    print_table(&["Job ID", "Kind", "Status", "Phase", "Host"], &job_rows);
    Ok(())
}

pub async fn run_runner_logs(cmd: &RunnersLogsCmd) -> Result<(), String> {
    let limit = cmd.limit.to_string();
    let path = with_query(
        "/runner-jobs",
        &[
            ("organization_id", cmd.organization_id.as_deref()),
            ("runner_host_id", cmd.runner_host_id.as_deref()),
            ("runner_id", cmd.runner_id.as_deref()),
            ("daemon_id", cmd.daemon_id.as_deref()),
            ("status", cmd.status.as_deref()),
            ("phase", cmd.phase.as_deref()),
            ("limit", Some(limit.as_str())),
        ],
    );
    let jobs: RunnerJobListResponse = api_json(Method::GET, &path, &cmd.target, None).await?;

    println!("{}", "Recent Runner Jobs".bold());
    for job in jobs.jobs {
        println!(
            "{} {} {} {}",
            job.id.cyan(),
            job.job_kind.to_string().bold(),
            color_status(job.status.as_str()),
            job.phase
                .map(|phase| phase.to_string())
                .unwrap_or_else(|| "-".to_string())
        );
        if let Some(error_text) = job.error_text.as_deref().filter(|value| !value.is_empty()) {
            println!("  {}", error_text.red());
        }
    }
    Ok(())
}

pub async fn run_runner_agent(cmd: &RunnersAgentServeCmd) -> Result<(), String> {
    loop {
        let host = fetch_runner_host_by_id(&cmd.runner_host_id, &cmd.target).await?;
        let preflight = build_local_preflight(&host)?;
        let preflight_body = json!({
            "runner_host_id": host.id,
            "platform": preflight.platform,
            "status": preflight.status,
            "docker_available": preflight.docker_available,
            "service_manager": preflight.service_manager,
            "checks": preflight.checks,
            "checked_at": preflight.checked_at,
        });
        let _: RunnerHostPreflightRecord = api_json(
            Method::POST,
            "/runner-host-preflights",
            &cmd.target,
            Some(preflight_body),
        )
        .await?;
        let _: RunnerHostRecord = api_json(
            Method::POST,
            &format!("/runner-hosts/{}/heartbeat", host.id),
            &cmd.target,
            Some(json!({
                "status": "online",
                "capabilities": {
                    "service_manager": preflight.service_manager,
                    "docker_available": preflight.docker_available,
                }
            })),
        )
        .await?;

        let claim_response = api_json_status(
            Method::POST,
            "/runner-jobs/claim",
            &cmd.target,
            Some(json!({
                "runner_host_id": host.id,
                "locked_by": host.hostname,
            })),
        )
        .await?;

        if claim_response.0 == StatusCode::NO_CONTENT {
            if cmd.once {
                println!("No runner jobs were available.");
                return Ok(());
            }
            sleep(Duration::from_secs(cmd.interval_seconds)).await;
            continue;
        }

        let job: RunnerJobRecord = serde_json::from_value(claim_response.1)
            .map_err(|error| format!("Failed to decode claimed runner job: {}", error))?;
        println!(
            "{} {} {}",
            "Claimed".green().bold(),
            job.job_kind.to_string().bold(),
            job.id.cyan()
        );
        let _ = api_json::<RunnerJobRecord>(
            Method::PATCH,
            &format!("/runner-jobs/{}", job.id),
            &cmd.target,
            Some(json!({
                "status": "running",
                "phase": "preflight",
                "details": {},
            })),
        )
        .await?;

        match execute_local_runner_job(&host, &job).await {
            Ok(result) => {
                let _ = api_json::<RunnerJobRecord>(
                    Method::PATCH,
                    &format!("/runner-jobs/{}", job.id),
                    &cmd.target,
                    Some(json!({
                        "status": "succeeded",
                        "phase": result.install_phase,
                        "details": {
                            "runtime_root": result.runtime_root,
                            "service_manager": result.service_manager,
                            "service_status": result.service_status,
                            "registration_state": result.registration_state,
                            "installed_version": result.installed_version,
                            "generated_files": result.generated_files,
                        },
                        "error_text": null,
                    })),
                )
                .await?;
            }
            Err(error) => {
                let _ = api_json::<RunnerJobRecord>(
                    Method::PATCH,
                    &format!("/runner-jobs/{}", job.id),
                    &cmd.target,
                    Some(json!({
                        "status": "failed",
                        "phase": "failed",
                        "details": {},
                        "error_text": error,
                    })),
                )
                .await?;
            }
        }

        if cmd.once {
            return Ok(());
        }

        sleep(Duration::from_secs(cmd.interval_seconds)).await;
    }
}

pub fn enrich_runner_job_payload(
    user_payload: Option<&str>,
    job_kind: &str,
) -> Result<Option<String>, String> {
    let mut payload = user_payload
        .map(|value| serde_json::from_str::<Value>(value).map_err(|error| error.to_string()))
        .transpose()?
        .unwrap_or_else(|| json!({}));

    if !payload.is_object() {
        return Err("Runner payload JSON must be an object.".to_string());
    }

    let defaults = load_project_runner_defaults()?;
    let config = SshConfig::load().unwrap_or_else(|_| SshConfig::new());
    let runtime = defaults
        .as_ref()
        .and_then(|defaults| defaults.preferred_runtime)
        .or_else(|| {
            config
                .runners
                .as_ref()
                .and_then(|runners| runners.preferred_runtime)
        })
        .unwrap_or(RunnerExecutionRuntime::Native);

    let object = payload.as_object_mut().expect("runner payload object");
    object
        .entry("runtime".to_string())
        .or_insert_with(|| Value::String(runtime.as_str().to_string()));
    if let Some(defaults) = defaults.as_ref() {
        if let Some(group) = defaults.default_runner_group.as_deref() {
            object
                .entry("runner_group".to_string())
                .or_insert_with(|| Value::String(group.to_string()));
        }
        if let Some(org) = defaults.default_organization.as_deref() {
            object
                .entry("organization_login".to_string())
                .or_insert_with(|| Value::String(org.to_string()));
        }
        if !defaults.default_labels.is_empty() {
            object.entry("labels".to_string()).or_insert_with(|| {
                Value::Array(
                    defaults
                        .default_labels
                        .iter()
                        .map(|label| Value::String(label.clone()))
                        .collect(),
                )
            });
        }
    }
    object
        .entry("requested_action".to_string())
        .or_insert_with(|| Value::String(job_kind.to_string()));

    serde_json::to_string(&payload)
        .map(Some)
        .map_err(|error| format!("Failed to serialize runner payload: {}", error))
}

async fn execute_local_runner_job(
    host: &RunnerHostRecord,
    job: &RunnerJobRecord,
) -> Result<LocalRunnerJobResult, String> {
    let payload = parse_job_payload(job.payload.clone())?;
    let runtime_root = runner_runtime_root(host, &payload)?;
    fs::create_dir_all(&runtime_root).map_err(|error| {
        format!(
            "Failed to create runner runtime directory {}: {}",
            runtime_root.display(),
            error
        )
    })?;

    match job.job_kind.as_str() {
        "deploy" | "sync" => install_runner_runtime(host, job, &payload, &runtime_root).await,
        "remove" => remove_runner_runtime(host, job, &payload, &runtime_root).await,
        other => Err(format!("Unsupported runner job kind `{other}`")),
    }
}

async fn install_runner_runtime(
    host: &RunnerHostRecord,
    job: &RunnerJobRecord,
    payload: &LocalRunnerJobPayload,
    runtime_root: &Path,
) -> Result<LocalRunnerJobResult, String> {
    let runner_name = resolve_runner_name(host, payload);
    let service_manager = detect_service_manager(host.platform);
    let organization_login = payload
        .organization_login
        .as_deref()
        .filter(|value| !value.trim().is_empty())
        .ok_or_else(|| "Runner payload is missing `organization_login`.".to_string())?;
    let installation_id = payload
        .github_installation_id
        .as_deref()
        .filter(|value| !value.trim().is_empty())
        .ok_or_else(|| "Runner payload is missing `github_installation_id`.".to_string())?;
    let mut generated_files = Vec::new();
    let work_dir = runtime_root.join("work");
    let config_dir = runtime_root.join("config");
    let runner_dir = runtime_root.join("runner");
    let log_path = runtime_root.join("runner.log");
    let service_path = service_file_path(runtime_root, service_manager);
    let cleanup;

    fs::create_dir_all(&work_dir).map_err(|error| error.to_string())?;
    fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?;

    write_job_phase(
        job,
        RunnerInstallPhase::Preflight,
        Some(json!({ "runtime_root": runtime_root.display().to_string() })),
    )
    .await?;
    upsert_runner_runtime_record(
        host,
        &runner_name,
        payload,
        None,
        RunnerRegistrationState::Pending,
        RunnerInstallPhase::Preflight,
        service_manager,
        RunnerServiceStatus::Missing,
        runtime_root,
        &work_dir,
        &config_dir,
        &log_path,
        Some(&service_path),
        None,
        None,
    )
    .await?;

    let registration_token = request_runner_token(
        "/runner-github/registration-token",
        organization_login,
        installation_id,
    )
    .await?;
    let release = if payload.runtime == RunnerExecutionRuntime::Native {
        Some(
            resolve_runner_release(
                host.platform,
                host.arch.as_deref().unwrap_or("x64"),
                payload.runner_version.as_deref(),
            )
            .await?,
        )
    } else {
        None
    };

    if payload.runtime == RunnerExecutionRuntime::Native {
        let release = release.clone().expect("native runner release");
        write_job_phase(
            job,
            RunnerInstallPhase::Downloading,
            Some(json!({
                "version": release.version,
                "download_url": release.download_url,
            })),
        )
        .await?;
        let archive_path = download_runner_release(runtime_root, &release).await?;
        generated_files.push(archive_path.display().to_string());

        write_job_phase(
            job,
            RunnerInstallPhase::Extracting,
            Some(json!({
                "archive": archive_path.display().to_string(),
            })),
        )
        .await?;
        extract_runner_archive(&archive_path, &runner_dir)?;

        write_job_phase(job, RunnerInstallPhase::Cleanup, None).await?;
        cleanup = cleanup_stale_runner_registration(
            host,
            payload,
            runtime_root,
            &runner_dir,
            &config_dir,
            organization_login,
            installation_id,
            Some(&runner_name),
        )
        .await?;

        write_job_phase(job, RunnerInstallPhase::Registering, None).await?;
        configure_native_runner(
            host,
            payload,
            &runner_dir,
            &work_dir,
            organization_login,
            &registration_token.token,
            &runner_name,
        )?;

        write_job_phase(job, RunnerInstallPhase::InstallingService, None).await?;
        install_native_service(host.platform, &runner_dir)?;

        write_job_phase(job, RunnerInstallPhase::StartingService, None).await?;
        start_native_service(host.platform, &runner_dir)?;
        generated_files.push(runner_dir.display().to_string());
    } else {
        write_job_phase(job, RunnerInstallPhase::Cleanup, None).await?;
        cleanup = cleanup_stale_runner_registration(
            host,
            payload,
            runtime_root,
            &runner_dir,
            &config_dir,
            organization_login,
            installation_id,
            Some(&runner_name),
        )
        .await?;

        let compose_path = runtime_root.join("compose.yaml");
        fs::write(
            &compose_path,
            docker_compose_contents(
                host,
                payload,
                runtime_root,
                &runner_name,
                &registration_token.token,
                organization_login,
            ),
        )
        .map_err(|error| format!("Failed to write {}: {}", compose_path.display(), error))?;
        generated_files.push(compose_path.display().to_string());

        let dockerfile_path = runtime_root.join("Dockerfile");
        fs::write(&dockerfile_path, dockerfile_contents())
            .map_err(|error| format!("Failed to write {}: {}", dockerfile_path.display(), error))?;
        generated_files.push(dockerfile_path.display().to_string());

        write_job_phase(job, RunnerInstallPhase::StartingService, None).await?;
        run_command(
            "docker",
            &[
                "compose",
                "-f",
                &compose_path.display().to_string(),
                "up",
                "-d",
                "--build",
            ],
            Some(runtime_root),
        )?;
    }

    let github_runner =
        list_remote_github_runners(organization_login, installation_id, Some(&runner_name))
            .await?
            .into_iter()
            .next();
    let runtime_metadata = LocalRunnerRuntimeMetadata {
        runner_host_id: Some(host.id.clone()),
        hostname: Some(host.hostname.clone()),
        runner_name_prefix: Some(host.runner_name_prefix.clone()),
        runner_name: Some(runner_name.clone()),
        runner_group: payload.runner_group.clone(),
        organization_login: Some(organization_login.to_string()),
        github_installation_id: Some(installation_id.to_string()),
        github_runner_id: github_runner.as_ref().map(|runner| runner.id),
        removed_runner_names: cleanup.removed_runner_names.clone(),
        removed_runner_ids: cleanup.removed_runner_ids.clone(),
    };
    let runtime_state = RunnerRuntimeState {
        runtime: payload.runtime,
        service_manager,
        service_status: RunnerServiceStatus::Running,
        root_directory: Some(runtime_root.display().to_string()),
        work_directory: Some(work_dir.display().to_string()),
        config_directory: Some(config_dir.display().to_string()),
        log_path: Some(log_path.display().to_string()),
        service_file_path: Some(service_path.display().to_string()),
        last_cleanup_at: None,
        metadata: json!({
            "runner_name": runner_name,
            "runner_group": payload.runner_group,
            "labels": payload.labels,
            "cleanup_removed_runner_names": cleanup.removed_runner_names,
            "cleanup_removed_runner_ids": cleanup.removed_runner_ids,
        }),
    };

    let config_path = runtime_root.join("runner-config.json");
    fs::write(
        &config_path,
        serde_json::to_string_pretty(&payload)
            .map_err(|error| format!("Failed to serialize runner config: {}", error))?,
    )
    .map_err(|error| format!("Failed to write {}: {}", config_path.display(), error))?;
    generated_files.push(config_path.display().to_string());

    let state_path = runtime_root.join("runtime-state.json");
    fs::write(
        &state_path,
        serde_json::to_string_pretty(&runtime_state)
            .map_err(|error| format!("Failed to serialize runtime state: {}", error))?,
    )
    .map_err(|error| format!("Failed to write {}: {}", state_path.display(), error))?;
    generated_files.push(state_path.display().to_string());

    let metadata_path = write_runtime_metadata(runtime_root, &runtime_metadata)?;
    generated_files.push(metadata_path.display().to_string());

    let startup_path = runtime_root.join("startup-summary.txt");
    fs::write(
        &startup_path,
        format_startup_summary(host, payload, &runtime_state, &runner_name),
    )
    .map_err(|error| format!("Failed to write {}: {}", startup_path.display(), error))?;
    generated_files.push(startup_path.display().to_string());

    let installed_version = release.as_ref().map(|value| value.version.clone());
    write_job_phase(
        job,
        RunnerInstallPhase::Running,
        Some(json!({
            "runtime_root": runtime_root.display().to_string(),
            "runner_name": runner_name,
            "installed_version": installed_version,
        })),
    )
    .await?;
    upsert_runner_runtime_record(
        host,
        &runner_name,
        payload,
        github_runner.as_ref().map(|runner| runner.id),
        RunnerRegistrationState::Registered,
        RunnerInstallPhase::Running,
        service_manager,
        RunnerServiceStatus::Running,
        runtime_root,
        &work_dir,
        &config_dir,
        &log_path,
        Some(&service_path),
        installed_version.clone(),
        None,
    )
    .await?;

    Ok(LocalRunnerJobResult {
        runtime_root: runtime_root.display().to_string(),
        service_manager,
        service_status: RunnerServiceStatus::Running,
        install_phase: RunnerInstallPhase::Running,
        registration_state: RunnerRegistrationState::Registered,
        installed_version,
        generated_files,
    })
}

async fn remove_runner_runtime(
    host: &RunnerHostRecord,
    job: &RunnerJobRecord,
    payload: &LocalRunnerJobPayload,
    runtime_root: &Path,
) -> Result<LocalRunnerJobResult, String> {
    let runner_name = resolve_runner_name(host, payload);
    let service_manager = detect_service_manager(host.platform);
    let organization_login = payload
        .organization_login
        .as_deref()
        .filter(|value| !value.trim().is_empty())
        .ok_or_else(|| "Runner payload is missing `organization_login`.".to_string())?;
    let installation_id = payload
        .github_installation_id
        .as_deref()
        .filter(|value| !value.trim().is_empty())
        .ok_or_else(|| "Runner payload is missing `github_installation_id`.".to_string())?;
    let runner_dir = runtime_root.join("runner");
    let config_dir = runtime_root.join("config");

    write_job_phase(job, RunnerInstallPhase::Removing, None).await?;
    let cleanup = cleanup_stale_runner_registration(
        host,
        payload,
        runtime_root,
        &runner_dir,
        &config_dir,
        organization_login,
        installation_id,
        Some(&runner_name),
    )
    .await?;

    if payload.runtime == RunnerExecutionRuntime::Docker {
        let compose_path = runtime_root.join("compose.yaml");
        if compose_path.exists() {
            let _ = run_command(
                "docker",
                &[
                    "compose",
                    "-f",
                    &compose_path.display().to_string(),
                    "down",
                    "--remove-orphans",
                ],
                Some(runtime_root),
            );
        }
    } else if runner_dir.exists() {
        let _ = stop_native_service(host.platform, &runner_dir);
        let _ = uninstall_native_service(host.platform, &runner_dir);
    }

    if runtime_root.exists() {
        fs::remove_dir_all(runtime_root).map_err(|error| {
            format!(
                "Failed to remove runner runtime directory {}: {}",
                runtime_root.display(),
                error
            )
        })?;
    }
    Ok(LocalRunnerJobResult {
        runtime_root: runtime_root.display().to_string(),
        service_manager,
        service_status: RunnerServiceStatus::Missing,
        install_phase: RunnerInstallPhase::Removing,
        registration_state: RunnerRegistrationState::Unregistered,
        installed_version: None,
        generated_files: cleanup
            .removed_runner_names
            .iter()
            .zip(
                cleanup
                    .removed_runner_ids
                    .iter()
                    .map(Some)
                    .chain(std::iter::repeat(None)),
            )
            .map(|(name, id)| match id {
                Some(id) => format!("{name} ({id})"),
                None => name.clone(),
            })
            .collect(),
    })
}

fn build_local_preflight(host: &RunnerHostRecord) -> Result<LocalPreflight, String> {
    let runtime_root = runner_runtime_root(
        host,
        &LocalRunnerJobPayload {
            runtime: RunnerExecutionRuntime::Native,
            organization_login: None,
            github_installation_id: None,
            runner_group: None,
            labels: Vec::new(),
            runner_name: None,
            runner_version: None,
            ephemeral: false,
        },
    )?;
    fs::create_dir_all(&runtime_root).map_err(|error| error.to_string())?;
    let docker_available =
        command_available("docker", &["version", "--format", "{{.Server.Version}}"]);
    let service_manager = detect_service_manager(host.platform);
    let writable = runtime_root.exists();
    let checks = json!({
        "runtime_root": runtime_root.display().to_string(),
        "runtime_root_writable": writable,
        "docker_available": docker_available,
        "service_manager": service_manager.as_str(),
        "platform": host.platform.as_str(),
    });
    Ok(LocalPreflight {
        platform: host.platform,
        status: if writable {
            RunnerSyncState::Synced
        } else {
            RunnerSyncState::Error
        },
        docker_available: Some(docker_available),
        service_manager: Some(service_manager),
        checks,
        checked_at: Some(Utc::now().to_rfc3339()),
    })
}

#[derive(Debug)]
struct LocalPreflight {
    platform: RunnerPlatform,
    status: RunnerSyncState,
    docker_available: Option<bool>,
    service_manager: Option<RunnerServiceManager>,
    checks: Value,
    checked_at: Option<String>,
}

fn print_preflight(host: &RunnerHostRecord, preflight: &LocalPreflight) {
    println!("{}", "Runner Host Preflight".bold());
    println!("  host: {} ({})", host.hostname, host.id);
    println!("  platform: {}", host.platform);
    println!("  status: {}", color_status(preflight.status.as_str()));
    println!(
        "  service manager: {}",
        preflight
            .service_manager
            .map(|value| value.to_string())
            .unwrap_or_else(|| "unknown".to_string())
    );
    println!(
        "  docker available: {}",
        preflight
            .docker_available
            .map(|value| if value { "yes" } else { "no" })
            .unwrap_or("unknown")
    );
}

async fn fetch_runner_host_by_id(
    runner_host_id: &str,
    target: &ApiTargetOptions,
) -> Result<RunnerHostRecord, String> {
    let response: RunnerHostListResponse =
        api_json(Method::GET, "/runner-hosts", target, None).await?;
    response
        .runner_hosts
        .into_iter()
        .find(|host| host.id == runner_host_id)
        .ok_or_else(|| format!("Runner host `{runner_host_id}` was not found."))
}

async fn api_json<T: for<'de> Deserialize<'de>>(
    method: Method,
    path: &str,
    target: &ApiTargetOptions,
    body: Option<Value>,
) -> Result<T, String> {
    let (_, value) = api_json_status(method, path, target, body).await?;
    serde_json::from_value(value)
        .map_err(|error| format!("Failed to decode API response for `{}`: {}", path, error))
}

async fn api_json_status(
    method: Method,
    path: &str,
    target: &ApiTargetOptions,
    body: Option<Value>,
) -> Result<(StatusCode, Value), String> {
    let url = resolve_request_url(path, target)?;
    let client = Client::builder()
        .timeout(Duration::from_secs(60))
        .build()
        .map_err(|error| format!("Failed to create HTTP client: {}", error))?;
    let mut request = client.request(method, url);
    if !target.no_auth {
        if let Some(token) = resolve_xbp_api_token() {
            request = request.bearer_auth(token);
        }
        if let Some(github_token) = resolve_github_oauth2_key() {
            request = request.header("X-XBP-GitHub-Token", github_token);
        }
    }
    if let Some(body) = body {
        request = request.json(&body);
    }
    let response = request
        .send()
        .await
        .map_err(|error| format!("Runner API request failed: {}", error))?;
    let status = response.status();
    if status == StatusCode::NO_CONTENT {
        return Ok((status, Value::Null));
    }
    let value = response
        .json::<Value>()
        .await
        .map_err(|error| format!("Failed to decode runner API response: {}", error))?;
    if !status.is_success() {
        return Err(format!(
            "Runner API request failed with status {}: {}",
            status,
            value
                .get("error")
                .and_then(Value::as_str)
                .unwrap_or("unknown error")
        ));
    }
    Ok((status, value))
}

fn load_project_runner_defaults() -> Result<Option<GitHubRunnersProjectConfig>, String> {
    let current_dir =
        env::current_dir().map_err(|error| format!("Failed to read cwd: {}", error))?;
    let Some(found) = find_xbp_config_upwards(&current_dir) else {
        return Ok(None);
    };
    let content = fs::read_to_string(&found.config_path)
        .map_err(|error| format!("Failed to read {}: {}", found.config_path.display(), error))?;
    let (config, _) = parse_config_with_auto_heal::<XbpConfig>(&content, found.kind)
        .map_err(|error| format!("Failed to parse {}: {}", found.config_path.display(), error))?;
    Ok(config.github.and_then(|github| github.runners))
}

fn runner_runtime_root(
    host: &RunnerHostRecord,
    payload: &LocalRunnerJobPayload,
) -> Result<PathBuf, String> {
    let base = global_runner_root_dir()?;
    let organization = payload
        .organization_login
        .clone()
        .unwrap_or_else(|| host.organization_id.clone());
    Ok(base
        .join(slugify(&organization))
        .join(slugify(&host.hostname)))
}

fn runtime_metadata_path(runtime_root: &Path) -> PathBuf {
    runtime_root.join("runtime-metadata.json")
}

fn read_runtime_metadata(runtime_root: &Path) -> Option<LocalRunnerRuntimeMetadata> {
    let path = runtime_metadata_path(runtime_root);
    let content = fs::read_to_string(path).ok()?;
    serde_json::from_str(&content).ok()
}

fn write_runtime_metadata(
    runtime_root: &Path,
    metadata: &LocalRunnerRuntimeMetadata,
) -> Result<PathBuf, String> {
    let path = runtime_metadata_path(runtime_root);
    fs::write(
        &path,
        serde_json::to_string_pretty(metadata)
            .map_err(|error| format!("Failed to serialize runtime metadata: {}", error))?,
    )
    .map_err(|error| format!("Failed to write {}: {}", path.display(), error))?;
    Ok(path)
}

fn slugify(value: &str) -> String {
    value
        .chars()
        .map(|character| {
            if character.is_ascii_alphanumeric() {
                character.to_ascii_lowercase()
            } else {
                '-'
            }
        })
        .collect::<String>()
        .trim_matches('-')
        .to_string()
}

fn with_query(path: &str, params: &[(&str, Option<&str>)]) -> String {
    let mut url = reqwest::Url::parse(&format!("http://localhost{path}"))
        .expect("static runner path should always parse");
    {
        let mut pairs = url.query_pairs_mut();
        for (key, value) in params {
            if let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) {
                pairs.append_pair(key, value);
            }
        }
    }
    match url.query() {
        Some(query) if !query.is_empty() => format!("{}?{}", url.path(), query),
        _ => url.path().to_string(),
    }
}

fn parse_job_payload(payload: Value) -> Result<LocalRunnerJobPayload, String> {
    let mut payload = if payload.is_null() {
        json!({})
    } else {
        payload
    };
    if !payload.is_object() {
        return Err("Runner job payload must be an object.".to_string());
    }
    if payload.get("runtime").is_none() {
        payload["runtime"] = Value::String("native".to_string());
    }
    serde_json::from_value(payload)
        .map_err(|error| format!("Invalid runner job payload: {}", error))
}

async fn write_job_phase(
    job: &RunnerJobRecord,
    phase: RunnerInstallPhase,
    details: Option<Value>,
) -> Result<(), String> {
    let detail_value = details.unwrap_or_else(|| json!({}));
    let _: RunnerJobRecord = api_json(
        Method::PATCH,
        &format!("/runner-jobs/{}", job.id),
        &ApiTargetOptions::default(),
        Some(json!({
            "status": "running",
            "phase": phase,
            "details": detail_value,
        })),
    )
    .await?;
    Ok(())
}

async fn upsert_runner_runtime_record(
    host: &RunnerHostRecord,
    runner_name: &str,
    payload: &LocalRunnerJobPayload,
    github_runner_id: Option<i64>,
    registration_state: RunnerRegistrationState,
    install_phase: RunnerInstallPhase,
    service_manager: RunnerServiceManager,
    service_status: RunnerServiceStatus,
    runtime_root: &Path,
    work_dir: &Path,
    config_dir: &Path,
    log_path: &Path,
    service_path: Option<&Path>,
    installed_version: Option<String>,
    registration_error: Option<String>,
) -> Result<(), String> {
    let runtime_state = json!({
        "runtime": payload.runtime,
        "service_manager": service_manager,
        "service_status": service_status,
        "root_directory": runtime_root.display().to_string(),
        "work_directory": work_dir.display().to_string(),
        "config_directory": config_dir.display().to_string(),
        "log_path": log_path.display().to_string(),
        "service_file_path": service_path.map(|path| path.display().to_string()),
        "metadata": {
            "runner_name": runner_name,
        }
    });
    let desired_config = json!({
        "organization_login": payload.organization_login.clone(),
        "runner_group": payload.runner_group.clone(),
        "labels": payload.labels.clone(),
        "runtime": payload.runtime,
        "work_directory": work_dir.display().to_string(),
        "config_directory": config_dir.display().to_string(),
        "ephemeral": payload.ephemeral,
    });
    let _: Value = api_json(
        Method::POST,
        &format!("/organizations/{}/runners", host.organization_id),
        &ApiTargetOptions::default(),
        Some(json!({
            "organization_id": host.organization_id.clone(),
            "runner_host_id": host.id.clone(),
            "github_runner_id": github_runner_id,
            "name": runner_name,
            "platform": host.platform,
            "os": host.platform.as_str(),
            "architecture": host.arch.clone(),
            "status": if service_status == RunnerServiceStatus::Running {
                "synced"
            } else {
                "pending"
            },
            "busy": false,
            "labels": payload.labels.clone(),
            "desired_config": desired_config,
            "runtime_state": runtime_state,
            "registration_state": registration_state,
            "install_phase": install_phase,
            "installed_version": installed_version,
            "registration_error": registration_error,
            "metadata": {
                "organization_login": payload.organization_login.clone(),
                "github_installation_id": payload.github_installation_id.clone(),
            },
            "last_registration_attempt_at": Utc::now().to_rfc3339(),
            "last_seen_at": Utc::now().to_rfc3339(),
        })),
    )
    .await?;
    Ok(())
}

async fn request_runner_token(
    endpoint: &str,
    organization_login: &str,
    installation_id: &str,
) -> Result<RunnerApiTokenResponse, String> {
    api_json(
        Method::POST,
        endpoint,
        &ApiTargetOptions::default(),
        Some(json!({
            "organization_login": organization_login,
            "github_installation_id": installation_id,
        })),
    )
    .await
}

async fn resolve_runner_release(
    platform: RunnerPlatform,
    arch: &str,
    version: Option<&str>,
) -> Result<RunnerApiReleaseResponse, String> {
    let mut path = format!(
        "/runner-github/releases/latest?platform={}&arch={}",
        platform.as_str(),
        arch
    );
    if let Some(version) = version.map(str::trim).filter(|value| !value.is_empty()) {
        path.push_str("&version=");
        path.push_str(version);
    }
    api_json(Method::GET, &path, &ApiTargetOptions::default(), None).await
}

async fn download_runner_release(
    runtime_root: &Path,
    release: &RunnerApiReleaseResponse,
) -> Result<PathBuf, String> {
    let downloads_dir = runtime_root.join("downloads");
    fs::create_dir_all(&downloads_dir).map_err(|error| error.to_string())?;
    let archive_path = downloads_dir.join(&release.file_name);
    let client = Client::builder()
        .timeout(Duration::from_secs(300))
        .build()
        .map_err(|error| format!("Failed to create download client: {}", error))?;
    let bytes = client
        .get(&release.download_url)
        .send()
        .await
        .map_err(|error| format!("Failed to download runner release: {}", error))?
        .error_for_status()
        .map_err(|error| format!("Runner release download failed: {}", error))?
        .bytes()
        .await
        .map_err(|error| format!("Failed to read runner release bytes: {}", error))?;
    fs::write(&archive_path, &bytes)
        .map_err(|error| format!("Failed to write {}: {}", archive_path.display(), error))?;
    Ok(archive_path)
}

fn extract_runner_archive(archive_path: &Path, runner_dir: &Path) -> Result<(), String> {
    if runner_dir.exists() {
        fs::remove_dir_all(runner_dir)
            .map_err(|error| format!("Failed to clear {}: {}", runner_dir.display(), error))?;
    }
    fs::create_dir_all(runner_dir).map_err(|error| error.to_string())?;

    let file_name = archive_path
        .file_name()
        .and_then(|value| value.to_str())
        .unwrap_or_default()
        .to_ascii_lowercase();
    if file_name.ends_with(".tar.gz") {
        let archive_file = fs::File::open(archive_path)
            .map_err(|error| format!("Failed to open archive: {}", error))?;
        let mut archive = Archive::new(GzDecoder::new(archive_file));
        archive
            .unpack(runner_dir)
            .map_err(|error| format!("Failed to unpack tar.gz archive: {}", error))?;
    } else if file_name.ends_with(".zip") {
        let bytes =
            fs::read(archive_path).map_err(|error| format!("Failed to read archive: {}", error))?;
        let reader = Cursor::new(bytes);
        let mut archive = ZipArchive::new(reader)
            .map_err(|error| format!("Failed to open zip archive: {}", error))?;
        for index in 0..archive.len() {
            let mut entry = archive
                .by_index(index)
                .map_err(|error| format!("Failed to read zip entry: {}", error))?;
            let output_path = runner_dir.join(entry.mangled_name());
            if entry.name().ends_with('/') {
                fs::create_dir_all(&output_path).map_err(|error| error.to_string())?;
                continue;
            }
            if let Some(parent) = output_path.parent() {
                fs::create_dir_all(parent).map_err(|error| error.to_string())?;
            }
            let mut output = fs::File::create(&output_path)
                .map_err(|error| format!("Failed to create file: {}", error))?;
            io::copy(&mut entry, &mut output)
                .map_err(|error| format!("Failed to extract zip entry: {}", error))?;
        }
    } else {
        return Err(format!(
            "Unsupported runner archive format for {}",
            archive_path.display()
        ));
    }
    ensure_executable(runner_dir.join("config.sh"))?;
    ensure_executable(runner_dir.join("svc.sh"))?;
    ensure_executable(runner_dir.join("run.sh"))?;
    Ok(())
}

async fn cleanup_stale_runner_registration(
    host: &RunnerHostRecord,
    payload: &LocalRunnerJobPayload,
    runtime_root: &Path,
    runner_dir: &Path,
    config_dir: &Path,
    organization_login: &str,
    installation_id: &str,
    fallback_runner_name: Option<&str>,
) -> Result<RunnerCleanupSummary, String> {
    if payload.runtime == RunnerExecutionRuntime::Docker {
        let compose_path = runtime_root.join("compose.yaml");
        if compose_path.exists() {
            let _ = run_command(
                "docker",
                &[
                    "compose",
                    "-f",
                    &compose_path.display().to_string(),
                    "down",
                    "--remove-orphans",
                ],
                Some(runtime_root),
            );
        }
    } else if runner_dir.exists() {
        let _ = stop_native_service(host.platform, runner_dir);
    }

    let runner_name = read_configured_runner_name(config_dir)
        .or_else(|| read_configured_runner_name(runner_dir))
        .or_else(|| read_runtime_metadata(runtime_root).and_then(|metadata| metadata.runner_name))
        .or_else(|| fallback_runner_name.map(ToString::to_string));
    let prior_metadata = read_runtime_metadata(runtime_root);
    let mut summary = RunnerCleanupSummary::default();
    let same_machine_runners =
        list_remote_github_runners(organization_login, installation_id, None)
            .await?
            .into_iter()
            .filter(|runner| {
                runner_matches_same_machine(host, payload, prior_metadata.as_ref(), runner)
            })
            .collect::<Vec<_>>();

    if let Some(name) = runner_name.as_deref() {
        if payload.runtime == RunnerExecutionRuntime::Native {
            if let Ok(remove_token) = request_runner_token(
                "/runner-github/remove-token",
                organization_login,
                installation_id,
            )
            .await
            {
                let remove_script = if cfg!(windows) {
                    runner_dir.join("config.cmd")
                } else {
                    runner_dir.join("config.sh")
                };
                if remove_script.exists() {
                    let remove_args = if cfg!(windows) {
                        vec![
                            "/c".to_string(),
                            remove_script.display().to_string(),
                            "remove".to_string(),
                            "--token".to_string(),
                            remove_token.token,
                        ]
                    } else {
                        vec![
                            remove_script.display().to_string(),
                            "remove".to_string(),
                            "--token".to_string(),
                            remove_token.token,
                        ]
                    };
                    let executable = if cfg!(windows) { "cmd" } else { "bash" };
                    let _ = run_command(
                        executable,
                        &remove_args.iter().map(String::as_str).collect::<Vec<_>>(),
                        Some(runner_dir),
                    );
                }
            }
        }

        if let Some(remote_runner) =
            list_remote_github_runners(organization_login, installation_id, Some(name))
                .await?
                .into_iter()
                .next()
        {
            delete_remote_runner(organization_login, installation_id, remote_runner.id).await?;
            summary.removed_runner_names.push(name.to_string());
            summary.removed_runner_ids.push(remote_runner.id);
        }
    }

    for remote_runner in same_machine_runners {
        if summary.removed_runner_ids.contains(&remote_runner.id) {
            continue;
        }
        delete_remote_runner(organization_login, installation_id, remote_runner.id).await?;
        summary.removed_runner_names.push(remote_runner.name);
        summary.removed_runner_ids.push(remote_runner.id);
    }

    if config_dir.exists() {
        let _ = fs::remove_dir_all(config_dir);
        let _ = fs::create_dir_all(config_dir);
    }
    let local_runner_file = runner_dir.join(".runner");
    if local_runner_file.exists() {
        let _ = fs::remove_file(local_runner_file);
    }
    Ok(summary)
}

fn runner_matches_same_machine(
    host: &RunnerHostRecord,
    payload: &LocalRunnerJobPayload,
    metadata: Option<&LocalRunnerRuntimeMetadata>,
    runner: &RunnerGitHubRunner,
) -> bool {
    let runner_name = runner.name.trim().to_ascii_lowercase();
    let host_prefix = slugify(&host.runner_name_prefix);
    let host_name = slugify(&host.hostname);
    let expected_name = resolve_runner_name(host, payload).to_ascii_lowercase();

    if runner_name == expected_name {
        return true;
    }
    if runner_name.starts_with(&format!("{host_prefix}-")) && runner_name.contains(&host_name) {
        return true;
    }
    metadata
        .and_then(|value| value.runner_name_prefix.as_deref().map(slugify))
        .is_some_and(|prefix| runner_name.starts_with(&format!("{prefix}-")))
}

fn configure_native_runner(
    host: &RunnerHostRecord,
    payload: &LocalRunnerJobPayload,
    runner_dir: &Path,
    work_dir: &Path,
    organization_login: &str,
    registration_token: &str,
    runner_name: &str,
) -> Result<(), String> {
    let github_url = format!("https://github.com/{}", organization_login);
    if cfg!(windows) {
        let config_cmd = runner_dir.join("config.cmd");
        let mut args = vec![
            "/c".to_string(),
            config_cmd.display().to_string(),
            "--unattended".to_string(),
            "--url".to_string(),
            github_url,
            "--token".to_string(),
            registration_token.to_string(),
            "--name".to_string(),
            runner_name.to_string(),
            "--work".to_string(),
            work_dir.display().to_string(),
            "--replace".to_string(),
            "--runasservice".to_string(),
        ];
        if let Some(group) = payload
            .runner_group
            .as_deref()
            .filter(|value| !value.trim().is_empty())
        {
            args.push("--runnergroup".to_string());
            args.push(group.to_string());
        }
        if !payload.labels.is_empty() {
            args.push("--labels".to_string());
            args.push(payload.labels.join(","));
        }
        if payload.ephemeral {
            args.push("--ephemeral".to_string());
        }
        run_command(
            "cmd",
            &args.iter().map(String::as_str).collect::<Vec<_>>(),
            Some(runner_dir),
        )
    } else {
        let config_sh = runner_dir.join("config.sh");
        let mut args = vec![
            config_sh.display().to_string(),
            "--unattended".to_string(),
            "--url".to_string(),
            github_url,
            "--token".to_string(),
            registration_token.to_string(),
            "--name".to_string(),
            runner_name.to_string(),
            "--work".to_string(),
            work_dir.display().to_string(),
            "--replace".to_string(),
        ];
        if let Some(group) = payload
            .runner_group
            .as_deref()
            .filter(|value| !value.trim().is_empty())
        {
            args.push("--runnergroup".to_string());
            args.push(group.to_string());
        }
        if !payload.labels.is_empty() {
            args.push("--labels".to_string());
            args.push(payload.labels.join(","));
        }
        if payload.ephemeral {
            args.push("--ephemeral".to_string());
        }
        run_command(
            "bash",
            &args.iter().map(String::as_str).collect::<Vec<_>>(),
            Some(runner_dir),
        )
    }?;

    let _ = host;
    Ok(())
}

fn install_native_service(platform: RunnerPlatform, runner_dir: &Path) -> Result<(), String> {
    if cfg!(windows) {
        return Ok(());
    }
    let svc = runner_dir.join("svc.sh");
    if !svc.exists() {
        return Err(format!("Missing {}", svc.display()));
    }
    if matches!(platform, RunnerPlatform::Linux | RunnerPlatform::Macos) {
        run_command(
            "bash",
            &[&svc.display().to_string(), "install"],
            Some(runner_dir),
        )
    } else {
        Ok(())
    }
}

fn start_native_service(platform: RunnerPlatform, runner_dir: &Path) -> Result<(), String> {
    if cfg!(windows) {
        return Ok(());
    }
    let svc = runner_dir.join("svc.sh");
    if matches!(platform, RunnerPlatform::Linux | RunnerPlatform::Macos) {
        run_command(
            "bash",
            &[&svc.display().to_string(), "start"],
            Some(runner_dir),
        )
    } else {
        Ok(())
    }
}

fn stop_native_service(platform: RunnerPlatform, runner_dir: &Path) -> Result<(), String> {
    if cfg!(windows) {
        return Ok(());
    }
    let svc = runner_dir.join("svc.sh");
    if svc.exists() && matches!(platform, RunnerPlatform::Linux | RunnerPlatform::Macos) {
        run_command(
            "bash",
            &[&svc.display().to_string(), "stop"],
            Some(runner_dir),
        )
    } else {
        Ok(())
    }
}

fn uninstall_native_service(platform: RunnerPlatform, runner_dir: &Path) -> Result<(), String> {
    if cfg!(windows) {
        return Ok(());
    }
    let svc = runner_dir.join("svc.sh");
    if svc.exists() && matches!(platform, RunnerPlatform::Linux | RunnerPlatform::Macos) {
        run_command(
            "bash",
            &[&svc.display().to_string(), "uninstall"],
            Some(runner_dir),
        )
    } else {
        Ok(())
    }
}

async fn list_remote_github_runners(
    organization_login: &str,
    installation_id: &str,
    name: Option<&str>,
) -> Result<Vec<RunnerGitHubRunner>, String> {
    let mut path = format!(
        "/runner-github/orgs/{}/runners?github_installation_id={}",
        organization_login, installation_id
    );
    if let Some(name) = name.map(str::trim).filter(|value| !value.is_empty()) {
        path.push_str("&name=");
        path.push_str(name);
    }
    let response: RunnerGitHubListResponse =
        api_json(Method::GET, &path, &ApiTargetOptions::default(), None).await?;
    Ok(response.runners)
}

async fn delete_remote_runner(
    organization_login: &str,
    installation_id: &str,
    runner_id: i64,
) -> Result<(), String> {
    let _ = api_json_status(
        Method::DELETE,
        &format!(
            "/runner-github/orgs/{}/runners/{}?github_installation_id={}",
            organization_login, runner_id, installation_id
        ),
        &ApiTargetOptions::default(),
        None,
    )
    .await?;
    Ok(())
}

fn run_command(executable: &str, args: &[&str], cwd: Option<&Path>) -> Result<(), String> {
    let mut command = Command::new(executable);
    command.args(args);
    if let Some(cwd) = cwd {
        command.current_dir(cwd);
    }
    command.stdout(Stdio::piped()).stderr(Stdio::piped());
    let output = command
        .output()
        .map_err(|error| format!("Failed to start `{}`: {}", executable, error))?;
    if output.status.success() {
        return Ok(());
    }
    Err(format!(
        "`{}` failed with status {}: {}",
        executable,
        output
            .status
            .code()
            .map(|value| value.to_string())
            .unwrap_or_else(|| "unknown".to_string()),
        String::from_utf8_lossy(&output.stderr).trim()
    ))
}

fn ensure_executable(path: PathBuf) -> Result<(), String> {
    if !path.exists() || cfg!(windows) {
        return Ok(());
    }
    let mut permissions = fs::metadata(&path)
        .map_err(|error| format!("Failed to read metadata for {}: {}", path.display(), error))?
        .permissions();
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        permissions.set_mode(0o755);
        fs::set_permissions(&path, permissions).map_err(|error| {
            format!(
                "Failed to set executable permissions on {}: {}",
                path.display(),
                error
            )
        })?;
    }
    Ok(())
}

fn read_configured_runner_name(root: &Path) -> Option<String> {
    let runner_file = root.join(".runner");
    let content = fs::read_to_string(runner_file).ok()?;
    let payload: Value = serde_json::from_str(&content).ok()?;
    payload
        .get("agentName")
        .and_then(Value::as_str)
        .map(ToString::to_string)
}

fn resolve_runner_name(host: &RunnerHostRecord, payload: &LocalRunnerJobPayload) -> String {
    payload
        .runner_name
        .clone()
        .filter(|value| !value.trim().is_empty())
        .unwrap_or_else(|| {
            format!(
                "{}-{}",
                slugify(&host.runner_name_prefix),
                slugify(&host.hostname)
            )
        })
}

fn command_available(executable: &str, args: &[&str]) -> bool {
    Command::new(executable)
        .args(args)
        .output()
        .map(|output| output.status.success())
        .unwrap_or(false)
}

fn detect_service_manager(platform: RunnerPlatform) -> RunnerServiceManager {
    match platform {
        RunnerPlatform::Linux => {
            if command_available("systemctl", &["--version"]) {
                RunnerServiceManager::Systemd
            } else {
                RunnerServiceManager::Unknown
            }
        }
        RunnerPlatform::Macos => {
            if command_available("launchctl", &["list"]) {
                RunnerServiceManager::Launchd
            } else {
                RunnerServiceManager::Unknown
            }
        }
        RunnerPlatform::Windows => RunnerServiceManager::WindowsService,
    }
}

fn service_file_path(root: &Path, service_manager: RunnerServiceManager) -> PathBuf {
    match service_manager {
        RunnerServiceManager::Systemd => root.join("systemd").join("xbp-github-runner.service"),
        RunnerServiceManager::Launchd => root.join("launchd").join("com.xbp.github-runner.plist"),
        RunnerServiceManager::WindowsService => {
            root.join("windows").join("install-runner-service.ps1")
        }
        RunnerServiceManager::DockerCompose => root.join("compose.yaml"),
        RunnerServiceManager::Unknown => root.join("runner-service.txt"),
    }
}

fn dockerfile_contents() -> &'static str {
    "FROM myoung34/github-runner:latest\n\n# Generated by xbp runners agent.\n"
}

fn docker_compose_contents(
    _host: &RunnerHostRecord,
    payload: &LocalRunnerJobPayload,
    runtime_root: &Path,
    runner_name: &str,
    registration_token: &str,
    organization_login: &str,
) -> String {
    format!(
        "services:\n  github-runner:\n    build: .\n    image: xbp-github-runner:latest\n    restart: unless-stopped\n    environment:\n      ORG_NAME: \"{org}\"\n      RUNNER_NAME: \"{runner_name}\"\n      RUNNER_SCOPE: \"org\"\n      RUNNER_GROUP: \"{group}\"\n      LABELS: \"{labels}\"\n      ACCESS_TOKEN: \"{token}\"\n      EPHEMERAL: \"{ephemeral}\"\n    volumes:\n      - \"{root}/work:/tmp/github-runner\"\n      - \"{root}/config:/runner/config\"\n",
        org = organization_login,
        runner_name = runner_name,
        group = payload.runner_group.clone().unwrap_or_else(|| "Default".to_string()),
        labels = payload.labels.join(","),
        token = registration_token,
        ephemeral = if payload.ephemeral { "true" } else { "false" },
        root = runtime_root.display(),
    )
}

#[allow(dead_code)]
fn native_service_contents(
    host: &RunnerHostRecord,
    payload: &LocalRunnerJobPayload,
    runtime_root: &Path,
    service_manager: RunnerServiceManager,
) -> String {
    match service_manager {
        RunnerServiceManager::Systemd => format!(
            "[Unit]\nDescription=XBP GitHub Runner ({host})\n\n[Service]\nWorkingDirectory={root}\nExecStart=/bin/echo xbp native runner placeholder for {group}\nRestart=always\n\n[Install]\nWantedBy=multi-user.target\n",
            host = host.hostname,
            root = runtime_root.display(),
            group = payload.runner_group.clone().unwrap_or_else(|| "Default".to_string()),
        ),
        RunnerServiceManager::Launchd => format!(
            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<plist version=\"1.0\"><dict><key>Label</key><string>com.xbp.github-runner</string><key>ProgramArguments</key><array><string>/bin/echo</string><string>xbp native runner placeholder</string></array><key>WorkingDirectory</key><string>{}</string></dict></plist>\n",
            runtime_root.display()
        ),
        RunnerServiceManager::WindowsService => format!(
            "Write-Host \"Install GitHub runner service for {} in {}\"\n",
            host.hostname,
            runtime_root.display()
        ),
        RunnerServiceManager::DockerCompose => docker_compose_contents(
            host,
            payload,
            runtime_root,
            "github-runner",
            "registration-token",
            payload
                .organization_login
                .as_deref()
                .unwrap_or(&host.organization_id),
        ),
        RunnerServiceManager::Unknown => "XBP runner service manager could not be detected on this host.\n".to_string(),
    }
}

fn format_startup_summary(
    host: &RunnerHostRecord,
    payload: &LocalRunnerJobPayload,
    runtime_state: &RunnerRuntimeState,
    runner_name: &str,
) -> String {
    format!(
        "Runner host: {host}\nRunner name: {runner_name}\nRuntime: {runtime}\nService manager: {manager}\nService status: {status}\nRunner group: {group}\nLabels: {labels}\n",
        host = host.hostname,
        runner_name = runner_name,
        runtime = payload.runtime,
        manager = runtime_state.service_manager,
        status = runtime_state.service_status,
        group = payload.runner_group.clone().unwrap_or_else(|| "Default".to_string()),
        labels = if payload.labels.is_empty() {
            "none".to_string()
        } else {
            payload.labels.join(", ")
        }
    )
}

fn color_status(status: &str) -> String {
    match status.trim().to_ascii_lowercase().as_str() {
        "online" | "running" | "succeeded" | "synced" | "allowed" => status.green().to_string(),
        "queued" | "claimed" | "enrolled" | "installed" => status.yellow().to_string(),
        "offline" | "failed" | "error" | "cancelled" => status.red().to_string(),
        _ => status.normal().to_string(),
    }
}

fn print_table(headers: &[&str], rows: &[Vec<String>]) {
    let mut widths = headers
        .iter()
        .map(|header| header.len())
        .collect::<Vec<_>>();
    for row in rows {
        for (index, cell) in row.iter().enumerate() {
            widths[index] = widths[index].max(strip_ansi(cell).len());
        }
    }

    let header_line = headers
        .iter()
        .enumerate()
        .map(|(index, header)| format!("{:width$}", header.bold(), width = widths[index]))
        .collect::<Vec<_>>()
        .join(" | ");
    println!("{}", header_line);
    println!(
        "{}",
        widths
            .iter()
            .map(|width| "-".repeat(*width))
            .collect::<Vec<_>>()
            .join("-+-")
    );
    for row in rows {
        let rendered = row
            .iter()
            .enumerate()
            .map(|(index, cell)| pad_ansi(cell, widths[index]))
            .collect::<Vec<_>>()
            .join(" | ");
        println!("{}", rendered);
    }
}

fn strip_ansi(value: &str) -> String {
    let mut out = String::with_capacity(value.len());
    let mut chars = value.chars().peekable();
    while let Some(ch) = chars.next() {
        if ch == '\u{1b}' && chars.peek() == Some(&'[') {
            let _ = chars.next();
            for next in chars.by_ref() {
                if ('@'..='~').contains(&next) {
                    break;
                }
            }
            continue;
        }
        out.push(ch);
    }
    out
}

fn pad_ansi(value: &str, width: usize) -> String {
    let visible = strip_ansi(value).len();
    if visible >= width {
        value.to_string()
    } else {
        format!("{value}{:pad$}", "", pad = width - visible)
    }
}

fn read_json_string(value: &Value, key: &str) -> String {
    value
        .get(key)
        .and_then(Value::as_str)
        .unwrap_or("-")
        .to_string()
}