rch 2.1.12

//! Rollback management for fleet deployments.
//!
//! Handles reverting workers to previous versions when deployments fail.
//!
//! ## Backup Registry
//!
//! The backup registry tracks all worker backups in a local JSON file at
//! `~/.local/share/rch/backups/registry.json`. Each backup entry includes:
//! - Worker ID and version
//! - SHA256 hash of the binary for verification
//! - Remote path where the backup is stored on the worker
//! - Timestamp of when the backup was created
//!
//! The registry uses atomic writes with file locking to handle concurrent access.

use crate::fleet::history::HistoryManager;
use crate::fleet::ssh::SshExecutor;
use crate::state::lock::ConfigLock;
use crate::ui::context::OutputContext;
use crate::ui::theme::StatusIndicator;
use anyhow::{Context, Result};

use crate::error::FleetError;
use futures::stream::{self, StreamExt};
use rch_common::WorkerConfig;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::{Read, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tracing::{debug, error, info, warn};

// =============================================================================
// Constants
// =============================================================================

/// Default remote path to the rch-wkr binary on workers.
#[allow(dead_code)] // Used by planned rollback commands
pub const REMOTE_RCH_PATH: &str = "~/.local/bin/rch-wkr";

/// Default remote backup directory on workers.
pub const REMOTE_BACKUP_DIR: &str = "~/.rch/backups";

/// Maximum number of backups to keep per worker.
#[allow(dead_code)] // Enforced when backup retention is implemented
pub const MAX_BACKUPS_PER_WORKER: usize = 3;

/// Registry file name.
const REGISTRY_FILE: &str = "registry.json";

/// Timeout for rollback operations.
#[allow(dead_code)] // Reserved for future timeout handling
const ROLLBACK_TIMEOUT: Duration = Duration::from_secs(60);

/// Timeout for acquiring the backup registry lock.
const REGISTRY_LOCK_TIMEOUT: Duration = Duration::from_secs(30);

/// Maximum concurrent rollback operations (cap to prevent resource exhaustion).
const MAX_CONCURRENT_ROLLBACKS: usize = 10;

/// Backup information for a worker's previous state.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerBackup {
    /// Worker identifier.
    pub worker_id: String,
    /// Version that was backed up.
    pub version: String,
    /// Local path to backup metadata.
    pub backup_path: PathBuf,
    /// Remote path where the backup binary is stored on the worker.
    pub remote_path: PathBuf,
    /// SHA256 hash of the backed up binary for verification.
    /// Set to "unknown" if hash could not be computed.
    pub binary_hash: String,
    /// When the backup was created (RFC 3339 format).
    pub created_at: String,
}

/// Result of a rollback operation.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RollbackResult {
    /// Worker that was rolled back.
    pub worker_id: String,
    /// Whether rollback succeeded.
    pub success: bool,
    /// Version rolled back to.
    pub rolled_back_to: Option<String>,
    /// Error message if failed.
    pub error: Option<String>,
}

/// Manages rollback operations for workers.
pub struct RollbackManager {
    history: HistoryManager,
    backup_dir: PathBuf,
}

impl RollbackManager {
    /// Create a new rollback manager.
    pub fn new() -> Result<Self> {
        let backup_dir = dirs::data_local_dir()
            .unwrap_or_else(|| PathBuf::from("."))
            .join("rch")
            .join("backups");

        std::fs::create_dir_all(&backup_dir)?;

        Ok(Self {
            history: HistoryManager::new()?,
            backup_dir,
        })
    }

    /// Create a rollback manager with a custom backup directory (for testing).
    /// The deploy/provenance history is isolated to a `history/` subdir so tests
    /// stay hermetic — the rollback-audit emission (bd-...-7.4) never touches the
    /// real provenance trail.
    #[cfg(test)]
    pub fn with_path(backup_dir: &std::path::Path) -> Result<Self> {
        std::fs::create_dir_all(backup_dir)?;
        Ok(Self {
            history: HistoryManager::with_dir(backup_dir.join("history"))?,
            backup_dir: backup_dir.to_path_buf(),
        })
    }

    /// Get the path to the registry file.
    fn registry_path(&self) -> PathBuf {
        self.backup_dir.join(REGISTRY_FILE)
    }

    /// Load the backup registry from disk.
    ///
    /// Returns an empty vector if the registry doesn't exist yet.
    /// Handles corrupted files by backing them up and returning empty.
    pub fn load_registry(&self) -> Result<Vec<WorkerBackup>> {
        let path = self.registry_path();
        if !path.exists() {
            debug!("Registry file does not exist, returning empty");
            return Ok(Vec::new());
        }

        let mut file = match std::fs::File::open(&path) {
            Ok(f) => f,
            Err(e) => {
                warn!(error = %e, "Failed to open registry file");
                return Ok(Vec::new());
            }
        };

        let mut content = String::new();
        if let Err(e) = file.read_to_string(&mut content) {
            warn!(error = %e, "Failed to read registry file");
            return Ok(Vec::new());
        }

        match serde_json::from_str::<Vec<WorkerBackup>>(&content) {
            Ok(registry) => {
                debug!(count = registry.len(), "Loaded backup registry");
                Ok(registry)
            }
            Err(e) => {
                error!(error = %e, "Corrupted backup registry, backing up and recreating");
                // Backup the corrupted file
                let backup_path = path.with_extension("json.corrupted");
                if let Err(rename_err) = std::fs::rename(&path, &backup_path) {
                    warn!(error = %rename_err, "Failed to backup corrupted registry");
                } else {
                    info!(path = %backup_path.display(), "Corrupted registry backed up");
                }
                Ok(Vec::new())
            }
        }
    }

    /// Save a backup entry to the registry.
    ///
    /// Uses atomic writes to prevent corruption from concurrent access.
    pub fn save_backup_entry(&self, backup: &WorkerBackup) -> Result<()> {
        let _lock = ConfigLock::acquire_with_timeout(
            "backup-registry",
            REGISTRY_LOCK_TIMEOUT,
            "save backup registry",
        )?;
        let path = self.registry_path();
        let temp_path = path.with_extension("json.tmp");

        // Load existing registry
        let mut registry = self.load_registry().unwrap_or_default();

        // Add the new backup
        registry.push(backup.clone());

        // Write atomically via temp file
        let content =
            serde_json::to_string_pretty(&registry).context("Failed to serialize registry")?;

        let mut file =
            std::fs::File::create(&temp_path).context("Failed to create temp registry file")?;
        file.write_all(content.as_bytes())
            .context("Failed to write registry")?;
        file.sync_all().context("Failed to sync registry file")?;
        drop(file);

        // Atomic rename
        std::fs::rename(&temp_path, &path)
            .context("Failed to rename temp registry to final path")?;

        info!(
            worker = %backup.worker_id,
            version = %backup.version,
            "Backup entry saved to registry"
        );

        Ok(())
    }

    /// Get the latest backup for a worker.
    pub fn get_latest_backup(&self, worker_id: &str) -> Option<WorkerBackup> {
        let registry = self.load_registry().ok()?;

        registry
            .into_iter()
            .filter(|b| b.worker_id == worker_id)
            .max_by(|a, b| a.created_at.cmp(&b.created_at))
    }

    /// Get a specific version's backup for a worker.
    pub fn get_backup(&self, worker_id: &str, version: &str) -> Option<WorkerBackup> {
        let registry = self.load_registry().ok()?;

        registry
            .into_iter()
            .find(|b| b.worker_id == worker_id && b.version == version)
    }

    /// Prune old backups, keeping only the most recent `max_per_worker` for each worker.
    ///
    /// Returns the list of removed backups so the caller can clean up remote files.
    pub fn prune_old_backups(&mut self, max_per_worker: usize) -> Result<Vec<WorkerBackup>> {
        let _lock = ConfigLock::acquire_with_timeout(
            "backup-registry",
            REGISTRY_LOCK_TIMEOUT,
            "prune backup registry",
        )?;
        let path = self.registry_path();
        let registry = self.load_registry()?;

        // Group by worker_id
        let mut by_worker: HashMap<String, Vec<WorkerBackup>> = HashMap::new();
        for backup in registry {
            by_worker
                .entry(backup.worker_id.clone())
                .or_default()
                .push(backup);
        }

        // Sort each worker's backups by created_at (newest first) and collect removed
        let mut kept = Vec::new();
        let mut removed = Vec::new();

        for (worker_id, mut backups) in by_worker {
            backups.sort_by(|a, b| b.created_at.cmp(&a.created_at));

            for (i, backup) in backups.into_iter().enumerate() {
                if i < max_per_worker {
                    kept.push(backup);
                } else {
                    debug!(
                        worker = %worker_id,
                        version = %backup.version,
                        "Pruning old backup"
                    );
                    removed.push(backup);
                }
            }
        }

        // Write the pruned registry
        if !removed.is_empty() {
            let temp_path = path.with_extension("json.tmp");
            let content = serde_json::to_string_pretty(&kept)?;
            let mut file = std::fs::File::create(&temp_path)?;
            file.write_all(content.as_bytes())?;
            file.sync_all()?;
            drop(file);
            std::fs::rename(&temp_path, &path)?;

            info!(
                removed = removed.len(),
                kept = kept.len(),
                "Pruned old backups from registry"
            );
        }

        Ok(removed)
    }

    /// Get the backup directory path.
    pub fn backup_dir(&self) -> &PathBuf {
        &self.backup_dir
    }

    /// Rollback workers to a specific or previous version.
    ///
    /// This function restores workers to a previous binary version by:
    /// 1. Looking up the backup for each worker (specific version or latest)
    /// 2. Gracefully stopping any running rch-wkr process
    /// 3. Copying the backup binary back to the main rch-wkr location via SSH
    /// 4. Optionally verifying the restored binary hash matches the backup record
    /// 5. Verifying the version after restoration
    /// 6. Restarting the rch-wkr service
    ///
    /// # Arguments
    ///
    /// * `workers` - List of workers to roll back
    /// * `to_version` - Optional specific version to roll back to (uses previous if None)
    /// * `parallelism` - Maximum number of concurrent rollback operations
    /// * `verify` - If true, verify binary hash after restoration
    /// * `ctx` - Output context for progress display
    ///
    /// # Returns
    ///
    /// A vector of `RollbackResult` for each worker, indicating success or failure.
    pub async fn rollback_workers(
        &self,
        workers: &[&WorkerConfig],
        to_version: Option<&str>,
        parallelism: usize,
        verify: bool,
        ctx: &OutputContext,
    ) -> Result<Vec<RollbackResult>> {
        if workers.is_empty() {
            return Ok(Vec::new());
        }

        // Cap parallelism to prevent resource exhaustion
        let max_concurrent = parallelism.clamp(1, MAX_CONCURRENT_ROLLBACKS);

        info!(
            workers = %workers.len(),
            parallelism = %max_concurrent,
            version = ?to_version,
            verify = %verify,
            "Starting parallel rollback operation"
        );

        // Phase 1: Look up backups sequentially (fast, no I/O)
        // This avoids capturing `self` in async closures
        type RollbackTask<'a> = (usize, &'a WorkerConfig, Option<(WorkerBackup, String)>);
        let mut rollback_tasks: Vec<RollbackTask<'_>> = Vec::with_capacity(workers.len());

        for (idx, worker) in workers.iter().enumerate() {
            let worker_id = &worker.id.0;

            // Determine target version
            let target_version = match to_version {
                Some(v) => v.to_string(),
                None => match self.history.get_previous_version(worker_id) {
                    Ok(Some(v)) => v,
                    Ok(None) => {
                        warn!(worker = %worker_id, "No previous version found for rollback");
                        rollback_tasks.push((idx, worker, None));
                        continue;
                    }
                    Err(e) => {
                        error!(worker = %worker_id, error = %e, "Failed to get previous version");
                        rollback_tasks.push((idx, worker, None));
                        continue;
                    }
                },
            };

            // Look up backup for this version
            let backup = match self.get_backup(worker_id, &target_version) {
                Some(b) => b,
                None => {
                    // Try getting the latest backup if specific version not found
                    match self.get_latest_backup(worker_id) {
                        Some(b) if b.version == target_version => b,
                        _ => {
                            error!(
                                worker = %worker_id,
                                version = %target_version,
                                "No backup found for target version"
                            );
                            rollback_tasks.push((idx, worker, None));
                            continue;
                        }
                    }
                }
            };

            rollback_tasks.push((idx, worker, Some((backup, target_version))));
        }

        let completed = Arc::new(AtomicUsize::new(0));
        let total = workers.len();

        // Phase 2: Execute rollbacks in parallel with bounded concurrency. Each
        // task is timed so the provenance audit record carries the rollback
        // duration (bd-session-history-remediation-ocv9i.7.4).
        let results: Vec<(usize, RollbackResult, u64)> = stream::iter(rollback_tasks)
            .map(|(idx, worker, backup_opt)| {
                let completed = completed.clone();

                async move {
                    let started = std::time::Instant::now();
                    let result = match backup_opt {
                        Some((backup, target_version)) => {
                            execute_single_rollback(worker, &backup, &target_version, verify, ctx)
                                .await
                        }
                        None => RollbackResult {
                            worker_id: worker.id.0.clone(),
                            success: false,
                            rolled_back_to: None,
                            error: Some(
                                "No backup found. Re-deploy the desired version instead."
                                    .to_string(),
                            ),
                        },
                    };
                    let duration_ms =
                        u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);

                    // Update progress
                    let done = completed.fetch_add(1, Ordering::Relaxed) + 1;
                    debug!(
                        progress = %format!("{}/{}", done, total),
                        worker = %worker.id,
                        success = %result.success,
                        "Worker rollback complete"
                    );

                    (idx, result, duration_ms)
                }
            })
            .buffer_unordered(max_concurrent)
            .collect()
            .await;

        // Sort by original index to preserve order.
        let mut sorted = results;
        sorted.sort_by_key(|(idx, _, _)| *idx);

        // Append a provenance audit row per worker (bd-...-7.4): flag each
        // worker's most recent deploy audit record rolled-back / rollback-failed
        // and re-persist it, preserving the artifact identity, remote user/path,
        // verification result, and trigger the rollback-history spec requires.
        for (_, result, duration_ms) in &sorted {
            record_rollback_audit(
                &self.history,
                &result.worker_id,
                result.success,
                *duration_ms,
            );
        }

        let results: Vec<RollbackResult> =
            sorted.into_iter().map(|(_, result, _)| result).collect();

        // Log summary
        let success_count = results.iter().filter(|r| r.success).count();
        let fail_count = results.len() - success_count;

        info!(
            total = %results.len(),
            success = %success_count,
            failed = %fail_count,
            "Rollback operation complete"
        );

        Ok(results)
    }

    /// Create a backup of a worker's current binary.
    ///
    /// This creates a local backup record. The actual remote backup creation
    /// (copying the binary on the worker) is done by `backup_before_deploy`
    /// in the executor module which uses SSH.
    pub async fn create_backup(
        &self,
        worker: &WorkerConfig,
        version: &str,
    ) -> Result<WorkerBackup> {
        let backup_path = self
            .backup_dir
            .join(&worker.id.0)
            .join(format!("{}.bak", version));

        std::fs::create_dir_all(backup_path.parent().unwrap())?;

        // Compute remote backup path
        let remote_path = PathBuf::from(format!("{}/rch-wkr-{}", REMOTE_BACKUP_DIR, version));

        // Note: binary_hash will be set by the actual SSH backup operation
        // This stub returns "unknown" - the real implementation in executor.rs
        // will compute the hash via `sha256sum` on the remote
        let backup = WorkerBackup {
            worker_id: worker.id.0.clone(),
            version: version.to_string(),
            backup_path,
            remote_path,
            binary_hash: "unknown".to_string(),
            created_at: chrono::Utc::now().to_rfc3339(),
        };

        debug!(
            worker = %worker.id,
            version = %version,
            remote_path = %backup.remote_path.display(),
            "Created backup record"
        );

        Ok(backup)
    }

    /// Create a backup entry with a known hash (for use by executor).
    pub fn create_backup_entry(
        &self,
        worker_id: &str,
        version: &str,
        remote_path: &str,
        binary_hash: &str,
    ) -> WorkerBackup {
        let backup_path = self
            .backup_dir
            .join(worker_id)
            .join(format!("{}.bak", version));

        WorkerBackup {
            worker_id: worker_id.to_string(),
            version: version.to_string(),
            backup_path,
            remote_path: PathBuf::from(remote_path),
            binary_hash: binary_hash.to_string(),
            created_at: chrono::Utc::now().to_rfc3339(),
        }
    }

    /// Restore a worker from backup.
    ///
    /// This function:
    /// 1. Copies the backup binary back to the main rch-wkr location
    /// 2. Sets executable permissions on the restored binary
    /// 3. Optionally verifies the hash matches the recorded value
    ///
    /// # Arguments
    ///
    /// * `backup` - The backup record containing remote path and hash
    /// * `worker` - The worker configuration for SSH connection
    /// * `verify_hash` - If true, verify the restored binary matches the recorded hash
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - SSH connection fails
    /// - The backup file doesn't exist on the worker
    /// - File copy or permission setting fails
    /// - Hash verification fails (when verify_hash is true)
    pub async fn restore_backup(&self, backup: &WorkerBackup, worker: &WorkerConfig) -> Result<()> {
        self.restore_backup_with_verification(backup, worker, false)
            .await
    }

    /// Restore a worker from backup with optional hash verification.
    pub async fn restore_backup_with_verification(
        &self,
        backup: &WorkerBackup,
        worker: &WorkerConfig,
        verify_hash: bool,
    ) -> Result<()> {
        let ssh = SshExecutor::with_timeout(worker, ROLLBACK_TIMEOUT);
        let worker_id = &worker.id.0;
        let remote_backup_path = backup.remote_path.to_string_lossy();

        info!(
            worker = %worker_id,
            backup_path = %remote_backup_path,
            version = %backup.version,
            "Restoring worker from backup"
        );

        // Step 1: Verify backup exists on remote
        let check_cmd = format!("test -f {}", remote_backup_path);
        let check_result = ssh
            .run_command(&check_cmd)
            .await
            .context("Failed to check if backup exists")?;

        if !check_result.success() {
            return Err(FleetError::NoBackupAvailable {
                worker_id: worker_id.to_string(),
            }
            .into());
        }

        debug!(
            worker = %worker_id,
            backup_path = %remote_backup_path,
            "Backup file exists, proceeding with restore"
        );

        // Step 2: Copy backup to main binary location
        // Use cp with --preserve to maintain permissions and timestamps
        let restore_cmd = format!(
            "cp --preserve=mode,timestamps {} {}",
            remote_backup_path, REMOTE_RCH_PATH
        );
        let restore_result = ssh
            .run_command(&restore_cmd)
            .await
            .context("Failed to copy backup to main location")?;

        if !restore_result.success() {
            return Err(FleetError::RollbackFailed {
                worker_id: worker_id.to_string(),
                reason: format!("cp command failed: {}", restore_result.stderr.trim()),
            }
            .into());
        }

        // Step 3: Ensure executable permissions (redundant with --preserve but safe)
        ssh.set_executable(REMOTE_RCH_PATH)
            .await
            .context("Failed to set executable permissions on restored binary")?;

        info!(
            worker = %worker_id,
            version = %backup.version,
            "Binary restored successfully"
        );

        // Step 4: Optionally verify hash
        if verify_hash && backup.binary_hash != "unknown" {
            debug!(
                worker = %worker_id,
                expected_hash = %backup.binary_hash,
                "Verifying restored binary hash"
            );

            let hash_cmd = format!("sha256sum {} | cut -d' ' -f1", REMOTE_RCH_PATH);
            let hash_result = ssh
                .run_command(&hash_cmd)
                .await
                .context("Failed to compute hash of restored binary")?;

            if !hash_result.success() {
                warn!(
                    worker = %worker_id,
                    stderr = %hash_result.stderr.trim(),
                    "Hash verification command failed, skipping verification"
                );
            } else {
                let actual_hash = hash_result.stdout.trim();
                if actual_hash != backup.binary_hash {
                    return Err(FleetError::RollbackFailed {
                        worker_id: worker_id.to_string(),
                        reason: format!(
                            "hash mismatch: expected {}, got {}",
                            backup.binary_hash, actual_hash
                        ),
                    }
                    .into());
                }

                info!(
                    worker = %worker_id,
                    hash = %actual_hash,
                    "Hash verification passed"
                );
            }
        }

        Ok(())
    }
}

// =============================================================================
// Standalone Functions (for use in async parallel execution)
// =============================================================================

/// Append a rollback entry to the provenance audit trail
/// (bd-session-history-remediation-ocv9i.7.4).
///
/// Loads the worker's most recent deploy provenance audit record, flags it
/// rolled-back (`success`) or rollback-failed (degraded — needs operator
/// attention), stamps the rollback duration, and re-persists it. This
/// mutate-and-re-append design preserves the original deploy's artifact identity,
/// remote user/path, verification result, and operator/remediation trigger — the
/// rollback-history fields the bead requires — while recording the rollback
/// outcome. When no prior deploy audit record exists (the deploy predates
/// provenance auditing), the rollback still happens; it simply leaves no
/// provenance row to flag, which is logged rather than fabricated.
fn record_rollback_audit(
    history: &HistoryManager,
    worker_id: &str,
    success: bool,
    duration_ms: u64,
) {
    // Newest-first, capped at 1 -> 0 or 1 record.
    let latest = match history.get_provenance_audit(1, Some(worker_id)) {
        Ok(mut records) => records.pop(),
        Err(e) => {
            debug!(
                worker = %worker_id,
                error = %e,
                "Could not read provenance audit trail to record rollback"
            );
            return;
        }
    };
    let Some(mut record) = latest else {
        debug!(
            worker = %worker_id,
            "No prior deploy audit record to flag for rollback (deploy predates provenance auditing)"
        );
        return;
    };

    if success {
        record.mark_rolled_back();
    } else {
        record.mark_rollback_failed();
    }
    record.set_duration_ms(duration_ms);

    if let Err(e) = history.record_provenance_audit(&record) {
        warn!(
            worker = %worker_id,
            error = %e,
            "Failed to append rollback provenance audit record"
        );
    }
}

/// Execute a rollback for a single worker.
///
/// This is a standalone function (not a method) to allow capture in async closures
/// without borrowing `self`.
///
/// Performs the complete rollback workflow:
/// 1. Gracefully stop running rch-wkr
/// 2. Restore the backup binary
/// 3. Verify the restored version
/// 4. Restart rch-wkr
async fn execute_single_rollback(
    worker: &WorkerConfig,
    backup: &WorkerBackup,
    target_version: &str,
    verify: bool,
    ctx: &OutputContext,
) -> RollbackResult {
    let style = ctx.theme();
    let worker_id = &worker.id.0;

    if !ctx.is_json() {
        println!(
            "  {} Rolling back {} to v{}...",
            StatusIndicator::Pending.display(style),
            style.highlight(worker_id),
            target_version
        );
    }

    let ssh = SshExecutor::new(worker);

    // Step 1: Gracefully stop running rch-wkr (if any)
    debug!(worker = %worker_id, "Stopping rch-wkr before rollback");
    if let Err(e) = ssh
        .run_command("pkill -TERM rch-wkr 2>/dev/null || true")
        .await
    {
        // Log but don't fail - process might not be running
        debug!(worker = %worker_id, error = %e, "pkill returned error (process may not be running)");
    }
    // Allow brief time for graceful shutdown
    tokio::time::sleep(Duration::from_millis(500)).await;

    // Step 2: Restore the backup via SSH
    let remote_backup_path = backup.remote_path.to_string_lossy();

    // Verify backup exists
    let check_cmd = format!("test -f {}", remote_backup_path);
    match ssh.run_command(&check_cmd).await {
        Ok(result) if result.success() => {
            debug!(worker = %worker_id, "Backup file exists, proceeding with restore");
        }
        Ok(_) => {
            error!(worker = %worker_id, path = %remote_backup_path, "Backup file does not exist");
            if !ctx.is_json() {
                println!(
                    "  {} {} rollback failed: backup file not found",
                    StatusIndicator::Error.display(style),
                    style.highlight(worker_id)
                );
            }
            return RollbackResult {
                worker_id: worker_id.clone(),
                success: false,
                rolled_back_to: None,
                error: Some(format!(
                    "Backup file {} not found on worker",
                    remote_backup_path
                )),
            };
        }
        Err(e) => {
            error!(worker = %worker_id, error = %e, "Failed to check backup file");
            if !ctx.is_json() {
                println!(
                    "  {} {} rollback failed: {}",
                    StatusIndicator::Error.display(style),
                    style.highlight(worker_id),
                    e
                );
            }
            return RollbackResult {
                worker_id: worker_id.clone(),
                success: false,
                rolled_back_to: None,
                error: Some(format!("SSH error checking backup: {}", e)),
            };
        }
    }

    // Copy backup to main binary location
    let restore_cmd = format!(
        "cp --preserve=mode,timestamps {} {}",
        remote_backup_path, REMOTE_RCH_PATH
    );
    match ssh.run_command(&restore_cmd).await {
        Ok(result) if result.success() => {
            debug!(worker = %worker_id, "Binary restored from backup");
        }
        Ok(result) => {
            error!(worker = %worker_id, stderr = %result.stderr.trim(), "Failed to copy backup");
            if !ctx.is_json() {
                println!(
                    "  {} {} rollback failed: {}",
                    StatusIndicator::Error.display(style),
                    style.highlight(worker_id),
                    result.stderr.trim()
                );
            }
            return RollbackResult {
                worker_id: worker_id.clone(),
                success: false,
                rolled_back_to: None,
                error: Some(format!("Failed to copy backup: {}", result.stderr.trim())),
            };
        }
        Err(e) => {
            error!(worker = %worker_id, error = %e, "SSH error during restore");
            if !ctx.is_json() {
                println!(
                    "  {} {} rollback failed: {}",
                    StatusIndicator::Error.display(style),
                    style.highlight(worker_id),
                    e
                );
            }
            return RollbackResult {
                worker_id: worker_id.clone(),
                success: false,
                rolled_back_to: None,
                error: Some(e.to_string()),
            };
        }
    }

    // Ensure executable permissions
    if let Err(e) = ssh.set_executable(REMOTE_RCH_PATH).await {
        warn!(worker = %worker_id, error = %e, "Failed to set executable permissions");
    }

    info!(worker = %worker_id, version = %target_version, "Binary restored successfully");

    // Step 3: Optionally verify hash
    if verify && backup.binary_hash != "unknown" {
        debug!(worker = %worker_id, expected_hash = %backup.binary_hash, "Verifying restored binary hash");

        let hash_cmd = format!("sha256sum {} | cut -d' ' -f1", REMOTE_RCH_PATH);
        match ssh.run_command(&hash_cmd).await {
            Ok(result) if result.success() => {
                let actual_hash = result.stdout.trim();
                if actual_hash != backup.binary_hash {
                    warn!(
                        worker = %worker_id,
                        expected = %backup.binary_hash,
                        actual = %actual_hash,
                        "Hash mismatch after restore (continuing anyway)"
                    );
                } else {
                    info!(worker = %worker_id, hash = %actual_hash, "Hash verification passed");
                }
            }
            Ok(result) => {
                warn!(worker = %worker_id, stderr = %result.stderr.trim(), "Hash verification command failed");
            }
            Err(e) => {
                warn!(worker = %worker_id, error = %e, "Could not verify hash after restore");
            }
        }
    }

    // Step 4: Verify the restored version
    debug!(worker = %worker_id, version = %target_version, "Verifying restored version");
    match ssh
        .run_command(&format!("{} --version", REMOTE_RCH_PATH))
        .await
    {
        Ok(output) if output.success() => {
            let reported_version = output.stdout.trim();
            if !reported_version.contains(target_version) {
                warn!(
                    worker = %worker_id,
                    expected = %target_version,
                    reported = %reported_version,
                    "Version mismatch after rollback (may be OK if format differs)"
                );
            } else {
                debug!(
                    worker = %worker_id,
                    version = %reported_version,
                    "Post-rollback version check passed"
                );
            }
        }
        Ok(output) => {
            warn!(
                worker = %worker_id,
                stderr = %output.stderr.trim(),
                "Version check failed after rollback"
            );
        }
        Err(e) => {
            warn!(worker = %worker_id, error = %e, "Could not verify version after rollback");
        }
    }

    // Step 5: Restart rch-wkr (best-effort)
    debug!(worker = %worker_id, "Restarting rch-wkr after rollback");
    if let Err(e) = ssh
        .run_command(&format!(
            "nohup {} serve >/dev/null 2>&1 &",
            REMOTE_RCH_PATH
        ))
        .await
    {
        warn!(
            worker = %worker_id,
            error = %e,
            "Failed to restart rch-wkr (may need manual intervention)"
        );
    }

    if !ctx.is_json() {
        println!(
            "  {} {} rolled back to v{}",
            StatusIndicator::Success.display(style),
            style.highlight(worker_id),
            target_version
        );
    }

    info!(
        worker = %worker_id,
        version = %target_version,
        "Rollback successful"
    );

    RollbackResult {
        worker_id: worker_id.clone(),
        success: true,
        rolled_back_to: Some(target_version.to_string()),
        error: None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // ========================
    // Helper functions
    // ========================

    #[allow(dead_code)] // Shared test helper for future rollback cases
    fn mock_backup(worker_id: &str, version: &str, timestamp: u64) -> WorkerBackup {
        WorkerBackup {
            worker_id: worker_id.to_string(),
            version: version.to_string(),
            backup_path: PathBuf::from(format!("/tmp/backups/{}/{}.bak", worker_id, version)),
            remote_path: PathBuf::from(format!("~/.rch/backups/rch-wkr-{}", version)),
            binary_hash: format!("hash_{}", version),
            created_at: format!("2024-01-15T12:00:{:02}Z", timestamp % 60),
        }
    }

    // ========================
    // WorkerBackup tests
    // ========================

    #[test]
    fn worker_backup_serializes() {
        let backup = WorkerBackup {
            worker_id: "worker-1".to_string(),
            version: "1.0.0".to_string(),
            backup_path: PathBuf::from("/tmp/backups/worker-1/1.0.0.bak"),
            remote_path: PathBuf::from("~/.rch/backups/rch-wkr-1.0.0"),
            binary_hash: "abc123".to_string(),
            created_at: "2024-01-15T12:00:00Z".to_string(),
        };
        let json = serde_json::to_string(&backup).unwrap();
        assert!(json.contains("worker-1"));
        assert!(json.contains("1.0.0"));
        assert!(json.contains("backups"));
        assert!(json.contains("abc123"));
        assert!(json.contains("2024-01-15T12:00:00Z"));
    }

    #[test]
    fn worker_backup_deserializes_roundtrip() {
        let backup = WorkerBackup {
            worker_id: "test-worker".to_string(),
            version: "2.0.0".to_string(),
            backup_path: PathBuf::from("/data/backups/test.bak"),
            remote_path: PathBuf::from("~/.rch/backups/rch-wkr-2.0.0"),
            binary_hash: "def456".to_string(),
            created_at: "2024-01-16T10:00:00Z".to_string(),
        };
        let json = serde_json::to_string(&backup).unwrap();
        let restored: WorkerBackup = serde_json::from_str(&json).unwrap();
        assert_eq!(restored.worker_id, "test-worker");
        assert_eq!(restored.version, "2.0.0");
        assert_eq!(
            restored.backup_path,
            PathBuf::from("/data/backups/test.bak")
        );
        assert_eq!(restored.binary_hash, "def456");
        assert_eq!(restored.created_at, "2024-01-16T10:00:00Z");
    }

    // ========================
    // RollbackResult tests
    // ========================

    #[test]
    fn rollback_result_success_serializes() {
        let result = RollbackResult {
            worker_id: "worker-1".to_string(),
            success: true,
            rolled_back_to: Some("1.0.0".to_string()),
            error: None,
        };
        let json = serde_json::to_string(&result).unwrap();
        assert!(json.contains("worker-1"));
        assert!(json.contains("\"success\":true"));
        assert!(json.contains("1.0.0"));
    }

    #[test]
    fn rollback_result_failure_serializes() {
        let result = RollbackResult {
            worker_id: "worker-2".to_string(),
            success: false,
            rolled_back_to: None,
            error: Some("No previous version found".to_string()),
        };
        let json = serde_json::to_string(&result).unwrap();
        assert!(json.contains("worker-2"));
        assert!(json.contains("\"success\":false"));
        assert!(json.contains("No previous version found"));
    }

    #[test]
    fn rollback_result_deserializes_roundtrip() {
        let result = RollbackResult {
            worker_id: "test-worker".to_string(),
            success: true,
            rolled_back_to: Some("3.0.0".to_string()),
            error: None,
        };
        let json = serde_json::to_string(&result).unwrap();
        let restored: RollbackResult = serde_json::from_str(&json).unwrap();
        assert_eq!(restored.worker_id, "test-worker");
        assert!(restored.success);
        assert_eq!(restored.rolled_back_to, Some("3.0.0".to_string()));
        assert!(restored.error.is_none());
    }

    #[test]
    fn rollback_result_failed_deserializes_roundtrip() {
        let result = RollbackResult {
            worker_id: "failed-worker".to_string(),
            success: false,
            rolled_back_to: None,
            error: Some("Connection refused".to_string()),
        };
        let json = serde_json::to_string(&result).unwrap();
        let restored: RollbackResult = serde_json::from_str(&json).unwrap();
        assert_eq!(restored.worker_id, "failed-worker");
        assert!(!restored.success);
        assert!(restored.rolled_back_to.is_none());
        assert_eq!(restored.error, Some("Connection refused".to_string()));
    }

    // ========================
    // RollbackManager tests
    // ========================

    #[test]
    fn rollback_manager_new_creates_backup_dir() {
        // RollbackManager::new() creates dirs in the user's data directory
        // This test verifies the constructor doesn't panic
        let manager = RollbackManager::new();
        assert!(manager.is_ok());
    }

    // ========================
    // WorkerBackup additional tests
    // ========================

    #[test]
    fn worker_backup_with_empty_strings() {
        let backup = WorkerBackup {
            worker_id: String::new(),
            version: String::new(),
            backup_path: PathBuf::new(),
            remote_path: PathBuf::new(),
            binary_hash: String::new(),
            created_at: String::new(),
        };
        let json = serde_json::to_string(&backup).unwrap();
        let restored: WorkerBackup = serde_json::from_str(&json).unwrap();
        assert!(restored.worker_id.is_empty());
        assert!(restored.version.is_empty());
    }

    #[test]
    fn worker_backup_with_special_characters() {
        let backup = WorkerBackup {
            worker_id: "worker/with:special\"chars".to_string(),
            version: "1.0.0-beta+build.123".to_string(),
            backup_path: PathBuf::from("/path/with spaces/and\"quotes"),
            remote_path: PathBuf::from("~/.rch/backups/rch-wkr-1.0.0"),
            binary_hash: "hash123".to_string(),
            created_at: "2024-01-15T12:00:00+05:30".to_string(),
        };
        let json = serde_json::to_string(&backup).unwrap();
        let restored: WorkerBackup = serde_json::from_str(&json).unwrap();
        assert!(restored.worker_id.contains("special"));
        assert!(restored.version.contains("beta"));
    }

    #[test]
    fn worker_backup_with_long_strings() {
        let backup = WorkerBackup {
            worker_id: "w".repeat(1000),
            version: "v".repeat(100),
            backup_path: PathBuf::from("/very/long/path".to_string() + &"/segment".repeat(100)),
            remote_path: PathBuf::from("~/.rch/backups/rch-wkr-v".to_string() + &"v".repeat(99)),
            binary_hash: "h".repeat(64),
            created_at: "2024-01-15T12:00:00Z".to_string(),
        };
        let json = serde_json::to_string(&backup).unwrap();
        let restored: WorkerBackup = serde_json::from_str(&json).unwrap();
        assert_eq!(restored.worker_id.len(), 1000);
        assert_eq!(restored.version.len(), 100);
    }

    #[test]
    fn worker_backup_path_variations() {
        let paths = vec![
            PathBuf::from("/absolute/path"),
            PathBuf::from("relative/path"),
            PathBuf::from("."),
            PathBuf::from(".."),
            PathBuf::from("./local"),
        ];
        for path in paths {
            let backup = WorkerBackup {
                worker_id: "test".to_string(),
                version: "1.0.0".to_string(),
                backup_path: path.clone(),
                remote_path: PathBuf::from("~/.rch/backups/rch-wkr-1.0.0"),
                binary_hash: "abc123".to_string(),
                created_at: "2024-01-15T12:00:00Z".to_string(),
            };
            let json = serde_json::to_string(&backup).unwrap();
            let restored: WorkerBackup = serde_json::from_str(&json).unwrap();
            assert_eq!(restored.backup_path, path);
        }
    }

    // ========================
    // RollbackResult additional tests
    // ========================

    #[test]
    fn rollback_result_both_version_and_error() {
        // Edge case: both rolled_back_to and error are Some (unlikely but valid struct)
        let result = RollbackResult {
            worker_id: "partial-worker".to_string(),
            success: false,
            rolled_back_to: Some("1.0.0".to_string()),
            error: Some("Verification failed after rollback".to_string()),
        };
        let json = serde_json::to_string(&result).unwrap();
        let restored: RollbackResult = serde_json::from_str(&json).unwrap();
        assert!(!restored.success);
        assert_eq!(restored.rolled_back_to, Some("1.0.0".to_string()));
        assert!(restored.error.is_some());
    }

    #[test]
    fn rollback_result_empty_worker_id() {
        let result = RollbackResult {
            worker_id: String::new(),
            success: true,
            rolled_back_to: Some("1.0.0".to_string()),
            error: None,
        };
        let json = serde_json::to_string(&result).unwrap();
        let restored: RollbackResult = serde_json::from_str(&json).unwrap();
        assert!(restored.worker_id.is_empty());
    }

    #[test]
    fn rollback_result_long_error_message() {
        let result = RollbackResult {
            worker_id: "worker-1".to_string(),
            success: false,
            rolled_back_to: None,
            error: Some("e".repeat(10000)),
        };
        let json = serde_json::to_string(&result).unwrap();
        let restored: RollbackResult = serde_json::from_str(&json).unwrap();
        assert_eq!(restored.error.unwrap().len(), 10000);
    }

    #[test]
    fn rollback_result_special_chars_in_error() {
        let result = RollbackResult {
            worker_id: "worker".to_string(),
            success: false,
            rolled_back_to: None,
            error: Some("Error: \"connection refused\" on host <remote>".to_string()),
        };
        let json = serde_json::to_string(&result).unwrap();
        let restored: RollbackResult = serde_json::from_str(&json).unwrap();
        assert!(restored.error.unwrap().contains("<remote>"));
    }

    #[test]
    fn rollback_result_version_formats() {
        let versions = vec![
            "1.0.0",
            "1.0.0-alpha",
            "1.0.0-alpha.1",
            "1.0.0+build.123",
            "1.0.0-beta+build.456",
            "0.0.1",
            "99.99.99",
        ];
        for v in versions {
            let result = RollbackResult {
                worker_id: "test".to_string(),
                success: true,
                rolled_back_to: Some(v.to_string()),
                error: None,
            };
            let json = serde_json::to_string(&result).unwrap();
            let restored: RollbackResult = serde_json::from_str(&json).unwrap();
            assert_eq!(restored.rolled_back_to, Some(v.to_string()));
        }
    }

    // ========================
    // RollbackManager tests
    // ========================

    #[test]
    fn rollback_manager_backup_dir_exists_after_new() {
        let manager = RollbackManager::new().unwrap();
        assert!(manager.backup_dir.exists() || !manager.backup_dir.to_string_lossy().is_empty());
    }

    #[tokio::test]
    async fn rollback_manager_create_backup_returns_valid_struct() {
        use rch_common::{WorkerConfig, WorkerId};

        let manager = RollbackManager::new().unwrap();
        let worker = WorkerConfig {
            id: WorkerId::new("test-worker"),
            host: "localhost".to_string(),
            user: "testuser".to_string(),
            identity_file: "~/.ssh/id_rsa".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let backup = manager.create_backup(&worker, "1.0.0").await.unwrap();
        assert_eq!(backup.worker_id, "test-worker");
        assert_eq!(backup.version, "1.0.0");
        assert!(backup.backup_path.to_string_lossy().contains("test-worker"));
        assert!(!backup.created_at.is_empty());
    }

    #[tokio::test]
    async fn rollback_manager_create_backup_different_versions() {
        use rch_common::{WorkerConfig, WorkerId};

        let manager = RollbackManager::new().unwrap();
        let worker = WorkerConfig {
            id: WorkerId::new("version-test"),
            host: "localhost".to_string(),
            user: "testuser".to_string(),
            identity_file: "~/.ssh/id_rsa".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        };

        let backup1 = manager.create_backup(&worker, "1.0.0").await.unwrap();
        let backup2 = manager.create_backup(&worker, "2.0.0").await.unwrap();

        assert_ne!(backup1.backup_path, backup2.backup_path);
        assert_ne!(backup1.version, backup2.version);
    }

    // Note: restore_backup tests require real SSH and are moved to integration tests

    // ========================
    // Registry Operations Tests
    // ========================

    #[test]
    fn test_backup_registry_add_entry() {
        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Create and save a backup entry
        let backup = manager.create_backup_entry("worker1", "0.9.0", "/path/to/backup", "abc123");
        manager.save_backup_entry(&backup).unwrap();

        // Verify it can be retrieved
        let latest = manager.get_latest_backup("worker1");
        assert!(latest.is_some());
        let latest = latest.unwrap();
        assert_eq!(latest.worker_id, "worker1");
        assert_eq!(latest.version, "0.9.0");
        assert_eq!(latest.binary_hash, "abc123");
    }

    #[test]
    fn test_backup_registry_multiple_workers() {
        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Save backups for different workers
        let backup1 = manager.create_backup_entry("worker1", "1.0.0", "/path/1", "hash1");
        let backup2 = manager.create_backup_entry("worker2", "1.0.0", "/path/2", "hash2");
        let backup3 = manager.create_backup_entry("worker1", "1.1.0", "/path/3", "hash3");

        manager.save_backup_entry(&backup1).unwrap();
        manager.save_backup_entry(&backup2).unwrap();
        // Small delay to ensure different timestamps
        std::thread::sleep(std::time::Duration::from_millis(10));
        manager.save_backup_entry(&backup3).unwrap();

        // Check worker1 has latest version 1.1.0
        let latest1 = manager.get_latest_backup("worker1").unwrap();
        assert_eq!(latest1.version, "1.1.0");

        // Check worker2 has version 1.0.0
        let latest2 = manager.get_latest_backup("worker2").unwrap();
        assert_eq!(latest2.version, "1.0.0");

        // Check unknown worker returns None
        assert!(manager.get_latest_backup("worker3").is_none());
    }

    #[test]
    fn test_backup_registry_get_specific_version() {
        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Save multiple versions
        let backup1 = manager.create_backup_entry("worker1", "0.8.0", "/path/1", "hash_0.8.0");
        let backup2 = manager.create_backup_entry("worker1", "0.9.0", "/path/2", "hash_0.9.0");
        let backup3 = manager.create_backup_entry("worker1", "1.0.0", "/path/3", "hash_1.0.0");

        manager.save_backup_entry(&backup1).unwrap();
        manager.save_backup_entry(&backup2).unwrap();
        manager.save_backup_entry(&backup3).unwrap();

        // Get specific version
        let backup = manager.get_backup("worker1", "0.9.0");
        assert!(backup.is_some());
        assert_eq!(backup.unwrap().binary_hash, "hash_0.9.0");

        // Non-existent version
        assert!(manager.get_backup("worker1", "0.7.0").is_none());

        // Non-existent worker
        assert!(manager.get_backup("worker2", "0.9.0").is_none());
    }

    #[test]
    fn test_backup_registry_persistence() {
        let temp_dir = tempfile::tempdir().unwrap();
        let registry_path = temp_dir.path();

        // Create and save with first manager instance
        {
            let manager = RollbackManager::with_path(registry_path).unwrap();
            let backup = manager.create_backup_entry("worker1", "1.0.0", "/path", "abc123");
            manager.save_backup_entry(&backup).unwrap();
        }

        // Reload with new manager instance and verify
        {
            let manager = RollbackManager::with_path(registry_path).unwrap();
            let latest = manager.get_latest_backup("worker1");
            assert!(latest.is_some());
            assert_eq!(latest.unwrap().version, "1.0.0");
        }
    }

    #[test]
    fn test_backup_registry_corrupted_recovery() {
        let temp_dir = tempfile::tempdir().unwrap();
        let registry_path = temp_dir.path().join("registry.json");

        // Write corrupted data
        std::fs::write(&registry_path, "{ invalid json").unwrap();

        // Create manager - should recover gracefully
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();
        let registry = manager.load_registry().unwrap();

        // Should return empty registry after recovery
        assert!(registry.is_empty());

        // Corrupted file should be backed up
        let backup_path = temp_dir.path().join("registry.json.corrupted");
        assert!(backup_path.exists());
    }

    #[test]
    fn test_backup_registry_prune() {
        let temp_dir = tempfile::tempdir().unwrap();
        let mut manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Add 4 backups for worker1 with different timestamps
        for i in 0..4 {
            let backup = WorkerBackup {
                worker_id: "worker1".to_string(),
                version: format!("0.{}.0", 7 + i),
                backup_path: PathBuf::from(format!("/tmp/backups/worker1/0.{}.0.bak", 7 + i)),
                remote_path: PathBuf::from(format!("~/.rch/backups/rch-wkr-0.{}.0", 7 + i)),
                binary_hash: format!("hash_{}", i),
                // Use sequential timestamps to ensure deterministic ordering
                created_at: format!("2024-01-15T12:00:{:02}Z", i * 10),
            };
            manager.save_backup_entry(&backup).unwrap();
        }

        // Prune to keep only 2 most recent
        let removed = manager.prune_old_backups(2).unwrap();

        // Should have removed 2 (oldest)
        assert_eq!(removed.len(), 2);
        assert!(removed.iter().any(|b| b.version == "0.7.0"));
        assert!(removed.iter().any(|b| b.version == "0.8.0"));

        // Verify only latest 2 remain
        let registry = manager.load_registry().unwrap();
        assert_eq!(registry.len(), 2);

        // Oldest should be gone
        assert!(manager.get_backup("worker1", "0.7.0").is_none());
        assert!(manager.get_backup("worker1", "0.8.0").is_none());

        // Newest should still exist
        assert!(manager.get_backup("worker1", "0.9.0").is_some());
        assert!(manager.get_backup("worker1", "0.10.0").is_some());
    }

    #[test]
    fn test_backup_registry_prune_multiple_workers() {
        let temp_dir = tempfile::tempdir().unwrap();
        let mut manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Add 3 backups each for 2 workers
        for worker in ["worker1", "worker2"] {
            for i in 0..3 {
                let backup = WorkerBackup {
                    worker_id: worker.to_string(),
                    version: format!("1.{}.0", i),
                    backup_path: PathBuf::from(format!("/tmp/{}/1.{}.0.bak", worker, i)),
                    remote_path: PathBuf::from(format!("~/.rch/backups/rch-wkr-1.{}.0", i)),
                    binary_hash: format!("hash_{}_{}", worker, i),
                    created_at: format!("2024-01-15T12:{}:00Z", i * 10),
                };
                manager.save_backup_entry(&backup).unwrap();
            }
        }

        // Prune to keep 2 per worker
        let removed = manager.prune_old_backups(2).unwrap();

        // Should remove 1 per worker (2 total)
        assert_eq!(removed.len(), 2);

        // Each worker should have 2 backups remaining
        let registry = manager.load_registry().unwrap();
        let worker1_count = registry.iter().filter(|b| b.worker_id == "worker1").count();
        let worker2_count = registry.iter().filter(|b| b.worker_id == "worker2").count();
        assert_eq!(worker1_count, 2);
        assert_eq!(worker2_count, 2);
    }

    #[test]
    fn test_backup_registry_empty_file() {
        let temp_dir = tempfile::tempdir().unwrap();
        let registry_path = temp_dir.path().join("registry.json");

        // Write empty JSON array
        std::fs::write(&registry_path, "[]").unwrap();

        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();
        let registry = manager.load_registry().unwrap();

        assert!(registry.is_empty());
    }

    #[test]
    fn test_backup_registry_missing_file() {
        let temp_dir = tempfile::tempdir().unwrap();

        // Don't create any registry file
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();
        let registry = manager.load_registry().unwrap();

        // Should return empty without error
        assert!(registry.is_empty());
    }

    #[test]
    fn test_backup_registry_atomic_write() {
        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Save multiple entries
        for i in 0..5 {
            let backup =
                manager.create_backup_entry(&format!("worker{}", i), "1.0.0", "/path", "hash");
            manager.save_backup_entry(&backup).unwrap();
        }

        // Verify no temp files left behind
        let entries: Vec<_> = std::fs::read_dir(temp_dir.path())
            .unwrap()
            .filter_map(|e| e.ok())
            .collect();

        // Should only have registry.json (and possibly worker subdirs)
        for entry in &entries {
            let name = entry.file_name();
            let name_str = name.to_string_lossy();
            assert!(
                !name_str.ends_with(".tmp"),
                "Temp file left behind: {}",
                name_str
            );
        }
    }

    #[test]
    fn test_backup_entry_timestamp_ordering() {
        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Save backups with explicit timestamps (out of order)
        let backup_old = WorkerBackup {
            worker_id: "worker1".to_string(),
            version: "0.8.0".to_string(),
            backup_path: PathBuf::from("/tmp/0.8.0.bak"),
            remote_path: PathBuf::from("~/.rch/backups/rch-wkr-0.8.0"),
            binary_hash: "old_hash".to_string(),
            created_at: "2024-01-15T10:00:00Z".to_string(), // Earlier
        };
        let backup_new = WorkerBackup {
            worker_id: "worker1".to_string(),
            version: "0.9.0".to_string(),
            backup_path: PathBuf::from("/tmp/0.9.0.bak"),
            remote_path: PathBuf::from("~/.rch/backups/rch-wkr-0.9.0"),
            binary_hash: "new_hash".to_string(),
            created_at: "2024-01-15T12:00:00Z".to_string(), // Later
        };

        // Save in reverse order
        manager.save_backup_entry(&backup_new).unwrap();
        manager.save_backup_entry(&backup_old).unwrap();

        // get_latest_backup should return the one with latest timestamp
        let latest = manager.get_latest_backup("worker1").unwrap();
        assert_eq!(latest.version, "0.9.0");
        assert_eq!(latest.binary_hash, "new_hash");
    }

    // ========================
    // Rollback Workers Tests (Empty Workers)
    // ========================

    #[tokio::test]
    async fn test_rollback_workers_empty_list() {
        use crate::ui::test_utils::OutputCapture;

        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();
        let capture = OutputCapture::json();
        let ctx = capture.context();

        let results = manager
            .rollback_workers(&[], None, 4, true, &ctx)
            .await
            .unwrap();
        assert!(results.is_empty());
    }

    // ========================
    // Create Backup Entry Tests
    // ========================

    #[test]
    fn test_create_backup_entry_fields() {
        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        let backup = manager.create_backup_entry(
            "test-worker",
            "1.2.3",
            "~/.rch/backups/rch-wkr-1.2.3",
            "sha256hash",
        );

        assert_eq!(backup.worker_id, "test-worker");
        assert_eq!(backup.version, "1.2.3");
        assert_eq!(backup.binary_hash, "sha256hash");
        assert!(backup.backup_path.to_string_lossy().contains("test-worker"));
        assert!(backup.backup_path.to_string_lossy().contains("1.2.3.bak"));
        assert!(!backup.created_at.is_empty());
    }

    #[test]
    fn test_create_backup_entry_unknown_hash() {
        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        let backup = manager.create_backup_entry("worker1", "1.0.0", "/path", "unknown");

        assert_eq!(backup.binary_hash, "unknown");
    }

    // ========================
    // Rollback Execution Tests (No SSH)
    // ========================

    /// Test helper to create a mock WorkerConfig
    fn mock_worker(id: &str) -> WorkerConfig {
        use rch_common::WorkerId;
        WorkerConfig {
            id: WorkerId::new(id),
            host: format!("mock://{}.local", id),
            user: "testuser".to_string(),
            identity_file: "~/.ssh/id_rsa".to_string(),
            total_slots: 4,
            priority: 100,
            tags: vec![],
            tools: Vec::new(),
        }
    }

    /// Test helper to create a mock backup with specific hash
    fn mock_backup_with_hash(
        worker_id: &str,
        version: &str,
        timestamp: u64,
        hash: &str,
    ) -> WorkerBackup {
        WorkerBackup {
            worker_id: worker_id.to_string(),
            version: version.to_string(),
            backup_path: PathBuf::from(format!("/tmp/backups/{}/{}.bak", worker_id, version)),
            remote_path: PathBuf::from(format!("~/.rch/backups/rch-wkr-{}", version)),
            binary_hash: hash.to_string(),
            created_at: format!("2024-01-15T12:00:{:02}Z", timestamp % 60),
        }
    }

    #[tokio::test]
    async fn test_rollback_no_backup() {
        use crate::ui::test_utils::OutputCapture;

        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();
        let worker = mock_worker("worker1");
        let workers = vec![&worker];
        let capture = OutputCapture::json();
        let ctx = capture.context();

        // Manager has no backups registered
        let results = manager
            .rollback_workers(&workers, None, 4, true, &ctx)
            .await
            .unwrap();

        assert_eq!(results.len(), 1);
        assert!(!results[0].success);
        assert!(
            results[0]
                .error
                .as_ref()
                .unwrap()
                .to_lowercase()
                .contains("no backup")
                || results[0].error.as_ref().unwrap().contains("Re-deploy")
        );
    }

    #[tokio::test]
    async fn test_rollback_specific_version_not_found() {
        use crate::ui::test_utils::OutputCapture;

        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();
        let worker = mock_worker("worker1");
        let workers = vec![&worker];

        // Save a backup for version 1.0.0
        let backup = mock_backup_with_hash("worker1", "1.0.0", 1000, "abc123");
        manager.save_backup_entry(&backup).unwrap();

        let capture = OutputCapture::json();
        let ctx = capture.context();

        // Request version 2.0.0 which doesn't exist
        let results = manager
            .rollback_workers(&workers, Some("2.0.0"), 4, true, &ctx)
            .await
            .unwrap();

        assert_eq!(results.len(), 1);
        assert!(!results[0].success);
        assert!(results[0].rolled_back_to.is_none());
    }

    // bd-session-history-remediation-ocv9i.7.4 — rollback audit trail.

    fn seed_deploy_audit(manager: &RollbackManager, worker: &str) -> String {
        use rch_common::fleet_provenance::{
            ArtifactProvenance, FleetDeployAuditRecord, ProvenancePolicy, SignatureCheck,
            verify_artifact_provenance,
        };
        let triple = "x86_64-unknown-linux-gnu";
        let p = ArtifactProvenance::dev_artifact("rch-wkr", triple);
        let verdict = verify_artifact_provenance(
            &p,
            triple,
            None,
            SignatureCheck::Absent,
            &ProvenancePolicy::dev_friendly(),
        );
        let rec = FleetDeployAuditRecord::from_verdict(
            "run-7",
            "bd-session-history-remediation-ocv9i.7.4",
            worker,
            "rch",
            "/home/rch/.local/bin/rch-wkr",
            &p,
            None,
            1000,
            &verdict,
            "operator",
            "deploy",
        );
        manager.history.record_provenance_audit(&rec).unwrap();
        verdict.verification_status().to_string()
    }

    #[tokio::test]
    async fn rollback_audit_flags_record_rolled_back_on_success() {
        use rch_common::fleet_provenance::rollback_status;
        let temp = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp.path()).unwrap();
        let verification = seed_deploy_audit(&manager, "css");

        // Simulate a successful rollback's audit emission.
        record_rollback_audit(&manager.history, "css", true, 4200);

        let after = manager
            .history
            .get_provenance_audit(10, Some("css"))
            .unwrap();
        let rolled = after
            .iter()
            .find(|r| r.rollback_status == rollback_status::ROLLED_BACK)
            .expect("a rolled_back audit record");
        assert_eq!(rolled.duration_ms, 4200);
        // The mutate-and-re-append preserves the original deploy context.
        assert_eq!(rolled.worker_id, "css");
        assert_eq!(rolled.remote_user, "rch");
        assert_eq!(rolled.trigger, "operator");
        assert_eq!(rolled.verification_status, verification);
    }

    #[tokio::test]
    async fn rollback_audit_flags_record_rollback_failed_on_failure() {
        use rch_common::fleet_provenance::rollback_status;
        let temp = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp.path()).unwrap();
        seed_deploy_audit(&manager, "css");

        // A failed rollback flags the record degraded (operator attention).
        record_rollback_audit(&manager.history, "css", false, 7);

        let after = manager
            .history
            .get_provenance_audit(10, Some("css"))
            .unwrap();
        let failed = after
            .iter()
            .find(|r| r.rollback_status == rollback_status::ROLLBACK_FAILED)
            .expect("a rollback_failed audit record");
        assert_eq!(failed.duration_ms, 7);
        assert_eq!(failed.worker_id, "css");
    }

    #[tokio::test]
    async fn rollback_audit_noop_without_prior_record() {
        let temp = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp.path()).unwrap();
        // No seeded deploy record -> nothing to flag, no panic, no write.
        record_rollback_audit(&manager.history, "ghost", true, 10);
        let after = manager
            .history
            .get_provenance_audit(10, Some("ghost"))
            .unwrap();
        assert!(after.is_empty());
    }

    // The full rollback_workers path actually emits the audit row (the wiring).
    // Outcome-agnostic: in the test harness the mock host's SSH may succeed or
    // fail, so we only assert the seeded record was flagged (no longer `none`) —
    // the per-outcome rolled_back/rollback_failed mapping is covered by the two
    // direct tests above.
    #[tokio::test]
    async fn rollback_workers_emits_audit_record() {
        use crate::ui::test_utils::OutputCapture;
        use rch_common::fleet_provenance::rollback_status;
        let temp = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp.path()).unwrap();
        let worker = mock_worker("css");
        let backup = mock_backup_with_hash("css", "1.0.0", 1000, "abc123");
        manager.save_backup_entry(&backup).unwrap();
        seed_deploy_audit(&manager, "css");

        let capture = OutputCapture::json();
        let ctx = capture.context();
        let results = manager
            .rollback_workers(&[&worker], Some("1.0.0"), 4, false, &ctx)
            .await
            .unwrap();
        assert_eq!(results.len(), 1);

        let after = manager
            .history
            .get_provenance_audit(10, Some("css"))
            .unwrap();
        assert!(
            after
                .iter()
                .any(|r| r.rollback_status != rollback_status::NONE),
            "rollback_workers must flag the worker's audit record"
        );
    }

    #[tokio::test]
    async fn test_rollback_parallelism_capped() {
        use crate::ui::test_utils::OutputCapture;

        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Create many workers
        let workers: Vec<WorkerConfig> = (0..20).map(|i| mock_worker(&format!("w{}", i))).collect();
        let worker_refs: Vec<&WorkerConfig> = workers.iter().collect();

        let capture = OutputCapture::json();
        let ctx = capture.context();

        // Even with parallelism=100, should be capped at MAX_CONCURRENT_ROLLBACKS
        let results = manager
            .rollback_workers(&worker_refs, None, 100, false, &ctx)
            .await
            .unwrap();

        // All should report "no backup" error since no backups registered
        assert_eq!(results.len(), 20);
        assert!(results.iter().all(|r| !r.success));
    }

    // ========================
    // MockSshExecutor-based Tests
    // These test the actual SSH execution logic using mocks
    // ========================

    /// Execute rollback with MockSshExecutor for testing.
    /// This mirrors execute_single_rollback but uses the mock.
    #[cfg(test)]
    async fn execute_rollback_with_mock(
        worker: &WorkerConfig,
        backup: &WorkerBackup,
        target_version: &str,
        verify: bool,
        mock: &crate::fleet::ssh::MockSshExecutor,
    ) -> RollbackResult {
        let worker_id = &worker.id.0;

        // Step 1: Graceful stop (pkill)
        if let Err(e) = mock
            .run_command("pkill -TERM rch-wkr 2>/dev/null || true")
            .await
        {
            debug!(worker = %worker_id, error = %e, "pkill returned error (expected)");
        }

        // Step 2: Verify backup exists
        let remote_backup_path = backup.remote_path.to_string_lossy();
        let check_cmd = format!("test -f {}", remote_backup_path);
        match mock.run_command(&check_cmd).await {
            Ok(result) if result.success() => {}
            Ok(_) => {
                return RollbackResult {
                    worker_id: worker_id.clone(),
                    success: false,
                    rolled_back_to: None,
                    error: Some(format!(
                        "Backup file {} not found on worker",
                        remote_backup_path
                    )),
                };
            }
            Err(e) => {
                return RollbackResult {
                    worker_id: worker_id.clone(),
                    success: false,
                    rolled_back_to: None,
                    error: Some(format!("SSH error checking backup: {}", e)),
                };
            }
        }

        // Step 3: Copy backup to main location
        let restore_cmd = format!(
            "cp --preserve=mode,timestamps {} {}",
            remote_backup_path, REMOTE_RCH_PATH
        );
        match mock.run_command(&restore_cmd).await {
            Ok(result) if result.success() => {}
            Ok(result) => {
                return RollbackResult {
                    worker_id: worker_id.clone(),
                    success: false,
                    rolled_back_to: None,
                    error: Some(format!("Failed to copy backup: {}", result.stderr.trim())),
                };
            }
            Err(e) => {
                return RollbackResult {
                    worker_id: worker_id.clone(),
                    success: false,
                    rolled_back_to: None,
                    error: Some(e.to_string()),
                };
            }
        }

        // Step 4: Set executable permissions
        let _ = mock
            .run_command(&format!("chmod +x {}", REMOTE_RCH_PATH))
            .await;

        // Step 5: Verify hash (if enabled and hash is known)
        if verify && backup.binary_hash != "unknown" {
            let hash_cmd = format!("sha256sum {} | cut -d' ' -f1", REMOTE_RCH_PATH);
            match mock.run_command(&hash_cmd).await {
                Ok(result) if result.success() => {
                    let actual_hash = result.stdout.trim();
                    if actual_hash != backup.binary_hash {
                        return RollbackResult {
                            worker_id: worker_id.clone(),
                            success: false,
                            rolled_back_to: Some(target_version.to_string()),
                            error: Some(format!(
                                "Hash mismatch: expected {}, got {}",
                                backup.binary_hash, actual_hash
                            )),
                        };
                    }
                }
                Ok(_) | Err(_) => {
                    // Hash check failed but we continue
                }
            }
        }

        // Step 6: Verify version
        let version_cmd = format!("{} --version", REMOTE_RCH_PATH);
        let _ = mock.run_command(&version_cmd).await;

        // Step 7: Restart service
        let restart_cmd = format!("nohup {} serve >/dev/null 2>&1 &", REMOTE_RCH_PATH);
        let _ = mock.run_command(&restart_cmd).await;

        RollbackResult {
            worker_id: worker_id.clone(),
            success: true,
            rolled_back_to: Some(target_version.to_string()),
            error: None,
        }
    }

    #[tokio::test]
    async fn test_rollback_success_with_mock() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "abc123");

        // Note: mock returns just the hash since the real command uses `| cut -d' ' -f1`
        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            .with_command("sha256sum", MockCommandResult::ok("abc123"))
            .with_command("--version", MockCommandResult::ok("rch-wkr 0.9.0"))
            .with_command("nohup", MockCommandResult::ok(""));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", true, &mock).await;

        assert!(result.success);
        assert_eq!(result.rolled_back_to, Some("0.9.0".to_string()));
        assert!(result.error.is_none());
    }

    #[tokio::test]
    async fn test_rollback_hash_mismatch_with_mock() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "expected_hash");

        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            // Hash mismatch!
            .with_command("sha256sum", MockCommandResult::ok("different_hash  /path"));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", true, &mock).await;

        assert!(!result.success);
        assert!(result.error.as_ref().unwrap().contains("Hash mismatch"));
    }

    #[tokio::test]
    async fn test_rollback_unknown_hash_skips_verification_with_mock() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        // Hash is "unknown" - should skip verification
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "unknown");

        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            // Would mismatch if checked, but should be ignored
            .with_command("sha256sum", MockCommandResult::ok("any_hash  /path"))
            .with_command("--version", MockCommandResult::ok("rch-wkr 0.9.0"))
            .with_command("nohup", MockCommandResult::ok(""));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", true, &mock).await;

        // Should succeed because hash="unknown" skips verification
        assert!(result.success);
        assert!(result.error.is_none());
    }

    #[tokio::test]
    async fn test_rollback_backup_file_missing_with_mock() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "abc123");

        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            // test -f returns non-zero (file doesn't exist)
            .with_command("test -f", MockCommandResult::err(1, ""));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", true, &mock).await;

        assert!(!result.success);
        assert!(result.error.as_ref().unwrap().contains("not found"));
    }

    #[tokio::test]
    async fn test_rollback_cp_fails_with_mock() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "abc123");

        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            // cp fails
            .with_command(
                "cp --preserve",
                MockCommandResult::err(1, "Permission denied"),
            );

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", true, &mock).await;

        assert!(!result.success);
        assert!(result.error.as_ref().unwrap().contains("Permission denied"));
    }

    #[tokio::test]
    async fn test_rollback_verifies_hash_when_enabled() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "correct_hash");

        // Note: mock returns just the hash since the real command uses `| cut -d' ' -f1`
        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            .with_command("sha256sum", MockCommandResult::ok("correct_hash"))
            .with_command("--version", MockCommandResult::ok("rch-wkr 0.9.0"))
            .with_command("nohup", MockCommandResult::ok(""));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", true, &mock).await;

        assert!(result.success);
        assert!(result.error.is_none());
    }

    #[tokio::test]
    async fn test_rollback_skips_hash_when_disabled() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "expected_hash");

        // Mock doesn't provide sha256sum result - would fail if hash check runs
        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            .with_command("--version", MockCommandResult::ok("rch-wkr 0.9.0"))
            .with_command("nohup", MockCommandResult::ok(""));

        // verify=false
        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", false, &mock).await;

        // Should succeed because verification is disabled
        assert!(result.success);
    }

    // ========================
    // Concurrent Write Tests
    // ========================

    #[tokio::test]
    async fn test_backup_registry_concurrent_writes() {
        use tokio::task::JoinSet;

        let temp_dir = tempfile::tempdir().unwrap();
        let registry_path = temp_dir.path().to_path_buf();

        // Spawn multiple concurrent writes
        let mut tasks = JoinSet::new();
        for i in 0..10 {
            let path = registry_path.clone();
            tasks.spawn(async move {
                let manager = RollbackManager::with_path(&path).unwrap();
                let backup =
                    manager.create_backup_entry(&format!("worker{}", i), "1.0.0", "/path", "hash");
                manager.save_backup_entry(&backup).unwrap();
                i
            });
        }

        // Wait for all to complete
        let mut completed = Vec::new();
        while let Some(result) = tasks.join_next().await {
            completed.push(result.unwrap());
        }

        // All 10 should have completed
        assert_eq!(completed.len(), 10);

        // Verify registry is not corrupted
        let manager = RollbackManager::with_path(&registry_path).unwrap();
        let registry = manager.load_registry().unwrap();

        // Should have all 10 entries (atomic writes should prevent corruption)
        assert_eq!(registry.len(), 10);

        // Verify each worker is present
        for i in 0..10 {
            let worker_id = format!("worker{}", i);
            assert!(
                manager.get_latest_backup(&worker_id).is_some(),
                "Worker {} should have backup",
                worker_id
            );
        }
    }

    // ========================
    // Graceful Shutdown Order Test
    // ========================

    #[tokio::test]
    async fn test_rollback_graceful_shutdown_order() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};
        use std::sync::{Arc, Mutex};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "abc123");

        // Track command order (variables intentionally unused - Arc/Mutex kept for future extension)
        let _call_order = Arc::new(Mutex::new(Vec::<String>::new()));

        // Note: mock returns just the hash since the real command uses `| cut -d' ' -f1`
        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            .with_command("sha256sum", MockCommandResult::ok("abc123"))
            .with_command("--version", MockCommandResult::ok("rch-wkr 0.9.0"))
            .with_command("nohup", MockCommandResult::ok(""));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", true, &mock).await;

        // Rollback should succeed
        assert!(result.success);

        // The execute_rollback_with_mock function calls pkill before cp before nohup
        // This is verified by the function's sequential execution order
        // (pkill -> test -f -> cp -> chmod -> sha256sum -> version -> nohup)
    }

    // ========================
    // Service Restart Verification
    // ========================

    #[tokio::test]
    async fn test_rollback_restarts_service() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "unknown");

        // Mock all commands as successful
        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            .with_command("--version", MockCommandResult::ok("rch-wkr 0.9.0"))
            // nohup with serve mode is expected
            .with_command("nohup", MockCommandResult::ok(""));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", false, &mock).await;

        // Service restart is part of the successful flow
        assert!(result.success);
        assert_eq!(result.rolled_back_to, Some("0.9.0".to_string()));
    }

    // ========================
    // Multiple Workers Parallel Test
    // ========================

    #[tokio::test]
    async fn test_rollback_multiple_workers_returns_all_results() {
        use crate::ui::test_utils::OutputCapture;

        let temp_dir = tempfile::tempdir().unwrap();
        let manager = RollbackManager::with_path(temp_dir.path()).unwrap();

        // Create 5 workers
        let workers: Vec<WorkerConfig> = (0..5).map(|i| mock_worker(&format!("w{}", i))).collect();
        let worker_refs: Vec<&WorkerConfig> = workers.iter().collect();

        let capture = OutputCapture::json();
        let ctx = capture.context();

        // No backups registered, so all should fail
        let results = manager
            .rollback_workers(&worker_refs, None, 4, false, &ctx)
            .await
            .unwrap();

        // Should get a result for each worker
        assert_eq!(results.len(), 5);

        // All should fail (no backups)
        assert!(results.iter().all(|r| !r.success));

        // Each worker should have its own result
        let worker_ids: Vec<&str> = results.iter().map(|r| r.worker_id.as_str()).collect();
        for i in 0..5 {
            let expected_id = format!("w{}", i);
            assert!(
                worker_ids.contains(&expected_id.as_str()),
                "Missing result for {}",
                expected_id
            );
        }
    }

    // ========================
    // Version Mismatch Warning Test
    // ========================

    #[tokio::test]
    async fn test_rollback_continues_on_version_mismatch_warning() {
        use crate::fleet::ssh::{MockCommandResult, MockSshExecutor};

        let worker = mock_worker("worker1");
        let backup = mock_backup_with_hash("worker1", "0.9.0", 1000, "unknown");

        // Version check returns different format but rollback should still succeed
        let mock = MockSshExecutor::new()
            .with_command("pkill", MockCommandResult::ok(""))
            .with_command("test -f", MockCommandResult::ok(""))
            .with_command("cp --preserve", MockCommandResult::ok(""))
            .with_command("chmod", MockCommandResult::ok(""))
            // Version reports different format
            .with_command(
                "--version",
                MockCommandResult::ok("rch-wkr version 0.9.0-dev"),
            )
            .with_command("nohup", MockCommandResult::ok(""));

        let result = execute_rollback_with_mock(&worker, &backup, "0.9.0", false, &mock).await;

        // Should still succeed (version mismatch is just a warning)
        assert!(result.success);
    }
}