mecha10-cli 0.6.3

//! Watch mode, process cleanup, and lifecycle mode changes for dev mode

use super::registry::NodeRegistry;
use super::spawn::spawn_nodes_by_name;
use crate::context::CliContext;
use crate::dev::node_selection::NodeToRun;
use crate::paths;
use crate::types::ProjectConfig;
use crate::ui::LogBuffer;
use anyhow::Result;
use mecha10_cli_core::utils::strip_node_scope;
use std::collections::HashMap;

/// Run watch mode - monitors processes and rebuilds on file changes
pub async fn run_watch_mode(ctx: &mut CliContext, _pids: HashMap<String, u32>) -> Result<()> {
    use std::sync::atomic::Ordering;
    use std::time::Duration;

    println!("Watch mode enabled (file changes will trigger rebuild)");
    println!();
    println!("Press Ctrl+C to stop all nodes");
    println!();

    // Set up the single process-wide Ctrl+C handler (see `crate::dev::termination`), replacing
    // what used to be this function's own independent `ctrlc::set_handler` registration.
    let running = crate::dev::termination::install_quit_signal()?;

    // Watch loop (simplified - full implementation would use file watching)
    while running.load(Ordering::SeqCst) {
        tokio::time::sleep(Duration::from_secs(1)).await;

        // Check process status
        let status = ctx.process().get_status();
        for (node, node_status) in &status {
            if !node_status.contains("running") {
                println!("⚠️  {} stopped unexpectedly", node);
            }
        }
    }

    println!();
    println!("Stopping all nodes...");
    // Use the same shutdown-order + timeout + force-kill sequence the interactive-mode exit path
    // uses (`cleanup` below), rather than the thinner `ProcessService::cleanup()` this used to
    // call directly - see this PR's description for why that's an intentional behavior change,
    // not just a refactor.
    cleanup(ctx)?;

    Ok(())
}

/// Cleanup dev mode - stop all processes in dependency order
pub fn cleanup(ctx: &mut CliContext) -> Result<()> {
    use std::time::Duration;
    use tracing::warn;

    println!();
    println!("Stopping all nodes in dependency order...");

    // Use shared process service for consistency with spawn
    let shared_process = ctx.shared_process();
    let mut process_service = shared_process.lock().unwrap();
    let shutdown_order = process_service.get_shutdown_order();

    if shutdown_order.is_empty() {
        println!("✅ No nodes to stop");
        return Ok(());
    }

    // Stop nodes in dependency order
    for node in shutdown_order {
        println!("  Stopping {}...", node);

        // Try graceful shutdown with 10 second timeout
        if let Err(e) = process_service.stop_with_timeout(&node, Duration::from_secs(10)) {
            warn!("Failed to gracefully stop {}: {}", node, e);
            // Try force kill as last resort
            if let Err(kill_err) = process_service.force_kill(&node) {
                warn!("Failed to force kill {}: {}", node, kill_err);
            } else {
                println!("  ✅ {} stopped (force killed)", node);
            }
        } else {
            println!("  ✅ {} stopped", node);
        }
    }

    println!("✅ All nodes stopped");

    Ok(())
}

/// Handle lifecycle mode change
///
/// Calculates node diff, stops old nodes, starts new nodes, updates lifecycle state.
///
/// # Returns
///
/// The list of `(node_name, error)` pairs for nodes that the target mode
/// requires but which failed to start. An empty vec means every node the
/// mode needs came up successfully. Callers (e.g. the `s`/`t` key handlers)
/// use this to know whether a specific node — such as `teleop` — actually
/// started, rather than assuming success just because this function
/// returned `Ok`.
#[allow(clippy::too_many_arguments)]
pub(super) async fn handle_mode_change(
    ctx: &mut CliContext,
    lifecycle: &mut crate::dev::lifecycle_adapter::CliLifecycleManager,
    target_mode: &str,
    registry: &mut NodeRegistry,
    project_name: &str,
    project_config: &ProjectConfig,
    nodes_to_run: &[NodeToRun],
    log_buffer: Option<&LogBuffer>,
) -> Result<Vec<(String, String)>> {
    println!();
    println!("⚡ Switching to {} mode...", target_mode);

    // Log mode change
    if let Some(buffer) = log_buffer {
        buffer.push(format!("MODE: Switching to {} mode", target_mode));
    }

    // Calculate what needs to change
    let diff = lifecycle.change_mode(target_mode)?;

    // Stop nodes that need to stop
    if !diff.stop.is_empty() {
        println!("🛑 Stopping: {}", diff.stop.join(", "));
        // Use shared process service (same one used by spawn_nodes_by_name)
        let shared_process = ctx.shared_process();
        let mut process_service = shared_process.lock().unwrap();
        for node_name in &diff.stop {
            if registry.mark_exited(node_name).is_some() {
                if let Err(e) = process_service.stop(node_name) {
                    eprintln!("  Warning: Failed to stop {}: {}", node_name, e);
                    if let Some(buffer) = log_buffer {
                        buffer.push(format!("ERROR: Failed to stop {}: {}", node_name, e));
                    }
                } else if let Some(buffer) = log_buffer {
                    buffer.push(format!("🛑 STOPPED: {}", node_name));
                }
            }
        }
        lifecycle.mark_nodes_stopped(&diff.stop);
    }

    // Start nodes that need to start
    let mut start_failures: Vec<(String, String)> = Vec::new();
    if !diff.start.is_empty() {
        println!("✅ Starting: {}", diff.start.join(", "));
        let outcome = spawn_nodes_by_name(ctx, &diff.start, project_name, project_config, log_buffer)?;
        // Mark only successfully spawned nodes as running
        let actually_started: Vec<String> = outcome.pids.keys().cloned().collect();
        for (node, pid) in &outcome.pids {
            registry.mark_started(node, *pid);
        }
        lifecycle.mark_nodes_running(&actually_started);
        start_failures = outcome.failures;

        // A node's OS process spawning successfully doesn't mean it's actually doing useful
        // work - some nodes (e.g. `simulator`, which supervises a Python subprocess that can
        // hit a permanent environment problem like a missing `mecha10_rl` install - see
        // `packages/nodes/simulator/src/lib.rs`) exit almost immediately after a fully
        // successful spawn, for reasons `spawn_nodes_by_name` has no way to see. Re-check
        // liveness after a short grace period so that class of failure reaches the TUI's System
        // Logs panel too, instead of the node silently vanishing with nothing to explain why
        // (see LAB-1795/LAB-1797 - this was exactly what made simulation mode's mujoco window
        // "not open" with zero visible error).
        start_failures.extend(recheck_liveness(ctx, &actually_started).await);
    }

    if start_failures.is_empty() {
        println!("✓ Mode changed to: {}", target_mode);
    } else {
        // Don't print "✓ Mode changed" as if everything came up — the mode
        // itself did switch, but some of the nodes it requires did not
        // start, and that needs to be visible at the moment it happens, not
        // just buried in the log buffer (see note above `enter_teleop_mode`
        // in `teleop.rs` about the TUI redraw wiping stdout).
        println!("⚠️  Mode changed to: {} — but some nodes failed to start:", target_mode);
        for (node, err) in &start_failures {
            println!("   ✗ {}: {}", node, err);
        }
    }
    println!();

    // Log mode change completion
    if let Some(buffer) = log_buffer {
        if start_failures.is_empty() {
            buffer.push(format!("MODE: Successfully changed to {} mode", target_mode));
        } else {
            let failed_names: Vec<&str> = start_failures.iter().map(|(name, _)| name.as_str()).collect();
            buffer.push(format!(
                "MODE: Changed to {} mode, but failed to start: {}",
                target_mode,
                failed_names.join(", ")
            ));
            for (node, err) in &start_failures {
                buffer.push(format!("ERROR: {} failed to start during mode change: {}", node, err));
            }
        }
    }

    // Display current mode status
    let running: Vec<String> = registry.running_node_names();
    crate::ui::dev_banners::print_mode_status(target_mode, &running, nodes_to_run);

    Ok(start_failures)
}

/// Re-check whether nodes that just spawned successfully (as an OS process) are still alive a
/// short grace period later, and fold any that already exited into the returned failure list -
/// same shape as [`crate::handlers::dev::ops::spawn::SpawnOutcome::failures`], so callers (in
/// particular the summary logging just above, and `keys.rs`'s `resolve_teleop_failure`) treat
/// this exactly like a spawn failure without needing to know the difference.
///
/// This exists because `spawn_nodes_by_name` only detects OS-level spawn failure (the binary
/// itself failing to fork/exec) - a node whose process starts fine and then exits almost
/// immediately once its own internal logic hits a fatal error (e.g. `simulator` discovering its
/// supervised Python subprocess can't import `mecha10_rl` - see `packages/nodes/simulator/src/
/// lib.rs::run`'s preflight check) is invisible to it: `spawn_nodes_by_name` reports `Ok(pid)`
/// and moves on, and the node's own stderr only ever reaches `logs/<node>.log`
/// (`packages/cli/src/services/process/spawn.rs::spawn_with_env`) or is discarded entirely, not
/// the CLI process itself - so nothing before this ever had a chance to notice or explain it.
async fn recheck_liveness(ctx: &mut CliContext, started: &[String]) -> Vec<(String, String)> {
    if started.is_empty() {
        return Vec::new();
    }

    // Long enough for a fail-fast permanent-environment-error exit (the case this targets) to
    // have already happened, short enough not to make every mode switch feel sluggish.
    tokio::time::sleep(std::time::Duration::from_millis(1500)).await;

    let statuses = {
        let shared_process = ctx.shared_process();
        let mut process_service = shared_process.lock().unwrap();
        process_service.get_status()
    };

    let mut died_early = Vec::new();
    for node_name in started {
        let Some(status) = statuses.get(node_name) else {
            continue;
        };
        if status == "running" {
            continue;
        }

        let detail = tail_node_log(node_name);
        let reason = match detail {
            Some(detail) => format!("exited shortly after starting ({status}): {detail}"),
            None => format!(
                "exited shortly after starting ({status}) - see logs/{}.log for details \
                 (enable `environments.<env>.logging.disk_logging_enabled` in mecha10.json if that file is empty)",
                strip_node_scope(node_name)
            ),
        };
        died_early.push((node_name.clone(), reason));
    }

    died_early
}

/// Best-effort read of the last non-empty line of a node's on-disk log file - used by
/// [`recheck_liveness`] to explain *why* a node that spawned successfully then exited almost
/// immediately, since (when disk logging is enabled) that's the only place the reason exists
/// outside the node's own now-dead process.
fn tail_node_log(node_name: &str) -> Option<String> {
    let log_path = std::path::Path::new(paths::project::LOGS_DIR).join(format!("{}.log", strip_node_scope(node_name)));
    let content = std::fs::read_to_string(log_path).ok()?;
    last_nonempty_line(&content)
}

/// Pure helper split out of [`tail_node_log`] so the "find the last non-empty, trimmed line"
/// logic is unit testable without touching the filesystem.
fn last_nonempty_line(content: &str) -> Option<String> {
    content
        .lines()
        .rev()
        .find(|line| !line.trim().is_empty())
        .map(|line| line.trim().to_string())
}

#[cfg(test)]
mod liveness_tests {
    use super::*;

    #[test]
    fn last_nonempty_line_skips_trailing_blank_lines() {
        let content = "line one\nline two\n\n   \n";
        assert_eq!(last_nonempty_line(content), Some("line two".to_string()));
    }

    #[test]
    fn last_nonempty_line_trims_whitespace() {
        let content = "first\n  last line with padding   \n";
        assert_eq!(last_nonempty_line(content), Some("last line with padding".to_string()));
    }

    #[test]
    fn last_nonempty_line_none_for_empty_content() {
        assert_eq!(last_nonempty_line(""), None);
        assert_eq!(last_nonempty_line("\n\n   \n"), None);
    }

    #[test]
    fn tail_node_log_returns_none_when_file_missing() {
        // Nonexistent node name -> nonexistent log file under whatever the CWD happens to be
        // during tests; this only exercises the "file doesn't exist" branch, not real I/O
        // content, so it's safe regardless of test execution order/CWD.
        assert_eq!(tail_node_log("@mecha10/definitely-not-a-real-node-xyz"), None);
    }
}