mati 0.1.4

//! End-to-end test for the `mati daemon start` lifecycle path.
//!
//! Validates that `cli::daemon::run_daemon_start`:
//!   1. Writes a `serve_start` lifecycle event on startup.
//!   2. Comes up cleanly (mati.sock + mati.pid present).
//!   3. Exits cleanly on SIGTERM and writes a `serve_shutdown` event.
//!
//! This is the path driven by `mati supervisor install`, which had no
//! end-to-end test before — and which previously had a hang bug where a
//! handler panic would cause `tokio::join!` to wait forever for an OS
//! signal that never arrived. The signaler now also wakes via
//! `shutdown.wait()`, so the daemon exits cleanly even when the accept
//! loop self-exits.
//!
//! Marked `#[ignore]` because subprocess tests are slow and depend on a
//! writable `~/`. Run with:
//!
//!     cargo test --test daemon_lifecycle -- --ignored

use std::path::Path;
use std::process::Stdio;
use std::time::{Duration, Instant};

use tempfile::TempDir;
use tokio::io::AsyncWriteExt;
use tokio::net::UnixStream;
use tokio::process::Command;

use mati_core::store::db::Store;
use mati_core::store::derive_slug;
use mati_core::store::enforcement::{scan_enforcement_events, EnforcementEventType};

mod common;

const READY_TIMEOUT: Duration = Duration::from_secs(20);
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(15);
const POLL: Duration = Duration::from_millis(100);

#[tokio::test]
#[ignore]
async fn daemon_start_writes_lifecycle_events_and_exits_cleanly_on_sigterm() {
    let project_temp = TempDir::new().expect("project tempdir");
    let project = std::fs::canonicalize(project_temp.path()).expect("canonicalize project");

    // ── 1. Spawn mati daemon start. ───────────────────────────────────────
    let bin = env!("CARGO_BIN_EXE_mati");
    let stderr_path = project.join("daemon.stderr");
    let stderr_file = std::fs::File::create(&stderr_path).unwrap();
    let mut child = Command::new(bin)
        .arg("daemon")
        .arg("start")
        .current_dir(&project)
        .env("RUST_LOG", "info")
        .env("MATI_HOME", common::mati_home()) // isolate store off real ~/.mati
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::from(stderr_file))
        .kill_on_drop(true) // safety net if the test panics before explicit kill
        .spawn()
        .expect("failed to spawn `mati daemon start`");
    let pid = child.id().expect("child pid available pre-wait");

    // ── 2. Wait for daemon-ready. ─────────────────────────────────────────
    let slug = derive_slug(&project);
    let mati_root = common::mati_home().join(&slug);
    let lifecycle_log = mati_root.join("lifecycle.log");
    let sock = mati_root.join("mati.sock");

    if wait_for_path(&sock, READY_TIMEOUT).is_err() {
        let _ = child.kill().await;
        let stderr = std::fs::read_to_string(&stderr_path).unwrap_or_default();
        panic!("mati.sock never appeared.\nstderr:\n{stderr}");
    }

    let start_seen = wait_for_log_event(&lifecycle_log, "serve_start", READY_TIMEOUT);
    assert!(
        start_seen,
        "lifecycle.log should contain serve_start; contents:\n{:?}",
        std::fs::read_to_string(&lifecycle_log).ok()
    );

    // ── 3. Send SIGTERM and await clean exit via tokio's async wait. ──────
    // Using `child.wait()` (not raw libc::kill(0)) ensures we observe the
    // *actual* process exit rather than the zombie state, AND it reaps so
    // the OS doesn't leak a zombie. `tokio::time::timeout` bounds the wait.
    unsafe {
        // SAFETY: SIGTERM to our own child is well-defined; this is the
        // daemon's documented graceful-shutdown signal.
        libc::kill(pid as libc::pid_t, libc::SIGTERM);
    }
    let exit = tokio::time::timeout(SHUTDOWN_TIMEOUT, child.wait()).await;
    match exit {
        Ok(Ok(status)) => {
            // Daemon should exit successfully on SIGTERM (graceful path).
            // Some platforms report 0, some 143 (128+15). Either is fine.
            let _ = status;
        }
        Ok(Err(e)) => {
            let _ = child.kill().await;
            panic!("child wait error: {e}");
        }
        Err(_) => {
            let _ = child.kill().await;
            let stderr = std::fs::read_to_string(&stderr_path).unwrap_or_default();
            panic!(
                "daemon did not exit within {SHUTDOWN_TIMEOUT:?} after SIGTERM\n\
                 stderr:\n{stderr}"
            );
        }
    }

    // ── 4. lifecycle.log must contain serve_shutdown. ─────────────────────
    let shutdown_seen =
        wait_for_log_event(&lifecycle_log, "serve_shutdown", Duration::from_secs(2));
    assert!(
        shutdown_seen,
        "lifecycle.log should contain serve_shutdown after SIGTERM; contents:\n{:?}",
        std::fs::read_to_string(&lifecycle_log).ok()
    );

    // The reason should specifically be signal_sigterm.
    let log_contents = std::fs::read_to_string(&lifecycle_log).unwrap();
    assert!(
        log_contents.contains("\tserve_shutdown\tsignal_sigterm"),
        "expected serve_shutdown reason 'signal_sigterm'; got:\n{log_contents}"
    );

    // Cleanup state on disk: sock + pid removed by the cleanup path.
    assert!(!sock.exists(), "mati.sock should be removed on shutdown");
    assert!(
        !mati_root.join("mati.pid").exists(),
        "mati.pid should be removed on shutdown"
    );
}

/// Test that `mati serve` (MCP stdio server) exits cleanly on SIGTERM after
/// the MCP client disconnects (idle-wait phase).
///
/// This is the path exercised by Fix 2: `spawn_signal_listener` owns the sole
/// SIGTERM subscription. When SIGTERM arrives it calls `shutdown.signal()`,
/// which resolves the `signal_shutdown.wait()` arm of the outer `select!` in
/// `serve()`. `wait_for_idle_or_signal` has no signal handler of its own —
/// the duplicate was removed. This test verifies the fix is complete: SIGTERM
/// during idle-wait reaches cleanup and the process exits cleanly.
///
/// Ready probe: we wait for `mati.sock` to appear. The socket is bound in a
/// task spawned before `spawn_signal_listener`; by the time the socket file
/// exists the signal registration (which has no I/O) has already completed.
// CI-QUARANTINED (WI-21): this races deterministically on the CI runners —
// dropping stdin before the MCP `initialize` completes makes the proxy report
// `serve_failed` ("connection closed: initialize request") instead of the clean
// idle-wait -> SIGTERM path, so `serve_shutdown` is never written. Excluded from
// the gating "Ignored integration tests" job via `-E` in ci.yml until the
// initialize handshake is completed before the simulated disconnect. Still runs
// (and passes) locally.
#[tokio::test]
#[ignore]
async fn serve_exits_cleanly_on_sigterm_after_client_disconnect() {
    let project_temp = TempDir::new().expect("project tempdir");
    let project = std::fs::canonicalize(project_temp.path()).expect("canonicalize project");
    // Unlike the `daemon start` tests, the child here is the proxy: it spawns
    // the daemon detached, so `kill_on_drop` never reaches it. Declared after
    // `project_temp` so it drops first, while its cwd still exists.
    let _daemon_guard = common::DaemonGuard::for_mati_home(&project, common::mati_home());

    let bin = env!("CARGO_BIN_EXE_mati");
    let stderr_path = project.join("serve.stderr");
    let stderr_file = std::fs::File::create(&stderr_path).unwrap();

    // ── 1. Spawn mati serve with piped stdin. ────────────────────────────────
    // Dropping the write end simulates MCP client disconnect: rmcp transport
    // sees EOF → service.waiting() returns → serve() enters idle-wait select.
    let mut child = Command::new(bin)
        .arg("serve")
        .current_dir(&project)
        .env("RUST_LOG", "info")
        .env("MATI_HOME", common::mati_home()) // isolate store off real ~/.mati
        .stdin(Stdio::piped())
        .stdout(Stdio::null())
        .stderr(Stdio::from(stderr_file))
        .kill_on_drop(true)
        .spawn()
        .expect("failed to spawn `mati serve`");
    let pid = child.id().expect("child pid available pre-wait");

    // Drop stdin write end → EOF → client disconnect → enters idle-wait.
    drop(child.stdin.take());

    // ── 2. Wait for mati.sock — confirms idle-wait + signal handler ready. ──
    let slug = derive_slug(&project);
    let mati_root = common::mati_home().join(&slug);
    let lifecycle_log = mati_root.join("lifecycle.log");
    let sock = mati_root.join("mati.sock");

    if wait_for_path(&sock, READY_TIMEOUT).is_err() {
        let _ = child.kill().await;
        let stderr = std::fs::read_to_string(&stderr_path).unwrap_or_default();
        panic!("mati.sock never appeared after stdin close.\nstderr:\n{stderr}");
    }

    let start_seen = wait_for_log_event(&lifecycle_log, "serve_start", Duration::from_secs(5));
    assert!(
        start_seen,
        "lifecycle.log should contain serve_start; contents:\n{:?}",
        std::fs::read_to_string(&lifecycle_log).ok()
    );

    // ── 3. Send SIGTERM and await clean exit. ────────────────────────────────
    // The signal goes to spawn_signal_listener's task → shutdown.signal() →
    // signal_shutdown.wait() in the inner select! resolves → cleanup runs.
    unsafe {
        libc::kill(pid as libc::pid_t, libc::SIGTERM);
    }
    let exit = tokio::time::timeout(SHUTDOWN_TIMEOUT, child.wait()).await;
    match exit {
        Ok(Ok(_status)) => { /* 0 or 143 (128+SIGTERM) are both acceptable */ }
        Ok(Err(e)) => {
            let _ = child.kill().await;
            panic!("child wait error: {e}");
        }
        Err(_) => {
            let _ = child.kill().await;
            let stderr = std::fs::read_to_string(&stderr_path).unwrap_or_default();
            panic!(
                "mati serve did not exit within {SHUTDOWN_TIMEOUT:?} after SIGTERM\n\
                 stderr:\n{stderr}"
            );
        }
    }

    // ── 4. Lifecycle events and cleanup. ────────────────────────────────────
    let shutdown_seen =
        wait_for_log_event(&lifecycle_log, "serve_shutdown", Duration::from_secs(2));
    assert!(
        shutdown_seen,
        "lifecycle.log should contain serve_shutdown after SIGTERM; contents:\n{:?}",
        std::fs::read_to_string(&lifecycle_log).ok()
    );

    // mcp/server.rs uses "signal_shutdown" (unlike cli/daemon.rs which uses
    // the REASONS-array string "signal_sigterm").
    let log_contents = std::fs::read_to_string(&lifecycle_log).unwrap();
    assert!(
        log_contents.contains("\tserve_shutdown\tsignal_shutdown"),
        "expected serve_shutdown reason 'signal_shutdown'; got:\n{log_contents}"
    );

    // Graceful shutdown must unlink both files so sibling processes can restart.
    assert!(
        !sock.exists(),
        "mati.sock should be removed on SIGTERM shutdown"
    );
    assert!(
        !mati_root.join("mati.pid").exists(),
        "mati.pid should be removed on SIGTERM shutdown"
    );
}

/// Regression test for the bug where `metrics::init()` was only ever called
/// on the `mati serve` proxy path (a different process's `OnceLock`), never
/// in `run_daemon_start`. Every `metrics::record` call in `dispatch_v2`
/// runs in the daemon process, so every sample died at `METRICS.get()` and
/// `mati doctor --internal` could never report a number.
///
/// Spawns a real daemon, drives one request through it, then asserts
/// `mati doctor --internal --json` reports a nonzero `total_calls` instead
/// of `null`/`(daemon has no metrics yet)`.
#[tokio::test]
#[ignore]
async fn daemon_start_initializes_metrics_for_doctor_internal() {
    let project_temp = TempDir::new().expect("project tempdir");
    let project = std::fs::canonicalize(project_temp.path()).expect("canonicalize project");

    let bin = env!("CARGO_BIN_EXE_mati");
    let stderr_path = project.join("daemon.stderr");
    let stderr_file = std::fs::File::create(&stderr_path).unwrap();
    let mut child = Command::new(bin)
        .arg("daemon")
        .arg("start")
        .current_dir(&project)
        .env("RUST_LOG", "info")
        .env("MATI_HOME", common::mati_home())
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::from(stderr_file))
        .kill_on_drop(true)
        .spawn()
        .expect("failed to spawn `mati daemon start`");
    let pid = child.id().expect("child pid available pre-wait");

    let slug = derive_slug(&project);
    let mati_root = common::mati_home().join(&slug);
    let lifecycle_log = mati_root.join("lifecycle.log");
    let sock = mati_root.join("mati.sock");

    if wait_for_path(&sock, READY_TIMEOUT).is_err() {
        let _ = child.kill().await;
        let stderr = std::fs::read_to_string(&stderr_path).unwrap_or_default();
        panic!("mati.sock never appeared.\nstderr:\n{stderr}");
    }
    // Wait past the "phase=ready" event — metrics::init() runs just before
    // it is logged, so this also guarantees init() has completed.
    let ready_seen = wait_for_log_event(&lifecycle_log, "startup", READY_TIMEOUT)
        && wait_for_phase_ready(&lifecycle_log, READY_TIMEOUT);
    assert!(
        ready_seen,
        "lifecycle.log should contain phase=ready; contents:\n{:?}",
        std::fs::read_to_string(&lifecycle_log).ok()
    );

    // Drive one request through the daemon so there is a sample to report.
    let ping = Command::new(bin)
        .arg("ping")
        .current_dir(&project)
        .env("MATI_HOME", common::mati_home())
        .output()
        .await
        .expect("failed to run `mati ping`");
    assert!(
        ping.status.success(),
        "mati ping should succeed against a live daemon: {}",
        String::from_utf8_lossy(&ping.stderr)
    );

    // `mati doctor --internal` must now report real numbers, not "no metrics".
    let doctor = Command::new(bin)
        .arg("doctor")
        .arg("--internal")
        .arg("--json")
        .current_dir(&project)
        .env("MATI_HOME", common::mati_home())
        .output()
        .await
        .expect("failed to run `mati doctor --internal --json`");
    assert!(
        doctor.status.success(),
        "mati doctor --internal --json should succeed against a live daemon: {}",
        String::from_utf8_lossy(&doctor.stderr)
    );
    let stdout = String::from_utf8_lossy(&doctor.stdout);
    let report: serde_json::Value =
        serde_json::from_str(stdout.trim()).expect("doctor --internal --json output valid JSON");
    assert!(
        !report.is_null(),
        "metrics snapshot should not be null once the daemon has served requests"
    );
    let total_calls = report
        .get("total_calls")
        .and_then(|v| v.as_u64())
        .unwrap_or(0);
    assert!(
        total_calls > 0,
        "expected total_calls > 0, got report: {report}"
    );

    // Cleanup: SIGTERM and reap.
    unsafe {
        libc::kill(pid as libc::pid_t, libc::SIGTERM);
    }
    let _ = tokio::time::timeout(SHUTDOWN_TIMEOUT, child.wait()).await;
}

/// Measures the graceful-shutdown wall clock when the drain has real in-flight
/// handlers to wait on — the busy case the design rests on that every prior
/// trial missed by running idle (see checkpoint `shutdown-terminator-shipped`).
///
/// Pins `N` handlers in-flight by opening sockets and writing a partial line
/// (no `\n`), so each blocks in `read_line` for its 3s `READ_TIMEOUT`. SIGTERM
/// then forces `serve_loop_graceful` to drain them under the 5s `DRAIN_TIMEOUT`.
/// Asserts:
///   1. exit is bounded well under any OS grace period (`< 8s`),
///   2. the drain actually engaged (`> 500ms`, vs the ~93ms idle path),
///   3. the `CleanShutdown` terminator is still the chain tail afterward.
///
/// Prints the measured wall clock on a `MEASURE` line. Run with `--nocapture`:
///
///     cargo test --test daemon_lifecycle -- --ignored --nocapture \
///         busy_daemon_shutdown_drains_inflight_then_records_terminator
#[tokio::test]
#[ignore]
async fn busy_daemon_shutdown_drains_inflight_then_records_terminator() {
    const IN_FLIGHT: usize = 4;

    // This test reopens the store in-process after the daemon exits; isolate
    // MATI_HOME off the real ~/.mati (also required by the store_home_guard).
    common::isolate_mati_home();

    let project_temp = TempDir::new().expect("project tempdir");
    let project = std::fs::canonicalize(project_temp.path()).expect("canonicalize project");

    let bin = env!("CARGO_BIN_EXE_mati");
    let stderr_path = project.join("daemon.stderr");
    let stderr_file = std::fs::File::create(&stderr_path).unwrap();
    let mut child = Command::new(bin)
        .arg("daemon")
        .arg("start")
        .current_dir(&project)
        .env("RUST_LOG", "info")
        .env("MATI_HOME", common::mati_home())
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::from(stderr_file))
        .kill_on_drop(true)
        .spawn()
        .expect("failed to spawn `mati daemon start`");
    let pid = child.id().expect("child pid available pre-wait");

    let slug = derive_slug(&project);
    let mati_root = common::mati_home().join(&slug);
    let lifecycle_log = mati_root.join("lifecycle.log");
    let sock = mati_root.join("mati.sock");

    if wait_for_path(&sock, READY_TIMEOUT).is_err() {
        let _ = child.kill().await;
        let stderr = std::fs::read_to_string(&stderr_path).unwrap_or_default();
        panic!("mati.sock never appeared.\nstderr:\n{stderr}");
    }
    let ready = wait_for_log_event(&lifecycle_log, "serve_start", READY_TIMEOUT)
        && wait_for_phase_ready(&lifecycle_log, READY_TIMEOUT);
    assert!(
        ready,
        "daemon never reached phase=ready; contents:\n{:?}",
        std::fs::read_to_string(&lifecycle_log).ok()
    );

    // Pin handlers in-flight: connect, write a partial line (no newline), and
    // hold the streams so each handler blocks in `read_line` until READ_TIMEOUT.
    let mut pinned = Vec::with_capacity(IN_FLIGHT);
    for _ in 0..IN_FLIGHT {
        let mut stream = UnixStream::connect(&sock)
            .await
            .expect("connect to daemon socket");
        stream.write_all(b"x").await.expect("partial write");
        stream.flush().await.expect("flush partial write");
        pinned.push(stream);
    }
    // Settle so the accept loop has spawned every handler into `in_flight`
    // before the accept loop stops. Without this the drain could see zero
    // in-flight and the measurement would silently fall back to the idle path.
    tokio::time::sleep(Duration::from_millis(300)).await;

    let t0 = Instant::now();
    unsafe {
        // SAFETY: SIGTERM to our own child — the documented graceful signal.
        libc::kill(pid as libc::pid_t, libc::SIGTERM);
    }
    let exit = tokio::time::timeout(SHUTDOWN_TIMEOUT, child.wait()).await;
    let elapsed = t0.elapsed();
    drop(pinned);

    match exit {
        Ok(Ok(_status)) => {}
        Ok(Err(e)) => {
            let _ = child.kill().await;
            panic!("child wait error: {e}");
        }
        Err(_) => {
            let _ = child.kill().await;
            let stderr = std::fs::read_to_string(&stderr_path).unwrap_or_default();
            panic!(
                "busy daemon did not exit within {SHUTDOWN_TIMEOUT:?} after SIGTERM\n\
                 stderr:\n{stderr}"
            );
        }
    }

    println!(
        "MEASURE busy_shutdown in_flight={IN_FLIGHT} sigterm_to_exit={:.3}s",
        elapsed.as_secs_f64()
    );

    assert!(
        elapsed < Duration::from_secs(8),
        "busy shutdown took {elapsed:?}; must stay well under any OS grace period"
    );
    assert!(
        elapsed > Duration::from_millis(500),
        "busy shutdown took only {elapsed:?} — drain never engaged, so this \
         measured the idle path, not the busy one"
    );

    // The terminator must still land after a non-trivial drain. Reopen the
    // released store and confirm `CleanShutdown` is the chain tail.
    let store = Store::open(&project)
        .await
        .expect("reopen store after exit");
    let events = scan_enforcement_events(&store, 0, u64::MAX)
        .await
        .expect("scan enforcement events");
    let tail = events.last().expect("chain has at least one event");
    match &tail.event_type {
        EnforcementEventType::CleanShutdown { reason } => {
            println!("MEASURE terminator seq={} reason={reason}", tail.seq_no);
        }
        other => panic!(
            "chain tail is {other:?} (seq {}), expected CleanShutdown terminator",
            tail.seq_no
        ),
    }
}

fn wait_for_phase_ready(log_path: &Path, timeout: Duration) -> bool {
    let start = Instant::now();
    while start.elapsed() < timeout {
        if let Ok(contents) = std::fs::read_to_string(log_path) {
            if contents.contains("phase=ready") {
                return true;
            }
        }
        std::thread::sleep(POLL);
    }
    false
}

/// The daemon's INFO output must land somewhere a human can find it, with
/// **no `RUST_LOG` set** — production never sets it.
///
/// Before `mcp::daemon_log`, `tracing` wrote to stderr at a `warn` default.
/// Stderr reached `~/.mati/daemon_start.log` only when `ensure_daemon` spawned
/// the daemon, and INFO was filtered out before it got there. So the lines that
/// answer "did this run?" — `mati daemon listening`, `staleness analysis
/// complete scanned=… updated=…` — existed in the source and nowhere on disk,
/// and three investigations read "nothing invoked it" as "it is broken".
///
/// Note the deliberately absent `.env("RUST_LOG", …)`: setting it is what made
/// the original problem invisible in every existing test.
#[tokio::test]
#[ignore]
async fn daemon_info_logging_is_readable_without_rust_log() {
    let project_temp = TempDir::new().expect("project tempdir");
    let project = std::fs::canonicalize(project_temp.path()).expect("canonicalize project");

    let bin = env!("CARGO_BIN_EXE_mati");
    let mut child = Command::new(bin)
        .arg("daemon")
        .arg("start")
        .current_dir(&project)
        .env("MATI_HOME", common::mati_home())
        .env_remove("RUST_LOG")
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::null())
        .kill_on_drop(true)
        .spawn()
        .expect("failed to spawn `mati daemon start`");
    let pid = child.id().expect("child pid available pre-wait");

    let slug = derive_slug(&project);
    let mati_root = common::mati_home().join(&slug);
    let daemon_log = mati_root.join("daemon.log");

    let seen = wait_for_file_containing(&daemon_log, "mati daemon listening", READY_TIMEOUT);

    unsafe {
        // SAFETY: SIGTERM to our own child — the documented shutdown signal.
        libc::kill(pid as libc::pid_t, libc::SIGTERM);
    }
    let _ = tokio::time::timeout(SHUTDOWN_TIMEOUT, child.wait()).await;

    assert!(
        seen,
        "daemon.log should carry the daemon's INFO lifecycle lines; contents:\n{:?}",
        std::fs::read_to_string(&daemon_log).ok()
    );
}

fn wait_for_file_containing(path: &Path, needle: &str, timeout: Duration) -> bool {
    let start = Instant::now();
    while start.elapsed() < timeout {
        if let Ok(contents) = std::fs::read_to_string(path) {
            if contents.contains(needle) {
                return true;
            }
        }
        std::thread::sleep(POLL);
    }
    false
}

fn wait_for_path(path: &Path, timeout: Duration) -> Result<(), &'static str> {
    let start = Instant::now();
    while start.elapsed() < timeout {
        if path.exists() {
            return Ok(());
        }
        std::thread::sleep(POLL);
    }
    Err("timeout")
}

fn wait_for_log_event(log_path: &Path, event: &str, timeout: Duration) -> bool {
    let needle = format!("\t{event}\t");
    let start = Instant::now();
    while start.elapsed() < timeout {
        if let Ok(contents) = std::fs::read_to_string(log_path) {
            if contents.contains(&needle) {
                return true;
            }
        }
        std::thread::sleep(POLL);
    }
    false
}