node-app-build 5.23.0

//! `--daemon monorepo` mode: launch the daemon from a checked-out copy of
//! `econ-v1/node`. Lets platform contributors edit BOTH app code AND
//! daemon code in a single inner-loop iteration.
//!
//! The CLI verifies the path is a workspace containing the `node-server`
//! crate, generates a per-(monorepo, instance) env file under
//! `$XDG_CACHE_HOME/node-app/monorepo-<hash>/`, and spawns
//! `cargo run -p node-server -- --env <env>` as a child process.
//!
//! The hash covers both the monorepo path AND the instance name so that
//! running alice and bob simultaneously does not collide on socket or db.
//!
//! When `log_tx` is `Some` (TUI active), the daemon and UI dev server
//! stdout/stderr are captured, prefixed with `[<instance>]`, and piped
//! into the split-pane log view instead of being inherited.

use std::collections::hash_map::DefaultHasher;
use std::fs;
use std::hash::{Hash, Hasher};
use std::io::BufRead;
use std::os::unix::net::UnixStream;
#[cfg(unix)]
use std::os::unix::process::CommandExt;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::Mutex;
use std::time::{Duration, Instant};

use anyhow::{anyhow, bail, Context, Result};
#[cfg(unix)]
use libc;
use serde_json;

use super::{DaemonHandle, DaemonHost, InstanceProfile};
use crate::tui::{self, LogSource, LogTx, ServiceStatus, TuiEvent};

// Bumped from 120s to 240s: cold cargo run can include a non-trivial link
// step + multi-app inotify scan, especially in multi-instance platform mode
// where alice and bob compete for the page cache.
const HEALTH_TIMEOUT: Duration = Duration::from_secs(240);
const HEALTH_POLL: Duration = Duration::from_secs(2);

/// Max time we'll wait for the daemon's HTTP server to answer /api/healthz
/// after the IPC socket comes up. Distinct from `HEALTH_TIMEOUT` because the
/// IPC socket can be live before axum binds the TCP port — a window where a
/// browser-loaded Vite proxy would fail with "connection refused".
const HTTP_READY_TIMEOUT: Duration = Duration::from_secs(60);

/// Max time we'll wait for Vite to bind its UI port and respond to a smoke
/// test through the proxy. Vite startup is typically 1–3s; 30s leaves margin
/// for cold node_modules / vite optimizer runs without hanging the TUI.
const VITE_READY_TIMEOUT: Duration = Duration::from_secs(30);

/// Return PIDs of processes listening on `port` (TCP, LISTEN state). Used to
/// reap stale Vite / daemon orphans before we try to bind. On non-unix or
/// when `lsof` is unavailable this returns an empty Vec (degrading to a
/// "best effort" cleanup — the subsequent spawn will simply fail loudly on
/// EADDRINUSE).
#[cfg(unix)]
fn pids_listening_on(port: u16) -> Vec<i32> {
    let arg = format!("-iTCP:{port}");
    Command::new("lsof")
        .args(["-sTCP:LISTEN", "-t", "-P", "-n", &arg])
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::null())
        .output()
        .ok()
        .filter(|o| o.status.success())
        .map(|o| {
            String::from_utf8_lossy(&o.stdout)
                .lines()
                .filter_map(|l| l.trim().parse::<i32>().ok())
                .collect()
        })
        .unwrap_or_default()
}

#[cfg(not(unix))]
fn pids_listening_on(_port: u16) -> Vec<i32> {
    Vec::new()
}

/// Tee a child stdio stream three ways: persistent log file (if open), TUI
/// sink (if active), and the controlling terminal (when no TUI). Each output
/// line is prefixed with the instance name so multi-instance log files stay
/// distinguishable on shared display paths.
fn tee_daemon_stream<R: std::io::Read + Send + 'static>(
    reader: R,
    prefix: String,
    log_file: Option<std::sync::Arc<std::sync::Mutex<std::fs::File>>>,
    tui_tx: Option<LogTx>,
    is_stderr: bool,
) {
    use std::io::Write as _;
    std::thread::spawn(move || {
        for line in std::io::BufReader::new(reader).lines().map_while(Result::ok) {
            let formatted = format!("{prefix}{line}");

            if let Some(file) = &log_file {
                if let Ok(mut f) = file.lock() {
                    let _ = writeln!(f, "{formatted}");
                }
            }
            if let Some(tx) = &tui_tx {
                let _ = tx.send(TuiEvent::Log(crate::tui::LogEntry {
                    source: LogSource::Daemon,
                    line: formatted,
                }));
            } else if is_stderr {
                eprintln!("{formatted}");
            } else {
                println!("{formatted}");
            }
        }
    });
}

pub struct MonorepoHost {
    monorepo_path: PathBuf,
    profile: InstanceProfile,
    socket_override: Option<PathBuf>,
    dev_dir_override: Option<PathBuf>,
    log_tx: Option<LogTx>,
    child: Mutex<Option<Child>>,
    ui_child: Mutex<Option<Child>>,
    /// Env file path saved by ensure_running() so restart() can respawn.
    env_file: Mutex<Option<PathBuf>>,
}

impl MonorepoHost {
    pub fn new(
        monorepo_path: PathBuf,
        profile: InstanceProfile,
        socket_override: Option<PathBuf>,
        dev_dir_override: Option<PathBuf>,
        log_tx: Option<LogTx>,
    ) -> Self {
        Self {
            monorepo_path,
            profile,
            socket_override,
            dev_dir_override,
            log_tx,
            child: Mutex::new(None),
            ui_child: Mutex::new(None),
            env_file: Mutex::new(None),
        }
    }

    fn syslog(&self, msg: impl Into<String>) {
        tui::sys_log(self.log_tx.as_ref(), msg);
    }

    /// Spawn daemon child with stdio wired to:
    ///   1. A persistent log file at `<env_dir>/daemon.log` (always)
    ///   2. The TUI sink (when active)
    ///   3. stdout/stderr inheritance (when no TUI is attached)
    ///
    /// Persistence is the important one — without it, daemon output is lost
    /// the moment the TUI exits, which makes diagnosing startup failures
    /// (the "daemon did not come up within Ns" error) almost impossible.
    ///
    /// We exec the built binary at `target/debug/node-server` directly
    /// instead of `cargo run -p node-server`. The earlier `cargo build`
    /// step already produced that binary, and `cargo run` would otherwise
    /// re-acquire the build lock — which contends with stray cargo
    /// processes (rust-analyzer, leftover daemons from cancelled runs) and
    /// is exactly what caused the silent "Blocking waiting for file lock
    /// on build directory" hangs that manifested as health-check timeouts.
    fn spawn_daemon(&self, env_file: &Path) -> Result<Child> {
        let binary_path = self.monorepo_path.join("target/debug/node-server");
        if !binary_path.exists() {
            bail!(
                "daemon binary missing at {} — the earlier `cargo build -p node-server` \
                 step should have produced it. Run `cargo build -p node-server \
                 --features agentic_payments` in {} to recover.",
                binary_path.display(),
                self.monorepo_path.display()
            );
        }
        let mut cmd = Command::new(&binary_path);
        cmd.args(["--env"])
            .arg(env_file)
            .current_dir(&self.monorepo_path)
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());

        #[cfg(unix)]
        cmd.process_group(0);

        let mut child = cmd
            .spawn()
            .with_context(|| format!("spawn {}", binary_path.display()))?;

        // Open daemon.log next to the env file. Best-effort: failure to open
        // (e.g. permission denied) only loses persistence; we still tee to
        // TUI/terminal. Truncate on each spawn so the file always reflects
        // the *current* run rather than accumulating across restarts.
        let log_file_path = env_file
            .parent()
            .map(|d| d.join("daemon.log"))
            .unwrap_or_else(|| PathBuf::from("daemon.log"));
        let log_file: Option<std::sync::Arc<std::sync::Mutex<std::fs::File>>> =
            std::fs::OpenOptions::new()
                .create(true)
                .write(true)
                .truncate(true)
                .open(&log_file_path)
                .ok()
                .map(|f| std::sync::Arc::new(std::sync::Mutex::new(f)));
        if log_file.is_some() {
            self.syslog(format!("→ daemon log: {}", log_file_path.display()));
        } else {
            self.syslog(format!(
                "⚠ could not open daemon log file at {} — output will only show in TUI/terminal",
                log_file_path.display()
            ));
        }

        let prefix = format!("[{}] ", self.profile.name);
        if let Some(stdout) = child.stdout.take() {
            tee_daemon_stream(
                stdout,
                prefix.clone(),
                log_file.clone(),
                self.log_tx.clone(),
                false,
            );
        }
        if let Some(stderr) = child.stderr.take() {
            tee_daemon_stream(stderr, prefix, log_file, self.log_tx.clone(), true);
        }

        Ok(child)
    }

    /// Spawn UI dev server with stdio wired to TUI or terminal.
    fn spawn_ui(&self) -> Option<Child> {
        let ui_dir = self.monorepo_path.join("system/ui");
        if !ui_dir.join("package.json").exists() {
            return None;
        }
        let yarn = if which_bin("yarn") { "yarn" } else { "npm" };
        let ui_port = self.profile.ui_port;
        let api_port = self.profile.http_port;
        self.syslog(format!(
            "→ starting UI dev server ({yarn} dev --port {ui_port}) in {}",
            ui_dir.display()
        ));

        let mut cmd = Command::new(yarn);
        cmd.args(["dev", "--port", &ui_port.to_string()])
            .env("VITE_BACKEND_PORT", api_port.to_string())
            .current_dir(&ui_dir)
            .stdin(Stdio::null());

        if self.log_tx.is_some() {
            cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
        } else {
            cmd.stdout(Stdio::inherit()).stderr(Stdio::inherit());
        }

        #[cfg(unix)]
        cmd.process_group(0);

        match cmd.spawn() {
            Ok(mut child) => {
                if let Some(tx) = &self.log_tx {
                    let prefix = format!("[{}] ", self.profile.name);
                    if let Some(stdout) = child.stdout.take() {
                        let tx = tx.clone();
                        let p = prefix.clone();
                        std::thread::spawn(move || {
                            for line in std::io::BufReader::new(stdout).lines().map_while(Result::ok) {
                                let _ = tx.send(TuiEvent::Log(crate::tui::LogEntry {
                                    source: LogSource::UiServer,
                                    line: format!("{p}{line}"),
                                }));
                            }
                        });
                    }
                    if let Some(stderr) = child.stderr.take() {
                        let tx = tx.clone();
                        let p = prefix.clone();
                        std::thread::spawn(move || {
                            for line in std::io::BufReader::new(stderr).lines().map_while(Result::ok) {
                                let _ = tx.send(TuiEvent::Log(crate::tui::LogEntry {
                                    source: LogSource::UiServer,
                                    line: format!("{p}{line}"),
                                }));
                            }
                        });
                    }
                }
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::UiServer,
                    ServiceStatus::Ready,
                    Some(format!("[{}] http://localhost:{ui_port}", self.profile.name)),
                );
                self.syslog(format!(
                    "✓ UI available at http://localhost:{ui_port} (Vite, hot reload) \
                     or http://localhost:{api_port} (backend-served, requires `yarn build` in system/ui)"
                ));
                Some(child)
            }
            Err(e) => {
                self.syslog(format!(
                    "⚠ could not start UI dev server: {e} — run manually: \
                     cd system/ui && VITE_BACKEND_PORT={api_port} {yarn} dev --port {ui_port}"
                ));
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::UiServer,
                    ServiceStatus::Disabled,
                    Some(format!("[{}]", self.profile.name)),
                );
                None
            }
        }
    }

    fn shutdown_daemon_only(&self) {
        if let Some(mut child) = self.child.lock().unwrap().take() {
            let pid = child.id() as i32;
            #[cfg(unix)]
            unsafe {
                libc::kill(-pid, libc::SIGTERM);
            }
            let t = Instant::now();
            loop {
                match child.try_wait() {
                    Ok(Some(_)) => break,
                    Ok(None) if t.elapsed() > Duration::from_secs(10) => {
                        #[cfg(unix)]
                        unsafe {
                            libc::kill(-pid, libc::SIGKILL);
                        }
                        let _ = child.wait();
                        break;
                    }
                    Ok(None) => std::thread::sleep(Duration::from_millis(200)),
                    Err(_) => break,
                }
            }
        }
    }
}

impl MonorepoHost {
    /// Reap any process listening on `port` (best-effort SIGTERM, escalating
    /// to SIGKILL after `grace`). Logged as "freeing <label>" so the TUI
    /// makes it obvious *which* port was contested.
    ///
    /// This is the antidote to the most common "node-app dev doesn't work"
    /// failure mode: a stale Vite or orphaned `node-server` from a previous
    /// run is still bound to the port the new launch wants, and the new
    /// process silently picks a different port (Vite) or fails to bind
    /// (axum) — either way the UI ends up calling a daemon that isn't there.
    #[cfg(unix)]
    fn free_port(&self, port: u16, label: &str) {
        let pids = pids_listening_on(port);
        if pids.is_empty() {
            return;
        }
        self.syslog(format!(
            "→ freeing {label} port {port} from stale pid(s) {}…",
            pids.iter()
                .map(i32::to_string)
                .collect::<Vec<_>>()
                .join(",")
        ));
        for pid in &pids {
            unsafe {
                libc::kill(*pid, libc::SIGTERM);
            }
        }
        let deadline = Instant::now() + Duration::from_secs(5);
        while Instant::now() < deadline {
            if pids_listening_on(port).is_empty() {
                self.syslog(format!("✓ {label} port {port} freed"));
                return;
            }
            std::thread::sleep(Duration::from_millis(200));
        }
        let remaining = pids_listening_on(port);
        if !remaining.is_empty() {
            self.syslog(format!(
                "⚠ {label} port {port} still held after SIGTERM — escalating to SIGKILL"
            ));
            for pid in &remaining {
                unsafe {
                    libc::kill(*pid, libc::SIGKILL);
                }
            }
        }
    }

    #[cfg(not(unix))]
    fn free_port(&self, _port: u16, _label: &str) {}

    /// Block until `http://127.0.0.1:<port>/api/healthz` answers (any status
    /// below 500 counts — we only care that the listener is up and routing).
    /// Returns Err on timeout so the caller can refuse to spawn Vite and
    /// surface a useful failure rather than letting the browser hit a dead
    /// proxy target.
    fn wait_http_ready(&self, port: u16) -> Result<()> {
        let url = format!("http://127.0.0.1:{port}/api/healthz");
        let started = Instant::now();
        self.syslog(format!("→ waiting for daemon HTTP on {url}…"));
        loop {
            // ureq treats any HTTP status >= 400 as an Err — but for a
            // readiness check we want to accept those too (the listener is
            // up, that's what matters). Match both Ok and Err(Status).
            let ready = match ureq::get(&url)
                .timeout(Duration::from_secs(2))
                .call()
            {
                Ok(_) => true,
                Err(ureq::Error::Status(code, _)) => code < 500,
                Err(_) => false,
            };
            if ready {
                self.syslog(format!(
                    "✓ daemon HTTP ready ({}ms)",
                    started.elapsed().as_millis()
                ));
                return Ok(());
            }
            if let Some(child) = self.child.lock().unwrap().as_mut() {
                if let Ok(Some(status)) = child.try_wait() {
                    bail!(
                        "daemon exited with {status} while waiting for HTTP readiness on {url}"
                    );
                }
            }
            if started.elapsed() >= HTTP_READY_TIMEOUT {
                bail!(
                    "daemon HTTP server not responding on {} after {}s",
                    url,
                    HTTP_READY_TIMEOUT.as_secs()
                );
            }
            std::thread::sleep(Duration::from_millis(300));
        }
    }

    /// Verify the Vite WebSocket proxy by speaking raw HTTP upgrade and
    /// checking for `HTTP/1.1 101`. This catches path-mismatch bugs
    /// (frontend `/ws` vs backend `/api/ws`) that otherwise only surface as
    /// noisy ECONNRESET loops in the Vite log.
    fn verify_ws_proxy(&self, ui_port: u16) -> Result<()> {
        use std::io::{Read, Write};
        use std::net::TcpStream;
        // Vite usually binds IPv6 on macOS; resolve via the OS so we hit
        // whichever family it picked.
        let addr_candidates: Vec<std::net::SocketAddr> = std::net::ToSocketAddrs::to_socket_addrs(
            &format!("localhost:{ui_port}"),
        )
        .map(|iter| iter.collect())
        .unwrap_or_default();
        if addr_candidates.is_empty() {
            bail!("could not resolve localhost:{ui_port} for WS probe");
        }
        let req = format!(
            "GET /api/ws HTTP/1.1\r\n\
             Host: localhost:{ui_port}\r\n\
             Upgrade: websocket\r\n\
             Connection: Upgrade\r\n\
             Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\
             Sec-WebSocket-Version: 13\r\n\
             \r\n"
        );
        let mut last_err = String::from("no candidate addresses tried");
        for addr in &addr_candidates {
            match TcpStream::connect_timeout(addr, Duration::from_secs(2)) {
                Ok(mut sock) => {
                    let _ = sock.set_read_timeout(Some(Duration::from_secs(2)));
                    let _ = sock.write_all(req.as_bytes());
                    let mut buf = [0u8; 256];
                    match sock.read(&mut buf) {
                        Ok(n) if n > 0 => {
                            let response = String::from_utf8_lossy(&buf[..n]);
                            let first_line =
                                response.lines().next().unwrap_or("").to_string();
                            if first_line.contains("101") {
                                self.syslog(format!(
                                    "✓ ws proxy verified: localhost:{ui_port}/api/ws → backend ({first_line})"
                                ));
                                return Ok(());
                            }
                            last_err = format!(
                                "{addr} replied {first_line} (expected 101 Switching Protocols)"
                            );
                        }
                        Ok(_) => last_err = format!("{addr} closed without responding"),
                        Err(e) => last_err = format!("{addr} read: {e}"),
                    }
                }
                Err(e) => last_err = format!("{addr} connect: {e}"),
            }
        }
        bail!("ws proxy probe failed: {last_err}");
    }

    /// Smoke-test the Vite proxy by issuing `/api/healthz` against the UI
    /// port and verifying we get a backend status (anything < 500 — the
    /// proxy forwarded successfully). Emits a clear ✓ or ⚠ line so the
    /// developer can tell at a glance whether the `/api` lane works.
    ///
    /// We try `localhost` first (which resolves to whichever family Vite
    /// bound — Vite 5+ defaults to IPv6 `::1` on macOS) and fall back to
    /// `127.0.0.1` then `[::1]` so a misconfigured /etc/hosts doesn't make
    /// us report a false negative.
    fn verify_proxy(&self, ui_port: u16, http_port: u16) -> Result<()> {
        let urls = [
            format!("http://localhost:{ui_port}/api/healthz"),
            format!("http://127.0.0.1:{ui_port}/api/healthz"),
            format!("http://[::1]:{ui_port}/api/healthz"),
        ];
        let started = Instant::now();
        let mut last_err = String::new();
        loop {
            for url in &urls {
                match ureq::get(url).timeout(Duration::from_secs(2)).call() {
                    Ok(resp) => {
                        self.syslog(format!(
                            "✓ proxy verified: {url} → 127.0.0.1:{http_port} (status {})",
                            resp.status()
                        ));
                        return Ok(());
                    }
                    Err(ureq::Error::Status(code, _)) if code < 500 => {
                        self.syslog(format!(
                            "✓ proxy verified: {url} → 127.0.0.1:{http_port} (status {code})"
                        ));
                        return Ok(());
                    }
                    Err(e) => last_err = format!("{url}: {e}"),
                }
            }
            if started.elapsed() >= VITE_READY_TIMEOUT {
                bail!(
                    "vite proxy unreachable on port {ui_port} after {}s (last: {last_err}) — \
                     check that VITE_BACKEND_PORT was set when vite started, \
                     and that the daemon is listening on 127.0.0.1:{http_port}",
                    VITE_READY_TIMEOUT.as_secs()
                );
            }
            std::thread::sleep(Duration::from_millis(400));
        }
    }

    /// Run a build command, streaming stderr to the TUI log (or to the
    /// terminal when no TUI is active).  Returns the exit status and all
    /// stderr lines collected — callers include them in error messages.
    fn run_build_cmd(
        &self,
        cmd: &mut Command,
        log_source: LogSource,
    ) -> Result<(std::process::ExitStatus, Vec<String>)> {
        use std::sync::mpsc;

        cmd.stdout(Stdio::null()).stderr(Stdio::piped());
        let mut child = cmd.spawn()?;
        let stderr = child.stderr.take().expect("piped");

        let log_tx = self.log_tx.clone();
        let (tx, rx) = mpsc::channel::<Vec<String>>();
        std::thread::spawn(move || {
            let mut lines = Vec::new();
            for line in std::io::BufReader::new(stderr).lines().map_while(Result::ok) {
                if let Some(ref ltx) = log_tx {
                    let _ = ltx.send(TuiEvent::Log(crate::tui::LogEntry {
                        source: log_source,
                        line: line.clone(),
                    }));
                } else {
                    eprintln!("{line}");
                }
                lines.push(line);
            }
            let _ = tx.send(lines);
        });

        let status = loop {
            match child.try_wait()? {
                Some(s) => break s,
                None => {
                    if crate::commands::dev::is_cancelled() {
                        let _ = child.kill();
                        let _ = child.wait();
                        let _lines = rx.recv().unwrap_or_default();
                        bail!("build cancelled");
                    }
                    std::thread::sleep(Duration::from_millis(50));
                }
            }
        };
        let lines = rx.recv().unwrap_or_default();
        Ok((status, lines))
    }
}

impl DaemonHost for MonorepoHost {
    fn ensure_running(&self) -> Result<DaemonHandle> {
        // 1. Validate the path looks like the node monorepo.
        let cargo_toml = self.monorepo_path.join("Cargo.toml");
        if !cargo_toml.exists() {
            bail!(
                "{} doesn't look like a monorepo root (no Cargo.toml found)",
                self.monorepo_path.display()
            );
        }
        let manifest = fs::read_to_string(&cargo_toml)
            .with_context(|| format!("read {}", cargo_toml.display()))?;
        let has_server = manifest.contains("\"system/server\"")
            || manifest.contains("system/server")
            || manifest.contains("\"apps/server\"")
            || manifest.contains("apps/server");
        if !has_server {
            bail!(
                "{}/Cargo.toml does not include 'system/server' — is this the right path?",
                self.monorepo_path.display()
            );
        }

        // 2. Resolve cache dir (namespaced by monorepo path + instance name).
        let env_dir = monorepo_env_dir(&self.monorepo_path, &self.profile.name)?;
        fs::create_dir_all(&env_dir).ok();
        let pid_file = env_dir.join("daemon.pid");

        // 2a. Kill any previous instance.
        if let Ok(contents) = fs::read_to_string(&pid_file) {
            if let Ok(old_pid) = contents.trim().parse::<i32>() {
                let alive = unsafe { libc::kill(old_pid, 0) == 0 };
                if alive {
                    self.syslog(format!(
                        "→ previous instance found (PID {old_pid}), sending SIGTERM…"
                    ));
                    unsafe {
                        libc::kill(-old_pid, libc::SIGTERM);
                    }
                    let deadline = Instant::now();
                    loop {
                        std::thread::sleep(Duration::from_millis(200));
                        if unsafe { libc::kill(old_pid, 0) } != 0 {
                            self.syslog(format!(
                                "✓ previous instance exited ({}ms)",
                                deadline.elapsed().as_millis()
                            ));
                            break;
                        }
                        if deadline.elapsed() > Duration::from_secs(15) {
                            self.syslog("⚠ previous instance did not exit after 15s, SIGKILL");
                            unsafe {
                                libc::kill(-old_pid, libc::SIGKILL);
                            }
                            break;
                        }
                    }
                }
                let _ = fs::remove_file(&pid_file);
            }
        }

        // 2b. Belt-and-suspenders: reap any stray process still holding our
        // ports. Covers daemons started without writing a pid file (e.g.
        // `cargo run` from a separate shell), Vite servers left behind by a
        // killed `node-app dev`, and old `make dev-*` runs.
        self.free_port(self.profile.http_port, "backend");
        self.free_port(self.profile.ui_port, "ui");

        // 2c. Stale Vite pid file (mirrors the daemon.pid path above but
        // tracks the UI dev server, which we spawn in step 6).
        let vite_pid_file = env_dir.join("vite.pid");
        if let Ok(contents) = fs::read_to_string(&vite_pid_file) {
            if let Ok(old_pid) = contents.trim().parse::<i32>() {
                let alive = unsafe { libc::kill(old_pid, 0) == 0 };
                if alive {
                    self.syslog(format!(
                        "→ previous vite found (PID {old_pid}), sending SIGTERM…"
                    ));
                    unsafe {
                        libc::kill(-old_pid, libc::SIGTERM);
                    }
                }
            }
            let _ = fs::remove_file(&vite_pid_file);
        }

        let socket_path = self
            .socket_override
            .clone()
            .unwrap_or_else(|| env_dir.join("control.sock"));
        let dev_dir = self
            .dev_dir_override
            .clone()
            .unwrap_or_else(|| env_dir.join("dev-apps"));
        let db_path = env_dir.join("dev.db");
        fs::create_dir_all(&dev_dir).ok();

        // 3. Generate env file.
        // The instance-specific base env (alice.env / bob.env) is the
        // source of truth for everything except the per-instance isolation
        // paths and ports — bail clearly if the file is missing rather than
        // silently fall back to a near-empty env that leads to confusing
        // downstream failures.
        let env_file = env_dir.join("daemon.env");
        let base_env_path = self
            .monorepo_path
            .join("system/server")
            .join(&self.profile.env_file_name);
        if !base_env_path.exists() {
            bail!(
                "missing base env file for instance '{}' — expected {}.\n\
                 Create it (typically a copy of an example env), or pick a different \
                 instance via --instances.",
                self.profile.name,
                base_env_path.display()
            );
        }
        let base_env = fs::read_to_string(&base_env_path)
            .with_context(|| format!("read base env {}", base_env_path.display()))?;
        let log_dir = env_dir.join("logs");
        fs::create_dir_all(&log_dir).ok();
        let ldk_dir = env_dir.join("ldk_data");
        fs::create_dir_all(&ldk_dir).ok();
        let api_port = self.profile.http_port;
        // Point APT_APPS_DIR at the monorepo's `modules/` so the daemon loads
        // the built-in apps (ldk-node, core-storage, …) whose .dylib/.so was
        // staged there by `make builtin-apps`. Without this, the daemon falls
        // back to the production `/usr/lib/node/apps` path which is absent in
        // dev (especially on macOS), so no builtin capabilities ever register
        // and dep apps like `discovery` hit `capability.not_found` on every
        // `core.lightning.*` call.
        // Overrides we *must* inject because they depend on runtime values
        // the base env file can't know (cache-dir hash, monorepo path,
        // per-profile ports). Anything else — TLS settings, log verbosity,
        // bitcoin RPC, JWT secrets — comes from the base env file as
        // written. dotenvy::from_path (the server's env loader at
        // core/foundation/src/config.rs:170) is first-wins and never
        // overrides an already-set key, so writing overrides BEFORE the
        // base content is the only way they take effect.
        let apt_apps_dir = self.monorepo_path.join("modules");
        let static_dir = self.monorepo_path.join("system/ui/dist");
        let pairs: [(&str, String, &str); 9] = [
            ("APT_APPS_DIR", apt_apps_dir.display().to_string(),
             "load builtin apps from the monorepo's modules/ instead of /usr/lib/node/apps"),
            ("NODE_DEV_APPS_DIR", dev_dir.display().to_string(),
             "stage dep apps into a per-instance cache dir so alice/bob don't collide"),
            ("DATABASE_URL", format!("sqlite://{}", db_path.display()),
             "per-instance dev DB under cache dir"),
            ("NODE_IPC_SOCKET", socket_path.display().to_string(),
             "per-instance IPC socket the orchestrator polls for readiness"),
            ("SERVER_ADDRESS", format!("127.0.0.1:{api_port}"),
             "profile-specific HTTP port (alice=3001, bob=3002, …)"),
            ("APPLICATION_LOG_DIR_PATH", log_dir.display().to_string(),
             "per-instance application logs under cache dir"),
            ("DATA_DIR_PATH", ldk_dir.display().to_string(),
             "per-instance LDK data dir under cache dir"),
            ("SIGNER_SEED_PATH", format!("{}/signer_seed.hex", ldk_dir.display()),
             "per-instance LDK signer seed under cache dir"),
            ("STATIC_DIR_PATH", static_dir.display().to_string(),
             "absolute path to monorepo/system/ui/dist so backend serves UI on its port"),
        ];

        let mut overrides = String::from(
            "# ============================================================\n\
             # node-app dev overrides (auto-generated — do not edit)\n\
             #\n\
             # These keys are computed at orchestrator startup and MUST win\n\
             # over the base env file below for per-instance isolation. The\n\
             # base env file (alice.env / bob.env) is the source of truth\n\
             # for everything else — edit it directly if you need to change\n\
             # other settings.\n\
             #\n\
             # dotenvy is first-wins, so this block has to come first.\n\
             # ============================================================\n",
        );
        for (key, _, reason) in &pairs {
            overrides.push_str(&format!("# {key}: {reason}\n"));
        }
        overrides.push('\n');
        for (key, value, _) in &pairs {
            overrides.push_str(&format!("{key}={value}\n"));
        }
        overrides.push_str(&format!(
            "\n# ============================================================\n\
             # Base config (system/server/{env_file_name})\n\
             # ============================================================\n",
            env_file_name = self.profile.env_file_name,
        ));
        fs::write(&env_file, overrides + &base_env)
            .with_context(|| format!("write {}", env_file.display()))?;
        *self.env_file.lock().unwrap() = Some(env_file.clone());
        // Publish the env file path to the TUI so developers can find the
        // generated config the running daemon is actually using.
        tui::update_daemon_env_file(
            self.log_tx.as_ref(),
            &self.profile.name,
            env_file.clone(),
        );

        // 3b. Remove stale socket.
        let _ = fs::remove_file(&socket_path);

        // 4a. Build mandatory builtin apps.
        let has_make = Command::new("make")
            .arg("--version")
            .stdout(Stdio::null())
            .stderr(Stdio::null())
            .status()
            .map(|s| s.success())
            .unwrap_or(false);
        if has_make {
            self.syslog("→ building mandatory builtin apps: make builtin-apps CARGO_PROFILE=debug");
            let t0 = Instant::now();
            let (status, _) = self.run_build_cmd(
                Command::new("make")
                    .args(["builtin-apps", "CARGO_PROFILE=debug"])
                    .current_dir(&self.monorepo_path)
                    .stdin(Stdio::null()),
                LogSource::System,
            )
            .with_context(|| "spawn `make builtin-apps`")?;
            let builtins_ms = t0.elapsed().as_millis() as u64;
            if !status.success() {
                self.syslog(format!(
                    "⚠ `make builtin-apps` failed (exit {}) — some platform apps may not load",
                    status
                ));
            }
            // Timing-only status update — no log lines go to the build tab for builtin-apps.
            tui::update_status(
                self.log_tx.as_ref(),
                LogSource::Build,
                ServiceStatus::Ready,
                Some(format!("builtins:{}ms", builtins_ms)),
            );
        } else {
            self.syslog("⚠ `make` not found — skipping builtin-apps build");
        }

        // 4b. Delete old binary to avoid macOS linker-replace issue.
        let binary_path = self.monorepo_path.join("target/debug/node-server");
        if binary_path.exists() {
            let _ = fs::remove_file(&binary_path);
        }

        self.syslog(format!(
            "→ building daemon: cargo build -p node-server (cwd={})",
            self.monorepo_path.display()
        ));
        tui::update_status(
            self.log_tx.as_ref(),
            LogSource::Daemon,
            ServiceStatus::Building,
            Some(format!("cargo build (cwd={})", self.monorepo_path.display())),
        );

        let (build_status, build_errors) = self
            .run_build_cmd(
                Command::new("cargo")
                    .args(["build", "-p", "node-server", "--features", "agentic_payments"])
                    .current_dir(&self.monorepo_path)
                    .stdin(Stdio::null()),
                LogSource::Daemon,
            )
            .with_context(|| "spawn `cargo build -p node-server`")?;
        if !build_status.success() {
            tui::update_status(
                self.log_tx.as_ref(),
                LogSource::Daemon,
                ServiceStatus::Failed("cargo build failed".into()),
                None,
            );
            let detail = if build_errors.is_empty() {
                String::new()
            } else {
                format!("\n\n{}", build_errors.join("\n"))
            };
            bail!(
                "cargo build -p node-server failed (exit {}){detail}",
                build_status
            );
        }

        // 4c. Launch daemon.
        self.syslog(format!(
            "→ spawning daemon (env={})",
            env_file.display()
        ));
        tui::update_status(
            self.log_tx.as_ref(),
            LogSource::Daemon,
            ServiceStatus::Starting,
            Some(format!("[{}] api=:{}  ipc={}", self.profile.name, self.profile.http_port, socket_path.display())),
        );

        let child = self.spawn_daemon(&env_file)?;
        let child_pid = child.id();
        if let Err(e) = fs::write(&pid_file, child_pid.to_string()) {
            self.syslog(format!("⚠ could not write PID file: {e}"));
        }
        *self.child.lock().unwrap() = Some(child);

        // 5. Wait for IPC socket.
        let started = Instant::now();
        loop {
            if UnixStream::connect(&socket_path).is_ok() {
                self.syslog(format!(
                    "✓ IPC socket up at {} ({}s) — waiting for apps…",
                    socket_path.display(),
                    started.elapsed().as_secs()
                ));
                break;
            }
            if started.elapsed() >= HEALTH_TIMEOUT {
                self.shutdown();
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::Daemon,
                    ServiceStatus::Failed(format!(
                        "socket not connectable after {}s",
                        HEALTH_TIMEOUT.as_secs()
                    )),
                    None,
                );
                return Err(anyhow!(
                    "daemon did not come up within {}s (socket at {} not connectable).",
                    HEALTH_TIMEOUT.as_secs(),
                    socket_path.display()
                ));
            }
            if let Some(child) = self.child.lock().unwrap().as_mut() {
                if let Ok(Some(status)) = child.try_wait() {
                    tui::update_status(
                        self.log_tx.as_ref(),
                        LogSource::Daemon,
                        ServiceStatus::Failed(format!("exited {}", status)),
                        None,
                    );
                    return Err(anyhow!("daemon exited with {} before becoming ready", status));
                }
            }
            std::thread::sleep(HEALTH_POLL);
        }

        // 5b. Wait for the HTTP listener to actually answer. The IPC socket
        // can come up before axum binds its TCP port; spawning Vite before
        // that point produces "ECONNREFUSED" through the proxy and looks
        // exactly like a broken proxy config.
        self.wait_http_ready(self.profile.http_port)?;

        // 6. Start UI dev server.
        let ui_child = self.spawn_ui();
        if let Some(ref c) = ui_child {
            // Persist the Vite PID so the next `node-app dev` run can reap
            // this process even if we get killed without running our
            // shutdown handler (kill -9, panic, OOM).
            let _ = fs::write(&vite_pid_file, c.id().to_string());
            // Smoke-test the proxy: hit /api/healthz through Vite and verify
            // we got a response from the backend. A failure here is far more
            // useful than letting the user discover the broken proxy at
            // first page load.
            if let Err(e) = self.verify_proxy(self.profile.ui_port, self.profile.http_port) {
                self.syslog(format!("⚠ proxy verification failed: {e:#}"));
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::UiServer,
                    ServiceStatus::Failed("proxy not forwarding /api".into()),
                    None,
                );
            } else if let Err(e) = self.verify_ws_proxy(self.profile.ui_port) {
                // /api proxy works but WS upgrade doesn't — most likely a
                // path mismatch between vite.config.ts and the backend
                // route. Don't fail the whole launch (the UI will still
                // mostly work without realtime), but flag it loudly.
                self.syslog(format!("⚠ ws proxy verification failed: {e:#}"));
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::UiServer,
                    ServiceStatus::Failed("ws proxy /api/ws not upgrading".into()),
                    None,
                );
            }
        }
        *self.ui_child.lock().unwrap() = ui_child;

        // 7. Wait for critical apps.
        let apps_started = Instant::now();
        self.syslog("→ waiting for critical apps (device-registry, core-storage)…");
        loop {
            let all_ready = ipc_list_app_statuses(&socket_path)
                .map(|statuses| {
                    ["device-registry", "core-storage"].iter().all(|name| {
                        statuses
                            .get(*name)
                            .map(|s| s == "active" || s == "running" || s == "lazy")
                            .unwrap_or(false)
                    })
                })
                .unwrap_or(false);

            if all_ready {
                self.syslog(format!(
                    "✓ critical apps ready ({}s)",
                    apps_started.elapsed().as_secs()
                ));
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::Daemon,
                    ServiceStatus::Ready,
                    Some(format!(
                        "[{}] api=http://localhost:{}  ui=http://localhost:{}",
                        self.profile.name, self.profile.http_port, self.profile.ui_port
                    )),
                );
                break;
            }
            if apps_started.elapsed() >= Duration::from_secs(60) {
                self.syslog("⚠ critical apps not fully loaded after 60s — proceeding anyway");
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::Daemon,
                    ServiceStatus::Ready,
                    Some("api ready (some apps slow)".into()),
                );
                break;
            }
            if let Some(child) = self.child.lock().unwrap().as_mut() {
                if let Ok(Some(status)) = child.try_wait() {
                    return Err(anyhow!("daemon exited with {} while waiting for apps", status));
                }
            }
            std::thread::sleep(Duration::from_secs(2));
        }

        Ok(DaemonHandle {
            name: self.profile.name.clone(),
            banner: format!(
                "monorepo daemon [{}] (cargo run from {}, api=http://localhost:{}, \
                 ui=http://localhost:{}, socket={})",
                self.profile.name,
                self.monorepo_path.display(),
                self.profile.http_port,
                self.profile.ui_port,
                socket_path.display()
            ),
            socket_path,
            dev_dir,
        })
    }

    fn tail_logs(&self, app_name: &str) {
        let log_tx = match &self.log_tx {
            Some(tx) => tx.clone(),
            None => return,
        };
        let env_dir = match monorepo_env_dir(&self.monorepo_path, &self.profile.name) {
            Ok(d) => d,
            Err(_) => return,
        };
        // App SDK logs: {DATA_DIR_PATH}/logs/apps/{app_name}/app.log
        let log_file = env_dir
            .join("ldk_data")
            .join("logs")
            .join("apps")
            .join(app_name)
            .join("app.log");
        // Prefix every log line with `[<node>][<app>] ` so AppState::push_log
        // routes it into both the per-node and per-app buffers. The outer
        // node prefix lets multi-instance platform mode show alice/bob's
        // logs separately; the inner app prefix preserves the existing
        // per-app filter. App-developer mode (no tracked_nodes) ignores
        // the node prefix since no name matches.
        let prefix = format!("[{node}][{app}] ", node = self.profile.name, app = app_name);

        std::thread::spawn(move || {
            use std::io::{BufRead, BufReader, Seek, SeekFrom};

            // Wait up to 5s for the log file to appear (created when daemon loads the app).
            let deadline = std::time::Instant::now();
            loop {
                if log_file.exists() {
                    break;
                }
                if deadline.elapsed() > std::time::Duration::from_secs(5) {
                    return;
                }
                std::thread::sleep(std::time::Duration::from_millis(200));
            }

            let file = match fs::File::open(&log_file) {
                Ok(f) => f,
                Err(_) => return,
            };
            let mut reader = BufReader::new(file);
            // Start from the beginning — app startup logs are already written by the time
            // tail_logs() is called (ipc_call for app.dev_load completes after init()).
            let _ = reader.seek(SeekFrom::Start(0));

            loop {
                let mut line = String::new();
                match reader.read_line(&mut line) {
                    Ok(0) => {
                        // EOF — file hasn't grown yet, poll.
                        std::thread::sleep(std::time::Duration::from_millis(100));
                    }
                    Ok(_) => {
                        let trimmed =
                            line.trim_end_matches('\n').trim_end_matches('\r');
                        if !trimmed.is_empty()
                            && log_tx
                                .send(TuiEvent::Log(crate::tui::LogEntry {
                                    source: LogSource::App,
                                    line: format!("{prefix}{trimmed}"),
                                }))
                                .is_err()
                        {
                            return; // TUI channel closed, exit cleanly
                        }
                    }
                    Err(_) => return,
                }
            }
        });
    }

    fn shutdown(&self) {
        if let Ok(env_dir) = monorepo_env_dir(&self.monorepo_path, &self.profile.name) {
            let _ = fs::remove_file(env_dir.join("daemon.pid"));
        }
        if let Some(mut child) = self.child.lock().unwrap().take() {
            let pid = child.id() as i32;
            self.syslog(format!("→ shutting down monorepo daemon (PID {pid})…"));
            #[cfg(unix)]
            unsafe {
                libc::kill(-pid, libc::SIGTERM);
            }
            let t = Instant::now();
            loop {
                match child.try_wait() {
                    Ok(Some(_)) => {
                        self.syslog(format!(
                            "✓ daemon exited ({}ms)",
                            t.elapsed().as_millis()
                        ));
                        break;
                    }
                    Ok(None) if t.elapsed() >= Duration::from_secs(10) => {
                        self.syslog("⚠ daemon did not exit after 10s, SIGKILL");
                        #[cfg(unix)]
                        unsafe {
                            libc::kill(-pid, libc::SIGKILL);
                        }
                        let _ = child.wait();
                        break;
                    }
                    Ok(None) => std::thread::sleep(Duration::from_millis(200)),
                    Err(e) => {
                        self.syslog(format!("⚠ daemon wait error: {e}"));
                        break;
                    }
                }
            }
        }
        if let Some(mut ui) = self.ui_child.lock().unwrap().take() {
            let pid = ui.id() as i32;
            self.syslog(format!("→ shutting down UI dev server (PID {pid})…"));
            #[cfg(unix)]
            unsafe {
                libc::kill(-pid, libc::SIGTERM);
            }
            let t = Instant::now();
            loop {
                match ui.try_wait() {
                    Ok(Some(_)) => break,
                    Ok(None) if t.elapsed() >= Duration::from_secs(5) => {
                        #[cfg(unix)]
                        unsafe {
                            libc::kill(-pid, libc::SIGKILL);
                        }
                        let _ = ui.wait();
                        break;
                    }
                    Ok(None) => std::thread::sleep(Duration::from_millis(200)),
                    Err(_) => break,
                }
            }
        }
    }

    fn restart(&self) -> Result<()> {
        let env_file = self
            .env_file
            .lock()
            .unwrap()
            .clone()
            .ok_or_else(|| anyhow!("daemon was never started — cannot restart"))?;

        self.syslog("→ restart: stopping daemon…");
        tui::update_status(
            self.log_tx.as_ref(),
            LogSource::Daemon,
            ServiceStatus::Starting,
            Some("restarting…".into()),
        );

        self.shutdown_daemon_only();

        // Reap any stale process still holding the backend port. Without
        // this, a daemon child that died unclean (panic, OOM) can leave its
        // socket in TIME_WAIT and the respawned daemon fails to bind.
        self.free_port(self.profile.http_port, "backend");

        // Remove stale socket.
        let socket_path = self
            .socket_override
            .clone()
            .unwrap_or_else(|| {
                monorepo_env_dir(&self.monorepo_path, &self.profile.name)
                    .ok()
                    .map(|d| d.join("control.sock"))
                    .unwrap_or_else(|| PathBuf::from("/tmp/node-control.sock"))
            });
        let _ = fs::remove_file(&socket_path);

        self.syslog("→ restart: spawning daemon…");
        let child = self.spawn_daemon(&env_file)?;
        let child_pid = child.id();
        *self.child.lock().unwrap() = Some(child);

        // Update PID file.
        if let Ok(env_dir) = monorepo_env_dir(&self.monorepo_path, &self.profile.name) {
            let _ = fs::write(env_dir.join("daemon.pid"), child_pid.to_string());
        }

        // Wait for socket.
        let t = Instant::now();
        loop {
            if UnixStream::connect(&socket_path).is_ok() {
                // Socket up — also confirm the HTTP listener answers before
                // declaring the restart successful. Otherwise the next /api
                // call from the browser races a half-booted daemon.
                if let Err(e) = self.wait_http_ready(self.profile.http_port) {
                    tui::update_status(
                        self.log_tx.as_ref(),
                        LogSource::Daemon,
                        ServiceStatus::Failed("HTTP not ready after restart".into()),
                        None,
                    );
                    bail!("daemon restart: HTTP readiness failed: {e:#}");
                }
                self.syslog(format!(
                    "✓ daemon restarted ({}s)",
                    t.elapsed().as_secs()
                ));
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::Daemon,
                    ServiceStatus::Ready,
                    Some("restarted".into()),
                );
                return Ok(());
            }
            if t.elapsed() > HEALTH_TIMEOUT {
                tui::update_status(
                    self.log_tx.as_ref(),
                    LogSource::Daemon,
                    ServiceStatus::Failed("socket not connectable after restart".into()),
                    None,
                );
                bail!("daemon did not come up after restart");
            }
            if let Some(child) = self.child.lock().unwrap().as_mut() {
                if let Ok(Some(status)) = child.try_wait() {
                    tui::update_status(
                        self.log_tx.as_ref(),
                        LogSource::Daemon,
                        ServiceStatus::Failed(format!("exited {}", status)),
                        None,
                    );
                    bail!("daemon exited with {} during restart", status);
                }
            }
            std::thread::sleep(HEALTH_POLL);
        }
    }

    fn pre_start_dev_dir(&self) -> Option<PathBuf> {
        monorepo_dev_dir(&self.monorepo_path, &self.profile.name, self.dev_dir_override.as_deref()).ok()
    }
}

fn monorepo_env_dir(monorepo_path: &Path, instance_name: &str) -> Result<PathBuf> {
    let cache_dir = cache_root()?;
    let path_hash = {
        let mut h = DefaultHasher::new();
        monorepo_path
            .canonicalize()
            .unwrap_or_else(|_| monorepo_path.to_path_buf())
            .hash(&mut h);
        instance_name.hash(&mut h);
        h.finish()
    };
    Ok(cache_dir.join(format!("monorepo-{:x}", path_hash)))
}

fn monorepo_dev_dir(monorepo_path: &Path, instance_name: &str, dev_dir_override: Option<&Path>) -> Result<PathBuf> {
    if let Some(override_path) = dev_dir_override {
        return Ok(override_path.to_path_buf());
    }
    Ok(monorepo_env_dir(monorepo_path, instance_name)?.join("dev-apps"))
}

fn ipc_list_app_statuses(
    socket_path: &std::path::Path,
) -> Result<std::collections::HashMap<String, String>> {
    use std::io::{BufRead, BufReader, Write};

    let mut stream = UnixStream::connect(socket_path)
        .with_context(|| "connect to IPC socket")?;
    stream.set_read_timeout(Some(Duration::from_secs(5))).ok();
    stream.set_write_timeout(Some(Duration::from_secs(5))).ok();

    let request = serde_json::json!({
        "jsonrpc": "2.0",
        "id": 1,
        "method": "app.list",
        "params": {}
    });
    let mut line = serde_json::to_string(&request).unwrap();
    line.push('\n');
    stream.write_all(line.as_bytes())?;

    let reader = BufReader::new(&stream);
    let response_line = reader
        .lines()
        .next()
        .ok_or_else(|| anyhow!("no response"))??;

    let v: serde_json::Value = serde_json::from_str(&response_line)?;
    let mut map = std::collections::HashMap::new();
    if let Some(apps) = v.pointer("/result/apps").and_then(|a| a.as_array()) {
        for app in apps {
            if let (Some(name), Some(status)) = (
                app.get("name").and_then(|n| n.as_str()),
                app.get("status").and_then(|s| s.as_str()),
            ) {
                map.insert(name.to_string(), status.to_string());
            }
        }
    }
    Ok(map)
}

fn which_bin(bin: &str) -> bool {
    std::env::var_os("PATH")
        .map(|path| std::env::split_paths(&path).any(|dir| dir.join(bin).is_file()))
        .unwrap_or(false)
}

fn cache_root() -> Result<PathBuf> {
    if let Ok(c) = std::env::var("XDG_CACHE_HOME") {
        if !c.is_empty() {
            return Ok(PathBuf::from(c).join("node-app"));
        }
    }
    let home = std::env::var_os("HOME").ok_or_else(|| anyhow!("$HOME not set"))?;
    Ok(PathBuf::from(home).join(".cache/node-app"))
}