openlatch-client 0.6.22

//! launchd LaunchAgent supervisor — macOS only.
//!
//! portability-ok: every path below is a launchd convention
//! (`~/Library/LaunchAgents`), and `select_supervisor_impl` only constructs
//! this type behind `#[cfg(target_os = "macos")]`. The module itself stays
//! uncfg'd so its plist-generation tests run on every host.

use std::path::{Path, PathBuf};

use crate::error::OlError;

use super::{
    Supervisor, SupervisorKind, SupervisorStatus, ERR_SUPERVISION_BOOTSTRAP_FAILED,
    ERR_SUPERVISION_CONTROL_FAILED, ERR_SUPERVISION_INSTALL_FAILED,
};

pub const LABEL: &str = "ai.openlatch.client";
pub const WATCHDOG_LABEL: &str = "ai.openlatch.client.watchdog";

#[cfg(unix)]
fn get_uid() -> u32 {
    unsafe { libc::getuid() }
}

#[cfg(not(unix))]
fn get_uid() -> u32 {
    0
}

enum BootstrapError {
    AlreadyBootstrapped,
    NoGuiSession,
    Other(String),
}

/// Invoke `launchctl bootstrap <domain> <plist>`, classifying the failure so
/// the caller can pick an alternate domain.
///
/// `gui/$UID` needs the Aqua session to be attached (a graphical login).
/// Over SSH or headless CI launchd returns `Input/output error` — that's
/// the signal to retry against `user/$UID`, which does not require it.
fn bootstrap(plist: &Path, domain: &str) -> Result<(), BootstrapError> {
    let out = std::process::Command::new("launchctl")
        .args(["bootstrap", domain, &plist.display().to_string()])
        .output();
    match out {
        Ok(o) if o.status.success() => Ok(()),
        Ok(o) => {
            let stderr = String::from_utf8_lossy(&o.stderr).to_string();
            if stderr.contains("already bootstrapped") {
                Err(BootstrapError::AlreadyBootstrapped)
            } else if stderr.contains("Input/output error")
                || stderr.contains("Could not find domain")
            {
                Err(BootstrapError::NoGuiSession)
            } else {
                Err(BootstrapError::Other(stderr))
            }
        }
        Err(e) => Err(BootstrapError::Other(format!("Cannot run launchctl: {e}"))),
    }
}

/// `bootstrap` into `gui/$UID`, falling back to `user/$UID` when there is no
/// GUI session. The message on failure is ready to wrap in whichever code the
/// caller owns.
fn bootstrap_with_fallback(plist: &Path) -> Result<(), String> {
    let uid = get_uid();
    match bootstrap(plist, &format!("gui/{uid}")) {
        Ok(()) | Err(BootstrapError::AlreadyBootstrapped) => Ok(()),
        Err(BootstrapError::NoGuiSession) => match bootstrap(plist, &format!("user/{uid}")) {
            Ok(()) | Err(BootstrapError::AlreadyBootstrapped) => Ok(()),
            Err(BootstrapError::NoGuiSession) => {
                Err("launchctl bootstrap failed in both gui/ and user/ domains (no session)".into())
            }
            Err(BootstrapError::Other(msg)) if msg.is_empty() => {
                Err("launchctl bootstrap failed in both gui/ and user/ domains".into())
            }
            Err(BootstrapError::Other(msg)) => Err(format!("launchctl bootstrap failed: {msg}")),
        },
        Err(BootstrapError::Other(msg)) => Err(format!("launchctl bootstrap failed: {msg}")),
    }
}

/// Best-effort `bootout` of `label` from both domains `bootstrap_with_fallback`
/// can have used.
fn bootout_everywhere(label: &str) {
    let uid = get_uid();
    for domain in [format!("gui/{uid}"), format!("user/{uid}")] {
        let _ = std::process::Command::new("launchctl")
            .args(["bootout", &format!("{domain}/{label}")])
            .output();
    }
}

/// Quote `s` as one POSIX shell word.
fn shell_quote(s: &str) -> String {
    format!("'{}'", s.replace('\'', r"'\''"))
}

pub struct LaunchdSupervisor {
    plist_path: PathBuf,
    watchdog_path: PathBuf,
}

impl Default for LaunchdSupervisor {
    fn default() -> Self {
        Self::new()
    }
}

impl LaunchdSupervisor {
    pub fn new() -> Self {
        let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"));
        let agents = home.join("Library/LaunchAgents");
        Self {
            plist_path: agents.join(format!("{LABEL}.plist")),
            watchdog_path: agents.join(format!("{WATCHDOG_LABEL}.plist")),
        }
    }

    /// The watchdog agent: every [`super::WATCHDOG_INTERVAL_SECS`] it loads the
    /// main job again if nothing has it loaded.
    ///
    /// `KeepAlive` covers every death but `launchctl bootout`, which is what
    /// `openlatch stop` does, and that stop used to last until the next login.
    /// Re-bootstrapping the main plist brings it back through its own
    /// `RunAtLoad`; the watchdog never runs the daemon or an `openlatch`
    /// command itself.
    ///
    /// The check comes first, in **both** domains, because the main job does
    /// not stay where it was installed: a job bootstrapped into `user/` over SSH
    /// is auto-loaded into `gui/` at the next graphical login. A bare bootstrap
    /// into a fixed domain would then load a second copy beside it. Unloaded
    /// everywhere, it takes the same gui → user fallback as `start`.
    ///
    /// A deleted main plist is rewritten first, from a base64 copy carried in
    /// this one — but only while the binary it names still exists. Without
    /// the binary the job could only fail to exec, every few seconds, forever;
    /// a removal that went that far is left alone.
    ///
    /// No `RunAtLoad`: loading the watchdog must not touch a running daemon.
    fn generate_watchdog_plist(&self, binary_path: &Path) -> String {
        use base64::Engine as _;
        let marker = super::unit_version_marker();
        let interval = super::WATCHDOG_INTERVAL_SECS;
        let uid = get_uid();
        let plist = shell_quote(&self.plist_path.display().to_string());
        let bin = shell_quote(&binary_path.display().to_string());
        let copy =
            base64::engine::general_purpose::STANDARD.encode(self.generate_plist(binary_path));
        let quiet = ">/dev/null 2>&1";
        let script = format!(
            "[ -f {plist} ] || {{ [ -x {bin} ] && echo {copy} | /usr/bin/base64 -D > {plist}; }}; \
             /bin/launchctl print gui/{uid}/{LABEL} {quiet} \
             || /bin/launchctl print user/{uid}/{LABEL} {quiet} \
             || /bin/launchctl bootstrap gui/{uid} {plist} {quiet} \
             || /bin/launchctl bootstrap user/{uid} {plist} {quiet}"
        )
        // `2>&1` is not valid plist text as written.
        .replace('&', "&amp;")
        .replace('<', "&lt;")
        .replace('>', "&gt;");
        format!(
            r#"<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<!-- {marker} -->
<dict>
    <key>Label</key>
    <string>{WATCHDOG_LABEL}</string>
    <key>ProgramArguments</key>
    <array>
        <string>/bin/sh</string>
        <string>-c</string>
        <string>{script}</string>
    </array>
    <key>StartInterval</key>
    <integer>{interval}</integer>
    <key>StandardOutPath</key>
    <string>/dev/null</string>
    <key>StandardErrorPath</key>
    <string>/dev/null</string>
</dict>
</plist>"#
        )
    }

    fn write(path: &Path, contents: &str) -> Result<(), OlError> {
        if let Some(parent) = path.parent() {
            std::fs::create_dir_all(parent).map_err(|e| {
                OlError::new(
                    ERR_SUPERVISION_INSTALL_FAILED,
                    format!("Cannot create LaunchAgents directory: {e}"),
                )
            })?;
        }
        std::fs::write(path, contents).map_err(|e| {
            OlError::new(
                ERR_SUPERVISION_INSTALL_FAILED,
                format!("Cannot write {}: {e}", path.display()),
            )
        })
    }

    /// Write the watchdog plist and (re)load it. Booted out first, so a loaded
    /// older watchdog does not keep running its old definition; it runs for a
    /// moment every 5 minutes, so there is nothing to interrupt.
    fn install_watchdog(&self, binary_path: &Path) -> Result<(), OlError> {
        Self::write(
            &self.watchdog_path,
            &self.generate_watchdog_plist(binary_path),
        )?;
        bootout_everywhere(WATCHDOG_LABEL);
        bootstrap_with_fallback(&self.watchdog_path)
            .map_err(|msg| OlError::new(ERR_SUPERVISION_BOOTSTRAP_FAILED, msg))
    }

    fn generate_plist(&self, binary_path: &Path) -> String {
        let bin = binary_path.display();
        let marker = super::unit_version_marker();
        // A bare `<true/>` KeepAlive, not the `SuccessfulExit=false` dict:
        // `openlatch stop` exits 0, and launchd read that dict as "any exit
        // launchd considers successful means don't restart" — so a daemon that
        // died in almost any way stayed dead. Agents route their model traffic
        // through this process; unconditional relaunch is the honest policy,
        // and `launchctl bootout` still stops it for good.
        //
        // ThrottleInterval 10 → 2 for the same reason systemd's RestartSec
        // dropped: it bounds the dead-port window the model relay leaves behind.
        // launchd clamps sub-second values, so 2 is the practical floor here.
        format!(
            r#"<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<!-- {marker} -->
<dict>
    <key>Label</key>
    <string>{LABEL}</string>
    <key>ProgramArguments</key>
    <array>
        <string>{bin}</string>
        <string>daemon</string>
        <string>start</string>
        <string>--foreground</string>
    </array>
    <key>KeepAlive</key>
    <true/>
    <key>ThrottleInterval</key>
    <integer>2</integer>
    <key>ExitTimeOut</key>
    <integer>20</integer>
    <key>RunAtLoad</key>
    <true/>
    <key>StandardOutPath</key>
    <string>/tmp/openlatch-stdout.log</string>
    <key>StandardErrorPath</key>
    <string>/tmp/openlatch-stderr.log</string>
</dict>
</plist>"#
        )
    }
}

impl Supervisor for LaunchdSupervisor {
    fn kind(&self) -> SupervisorKind {
        SupervisorKind::Launchd
    }

    fn install(&self, binary_path: &Path) -> Result<(), OlError> {
        Self::write(&self.plist_path, &self.generate_plist(binary_path))?;

        // Try gui/$UID first (the canonical per-user domain for LaunchAgents
        // started by the Aqua login session). If it fails with the specific
        // "Input/output error" that launchd returns when no GUI session is
        // attached — e.g. over SSH or in a headless CI runner — retry with
        // user/$UID, which does not require a logged-in Aqua session.
        bootstrap_with_fallback(&self.plist_path)
            .map_err(|msg| OlError::new(ERR_SUPERVISION_BOOTSTRAP_FAILED, msg))?;

        // The main job is loaded and running, so a watchdog that failed to load
        // must not fail the install — `init` would then spawn a second,
        // unsupervised daemon beside it. The plist is on disk either way and
        // loads at the next login.
        if let Err(e) = self.install_watchdog(binary_path) {
            tracing::warn!(
                error = %e.message, code = e.code,
                "daemon supervised, but the watchdog agent did not load; it loads at the next login"
            );
        }
        Ok(())
    }

    fn refresh_in_place(&self, binary_path: &Path) -> Result<(), OlError> {
        // The main plist is rewritten, never reloaded: a `bootout` would end
        // the daemon calling this. The running job keeps its loaded definition
        // and the next load reads the new file.
        Self::write(&self.plist_path, &self.generate_plist(binary_path))?;
        self.install_watchdog(binary_path)
    }

    fn uninstall(&self) -> Result<(), OlError> {
        // The watchdog first, or it could load the job this is removing.
        //
        // Bootout from BOTH domains — install() falls back from gui/ to
        // user/ when no GUI session is attached, so uninstall has to undo
        // whichever one succeeded. Each call is best-effort: "not
        // bootstrapped" in one domain is benign.
        bootout_everywhere(WATCHDOG_LABEL);
        let _ = std::fs::remove_file(&self.watchdog_path);
        bootout_everywhere(LABEL);
        let _ = std::fs::remove_file(&self.plist_path);
        Ok(())
    }

    fn status(&self) -> Result<SupervisorStatus, OlError> {
        if !self.plist_path.exists() {
            return Ok(SupervisorStatus {
                installed: false,
                running: false,
                unit_current: false,
                description: "not installed".into(),
            });
        }

        // A v4 host has the main plist but no watchdog, and reads as outdated
        // until the watchdog is there.
        let unit_current = [&self.plist_path, &self.watchdog_path].iter().all(|p| {
            std::fs::read_to_string(p)
                .map(|c| super::unit_is_current(&c))
                .unwrap_or(false)
        });

        let output = std::process::Command::new("launchctl")
            .args(["list", LABEL])
            .output();

        match output {
            Ok(o) if o.status.success() => Ok(SupervisorStatus {
                installed: true,
                running: true,
                unit_current,
                description: if unit_current {
                    "launchd (KeepAlive and watchdog active)".into()
                } else {
                    "launchd (running an outdated plist — reinstall to get the watchdog)".into()
                },
            }),
            _ => Ok(SupervisorStatus {
                installed: true,
                running: false,
                unit_current,
                description: "launchd (plist present, not running)".into(),
            }),
        }
    }

    fn start(&self) -> Result<(), OlError> {
        // `bootstrap` both loads the job and, with `RunAtLoad`, starts it —
        // and it is already the gui/ → user/ fallback `install` relies on, so
        // starting after a `stop` takes the same path that installing does.
        // "Already bootstrapped" is the idempotent success case: the job is
        // loaded, which is exactly what was asked for.
        bootstrap_with_fallback(&self.plist_path)
            .map_err(|msg| OlError::new(ERR_SUPERVISION_CONTROL_FAILED, msg))
    }

    fn stop(&self) -> Result<(), OlError> {
        // `bootout`, not `launchctl kill`: an unconditional `KeepAlive` means
        // a signalled job is relaunched after `ThrottleInterval`, so signalling
        // it is not a stop at all. Booting the job out of the domain unloads
        // it. The watchdog agent stays loaded and bootstraps the job again
        // within `WATCHDOG_INTERVAL_SECS`, matching systemd's watchdog timer.
        //
        // Both domains, because `start`/`install` fall back from gui/ to user/
        // and only one of them holds the job. Success in EITHER is a stop;
        // failure in both is a real failure.
        let uid = get_uid();
        let mut last_err = String::new();
        for domain in [format!("gui/{uid}"), format!("user/{uid}")] {
            let out = std::process::Command::new("launchctl")
                .args(["bootout", &format!("{domain}/{LABEL}")])
                .output();
            match out {
                Ok(o) if o.status.success() => return Ok(()),
                Ok(o) => last_err = String::from_utf8_lossy(&o.stderr).trim().to_string(),
                Err(e) => last_err = format!("cannot run launchctl: {e}"),
            }
        }
        Err(OlError::new(
            ERR_SUPERVISION_CONTROL_FAILED,
            format!("launchctl bootout failed in gui/ and user/ domains: {last_err}"),
        ))
    }

    fn restart(&self) -> Result<(), OlError> {
        // `kickstart -k` kills the running instance and starts a fresh one in
        // a single launchd-mediated step — no window for a caller-spawned
        // replacement to race the relaunch.
        let uid = get_uid();
        for domain in [format!("gui/{uid}"), format!("user/{uid}")] {
            let out = std::process::Command::new("launchctl")
                .args(["kickstart", "-k", &format!("{domain}/{LABEL}")])
                .output();
            if out.is_ok_and(|o| o.status.success()) {
                return Ok(());
            }
        }
        // Not loaded in either domain — a restart of something that is not
        // running is a start.
        self.start()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const BIN: &str = "/opt/openlatch/bin/openlatch";

    /// The `SuccessfulExit=false` dict was the launchd analogue of systemd's
    /// `Restart=on-failure`: the daemon exited 0 even when it crashed, so
    /// launchd read every death as intentional and never relaunched. A bare
    /// `<true/>` relaunches unconditionally; `launchctl bootout` still stops it
    /// for good, so the user has not lost the off switch.
    #[test]
    fn plist_keep_alive_is_unconditional() {
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_plist(Path::new("/opt/openlatch/bin/openlatch"));
        assert!(
            plist.contains("<key>KeepAlive</key>\n    <true/>"),
            "{plist}"
        );
        assert!(
            !plist.contains("<key>SuccessfulExit</key>"),
            "the conditional KeepAlive dict is what stopped relaunches:\n{plist}"
        );
    }

    #[test]
    fn plist_has_throttle_interval() {
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_plist(Path::new("/opt/openlatch/bin/openlatch"));
        assert!(plist.contains("<key>ThrottleInterval</key>"));
        // 2s, not 10s — bounds the model relay listener's dead-port window.
        assert!(plist.contains("<integer>2</integer>"), "{plist}");
    }

    #[test]
    fn plist_carries_the_version_marker() {
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_plist(Path::new("/opt/openlatch/bin/openlatch"));
        assert!(super::super::unit_is_current(&plist), "{plist}");
    }

    #[test]
    fn plist_contains_label() {
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_plist(Path::new("/opt/openlatch/bin/openlatch"));
        assert!(plist.contains(LABEL));
    }

    #[test]
    fn watchdog_reloads_the_main_job_every_five_minutes() {
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_watchdog_plist(Path::new(BIN));
        assert!(
            plist.contains(&format!("<string>{WATCHDOG_LABEL}</string>")),
            "{plist}"
        );
        assert!(
            plist.contains("<key>StartInterval</key>\n    <integer>300</integer>"),
            "{plist}"
        );
        // Loading the watchdog must never touch a running daemon.
        assert!(!plist.contains("RunAtLoad"), "{plist}");
        assert!(!plist.contains("KeepAlive"), "{plist}");
        assert!(super::super::unit_is_current(&plist), "{plist}");

        let uid = get_uid();
        let main = sup.plist_path.display().to_string();
        assert!(
            plist.contains(&format!("/bin/launchctl bootstrap gui/{uid} '{main}'")),
            "{plist}"
        );
        assert!(
            plist.contains(&format!("/bin/launchctl bootstrap user/{uid} '{main}'")),
            "{plist}"
        );
        // XML-escaped, or launchd rejects the plist outright.
        assert!(plist.contains("2&gt;&amp;1"), "{plist}");
        assert!(!plist.contains("2>&1"), "{plist}");
        // The daemon is never run by the watchdog itself.
        assert!(!plist.contains("--foreground"), "{plist}");
    }

    /// A job installed into `user/` is auto-loaded into `gui/` at the next
    /// graphical login. Bootstrapping without looking first would then load a
    /// second copy, so both domains are checked before either bootstrap.
    #[test]
    fn watchdog_checks_both_domains_before_bootstrapping() {
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_watchdog_plist(Path::new(BIN));
        let uid = get_uid();
        let first_bootstrap = plist.find("launchctl bootstrap").expect("bootstraps");
        for domain in ["gui", "user"] {
            let check = plist
                .find(&format!("launchctl print {domain}/{uid}/{LABEL} "))
                .unwrap_or_else(|| panic!("no {domain}/ check:\n{plist}"));
            assert!(check < first_bootstrap, "{plist}");
        }
    }

    /// A deleted main plist comes back byte-identical, and only while the
    /// binary it would run is still there.
    #[test]
    fn watchdog_restores_a_deleted_main_plist_while_the_binary_exists() {
        use base64::Engine as _;
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_watchdog_plist(Path::new(BIN));
        let main = shell_quote(&sup.plist_path.display().to_string());
        let guard = format!("[ -f {main} ] || {{ [ -x '{BIN}' ] &amp;&amp; echo ");
        let start = plist.find(&guard).unwrap_or_else(|| panic!("{plist}"));
        let copy = plist[start + guard.len()..]
            .split_whitespace()
            .next()
            .expect("base64 copy");
        let decoded = base64::engine::general_purpose::STANDARD
            .decode(copy)
            .expect("valid base64");
        assert_eq!(
            String::from_utf8(decoded).unwrap(),
            sup.generate_plist(Path::new(BIN))
        );
        // Restored before anything tries to load it.
        assert!(start < plist.find("launchctl print").unwrap(), "{plist}");
    }

    #[test]
    fn shell_quote_survives_an_apostrophe() {
        assert_eq!(shell_quote("/Users/o'neil/x"), r"'/Users/o'\''neil/x'");
    }

    #[test]
    fn plist_declares_utf8() {
        // plist bytes are written UTF-8; the XML prolog must say UTF-8 so
        // strict parsers (and future OS validators) don't reject it — same
        // class of failure as the Windows Task Scheduler UTF-16 mismatch.
        let sup = LaunchdSupervisor::new();
        let plist = sup.generate_plist(Path::new("/opt/openlatch/bin/openlatch"));
        assert!(plist.starts_with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));
    }
}