zc2 0.0.50

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! The host-runner watchdog: keeps a host runner unnoticed. When the owner
//! comes back (unplugged with `on_ac_only`, or no longer idle with
//! `when = "idle"`), when the Mac reports memory pressure, or when Zakuro's
//! processes outgrow their reservation, the runner stops taking jobs
//! (`POST /drain`) and every running job is wrapped up
//! (`POST /jobs/{id}/wrap-up`: keep the best checkpoint, end early). Jobs
//! are never killed and never left running to the end.

use crate::agent::hosting::Hosting;

/// What the watchdog reads about the machine.
pub trait Signals {
    fn on_ac(&self) -> bool;
    /// The owner has not touched keyboard or mouse for [`IDLE_AFTER_SECS`].
    fn idle(&self) -> bool;
    fn memory_pressure(&self) -> bool;
    /// The owner's wall clock, for `when = "schedule"`.
    fn local_time(&self) -> crate::policy::schedule::LocalTime {
        crate::policy::schedule::LocalTime::now()
    }
    /// Resident memory of the host runner's process tree.
    fn zakuro_rss(&self) -> u64;
}

pub const IDLE_AFTER_SECS: u64 = 300;
pub const TICK_SECS: u64 = 15;

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Action {
    None,
    StopAccepting,
    WrapUp(Vec<String>),
}

/// Why the runner must yield now, if it must.
pub fn yield_reason(s: &dyn Signals, h: &Hosting, reserved_bytes: u64) -> Option<&'static str> {
    if h.budget.on_ac_only && !s.on_ac() {
        return Some("on-battery");
    }
    if h.sharing.when == "schedule"
        && !crate::policy::schedule::is_open(&h.sharing.schedule, s.local_time())
    {
        return Some("outside-schedule");
    }
    if h.sharing.when == "idle" && !s.idle() {
        return Some("owner-active");
    }
    if s.memory_pressure() {
        return Some("memory-pressure");
    }
    if s.zakuro_rss() > reserved_bytes {
        return Some("over-reservation");
    }
    None
}

pub fn decide(
    s: &dyn Signals,
    h: &Hosting,
    reserved_bytes: u64,
    running_jobs: &[String],
) -> Action {
    match yield_reason(s, h, reserved_bytes) {
        None => Action::None,
        Some(_) if running_jobs.is_empty() => Action::StopAccepting,
        Some(_) => Action::WrapUp(running_jobs.to_vec()),
    }
}

/// The real signals. macOS reads `pmset`, `ioreg`, `sysctl` and `ps`;
/// elsewhere the machine counts as plugged in, idle and unpressured (Linux
/// host runners are not shipped).
pub struct MacSignals {
    /// Leader of the host runner's process group.
    pub runner_pid: Option<u32>,
}

/// Absolute: launchd starts the agent with PATH=/usr/bin:/bin, and ioreg and
/// sysctl live in /usr/sbin (a bare name failed silently: never idle-aware,
/// never pressure-aware).
pub const PMSET: &str = "/usr/bin/pmset";
pub const IOREG: &str = "/usr/sbin/ioreg";
pub const SYSCTL: &str = "/usr/sbin/sysctl";
pub const PS: &str = "/bin/ps";

fn run(prog: &str, args: &[&str]) -> Option<String> {
    let out = std::process::Command::new(prog).args(args).output().ok()?;
    out.status
        .success()
        .then(|| String::from_utf8_lossy(&out.stdout).into_owned())
}

/// `kern.memorystatus_vm_pressure_level`: 1 normal, 2 warn, 4 critical.
/// Only critical counts: warn is routine while a job trains on a 16 GB Mac,
/// and yielding there would cut every job short.
pub fn pressure_is_critical(level: u32) -> bool {
    level >= 4
}

/// `ioreg -c IOHIDSystem` → `"HIDIdleTime" = <nanoseconds>`.
pub fn parse_hid_idle_secs(ioreg: &str) -> Option<u64> {
    ioreg
        .lines()
        .find(|l| l.contains("\"HIDIdleTime\""))?
        .rsplit('=')
        .next()?
        .trim()
        .parse::<u64>()
        .ok()
        .map(|ns| ns / 1_000_000_000)
}

/// `ps -o rss= -g <pgid>` (KiB per line) → bytes.
pub fn parse_ps_rss(ps: &str) -> u64 {
    ps.lines()
        .filter_map(|l| l.trim().parse::<u64>().ok())
        .sum::<u64>()
        * 1024
}

impl Signals for MacSignals {
    fn on_ac(&self) -> bool {
        if !cfg!(target_os = "macos") {
            return true;
        }
        run(PMSET, &["-g", "batt"]).is_none_or(|s| s.contains("AC Power"))
    }

    fn idle(&self) -> bool {
        if !cfg!(target_os = "macos") {
            return true;
        }
        run(IOREG, &["-c", "IOHIDSystem"])
            .and_then(|s| parse_hid_idle_secs(&s))
            .is_none_or(|secs| secs >= IDLE_AFTER_SECS)
    }

    fn memory_pressure(&self) -> bool {
        if !cfg!(target_os = "macos") {
            return false;
        }
        run(SYSCTL, &["-n", "kern.memorystatus_vm_pressure_level"])
            .and_then(|s| s.trim().parse::<u32>().ok())
            .is_some_and(pressure_is_critical)
    }

    fn zakuro_rss(&self) -> u64 {
        let Some(pid) = self.runner_pid else { return 0 };
        run(PS, &["-o", "rss=", "-g", &pid.to_string()]).map_or(0, |s| parse_ps_rss(&s))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::agent::hosting::defaults;

    struct S {
        ac: bool,
        idle: bool,
        pressure: bool,
        rss: u64,
    }

    impl Signals for S {
        fn on_ac(&self) -> bool {
            self.ac
        }
        fn idle(&self) -> bool {
            self.idle
        }
        fn memory_pressure(&self) -> bool {
            self.pressure
        }
        fn zakuro_rss(&self) -> u64 {
            self.rss
        }
    }

    fn calm() -> S {
        S {
            ac: true,
            idle: true,
            pressure: false,
            rss: 1 << 30,
        }
    }

    const RESERVED: u64 = 8 << 30;

    fn h() -> crate::agent::hosting::Hosting {
        defaults(&[])
    }

    fn jobs() -> Vec<String> {
        vec!["j1".to_string()]
    }

    #[test]
    fn owner_back_wraps_up() {
        let s = S {
            ac: false,
            ..calm()
        };
        assert_eq!(decide(&s, &h(), RESERVED, &jobs()), Action::WrapUp(jobs()));
        let s = S {
            idle: false,
            ..calm()
        };
        assert_eq!(decide(&s, &h(), RESERVED, &jobs()), Action::WrapUp(jobs()));
    }

    #[test]
    fn pressure_wraps_up() {
        let s = S {
            pressure: true,
            ..calm()
        };
        assert_eq!(decide(&s, &h(), RESERVED, &jobs()), Action::WrapUp(jobs()));
    }

    #[test]
    fn over_reservation_wraps_up() {
        let s = S {
            rss: RESERVED + 1,
            ..calm()
        };
        assert_eq!(decide(&s, &h(), RESERVED, &jobs()), Action::WrapUp(jobs()));
    }

    #[test]
    fn idle_and_on_ac_does_nothing() {
        assert_eq!(decide(&calm(), &h(), RESERVED, &jobs()), Action::None);
    }

    #[test]
    fn owner_back_with_no_jobs_stops_accepting() {
        let s = S {
            ac: false,
            ..calm()
        };
        assert_eq!(decide(&s, &h(), RESERVED, &[]), Action::StopAccepting);
    }

    #[test]
    fn hid_idle_time_is_read_in_seconds() {
        let ioreg = "    | |   \"HIDIdleTime\" = 301000000000\n    | |   \"Other\" = 1\n";
        assert_eq!(parse_hid_idle_secs(ioreg), Some(301));
        assert_eq!(parse_hid_idle_secs("nothing"), None);
    }

    #[test]
    fn system_tools_are_called_by_absolute_path() {
        // launchd starts the agent with PATH=/usr/bin:/bin (no /usr/sbin).
        for t in [PMSET, IOREG, SYSCTL, PS] {
            assert!(t.starts_with('/'), "{t}");
        }
        assert_eq!(IOREG, "/usr/sbin/ioreg");
        assert_eq!(SYSCTL, "/usr/sbin/sysctl");
    }

    #[test]
    fn only_critical_pressure_counts() {
        // 1 normal, 2 warn (routine while training on a 16 GB Mac), 4 critical.
        assert!(!pressure_is_critical(1));
        assert!(!pressure_is_critical(2));
        assert!(pressure_is_critical(4));
    }

    #[test]
    fn ps_rss_sums_kib_lines() {
        assert_eq!(parse_ps_rss(" 1024\n 2048\n\n"), 3 * 1024 * 1024);
    }

    #[test]
    fn battery_is_fine_when_not_ac_only() {
        let mut h = h();
        h.budget.on_ac_only = false;
        let s = S {
            ac: false,
            ..calm()
        };
        assert_eq!(decide(&s, &h, RESERVED, &jobs()), Action::None);
    }

    #[test]
    fn busy_owner_is_fine_when_always() {
        let mut h = h();
        h.sharing.when = "always".into();
        let s = S {
            idle: false,
            ..calm()
        };
        assert_eq!(decide(&s, &h, RESERVED, &jobs()), Action::None);
    }

    #[test]
    fn when_schedule_yields_outside_its_windows() {
        use crate::policy::schedule::LocalTime;
        struct Clock(LocalTime);
        impl Signals for Clock {
            fn on_ac(&self) -> bool {
                true
            }
            fn idle(&self) -> bool {
                true
            }
            fn memory_pressure(&self) -> bool {
                false
            }
            fn zakuro_rss(&self) -> u64 {
                0
            }
            fn local_time(&self) -> LocalTime {
                self.0
            }
        }
        let mut h = crate::agent::hosting::defaults(&[]);
        h.sharing.when = "schedule".into();
        h.sharing.schedule = vec!["mon-fri 19:00-08:00".into()];
        let noon = Clock(LocalTime {
            weekday: 2,
            minute: 720,
        });
        let night = Clock(LocalTime {
            weekday: 2,
            minute: 1320,
        });
        assert_eq!(yield_reason(&noon, &h, u64::MAX), Some("outside-schedule"));
        assert_eq!(yield_reason(&night, &h, u64::MAX), None);
    }
}