runner_manager_platform/
daemon_heartbeat.rs

1//! Whether the daemon is making progress at all.
2//!
3//! A daemon blocked in a file system call answers nothing and logs nothing: on
4//! a Mac after an update, one sat at 0% CPU for half an hour behind a pending
5//! "would like to access files on a removable volume" question, while `service
6//! status` said `verdict healthy` (launchd reported it running) and the
7//! host-unfit record it could no longer re-stamp expired from `status`.
8//!
9//! So the daemon writes a heartbeat from its own async loop every
10//! [`HEARTBEAT_INTERVAL`]. The loop runs on one thread, so any task that blocks
11//! it stops the heartbeat too. A heartbeat older than [`STALLED_AFTER`] whose
12//! process is still running is a stalled daemon. The record holds the
13//! process's identity, so a daemon that crashed is told apart from one that is
14//! stuck, and a graceful exit removes it.
15
16use std::path::PathBuf;
17use std::time::Duration;
18
19use chrono::{DateTime, Utc};
20use serde::{Deserialize, Serialize};
21
22use crate::paths::AppPaths;
23use crate::process::{Adoption, ProcessIdentity};
24use crate::service::{ServiceError, read_state_record, remove_state_record, write_state_record};
25
26/// The record, inside `state/`.
27pub const HEARTBEAT_FILE: &str = "daemon-heartbeat.toml";
28
29/// How often the daemon writes it.
30pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60);
31
32/// How old it may get before a running daemon counts as stalled: fifteen
33/// missed beats. Long enough that legitimate work on the loop (removing a
34/// large workspace on a slow disk) or the Mac sleeping does not trip it.
35pub const STALLED_AFTER: Duration = Duration::from_secs(15 * HEARTBEAT_INTERVAL.as_secs());
36
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38struct Heartbeat {
39    schema_version: u32,
40    process: ProcessIdentity,
41    at: DateTime<Utc>,
42}
43
44/// What the heartbeat says about the daemon.
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub enum Liveness {
47    /// No heartbeat: no daemon, one that exited cleanly, or one older than
48    /// heartbeats.
49    Unknown,
50    /// Beating within [`STALLED_AFTER`].
51    Beating { at: DateTime<Utc> },
52    /// Its process still runs and has not beaten since `since`.
53    Stalled { since: DateTime<Utc>, pid: u32 },
54    /// Its process is gone.
55    Exited,
56}
57
58impl Liveness {
59    /// Why a stalled daemon reads unhealthy, with what to do. `None`
60    /// otherwise. `restart` is the service manager's restart command, from
61    /// [`crate::service::ServiceIdentity::restart_command`].
62    #[must_use]
63    pub fn stall(
64        &self,
65        last_github_contact: Option<DateTime<Utc>>,
66        restart: &str,
67    ) -> Option<String> {
68        let Self::Stalled { since, pid } = self else {
69            return None;
70        };
71        let contact = last_github_contact.map_or_else(
72            || "it has never reached GitHub".to_owned(),
73            |at| format!("it last reached GitHub at {}", at.to_rfc3339()),
74        );
75        Some(format!(
76            "the service (process {pid}) has made no progress since {}: its loop has not \
77             beaten for over {} minutes, and {contact}. A daemon blocked in a file system call \
78             looks like this, for example on a runner root behind a macOS question nobody has \
79             answered (look for a dialog on the Mac's desktop). If nothing is waiting and the \
80             machine has not just woken from sleep, restart it: `{restart}`",
81            since.to_rfc3339(),
82            STALLED_AFTER.as_secs() / 60
83        ))
84    }
85}
86
87fn path(paths: &AppPaths) -> PathBuf {
88    paths.state_dir().join(HEARTBEAT_FILE)
89}
90
91/// Records that this process's loop ran at `at`.
92///
93/// # Errors
94/// The record could not be written, or this process's identity read.
95pub fn beat(paths: &AppPaths, at: DateTime<Utc>) -> Result<(), ServiceError> {
96    let process = ProcessIdentity::of_current_process().map_err(|error| ServiceError::Record {
97        operation: "write",
98        path: path(paths),
99        detail: error.to_string(),
100    })?;
101    write_state_record(
102        &path(paths),
103        &Heartbeat {
104            schema_version: 1,
105            process,
106            at,
107        },
108    )
109}
110
111/// Removes the record, on a graceful exit.
112///
113/// # Errors
114/// The record exists and cannot be removed.
115pub fn stop(paths: &AppPaths) -> Result<(), ServiceError> {
116    remove_state_record(&path(paths))
117}
118
119/// What the record says at `now`.
120#[must_use]
121pub fn liveness(paths: &AppPaths, now: DateTime<Utc>) -> Liveness {
122    let Ok(Some(heartbeat)) = read_state_record::<Heartbeat>(&path(paths)) else {
123        return Liveness::Unknown;
124    };
125    // A process this account may not inspect (a boot-mode daemon seen by an
126    // operator) is taken to be running: a stale heartbeat from it is the one
127    // thing worth saying, and a crashed one leaves launchd or the SCM to say
128    // it stopped.
129    let running = !matches!(
130        heartbeat.process.recheck(),
131        Ok(Adoption::Gone | Adoption::PidRecycled { .. })
132    );
133    judge(heartbeat.at, heartbeat.process.pid(), running, now)
134}
135
136fn judge(at: DateTime<Utc>, pid: u32, running: bool, now: DateTime<Utc>) -> Liveness {
137    if !running {
138        Liveness::Exited
139    } else if crate::wsl::fence::elapsed_at_least(at, now, STALLED_AFTER) {
140        Liveness::Stalled { since: at, pid }
141    } else {
142        Liveness::Beating { at }
143    }
144}
145
146#[cfg(test)]
147mod tests {
148    use super::*;
149
150    fn at(minute: i64) -> DateTime<Utc> {
151        DateTime::from_timestamp(1_790_000_000 + minute * 60, 0).unwrap()
152    }
153
154    #[test]
155    fn a_running_daemon_that_stopped_beating_is_stalled() {
156        assert_eq!(
157            judge(at(0), 7, true, at(14)),
158            Liveness::Beating { at: at(0) }
159        );
160        assert_eq!(
161            judge(at(0), 7, true, at(15)),
162            Liveness::Stalled {
163                since: at(0),
164                pid: 7
165            }
166        );
167        assert_eq!(judge(at(0), 7, false, at(30)), Liveness::Exited);
168    }
169
170    #[test]
171    fn this_process_beats_and_a_graceful_exit_removes_the_record() {
172        let root = tempfile::tempdir().unwrap();
173        let paths = AppPaths::rooted_at(root.path());
174        paths.create_all().unwrap();
175        assert_eq!(liveness(&paths, Utc::now()), Liveness::Unknown);
176
177        beat(&paths, Utc::now()).unwrap();
178        assert!(matches!(
179            liveness(&paths, Utc::now()),
180            Liveness::Beating { .. }
181        ));
182        // The same running process, not heard from for long enough.
183        let stalled = liveness(&paths, Utc::now() + chrono::Duration::minutes(16));
184        assert!(matches!(stalled, Liveness::Stalled { pid, .. } if pid == std::process::id()));
185        let said = stalled
186            .stall(None, "the-restart-command")
187            .expect("a stall is a problem");
188        assert!(said.contains("made no progress since"), "{said}");
189        assert!(said.contains("`the-restart-command`"), "{said}");
190        assert!(said.contains("never reached GitHub"), "{said}");
191
192        stop(&paths).unwrap();
193        assert_eq!(liveness(&paths, Utc::now()), Liveness::Unknown);
194    }
195}
runner_manager_platform/daemon_heartbeat.rs

runner_manager_platform/
daemon_heartbeat.rs
