runner_manager_platform/
daemon_heartbeat.rs

1//! Whether the daemon is making progress at all.
2//!
3//! A daemon blocked in a file system call answers nothing and logs nothing: on
4//! a Mac after an update, one sat at 0% CPU for half an hour behind a pending
5//! "would like to access files on a removable volume" question, while `service
6//! status` said `verdict healthy` (launchd reported it running) and the
7//! host-unfit record it could no longer re-stamp expired from `status`.
8//!
9//! So the daemon writes a heartbeat from its own async loop every
10//! [`HEARTBEAT_INTERVAL`]. The loop runs on one thread, so any task that blocks
11//! it stops the heartbeat too. A heartbeat older than [`STALLED_AFTER`] whose
12//! process is still running is a stalled daemon. The record holds the
13//! process's identity, so a daemon that crashed is told apart from one that is
14//! stuck, and a graceful exit removes it.
15
16use std::path::PathBuf;
17use std::time::Duration;
18
19use chrono::{DateTime, Utc};
20use serde::{Deserialize, Serialize};
21
22use crate::paths::AppPaths;
23use crate::process::{Adoption, ProcessIdentity};
24use crate::service::{ServiceError, read_state_record, remove_state_record, write_state_record};
25
26/// The record, inside `state/`.
27pub const HEARTBEAT_FILE: &str = "daemon-heartbeat.toml";
28
29/// How often the daemon writes it.
30pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60);
31
32/// How old it may get before a running daemon counts as stalled: fifteen
33/// missed beats. Long enough that legitimate work on the loop (removing a
34/// large workspace on a slow disk) or the Mac sleeping does not trip it.
35pub const STALLED_AFTER: Duration = Duration::from_secs(15 * HEARTBEAT_INTERVAL.as_secs());
36
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38struct Heartbeat {
39    schema_version: u32,
40    process: ProcessIdentity,
41    at: DateTime<Utc>,
42}
43
44/// What the heartbeat says about the daemon.
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub enum Liveness {
47    /// No heartbeat: no daemon, one that exited cleanly, or one older than
48    /// heartbeats.
49    Unknown,
50    /// Beating within [`STALLED_AFTER`].
51    Beating { at: DateTime<Utc> },
52    /// Its process still runs and has not beaten since `since`.
53    Stalled { since: DateTime<Utc>, pid: u32 },
54    /// Its process is gone.
55    Exited,
56}
57
58impl Liveness {
59    /// Why a stalled daemon reads unhealthy, with what to do. `None`
60    /// otherwise.
61    #[must_use]
62    pub fn stall(&self, last_github_contact: Option<DateTime<Utc>>) -> Option<String> {
63        let Self::Stalled { since, pid } = self else {
64            return None;
65        };
66        let contact = last_github_contact.map_or_else(
67            || "it has never reached GitHub".to_owned(),
68            |at| format!("it last reached GitHub at {}", at.to_rfc3339()),
69        );
70        Some(format!(
71            "the service (process {pid}) has made no progress since {}: its loop has not \
72             beaten for over {} minutes, and {contact}. A daemon blocked in a file system call \
73             looks like this, for example on a runner root behind a macOS question nobody has \
74             answered (look for a dialog on the Mac's desktop). If nothing is waiting and the \
75             machine has not just woken from sleep, restart it: `runner-manager service stop` \
76             then `runner-manager service start`",
77            since.to_rfc3339(),
78            STALLED_AFTER.as_secs() / 60
79        ))
80    }
81}
82
83fn path(paths: &AppPaths) -> PathBuf {
84    paths.state_dir().join(HEARTBEAT_FILE)
85}
86
87/// Records that this process's loop ran at `at`.
88///
89/// # Errors
90/// The record could not be written, or this process's identity read.
91pub fn beat(paths: &AppPaths, at: DateTime<Utc>) -> Result<(), ServiceError> {
92    let process = ProcessIdentity::of_current_process().map_err(|error| ServiceError::Record {
93        operation: "write",
94        path: path(paths),
95        detail: error.to_string(),
96    })?;
97    write_state_record(
98        &path(paths),
99        &Heartbeat {
100            schema_version: 1,
101            process,
102            at,
103        },
104    )
105}
106
107/// Removes the record, on a graceful exit.
108///
109/// # Errors
110/// The record exists and cannot be removed.
111pub fn stop(paths: &AppPaths) -> Result<(), ServiceError> {
112    remove_state_record(&path(paths))
113}
114
115/// What the record says at `now`.
116#[must_use]
117pub fn liveness(paths: &AppPaths, now: DateTime<Utc>) -> Liveness {
118    let Ok(Some(heartbeat)) = read_state_record::<Heartbeat>(&path(paths)) else {
119        return Liveness::Unknown;
120    };
121    // A process this account may not inspect (a boot-mode daemon seen by an
122    // operator) is taken to be running: a stale heartbeat from it is the one
123    // thing worth saying, and a crashed one leaves launchd or the SCM to say
124    // it stopped.
125    let running = !matches!(
126        heartbeat.process.recheck(),
127        Ok(Adoption::Gone | Adoption::PidRecycled { .. })
128    );
129    judge(heartbeat.at, heartbeat.process.pid(), running, now)
130}
131
132fn judge(at: DateTime<Utc>, pid: u32, running: bool, now: DateTime<Utc>) -> Liveness {
133    if !running {
134        Liveness::Exited
135    } else if crate::wsl::fence::elapsed_at_least(at, now, STALLED_AFTER) {
136        Liveness::Stalled { since: at, pid }
137    } else {
138        Liveness::Beating { at }
139    }
140}
141
142#[cfg(test)]
143mod tests {
144    use super::*;
145
146    fn at(minute: i64) -> DateTime<Utc> {
147        DateTime::from_timestamp(1_790_000_000 + minute * 60, 0).unwrap()
148    }
149
150    #[test]
151    fn a_running_daemon_that_stopped_beating_is_stalled() {
152        assert_eq!(
153            judge(at(0), 7, true, at(14)),
154            Liveness::Beating { at: at(0) }
155        );
156        assert_eq!(
157            judge(at(0), 7, true, at(15)),
158            Liveness::Stalled {
159                since: at(0),
160                pid: 7
161            }
162        );
163        assert_eq!(judge(at(0), 7, false, at(30)), Liveness::Exited);
164    }
165
166    #[test]
167    fn this_process_beats_and_a_graceful_exit_removes_the_record() {
168        let root = tempfile::tempdir().unwrap();
169        let paths = AppPaths::rooted_at(root.path());
170        paths.create_all().unwrap();
171        assert_eq!(liveness(&paths, Utc::now()), Liveness::Unknown);
172
173        beat(&paths, Utc::now()).unwrap();
174        assert!(matches!(
175            liveness(&paths, Utc::now()),
176            Liveness::Beating { .. }
177        ));
178        // The same running process, not heard from for long enough.
179        let stalled = liveness(&paths, Utc::now() + chrono::Duration::minutes(16));
180        assert!(matches!(stalled, Liveness::Stalled { pid, .. } if pid == std::process::id()));
181        let said = stalled.stall(None).expect("a stall is a problem");
182        assert!(said.contains("made no progress since"), "{said}");
183        assert!(said.contains("never reached GitHub"), "{said}");
184
185        stop(&paths).unwrap();
186        assert_eq!(liveness(&paths, Utc::now()), Liveness::Unknown);
187    }
188}
runner_manager_platform/daemon_heartbeat.rs

runner_manager_platform/
daemon_heartbeat.rs
