Skip to main content

runnerd_core/scheduler/engine/
unsynced.rs

1//! Whether what runners wrote without syncing may be lost to a host crash (ADR 0141): the one value that says when
2//! the slots' volumes and the shared pnpm and Maven stores must be emptied before the next runner starts. It owns the
3//! file that keeps that value across restarts and the kernel's boot ID it compares, and asks `Durability` whether a
4//! past boot ended in an orderly shutdown; it never calls Docker.
5
6use std::fmt;
7use std::io::{self, Write as _};
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10
11/// How the host makes what runners wrote durable, and tells whether it did: `sync(2)` and the systemd journal on the
12/// host (`boots`), a counter and a script in tests.
13#[derive(Clone)]
14pub struct Durability(Arc<Hooks>);
15
16/// What `Durability` calls, behind one pointer: the engine is held across every await of its loop.
17struct Hooks {
18    flush: Box<dyn Fn() + Send + Sync>,
19    ended_synced: Box<dyn Fn(&str) -> bool + Send + Sync>,
20}
21
22impl Durability {
23    /// `flush` writes what the kernel holds in memory to disk; `ended_synced` says whether the boot of the boot ID it
24    /// is given ended in an orderly shutdown whose final sync finished.
25    pub fn new(
26        flush: impl Fn() + Send + Sync + 'static,
27        ended_synced: impl Fn(&str) -> bool + Send + Sync + 'static,
28    ) -> Durability {
29        Durability(Arc::new(Hooks {
30            flush: Box::new(flush),
31            ended_synced: Box::new(ended_synced),
32        }))
33    }
34
35    /// The host's `sync(2)`, which every user may call, and its journal (`boots::ended_synced`).
36    pub fn host() -> Durability {
37        Durability::new(rustix::fs::sync, crate::boots::ended_synced)
38    }
39}
40
41impl fmt::Debug for Durability {
42    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
43        f.write_str("Durability")
44    }
45}
46
47/// What runners that skip their syncs may have left unwritten. Kept in a file of the daemon's, written before it
48/// changes, so it survives a restart and a host crash; a reload does not change it, as a runner started before keeps
49/// what it wrote whatever `skip_syncs` says now. The count of refused emptyings is not kept: a restart tries
50/// `REFUSALS` times again.
51#[derive(Debug)]
52pub(crate) struct Unsynced {
53    /// Where the state is kept: absent while `Synced`.
54    file: PathBuf,
55    /// The kernel's boot ID, which changes with every boot: `/proc/sys/kernel/random/boot_id`.
56    boot_id: PathBuf,
57    durability: Durability,
58    state: Owed,
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
62enum Owed {
63    /// Everything runners wrote is on disk, or none skipped its syncs since it was.
64    Synced,
65    /// A runner that skips its syncs started during the boot named; what it wrote may be only in memory.
66    Since { boot: String },
67    /// The host booted again after `Since` with no clean stop between and no orderly shutdown that ended it: what
68    /// runners wrote may be torn, so the slots' volumes and the shared stores are emptied before any runner starts.
69    /// `refusals` counts the ticks since this daemon started that found a container left on a torn volume.
70    Torn { refusals: u32 },
71}
72
73/// How many emptyings Docker may refuse before the engine gives up on them and starts runners on what the caches
74/// hold, rather than start none.
75pub(crate) const REFUSALS: u32 = 5;
76
77/// What `recovered` found.
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub(crate) enum Recovered {
80    /// Nothing is owed, or the host has not booted since runners skipped their syncs.
81    Unchanged,
82    /// The host booted since, after an orderly shutdown that wrote everything: nothing is owed.
83    ShutDownInOrder,
84    /// The host booted since with neither a clean stop nor an orderly shutdown: the caches are torn.
85    Torn,
86}
87
88/// What `refused` came to.
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub(crate) enum AfterRefusal {
91    /// The caches stay torn, and the next tick tries again: `tries` of `REFUSALS` were refused.
92    Again { tries: u32 },
93    /// That was the last try: nothing is owed, and runners start on what the caches hold.
94    GaveUp,
95}
96
97/// What a file that is not one this module writes reads as: what it held cannot be told, so it is kept as torn.
98const TORN: &str = "torn";
99const SINCE: &str = "since ";
100
101impl Unsynced {
102    /// The state `file` keeps. A file that cannot be read, or holds what this module never writes, reads as torn:
103    /// emptying the caches once costs less than a job on a torn one.
104    pub(crate) fn open(file: PathBuf, proc: &Path, durability: Durability) -> Unsynced {
105        let state = match std::fs::read_to_string(&file) {
106            Err(error) if error.kind() == io::ErrorKind::NotFound => Owed::Synced,
107            Err(_) => Owed::Torn { refusals: 0 },
108            Ok(text) => match text.trim_end().strip_prefix(SINCE) {
109                Some(boot) if !boot.is_empty() => Owed::Since {
110                    boot: boot.to_owned(),
111                },
112                _ => Owed::Torn { refusals: 0 },
113            },
114        };
115        Unsynced {
116            file,
117            boot_id: proc.join("sys/kernel/random/boot_id"),
118            durability,
119            state,
120        }
121    }
122
123    fn boot(&self) -> Option<String> {
124        let boot = std::fs::read_to_string(&self.boot_id).ok()?;
125        let boot = boot.trim();
126        (!boot.is_empty()).then(|| boot.to_owned())
127    }
128
129    /// At start: a host that booted since runners skipped their syncs, with no clean stop between, may have lost what
130    /// they wrote, unless that boot ended in an orderly shutdown, whose final sync wrote everything once every runner
131    /// had stopped. A boot ID that cannot be read now cannot be told from another boot.
132    pub(crate) fn recovered(&mut self) -> io::Result<Recovered> {
133        let Owed::Since { boot } = &self.state else {
134            return Ok(Recovered::Unchanged);
135        };
136        if self.boot().as_deref() == Some(boot.as_str()) {
137            return Ok(Recovered::Unchanged);
138        }
139        if (self.durability.0.ended_synced)(boot) {
140            self.settle()?;
141            return Ok(Recovered::ShutDownInOrder);
142        }
143        self.state = Owed::Torn { refusals: 0 };
144        write(&self.file, TORN)?;
145        Ok(Recovered::Torn)
146    }
147
148    /// Whether the caches must be emptied before any runner starts.
149    pub(crate) fn is_torn(&self) -> bool {
150        matches!(self.state, Owed::Torn { .. })
151    }
152
153    /// Before a runner that may skip its syncs starts: whether it may, once that is kept on disk. A runner that may not
154    /// keeps its syncs: while the caches are torn none starts, and where the boot ID or the file cannot be had, a
155    /// later crash could not be told.
156    pub(crate) fn runner_skips(&mut self) -> bool {
157        let Some(boot) = self.boot() else {
158            return false;
159        };
160        match &self.state {
161            Owed::Torn { .. } => false,
162            Owed::Since { boot: since } if *since == boot => true,
163            Owed::Synced | Owed::Since { .. } => {
164                if write(&self.file, &format!("{SINCE}{boot}")).is_err() {
165                    return false;
166                }
167                self.state = Owed::Since { boot };
168                true
169            }
170        }
171    }
172
173    /// The torn caches are emptied: once that is on disk, nothing is owed.
174    pub(crate) fn emptied(&mut self) -> io::Result<()> {
175        if !self.is_torn() {
176            return Ok(());
177        }
178        self.settle()
179    }
180
181    /// A container left on a torn cache kept it from being emptied. After `REFUSALS` of them nothing is owed, and runners start on what the
182    /// caches hold: a cache that may be torn costs less than no runner at all.
183    pub(crate) fn refused(&mut self) -> io::Result<AfterRefusal> {
184        let Owed::Torn { refusals } = &mut self.state else {
185            return Ok(AfterRefusal::GaveUp);
186        };
187        *refusals = refusals.saturating_add(1);
188        let tries = *refusals;
189        if tries < REFUSALS {
190            return Ok(AfterRefusal::Again { tries });
191        }
192        self.settle()?;
193        Ok(AfterRefusal::GaveUp)
194    }
195
196    /// A stop that leaves no runner running: once what runners wrote is on disk, nothing is owed. Torn caches stay
197    /// owed: they are emptied at the next start.
198    pub(crate) fn stopped_clean(&mut self) -> io::Result<()> {
199        if !matches!(self.state, Owed::Since { .. }) {
200            return Ok(());
201        }
202        self.settle()
203    }
204
205    fn settle(&mut self) -> io::Result<()> {
206        (self.durability.0.flush)();
207        match std::fs::remove_file(&self.file) {
208            Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
209            _ => {}
210        }
211        sync_parent(&self.file)?;
212        self.state = Owed::Synced;
213        Ok(())
214    }
215}
216
217/// Replaces `file` with `text`, on disk before it returns: a temporary file synced, renamed over it, and the directory
218/// synced.
219fn write(file: &Path, text: &str) -> io::Result<()> {
220    let temporary = file.with_extension("new");
221    {
222        let mut out = std::fs::File::create(&temporary)?;
223        out.write_all(text.as_bytes())?;
224        out.write_all(b"\n")?;
225        out.sync_all()?;
226    }
227    std::fs::rename(&temporary, file)?;
228    sync_parent(file)
229}
230
231fn sync_parent(file: &Path) -> io::Result<()> {
232    match file.parent() {
233        Some(dir) if !dir.as_os_str().is_empty() => std::fs::File::open(dir)?.sync_all(),
234        _ => Ok(()),
235    }
236}