runnerd_core/scheduler/engine/unsynced.rs
1//! Whether what runners wrote without syncing may be lost to a host crash (ADR 0141): the one value that says when
2//! the slots' volumes and the shared pnpm and Maven stores must be emptied before the next runner starts. It owns the
3//! file that keeps that value across restarts and the kernel's boot ID it compares, and asks `Durability` whether a
4//! past boot ended in an orderly shutdown; it never calls Docker.
5
6use std::fmt;
7use std::io::{self, Write as _};
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10
11/// How the host makes what runners wrote durable, and tells whether it did: `sync(2)` and the systemd journal on the
12/// host (`boots`), a counter and a script in tests.
13#[derive(Clone)]
14pub struct Durability(Arc<Hooks>);
15
16/// What `Durability` calls, behind one pointer: the engine is held across every await of its loop.
17struct Hooks {
18 flush: Box<dyn Fn() + Send + Sync>,
19 ended_synced: Box<dyn Fn(&str) -> bool + Send + Sync>,
20}
21
22impl Durability {
23 /// `flush` writes what the kernel holds in memory to disk; `ended_synced` says whether the boot of the boot ID it
24 /// is given ended in an orderly shutdown whose final sync finished.
25 pub fn new(
26 flush: impl Fn() + Send + Sync + 'static,
27 ended_synced: impl Fn(&str) -> bool + Send + Sync + 'static,
28 ) -> Durability {
29 Durability(Arc::new(Hooks {
30 flush: Box::new(flush),
31 ended_synced: Box::new(ended_synced),
32 }))
33 }
34
35 /// The host's `sync(2)`, which every user may call, and its journal (`boots::ended_synced`).
36 pub fn host() -> Durability {
37 Durability::new(rustix::fs::sync, crate::boots::ended_synced)
38 }
39}
40
41impl fmt::Debug for Durability {
42 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
43 f.write_str("Durability")
44 }
45}
46
47/// What runners that skip their syncs may have left unwritten. Kept in a file of the daemon's, written before it
48/// changes, so it survives a restart and a host crash; a reload does not change it, as a runner started before keeps
49/// what it wrote whatever `skip_syncs` says now. The count of refused emptyings is not kept: a restart tries
50/// `REFUSALS` times again.
51#[derive(Debug)]
52pub(crate) struct Unsynced {
53 /// Where the state is kept: absent while `Synced`.
54 file: PathBuf,
55 /// The kernel's boot ID, which changes with every boot: `/proc/sys/kernel/random/boot_id`.
56 boot_id: PathBuf,
57 durability: Durability,
58 state: Owed,
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
62enum Owed {
63 /// Everything runners wrote is on disk, or none skipped its syncs since it was.
64 Synced,
65 /// A runner that skips its syncs started during the boot named; what it wrote may be only in memory.
66 Since { boot: String },
67 /// The host booted again after `Since` with no clean stop between and no orderly shutdown that ended it: what
68 /// runners wrote may be torn, so the slots' volumes and the shared stores are emptied before any runner starts.
69 /// `refusals` counts the ticks since this daemon started that found a container left on a torn volume.
70 Torn { refusals: u32 },
71}
72
73/// How many emptyings Docker may refuse before the engine gives up on them and starts runners on what the caches
74/// hold, rather than start none.
75pub(crate) const REFUSALS: u32 = 5;
76
77/// What `recovered` found.
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub(crate) enum Recovered {
80 /// Nothing is owed, or the host has not booted since runners skipped their syncs.
81 Unchanged,
82 /// The host booted since, after an orderly shutdown that wrote everything: nothing is owed.
83 ShutDownInOrder,
84 /// The host booted since with neither a clean stop nor an orderly shutdown: the caches are torn.
85 Torn,
86}
87
88/// What `refused` came to.
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub(crate) enum AfterRefusal {
91 /// The caches stay torn, and the next tick tries again: `tries` of `REFUSALS` were refused.
92 Again { tries: u32 },
93 /// That was the last try: nothing is owed, and runners start on what the caches hold.
94 GaveUp,
95}
96
97/// What a file that is not one this module writes reads as: what it held cannot be told, so it is kept as torn.
98const TORN: &str = "torn";
99const SINCE: &str = "since ";
100
101impl Unsynced {
102 /// The state `file` keeps. A file that cannot be read, or holds what this module never writes, reads as torn:
103 /// emptying the caches once costs less than a job on a torn one.
104 pub(crate) fn open(file: PathBuf, proc: &Path, durability: Durability) -> Unsynced {
105 let state = match std::fs::read_to_string(&file) {
106 Err(error) if error.kind() == io::ErrorKind::NotFound => Owed::Synced,
107 Err(_) => Owed::Torn { refusals: 0 },
108 Ok(text) => match text.trim_end().strip_prefix(SINCE) {
109 Some(boot) if !boot.is_empty() => Owed::Since {
110 boot: boot.to_owned(),
111 },
112 _ => Owed::Torn { refusals: 0 },
113 },
114 };
115 Unsynced {
116 file,
117 boot_id: proc.join("sys/kernel/random/boot_id"),
118 durability,
119 state,
120 }
121 }
122
123 fn boot(&self) -> Option<String> {
124 let boot = std::fs::read_to_string(&self.boot_id).ok()?;
125 let boot = boot.trim();
126 (!boot.is_empty()).then(|| boot.to_owned())
127 }
128
129 /// At start: a host that booted since runners skipped their syncs, with no clean stop between, may have lost what
130 /// they wrote, unless that boot ended in an orderly shutdown, whose final sync wrote everything once every runner
131 /// had stopped. A boot ID that cannot be read now cannot be told from another boot.
132 pub(crate) fn recovered(&mut self) -> io::Result<Recovered> {
133 let Owed::Since { boot } = &self.state else {
134 return Ok(Recovered::Unchanged);
135 };
136 if self.boot().as_deref() == Some(boot.as_str()) {
137 return Ok(Recovered::Unchanged);
138 }
139 if (self.durability.0.ended_synced)(boot) {
140 self.settle()?;
141 return Ok(Recovered::ShutDownInOrder);
142 }
143 self.state = Owed::Torn { refusals: 0 };
144 write(&self.file, TORN)?;
145 Ok(Recovered::Torn)
146 }
147
148 /// Whether the caches must be emptied before any runner starts.
149 pub(crate) fn is_torn(&self) -> bool {
150 matches!(self.state, Owed::Torn { .. })
151 }
152
153 /// Before a runner that may skip its syncs starts: whether it may, once that is kept on disk. A runner that may not
154 /// keeps its syncs: while the caches are torn none starts, and where the boot ID or the file cannot be had, a
155 /// later crash could not be told.
156 pub(crate) fn runner_skips(&mut self) -> bool {
157 let Some(boot) = self.boot() else {
158 return false;
159 };
160 match &self.state {
161 Owed::Torn { .. } => false,
162 Owed::Since { boot: since } if *since == boot => true,
163 Owed::Synced | Owed::Since { .. } => {
164 if write(&self.file, &format!("{SINCE}{boot}")).is_err() {
165 return false;
166 }
167 self.state = Owed::Since { boot };
168 true
169 }
170 }
171 }
172
173 /// The torn caches are emptied: once that is on disk, nothing is owed.
174 pub(crate) fn emptied(&mut self) -> io::Result<()> {
175 if !self.is_torn() {
176 return Ok(());
177 }
178 self.settle()
179 }
180
181 /// A container left on a torn cache kept it from being emptied. After `REFUSALS` of them nothing is owed, and runners start on what the
182 /// caches hold: a cache that may be torn costs less than no runner at all.
183 pub(crate) fn refused(&mut self) -> io::Result<AfterRefusal> {
184 let Owed::Torn { refusals } = &mut self.state else {
185 return Ok(AfterRefusal::GaveUp);
186 };
187 *refusals = refusals.saturating_add(1);
188 let tries = *refusals;
189 if tries < REFUSALS {
190 return Ok(AfterRefusal::Again { tries });
191 }
192 self.settle()?;
193 Ok(AfterRefusal::GaveUp)
194 }
195
196 /// A stop that leaves no runner running: once what runners wrote is on disk, nothing is owed. Torn caches stay
197 /// owed: they are emptied at the next start.
198 pub(crate) fn stopped_clean(&mut self) -> io::Result<()> {
199 if !matches!(self.state, Owed::Since { .. }) {
200 return Ok(());
201 }
202 self.settle()
203 }
204
205 fn settle(&mut self) -> io::Result<()> {
206 (self.durability.0.flush)();
207 match std::fs::remove_file(&self.file) {
208 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
209 _ => {}
210 }
211 sync_parent(&self.file)?;
212 self.state = Owed::Synced;
213 Ok(())
214 }
215}
216
217/// Replaces `file` with `text`, on disk before it returns: a temporary file synced, renamed over it, and the directory
218/// synced.
219fn write(file: &Path, text: &str) -> io::Result<()> {
220 let temporary = file.with_extension("new");
221 {
222 let mut out = std::fs::File::create(&temporary)?;
223 out.write_all(text.as_bytes())?;
224 out.write_all(b"\n")?;
225 out.sync_all()?;
226 }
227 std::fs::rename(&temporary, file)?;
228 sync_parent(file)
229}
230
231fn sync_parent(file: &Path) -> io::Result<()> {
232 match file.parent() {
233 Some(dir) if !dir.as_os_str().is_empty() => std::fs::File::open(dir)?.sync_all(),
234 _ => Ok(()),
235 }
236}