runnerd_core/scheduler/engine/unsynced.rs
1//! Whether what runners wrote without syncing may be lost to a host crash (ADR 0141): the one value that says when
2//! the slots' volumes and the shared pnpm and Maven stores must be emptied before the next runner starts. It owns the
3//! file that keeps that value across restarts and the kernel's boot ID it compares, and asks `Durability` whether a
4//! past boot ended in an orderly shutdown; it never calls Docker.
5
6use std::fmt;
7use std::io::{self, Write as _};
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10
11/// How the host makes what runners wrote durable, and tells whether it did: `sync(2)` and the systemd journal on the
12/// host (`boots`), a counter and a script in tests.
13#[derive(Clone)]
14pub struct Durability(Arc<Hooks>);
15
16/// What `Durability` calls, behind one pointer: the engine is held across every await of its loop.
17struct Hooks {
18 flush: Box<dyn Fn() + Send + Sync>,
19 ended_synced: Box<dyn Fn(&str) -> bool + Send + Sync>,
20}
21
22impl Durability {
23 /// `flush` writes what the kernel holds in memory to disk; `ended_synced` says whether the boot of the boot ID it
24 /// is given ended in an orderly shutdown whose final sync finished.
25 pub fn new(
26 flush: impl Fn() + Send + Sync + 'static,
27 ended_synced: impl Fn(&str) -> bool + Send + Sync + 'static,
28 ) -> Durability {
29 Durability(Arc::new(Hooks {
30 flush: Box::new(flush),
31 ended_synced: Box::new(ended_synced),
32 }))
33 }
34
35 /// The host's `sync(2)`, which every user may call, and its journal (`boots::ended_synced`).
36 pub fn host() -> Durability {
37 Durability::new(rustix::fs::sync, crate::boots::ended_synced)
38 }
39}
40
41impl fmt::Debug for Durability {
42 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
43 f.write_str("Durability")
44 }
45}
46
47/// What runners that skip their syncs may have left unwritten. Kept in a file of the daemon's, written before it
48/// changes, so it survives a restart and a host crash; a reload does not change it, as a runner started before keeps
49/// what it wrote whatever `skip_syncs` says now. The count of refused emptyings is not kept: a restart tries
50/// `REFUSALS` times again.
51#[derive(Debug)]
52pub(crate) struct Unsynced {
53 /// Where the state is kept: absent while `Synced`.
54 file: PathBuf,
55 /// The kernel's boot ID, which changes with every boot: `/proc/sys/kernel/random/boot_id`.
56 boot_id: PathBuf,
57 durability: Durability,
58 state: Owed,
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
62enum Owed {
63 /// Everything runners wrote is on disk, or none skipped its syncs since it was.
64 Synced,
65 /// A runner that skips its syncs started during the boot named; what it wrote may be only in memory.
66 Since { boot: String },
67 /// The host booted again after `Since` with no clean stop between and no orderly shutdown that ended it: what
68 /// runners wrote may be torn, so the slots' volumes and the shared stores are emptied before any runner starts.
69 /// `refusals` counts the ticks since this daemon started that found a container left on a torn volume; `held`
70 /// are the volumes the last of them could not empty.
71 Torn { refusals: u32, held: Vec<String> },
72}
73
74/// How many emptyings Docker may refuse before the engine gives up on them and starts runners on what the caches
75/// hold, rather than start none.
76pub(crate) const REFUSALS: u32 = 5;
77
78/// What `recovered` found.
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub(crate) enum Recovered {
81 /// Nothing is owed, or the host has not booted since runners skipped their syncs.
82 Unchanged,
83 /// The host booted since, after an orderly shutdown that wrote everything: nothing is owed.
84 ShutDownInOrder,
85 /// The host booted since with neither a clean stop nor an orderly shutdown: the caches are torn.
86 Torn,
87}
88
89/// What `refused` came to.
90#[derive(Debug, Clone, Copy, PartialEq, Eq)]
91pub(crate) enum AfterRefusal {
92 /// The caches stay torn, and the next tick tries again: `tries` of `REFUSALS` were refused.
93 Again { tries: u32 },
94 /// That was the last try: the engine may give up (`gave_up`) once no slot's layers it could not empty are left
95 /// as they are. Until then the caches stay torn, on disk too, so a restart tries again.
96 GiveUp,
97}
98
99/// What a file that is not one this module writes reads as: what it held cannot be told, so it is kept as torn.
100const TORN: &str = "torn";
101const SINCE: &str = "since ";
102
103impl Unsynced {
104 /// The state `file` keeps. A file that cannot be read, or holds what this module never writes, reads as torn:
105 /// emptying the caches once costs less than a job on a torn one.
106 pub(crate) fn open(file: PathBuf, proc: &Path, durability: Durability) -> Unsynced {
107 let state = match std::fs::read_to_string(&file) {
108 Err(error) if error.kind() == io::ErrorKind::NotFound => Owed::Synced,
109 Err(_) => Owed::Torn {
110 refusals: 0,
111 held: Vec::new(),
112 },
113 Ok(text) => match text.trim_end().strip_prefix(SINCE) {
114 Some(boot) if !boot.is_empty() => Owed::Since {
115 boot: boot.to_owned(),
116 },
117 _ => Owed::Torn {
118 refusals: 0,
119 held: Vec::new(),
120 },
121 },
122 };
123 Unsynced {
124 file,
125 boot_id: proc.join("sys/kernel/random/boot_id"),
126 durability,
127 state,
128 }
129 }
130
131 fn boot(&self) -> Option<String> {
132 let boot = std::fs::read_to_string(&self.boot_id).ok()?;
133 let boot = boot.trim();
134 (!boot.is_empty()).then(|| boot.to_owned())
135 }
136
137 /// At start: a host that booted since runners skipped their syncs, with no clean stop between, may have lost what
138 /// they wrote, unless that boot ended in an orderly shutdown, whose final sync wrote everything once every runner
139 /// had stopped. A boot ID that cannot be read now cannot be told from another boot.
140 pub(crate) fn recovered(&mut self) -> io::Result<Recovered> {
141 let Owed::Since { boot } = &self.state else {
142 return Ok(Recovered::Unchanged);
143 };
144 if self.boot().as_deref() == Some(boot.as_str()) {
145 return Ok(Recovered::Unchanged);
146 }
147 if (self.durability.0.ended_synced)(boot) {
148 self.settle()?;
149 return Ok(Recovered::ShutDownInOrder);
150 }
151 self.state = Owed::Torn {
152 refusals: 0,
153 held: Vec::new(),
154 };
155 write(&self.file, TORN)?;
156 Ok(Recovered::Torn)
157 }
158
159 /// Whether the caches must be emptied before any runner starts.
160 pub(crate) fn is_torn(&self) -> bool {
161 matches!(self.state, Owed::Torn { .. })
162 }
163
164 /// While the caches are torn, the volumes a container kept the last try from emptying: none before a try, or
165 /// after one Docker failed.
166 pub(crate) fn torn(&self) -> Option<&[String]> {
167 match &self.state {
168 Owed::Torn { held, .. } => Some(held),
169 Owed::Synced | Owed::Since { .. } => None,
170 }
171 }
172
173 /// Before a runner that may skip its syncs starts: whether it may, once that is kept on disk. A runner that may not
174 /// keeps its syncs: while the caches are torn none starts, and where the boot ID or the file cannot be had, a
175 /// later crash could not be told.
176 pub(crate) fn runner_skips(&mut self) -> bool {
177 let Some(boot) = self.boot() else {
178 return false;
179 };
180 match &self.state {
181 Owed::Torn { .. } => false,
182 Owed::Since { boot: since } if *since == boot => true,
183 Owed::Synced | Owed::Since { .. } => {
184 if write(&self.file, &format!("{SINCE}{boot}")).is_err() {
185 return false;
186 }
187 self.state = Owed::Since { boot };
188 true
189 }
190 }
191 }
192
193 /// The torn caches are emptied: once that is on disk, nothing is owed.
194 pub(crate) fn emptied(&mut self) -> io::Result<()> {
195 if !self.is_torn() {
196 return Ok(());
197 }
198 self.settle()
199 }
200
201 /// A container left on each of `volumes`, torn caches, kept them from being emptied. After `REFUSALS` of them the engine may give up
202 /// and start runners on what the caches hold, as a cache that may be torn costs less than no runner at all; it
203 /// stays torn until it does (`gave_up`).
204 pub(crate) fn refused(&mut self, volumes: Vec<String>) -> AfterRefusal {
205 let Owed::Torn { refusals, held } = &mut self.state else {
206 return AfterRefusal::GiveUp;
207 };
208 *refusals = refusals.saturating_add(1);
209 *held = volumes;
210 let tries = *refusals;
211 if tries < REFUSALS {
212 return AfterRefusal::Again { tries };
213 }
214 AfterRefusal::GiveUp
215 }
216
217 /// The engine gave up on the caches it could not empty, once no slot's layers among them are left as they are:
218 /// once that is on disk, nothing is owed.
219 pub(crate) fn gave_up(&mut self) -> io::Result<()> {
220 if !self.is_torn() {
221 return Ok(());
222 }
223 self.settle()
224 }
225
226 /// A stop that leaves no runner running: once what runners wrote is on disk, nothing is owed. Torn caches stay
227 /// owed: they are emptied at the next start.
228 pub(crate) fn stopped_clean(&mut self) -> io::Result<()> {
229 if !matches!(self.state, Owed::Since { .. }) {
230 return Ok(());
231 }
232 self.settle()
233 }
234
235 fn settle(&mut self) -> io::Result<()> {
236 (self.durability.0.flush)();
237 match std::fs::remove_file(&self.file) {
238 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
239 _ => {}
240 }
241 sync_parent(&self.file)?;
242 self.state = Owed::Synced;
243 Ok(())
244 }
245}
246
247/// Replaces `file` with `text`, on disk before it returns: a temporary file synced, renamed over it, and the directory
248/// synced.
249fn write(file: &Path, text: &str) -> io::Result<()> {
250 let temporary = file.with_extension("new");
251 {
252 let mut out = std::fs::File::create(&temporary)?;
253 out.write_all(text.as_bytes())?;
254 out.write_all(b"\n")?;
255 out.sync_all()?;
256 }
257 std::fs::rename(&temporary, file)?;
258 sync_parent(file)
259}
260
261fn sync_parent(file: &Path) -> io::Result<()> {
262 match file.parent() {
263 Some(dir) if !dir.as_os_str().is_empty() => std::fs::File::open(dir)?.sync_all(),
264 _ => Ok(()),
265 }
266}