Skip to main content

runnerd_core/scheduler/engine/
unsynced.rs

1//! Whether what runners wrote without syncing may be lost to a host crash (ADR 0141): the one value that says when
2//! the slots' volumes and the shared pnpm and Maven stores must be emptied before the next runner starts. It owns the
3//! file that keeps that value across restarts and the kernel's boot ID it compares, and asks `Durability` whether a
4//! past boot ended in an orderly shutdown; it never calls Docker.
5
6use std::fmt;
7use std::io::{self, Write as _};
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10
11/// How the host makes what runners wrote durable, and tells whether it did: `sync(2)` and the systemd journal on the
12/// host (`boots`), a counter and a script in tests.
13#[derive(Clone)]
14pub struct Durability(Arc<Hooks>);
15
16/// What `Durability` calls, behind one pointer: the engine is held across every await of its loop.
17struct Hooks {
18    flush: Box<dyn Fn() + Send + Sync>,
19    ended_synced: Box<dyn Fn(&str) -> bool + Send + Sync>,
20}
21
22impl Durability {
23    /// `flush` writes what the kernel holds in memory to disk; `ended_synced` says whether the boot of the boot ID it
24    /// is given ended in an orderly shutdown whose final sync finished.
25    pub fn new(
26        flush: impl Fn() + Send + Sync + 'static,
27        ended_synced: impl Fn(&str) -> bool + Send + Sync + 'static,
28    ) -> Durability {
29        Durability(Arc::new(Hooks {
30            flush: Box::new(flush),
31            ended_synced: Box::new(ended_synced),
32        }))
33    }
34
35    /// The host's `sync(2)`, which every user may call, and its journal (`boots::ended_synced`).
36    pub fn host() -> Durability {
37        Durability::new(rustix::fs::sync, crate::boots::ended_synced)
38    }
39}
40
41impl fmt::Debug for Durability {
42    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
43        f.write_str("Durability")
44    }
45}
46
47/// What runners that skip their syncs may have left unwritten. Kept in a file of the daemon's, written before it
48/// changes, so it survives a restart and a host crash; a reload does not change it, as a runner started before keeps
49/// what it wrote whatever `skip_syncs` says now. The count of refused emptyings is not kept: a restart tries
50/// `REFUSALS` times again.
51#[derive(Debug)]
52pub(crate) struct Unsynced {
53    /// Where the state is kept: absent while `Synced`.
54    file: PathBuf,
55    /// The kernel's boot ID, which changes with every boot: `/proc/sys/kernel/random/boot_id`.
56    boot_id: PathBuf,
57    durability: Durability,
58    state: Owed,
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
62enum Owed {
63    /// Everything runners wrote is on disk, or none skipped its syncs since it was.
64    Synced,
65    /// A runner that skips its syncs started during the boot named; what it wrote may be only in memory.
66    Since { boot: String },
67    /// The host booted again after `Since` with no clean stop between and no orderly shutdown that ended it: what
68    /// runners wrote may be torn, so the slots' volumes and the shared stores are emptied before any runner starts.
69    /// `refusals` counts the ticks since this daemon started that found a container left on a torn volume; `held`
70    /// are the volumes the last of them could not empty.
71    Torn { refusals: u32, held: Vec<String> },
72}
73
74/// How many emptyings Docker may refuse before the engine gives up on them and starts runners on what the caches
75/// hold, rather than start none.
76pub(crate) const REFUSALS: u32 = 5;
77
78/// What `recovered` found.
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub(crate) enum Recovered {
81    /// Nothing is owed, or the host has not booted since runners skipped their syncs.
82    Unchanged,
83    /// The host booted since, after an orderly shutdown that wrote everything: nothing is owed.
84    ShutDownInOrder,
85    /// The host booted since with neither a clean stop nor an orderly shutdown: the caches are torn.
86    Torn,
87}
88
89/// What `refused` came to.
90#[derive(Debug, Clone, Copy, PartialEq, Eq)]
91pub(crate) enum AfterRefusal {
92    /// The caches stay torn, and the next tick tries again: `tries` of `REFUSALS` were refused.
93    Again { tries: u32 },
94    /// That was the last try: the engine may give up (`gave_up`) once no slot's layers it could not empty are left
95    /// as they are. Until then the caches stay torn, on disk too, so a restart tries again.
96    GiveUp,
97}
98
99/// What a file that is not one this module writes reads as: what it held cannot be told, so it is kept as torn.
100const TORN: &str = "torn";
101const SINCE: &str = "since ";
102
103impl Unsynced {
104    /// The state `file` keeps. A file that cannot be read, or holds what this module never writes, reads as torn:
105    /// emptying the caches once costs less than a job on a torn one.
106    pub(crate) fn open(file: PathBuf, proc: &Path, durability: Durability) -> Unsynced {
107        let state = match std::fs::read_to_string(&file) {
108            Err(error) if error.kind() == io::ErrorKind::NotFound => Owed::Synced,
109            Err(_) => Owed::Torn {
110                refusals: 0,
111                held: Vec::new(),
112            },
113            Ok(text) => match text.trim_end().strip_prefix(SINCE) {
114                Some(boot) if !boot.is_empty() => Owed::Since {
115                    boot: boot.to_owned(),
116                },
117                _ => Owed::Torn {
118                    refusals: 0,
119                    held: Vec::new(),
120                },
121            },
122        };
123        Unsynced {
124            file,
125            boot_id: proc.join("sys/kernel/random/boot_id"),
126            durability,
127            state,
128        }
129    }
130
131    fn boot(&self) -> Option<String> {
132        let boot = std::fs::read_to_string(&self.boot_id).ok()?;
133        let boot = boot.trim();
134        (!boot.is_empty()).then(|| boot.to_owned())
135    }
136
137    /// At start: a host that booted since runners skipped their syncs, with no clean stop between, may have lost what
138    /// they wrote, unless that boot ended in an orderly shutdown, whose final sync wrote everything once every runner
139    /// had stopped. A boot ID that cannot be read now cannot be told from another boot.
140    pub(crate) fn recovered(&mut self) -> io::Result<Recovered> {
141        let Owed::Since { boot } = &self.state else {
142            return Ok(Recovered::Unchanged);
143        };
144        if self.boot().as_deref() == Some(boot.as_str()) {
145            return Ok(Recovered::Unchanged);
146        }
147        if (self.durability.0.ended_synced)(boot) {
148            self.settle()?;
149            return Ok(Recovered::ShutDownInOrder);
150        }
151        self.state = Owed::Torn {
152            refusals: 0,
153            held: Vec::new(),
154        };
155        write(&self.file, TORN)?;
156        Ok(Recovered::Torn)
157    }
158
159    /// Whether the caches must be emptied before any runner starts.
160    pub(crate) fn is_torn(&self) -> bool {
161        matches!(self.state, Owed::Torn { .. })
162    }
163
164    /// While the caches are torn, the volumes a container kept the last try from emptying: none before a try, or
165    /// after one Docker failed.
166    pub(crate) fn torn(&self) -> Option<&[String]> {
167        match &self.state {
168            Owed::Torn { held, .. } => Some(held),
169            Owed::Synced | Owed::Since { .. } => None,
170        }
171    }
172
173    /// Before a runner that may skip its syncs starts: whether it may, once that is kept on disk. A runner that may not
174    /// keeps its syncs: while the caches are torn none starts, and where the boot ID or the file cannot be had, a
175    /// later crash could not be told.
176    pub(crate) fn runner_skips(&mut self) -> bool {
177        let Some(boot) = self.boot() else {
178            return false;
179        };
180        match &self.state {
181            Owed::Torn { .. } => false,
182            Owed::Since { boot: since } if *since == boot => true,
183            Owed::Synced | Owed::Since { .. } => {
184                if write(&self.file, &format!("{SINCE}{boot}")).is_err() {
185                    return false;
186                }
187                self.state = Owed::Since { boot };
188                true
189            }
190        }
191    }
192
193    /// The torn caches are emptied: once that is on disk, nothing is owed.
194    pub(crate) fn emptied(&mut self) -> io::Result<()> {
195        if !self.is_torn() {
196            return Ok(());
197        }
198        self.settle()
199    }
200
201    /// A container left on each of `volumes`, torn caches, kept them from being emptied. After `REFUSALS` of them the engine may give up
202    /// and start runners on what the caches hold, as a cache that may be torn costs less than no runner at all; it
203    /// stays torn until it does (`gave_up`).
204    pub(crate) fn refused(&mut self, volumes: Vec<String>) -> AfterRefusal {
205        let Owed::Torn { refusals, held } = &mut self.state else {
206            return AfterRefusal::GiveUp;
207        };
208        *refusals = refusals.saturating_add(1);
209        *held = volumes;
210        let tries = *refusals;
211        if tries < REFUSALS {
212            return AfterRefusal::Again { tries };
213        }
214        AfterRefusal::GiveUp
215    }
216
217    /// The engine gave up on the caches it could not empty, once no slot's layers among them are left as they are:
218    /// once that is on disk, nothing is owed.
219    pub(crate) fn gave_up(&mut self) -> io::Result<()> {
220        if !self.is_torn() {
221            return Ok(());
222        }
223        self.settle()
224    }
225
226    /// A stop that leaves no runner running: once what runners wrote is on disk, nothing is owed. Torn caches stay
227    /// owed: they are emptied at the next start.
228    pub(crate) fn stopped_clean(&mut self) -> io::Result<()> {
229        if !matches!(self.state, Owed::Since { .. }) {
230            return Ok(());
231        }
232        self.settle()
233    }
234
235    fn settle(&mut self) -> io::Result<()> {
236        (self.durability.0.flush)();
237        match std::fs::remove_file(&self.file) {
238            Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
239            _ => {}
240        }
241        sync_parent(&self.file)?;
242        self.state = Owed::Synced;
243        Ok(())
244    }
245}
246
247/// Replaces `file` with `text`, on disk before it returns: a temporary file synced, renamed over it, and the directory
248/// synced.
249fn write(file: &Path, text: &str) -> io::Result<()> {
250    let temporary = file.with_extension("new");
251    {
252        let mut out = std::fs::File::create(&temporary)?;
253        out.write_all(text.as_bytes())?;
254        out.write_all(b"\n")?;
255        out.sync_all()?;
256    }
257    std::fs::rename(&temporary, file)?;
258    sync_parent(file)
259}
260
261fn sync_parent(file: &Path) -> io::Result<()> {
262    match file.parent() {
263        Some(dir) if !dir.as_os_str().is_empty() => std::fs::File::open(dir)?.sync_all(),
264        _ => Ok(()),
265    }
266}