zond_engine/journal/lock.rs
1// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
2//
3// This file is part of Zond Engine, licensed under the GNU Affero General
4// Public License, version 3 or later. See the LICENSE file for details, or
5// <https://www.gnu.org/licenses/agpl-3.0.html>.
6//
7// SPDX-License-Identifier: AGPL-3.0-or-later
8
9//! # Telling a running scan from a crashed one
10//!
11//! A journal that is being written to must not be resumed, and a journal whose writer
12//! died must not stay locked forever. Resuming a live scan corrupts both sittings' cursors;
13//! a lock nobody can clear makes a crashed scan impossible to continue.
14//!
15//! ## Three facts
16//!
17//! The lock file records a process id, a boot identity and a heartbeat, because each alone
18//! is wrong in some case:
19//!
20//! - **The pid alone is wrong after a reboot.** Process ids restart from a low number, so
21//! the pid in a lock written before a crash-and-reboot often belongs to something live
22//! and unrelated, such as `launchd` or a shell.
23//! - **The boot identity alone is wrong within one boot.** A pid freed by a crash can be
24//! reissued to an unrelated process minutes later.
25//! - **The heartbeat alone is wrong about a slow scan.** A scan may legitimately do
26//! nothing for a while, under a long silence tolerance or against a tarpit.
27//!
28//! The boot identity rules out everything before the last boot; within one boot the pid
29//! says whether something holds that number, and the heartbeat says whether it is still
30//! this scan.
31//!
32//! ## The policy is a pure function
33//!
34//! [`classify`] takes the record, the current boot identity, whether the pid is alive and
35//! the time, and returns a [`LockState`], so a reboot, a reused pid and a hung writer can be
36//! tested in CI. The syscalls live in [`inspect`], a thin wrapper over it.
37//!
38//! ## What is refused
39//!
40//! A scan that is or might be running is refused; one that certainly is not may be
41//! resumed. A live pid that has stopped touching the lock is refused, since it may be a hung
42//! writer that will wake up, and two writers on one journal is what this module prevents.
43//!
44//! Every refusal is overridable, so a stale lock left by an unforeseen defect cannot make a
45//! journal unusable forever. [`Lock::take_over`] overrides only that ambiguous case, where
46//! the user can judge whether the process holding the number is the scan.
47//! [`Lock::force`] also overrides a writer that is checkpointing now.
48
49use std::time::{Duration, SystemTime};
50
51use serde::{Deserialize, Serialize};
52
53/// How long a heartbeat may go untouched before the writer holding a lock is
54/// treated as no longer obviously alive.
55///
56/// A scan checkpoints every few seconds, so a minute of silence is many missed beats.
57/// Erring long costs an overridable refusal; erring short costs two writers on one
58/// journal.
59pub const HEARTBEAT_STALE_AFTER: Duration = Duration::from_secs(60);
60
61/// What a lock file says about the process that wrote it.
62#[non_exhaustive]
63#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
64pub struct LockRecord {
65 /// The process holding the journal.
66 pub pid: u32,
67 /// Which boot that pid belongs to. A pid is only meaningful within one.
68 pub boot: String,
69 /// When the scan started, so a listing can say how old each journal is.
70 pub started_at: SystemTime,
71 /// Last touched by the writer, once per checkpoint.
72 pub heartbeat: SystemTime,
73}
74
75impl LockRecord {
76 /// The record this process would write now.
77 pub fn current() -> Self {
78 let now = SystemTime::now();
79 Self {
80 pid: std::process::id(),
81 boot: boot_identity(),
82 started_at: now,
83 heartbeat: now,
84 }
85 }
86
87 /// The same record with its heartbeat moved to `now`.
88 pub fn beating_at(&self, now: SystemTime) -> Self {
89 Self {
90 heartbeat: now,
91 ..self.clone()
92 }
93 }
94}
95
96/// What is known about a journal's writer.
97#[non_exhaustive]
98#[derive(Debug, Clone, PartialEq, Eq)]
99pub enum LockState {
100 /// No lock file. The journal is free.
101 Free,
102
103 /// A live process is writing this journal now. Refuse.
104 Held {
105 /// The process holding it, so a refusal can name it.
106 pid: u32,
107 /// How long ago it last checkpointed.
108 last_beat: Duration,
109 },
110
111 /// The pid is gone and the writer died without releasing. Resume, noting that the
112 /// last checkpoint interval may be missing.
113 Crashed {
114 /// The process that held it, for the note.
115 pid: u32,
116 },
117
118 /// The machine rebooted while this journal was locked, so the pid in the file is not
119 /// the writer. Resume, and warn that a reboot loses more than a crash: anything the page
120 /// cache had not flushed is gone.
121 RebootedUnder {
122 /// The process that held it, before the reboot.
123 pid: u32,
124 },
125
126 /// Something holds that pid and it has stopped touching the lock: either the writer
127 /// is hung or it died and the number was reissued. Refused, overridably.
128 Stale {
129 /// The process that number now belongs to, whoever that is.
130 pid: u32,
131 /// How long the heartbeat has been untouched.
132 last_beat: Duration,
133 },
134}
135
136impl LockState {
137 /// Whether the journal may be resumed without the user overriding anything.
138 pub fn is_resumable(&self) -> bool {
139 matches!(
140 self,
141 LockState::Free | LockState::Crashed { .. } | LockState::RebootedUnder { .. }
142 )
143 }
144
145 /// Whether resuming loses more than one checkpoint interval.
146 ///
147 /// True only after a reboot: every other stop is a process death, and the page cache
148 /// outlives a process. See [`journal`](crate::journal) for the survival table.
149 pub fn may_have_lost_the_tail(&self) -> bool {
150 matches!(self, LockState::RebootedUnder { .. })
151 }
152
153 /// Why a resume was refused, phrased for a user deciding what to do about it. `None`
154 /// where nothing was refused.
155 pub fn refusal(&self) -> Option<String> {
156 match self {
157 LockState::Free | LockState::Crashed { .. } | LockState::RebootedUnder { .. } => None,
158 LockState::Held { pid, last_beat } => Some(format!(
159 "process {pid} is scanning this journal now (last checkpoint {}s ago); \
160 wait for it to finish, or stop it first",
161 last_beat.as_secs()
162 )),
163 LockState::Stale { pid, last_beat } => Some(format!(
164 "this journal is locked by process {pid}, which has not checkpointed for {}s — \
165 it is either hung or the number was reissued to something else. \
166 Stop it, or take the journal over if you are sure it is not scanning",
167 last_beat.as_secs()
168 )),
169 }
170 }
171}
172
173/// Decides what a lock record means, given everything observed about it. Pure.
174///
175/// The boot identity is checked first: a pid from a previous boot says nothing, and asking
176/// whether it is alive first would leave every journal locked behind an unrelated process
177/// after a reboot.
178pub fn classify(
179 record: &LockRecord,
180 boot: &str,
181 pid_is_alive: bool,
182 now: SystemTime,
183 stale_after: Duration,
184) -> LockState {
185 if record.boot != boot {
186 return LockState::RebootedUnder { pid: record.pid };
187 }
188
189 if !pid_is_alive {
190 return LockState::Crashed { pid: record.pid };
191 }
192
193 // A clock that went backwards yields no elapsed time, read as a fresh beat: the safe
194 // reading of an unknowable age is that the writer is alive.
195 let last_beat = now.duration_since(record.heartbeat).unwrap_or_default();
196
197 if last_beat > stale_after {
198 LockState::Stale {
199 pid: record.pid,
200 last_beat,
201 }
202 } else {
203 LockState::Held {
204 pid: record.pid,
205 last_beat,
206 }
207 }
208}
209
210/// A value that changes every boot, so a pid from before one can be disregarded.
211///
212/// Linux has a boot id; macOS and the BSDs use the boot time. Where neither can be read the
213/// identity is empty, which differs from any recorded one and so reads as a reboot,
214/// releasing the lock.
215pub fn boot_identity() -> String {
216 imp::boot_identity()
217}
218
219/// Whether any process currently holds `pid`.
220///
221/// Says nothing about which process. Within one boot the heartbeat distinguishes this
222/// scan from a reused number.
223pub fn pid_is_alive(pid: u32) -> bool {
224 imp::pid_is_alive(pid)
225}
226
227#[cfg(unix)]
228mod imp {
229 pub fn boot_identity() -> String {
230 // Linux: a UUID minted per boot.
231 if let Ok(id) = std::fs::read_to_string("/proc/sys/kernel/random/boot_id") {
232 return id.trim().to_string();
233 }
234
235 // macOS and the BSDs: the boot instant from `KERN_BOOTTIME`.
236 #[cfg(any(target_os = "macos", target_os = "ios", target_vendor = "apple"))]
237 {
238 let mut boot = libc::timeval {
239 tv_sec: 0,
240 tv_usec: 0,
241 };
242 let mut size = std::mem::size_of::<libc::timeval>();
243 let mut mib = [libc::CTL_KERN, libc::KERN_BOOTTIME];
244
245 // SAFETY: `mib` is a live array of the length passed, `boot` is a
246 // live `timeval` and `size` names its exact size, which is what
247 // `KERN_BOOTTIME` writes. The call reads and writes only those.
248 let code = unsafe {
249 libc::sysctl(
250 mib.as_mut_ptr(),
251 mib.len() as libc::c_uint,
252 (&raw mut boot).cast::<libc::c_void>(),
253 &mut size,
254 std::ptr::null_mut(),
255 0,
256 )
257 };
258
259 if code == 0 {
260 return format!("{}.{}", boot.tv_sec, boot.tv_usec);
261 }
262 }
263
264 String::new()
265 }
266
267 pub fn pid_is_alive(pid: u32) -> bool {
268 // `pid_t` is signed and `kill` reads a negative argument as a process group:
269 // `u32::MAX` casts to `-1`, which asks about every process the caller may signal and is
270 // always answered yes. A lock naming it would then be refused forever.
271 let Ok(pid) = libc::pid_t::try_from(pid) else {
272 return false;
273 };
274 if pid <= 0 {
275 return false;
276 }
277
278 // SAFETY: `kill` with signal 0 sends nothing. It performs the existence
279 // and permission checks and returns, dereferencing nothing.
280 let code = unsafe { libc::kill(pid, 0) };
281 if code == 0 {
282 return true;
283 }
284
285 // `EPERM` means the process exists and belongs to somebody else, the ordinary case
286 // for a scan started under `sudo` and inspected without it.
287 std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
288 }
289}
290
291#[cfg(windows)]
292mod imp {
293 /// Empty on Windows, so the process id alone decides.
294 ///
295 /// Wall clock minus `GetTickCount64` is not constant within a boot: the two are read at
296 /// different instants, the tick counter has about 15 ms resolution, and it stops across
297 /// some suspend states. A value that moved within a boot would read every lock as
298 /// `RebootedUnder`, which is resumable.
299 ///
300 /// Two empty identities compare equal, so [`pid_is_alive`] decides every lock. The cost is
301 /// that a lock left by a crash before a reboot, whose number another process has taken
302 /// since, reads as held and is refused.
303 pub fn boot_identity() -> String {
304 String::new()
305 }
306
307 /// Whether a process with this id is running, as the unix arm asks it.
308 ///
309 /// Access denied means the process exists and belongs to somebody else, as `EPERM`
310 /// does on unix, so it reads as alive. A query that fails after the process was opened also
311 /// reads as alive: a refused resume only waits, while a resume under a live writer
312 /// corrupts its journal.
313 pub fn pid_is_alive(pid: u32) -> bool {
314 use windows_sys::Win32::Foundation::{
315 CloseHandle, ERROR_ACCESS_DENIED, GetLastError, STILL_ACTIVE,
316 };
317 use windows_sys::Win32::System::Threading::{
318 GetExitCodeProcess, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
319 };
320
321 // Zero is the idle process, which no scan runs as.
322 if pid == 0 {
323 return false;
324 }
325
326 // SAFETY: `OpenProcess` takes no pointers and returns a null handle
327 // on failure, which is checked before any use.
328 let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) };
329 if handle.is_null() {
330 // SAFETY: reads the calling thread's last-error value.
331 return unsafe { GetLastError() } == ERROR_ACCESS_DENIED;
332 }
333
334 let mut code = 0u32;
335 // SAFETY: `handle` was opened above with the right to query, and `code`
336 // is a live `u32` the call writes the exit code into.
337 let answered = unsafe { GetExitCodeProcess(handle, &mut code) } != 0;
338 // SAFETY: `handle` was opened above and is not used again.
339 unsafe { CloseHandle(handle) };
340
341 !answered || code == STILL_ACTIVE as u32
342 }
343}
344
345#[cfg(feature = "journal-format")]
346mod persistence {
347 use std::fs;
348 use std::io::Write;
349 use std::path::{Path, PathBuf};
350 use std::time::SystemTime;
351
352 use super::{HEARTBEAT_STALE_AFTER, LockRecord, LockState, boot_identity, classify};
353 use crate::journal::file::{link_new, open_or_create_private, open_to_read, remove, replace};
354 use crate::journal::format::JournalError;
355
356 /// Reads a lock file and says what it means.
357 ///
358 /// A missing file is [`LockState::Free`], and so is a file that cannot be parsed or a
359 /// link at the name, which is not followed: nothing that takes a lock leaves one there. A
360 /// truncated or corrupt lock was left by a writer that crashed mid-write, and refusing it
361 /// would make the crash unrecoverable. The next holder's heartbeat protects the journal.
362 pub fn inspect(path: &Path) -> LockState {
363 let text = open_to_read(path).and_then(|mut file| {
364 let mut text = String::new();
365 std::io::Read::read_to_string(&mut file, &mut text).map(|_| text)
366 });
367 let Ok(text) = text else {
368 return LockState::Free;
369 };
370
371 let Ok(record) = serde_json::from_str::<LockRecord>(&text) else {
372 return LockState::Free;
373 };
374
375 classify(
376 &record,
377 &boot_identity(),
378 super::pid_is_alive(record.pid),
379 SystemTime::now(),
380 HEARTBEAT_STALE_AFTER,
381 )
382 }
383
384 /// A held lock, which releases itself when dropped.
385 #[derive(Debug)]
386 pub struct Lock {
387 path: PathBuf,
388 record: LockRecord,
389 /// Set by [`Lock::release`] so the `Drop` path does not try again. A flag, since
390 /// `mem::forget` would leak the path and the record.
391 released: bool,
392 }
393
394 impl Lock {
395 /// Takes the lock, or explains why it could not be taken.
396 ///
397 /// Creating the file is the exclusion: it fails if anything is there, so two
398 /// processes racing for one journal cannot both succeed. The existing lock is inspected
399 /// only after that fails, to decide whether it may be broken.
400 pub fn acquire(path: &Path) -> Result<Self, LockRefused> {
401 Self::acquire_inner(path, Breaks::Dead)
402 }
403
404 /// [`acquire`](Self::acquire), breaking a [`Stale`](LockState::Stale) lock as well
405 /// as a dead one.
406 ///
407 /// For a caller who has judged that the process holding the number in a lock that stopped
408 /// beating is not the scan, usually because the number was reissued after a crash. A lock
409 /// whose writer is checkpointing now is still refused.
410 pub fn take_over(path: &Path) -> Result<Self, LockRefused> {
411 Self::acquire_inner(path, Breaks::Stale)
412 }
413
414 /// [`acquire`](Self::acquire), overriding any refusal.
415 ///
416 /// For a lock left by a defect nothing here anticipated, which would otherwise make the
417 /// journal unusable forever.
418 pub fn force(path: &Path) -> Result<Self, LockRefused> {
419 Self::acquire_inner(path, Breaks::Anything)
420 }
421
422 /// How many times a break is retried before giving up.
423 ///
424 /// A retry happens only when another process won the create between this one removing a
425 /// dead lock and replacing it; the winner's lock is live, so the next round refuses. More
426 /// rounds than that means two processes are breaking each other's locks in a loop.
427 const BREAK_ATTEMPTS: usize = 3;
428
429 fn acquire_inner(path: &Path, breaks: Breaks) -> Result<Self, LockRefused> {
430 // Built only after the create succeeds: `Lock` removes its file on drop, so one
431 // dropped after losing the create would delete the winner's lock.
432 let held = |record| Self {
433 path: path.to_path_buf(),
434 record,
435 released: false,
436 };
437 let mut refusal = None;
438
439 for _ in 0..Self::BREAK_ATTEMPTS {
440 let record = LockRecord::current();
441 match Self::create_exclusively(path, &record) {
442 Ok(()) => return Ok(held(record)),
443 Err(error) if error.kind() != std::io::ErrorKind::AlreadyExists => {
444 return Err(LockRefused::Io(error.to_string()));
445 }
446 Err(_) => {}
447 }
448
449 // Deciding a lock is dead and replacing it must be one operation. `create_new`
450 // only excludes racers for a free journal. Replacing a dead lock by rename
451 // bypasses the create, and removing it first lets every racer remove whatever is
452 // at the name, including the lock the last winner just created: eight processes
453 // on one crashed journal yielded two to four holders.
454 //
455 // So the inspect, removal and create run under an advisory lock on a sibling
456 // file, held only for those three steps. The kernel drops it when its holder
457 // exits, so it cannot go stale and `force` never needs to clear it.
458 let _breaking = Breaking::take(path).map_err(|e| LockRefused::Io(e.to_string()))?;
459
460 // Asked again under the guard: the previous guard holder may have taken the
461 // journal in the meantime.
462 let state = inspect(path);
463 if !breaks.permits(&state) {
464 return Err(LockRefused::Held(state));
465 }
466
467 match remove(path) {
468 Ok(()) => {}
469 // Removed between the two inspections; the create below decides either way.
470 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
471 Err(error) => return Err(LockRefused::Io(error.to_string())),
472 }
473
474 let record = LockRecord::current();
475 match Self::create_exclusively(path, &record) {
476 Ok(()) => return Ok(held(record)),
477 Err(error) if error.kind() != std::io::ErrorKind::AlreadyExists => {
478 return Err(LockRefused::Io(error.to_string()));
479 }
480 // A process arriving after this one removed the lock sees nothing to break,
481 // takes no guard, and can win the create in that gap. The next round then
482 // finds a beating lock and refuses by name.
483 Err(_) => refusal = Some(state),
484 }
485 }
486
487 Err(match refusal {
488 Some(state) => LockRefused::Held(state),
489 None => LockRefused::Io("the lock could not be taken".to_string()),
490 })
491 }
492
493 /// Moves the heartbeat forward. Called once per checkpoint.
494 pub fn beat(&mut self) -> Result<(), JournalError> {
495 self.record = self.record.beating_at(SystemTime::now());
496 Self::write(&self.path, &self.record)?;
497 Ok(())
498 }
499
500 /// What this lock claims, as written.
501 pub fn record(&self) -> &LockRecord {
502 &self.record
503 }
504
505 /// Releases the lock, reporting a failure the `Drop` path would swallow.
506 pub fn release(mut self) -> Result<(), JournalError> {
507 self.released = true;
508 remove(&self.path)?;
509 Ok(())
510 }
511
512 /// Takes the lock, or fails with
513 /// [`AlreadyExists`](std::io::ErrorKind::AlreadyExists) if somebody has it.
514 ///
515 /// The name appears already holding the record. Creating then writing would let a racer
516 /// read an empty lock between the two steps, which [`inspect`] reports as `Free` (that is
517 /// what a writer killed mid-write leaves), and delete a lock taken a microsecond earlier.
518 /// So the record is written to a staged file and linked into place: `link` refuses an
519 /// existing name, as `create_new` does, over a file that already has its contents.
520 ///
521 /// Created like every other journal file, so under `sudo` it belongs to whoever invoked the
522 /// scan and they can release it.
523 ///
524 /// The staged name carries a counter as well as the pid, because this is a library and two
525 /// threads of one caller may take the same journal. With the pid alone they would share the
526 /// staged name, and since [`link_new`] removes a staged name it finds occupied, one thread
527 /// could delete the other's file (failing its link with `NotFound`) or link an empty file
528 /// that [`inspect`] reads as `Free`. Either way the lock would get two holders.
529 fn create_exclusively(path: &Path, record: &LockRecord) -> std::io::Result<()> {
530 static STAGING: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
531 let staged = path.with_extension(format!(
532 "lock-{}-{}",
533 std::process::id(),
534 STAGING.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
535 ));
536
537 let text = serde_json::to_string(record).map_err(std::io::Error::other)?;
538 // Staged with a retry, since a reused pid can find a staged file left by a run
539 // that died between the create and the link.
540 link_new(path, &staged, |mut file| file.write_all(text.as_bytes()))
541 }
542
543 /// Replaces the lock file in place, for the holder moving its own heartbeat forward.
544 /// Taking a lock goes through [`create_exclusively`](Self::create_exclusively).
545 fn write(path: &Path, record: &LockRecord) -> std::io::Result<()> {
546 let text = serde_json::to_string(record).map_err(std::io::Error::other)?;
547 // Private from creation like the original, so a heartbeat never widens the mode.
548 // The lock becomes the staged file's inode, ownership included.
549 replace(path, &path.with_extension("lock-tmp"), |mut file| {
550 file.write_all(text.as_bytes())
551 })
552 }
553 }
554
555 /// Which locks an attempt to take one may break.
556 #[derive(Debug, Clone, Copy)]
557 enum Breaks {
558 /// Only one nothing holds: free, crashed, or from before a reboot.
559 Dead,
560 /// Those, and one whose holder stopped checkpointing.
561 Stale,
562 /// Whatever is there.
563 Anything,
564 }
565
566 impl Breaks {
567 fn permits(self, state: &LockState) -> bool {
568 match self {
569 Breaks::Dead => state.is_resumable(),
570 Breaks::Stale => state.is_resumable() || matches!(state, LockState::Stale { .. }),
571 Breaks::Anything => true,
572 }
573 }
574 }
575
576 /// Serialises deciding that a lock is dead and replacing it.
577 ///
578 /// See the comment at the call site. Dropped as soon as the three steps are done.
579 struct Breaking {
580 /// Held only for the handle: the lock lives on the open file and is released when
581 /// it closes.
582 _file: fs::File,
583 }
584
585 impl Breaking {
586 fn take(lock: &Path) -> std::io::Result<Self> {
587 // A sibling file, because the lock file is about to be removed and an advisory
588 // lock follows the open file, not the name. Opened, not created, so every racer reaches
589 // the same file.
590 let file = open_or_create_private(&lock.with_extension("break"))?;
591
592 // Blocks for the exclusive lock: `flock` on unix, `LockFileEx` on Windows.
593 file.lock()?;
594 Ok(Self { _file: file })
595 }
596 }
597
598 impl Drop for Lock {
599 fn drop(&mut self) {
600 if self.released {
601 return;
602 }
603
604 // Best effort. A lock left by a failed removal reads as `Crashed` to the next
605 // reader, which is resumable.
606 let _ = remove(&self.path);
607 }
608 }
609
610 /// Why a lock could not be taken.
611 #[non_exhaustive]
612 #[derive(Debug, Clone, PartialEq, Eq)]
613 pub enum LockRefused {
614 /// Somebody else has it. Carries the state so a caller can render
615 /// [`LockState::refusal`].
616 Held(LockState),
617 /// The lock file could not be written.
618 Io(String),
619 }
620
621 impl std::fmt::Display for LockRefused {
622 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
623 match self {
624 LockRefused::Held(state) => match state.refusal() {
625 Some(reason) => write!(f, "{reason}"),
626 None => write!(f, "the journal is locked"),
627 },
628 LockRefused::Io(error) => write!(f, "could not write the lock: {error}"),
629 }
630 }
631 }
632
633 impl std::error::Error for LockRefused {}
634}
635
636#[cfg(feature = "journal-format")]
637pub use persistence::{Lock, LockRefused, inspect};
638
639// ╔════════════════════════════════════════════╗
640// ║ ████████╗███████╗███████╗████████╗███████╗ ║
641// ║ ╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝ ║
642// ║ ██║ █████╗ ███████╗ ██║ ███████╗ ║
643// ║ ██║ ██╔══╝ ╚════██║ ██║ ╚════██║ ║
644// ║ ██║ ███████╗███████║ ██║ ███████║ ║
645// ║ ╚═╝ ╚══════╝╚══════╝ ╚═╝ ╚══════╝ ║
646// ╚════════════════════════════════════════════╝
647
648#[cfg(test)]
649mod tests {
650 use super::*;
651
652 const BOOT: &str = "boot-a";
653 const OTHER_BOOT: &str = "boot-b";
654
655 fn at(seconds: u64) -> SystemTime {
656 SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
657 }
658
659 fn record(boot: &str, beat: u64) -> LockRecord {
660 LockRecord {
661 pid: 4_242,
662 boot: boot.to_string(),
663 started_at: at(0),
664 heartbeat: at(beat),
665 }
666 }
667
668 /// A live writer checkpointing normally is always refused.
669 #[test]
670 fn a_beating_lock_on_a_live_pid_is_held() {
671 let state = classify(
672 &record(BOOT, 100),
673 BOOT,
674 true,
675 at(105),
676 HEARTBEAT_STALE_AFTER,
677 );
678
679 assert!(matches!(state, LockState::Held { pid: 4_242, .. }));
680 assert!(!state.is_resumable());
681 assert!(state.refusal().is_some_and(|r| r.contains("4242")));
682 }
683
684 /// The ordinary crash: the process is gone and the journal is free.
685 #[test]
686 fn a_lock_whose_pid_is_gone_is_a_crash() {
687 let state = classify(
688 &record(BOOT, 100),
689 BOOT,
690 false,
691 at(105),
692 HEARTBEAT_STALE_AFTER,
693 );
694
695 assert_eq!(state, LockState::Crashed { pid: 4_242 });
696 assert!(state.is_resumable());
697 assert!(!state.may_have_lost_the_tail());
698 assert_eq!(state.refusal(), None);
699 }
700
701 /// The case the pid alone gets wrong. After a reboot low pids go to init and its
702 /// children, so the pid in an old lock is often alive and unrelated.
703 #[test]
704 fn a_lock_from_a_previous_boot_is_released_even_though_its_pid_is_alive() {
705 let state = classify(
706 &record(OTHER_BOOT, 100),
707 BOOT,
708 true, // alive, but not the same process
709 at(105),
710 HEARTBEAT_STALE_AFTER,
711 );
712
713 assert_eq!(state, LockState::RebootedUnder { pid: 4_242 });
714 assert!(state.is_resumable());
715 assert!(
716 state.may_have_lost_the_tail(),
717 "a reboot loses what the page cache had not flushed"
718 );
719 }
720
721 /// The case the boot identity alone gets wrong. Within one boot a freed pid can be
722 /// reissued, and only the heartbeat tells the two apart, so a stale beat is refused.
723 #[test]
724 fn a_live_pid_that_stopped_beating_is_stale_and_refused() {
725 let state = classify(
726 &record(BOOT, 100),
727 BOOT,
728 true,
729 at(100 + HEARTBEAT_STALE_AFTER.as_secs() + 1),
730 HEARTBEAT_STALE_AFTER,
731 );
732
733 assert!(matches!(state, LockState::Stale { pid: 4_242, .. }));
734 assert!(!state.is_resumable(), "hung and reused look the same here");
735 assert!(
736 state
737 .refusal()
738 .is_some_and(|r| r.contains("take the journal over"))
739 );
740 }
741
742 /// A beat exactly at the threshold is not yet stale. The boundary decides between
743 /// refusing a live scan and permitting a second writer.
744 #[test]
745 fn the_staleness_boundary_is_exclusive() {
746 let held = classify(
747 &record(BOOT, 100),
748 BOOT,
749 true,
750 at(100) + HEARTBEAT_STALE_AFTER,
751 HEARTBEAT_STALE_AFTER,
752 );
753 assert!(matches!(held, LockState::Held { .. }), "{held:?}");
754
755 let stale = classify(
756 &record(BOOT, 100),
757 BOOT,
758 true,
759 at(100) + HEARTBEAT_STALE_AFTER + Duration::from_nanos(1),
760 HEARTBEAT_STALE_AFTER,
761 );
762 assert!(matches!(stale, LockState::Stale { .. }), "{stale:?}");
763 }
764
765 /// A clock that moved backwards does not turn a live scan into a stale one.
766 #[test]
767 fn a_heartbeat_from_the_future_reads_as_fresh() {
768 let state = classify(
769 &record(BOOT, 500),
770 BOOT,
771 true,
772 at(100),
773 HEARTBEAT_STALE_AFTER,
774 );
775
776 assert!(matches!(state, LockState::Held { .. }), "{state:?}");
777 }
778
779 /// An unreadable boot identity compares unequal to any recorded one, so it releases
780 /// locks.
781 #[test]
782 fn an_unknown_boot_identity_releases_rather_than_holds() {
783 let state = classify(&record(BOOT, 100), "", true, at(105), HEARTBEAT_STALE_AFTER);
784
785 assert!(state.is_resumable(), "{state:?}");
786 }
787
788 /// This host reports its boot identity, and the same one twice.
789 #[test]
790 fn the_boot_identity_is_readable_and_stable() {
791 let first = boot_identity();
792 assert!(!first.is_empty(), "no boot identity on this platform");
793 assert_eq!(first, boot_identity(), "it must not change while running");
794 }
795
796 /// The liveness check is right about the one process a test can be sure of:
797 /// itself.
798 #[test]
799 fn this_process_is_alive() {
800 assert!(pid_is_alive(std::process::id()));
801 assert!(!pid_is_alive(0), "pid 0 is not a process this can signal");
802 }
803}
804
805#[cfg(all(test, feature = "journal-format"))]
806mod file_tests {
807 use super::*;
808 use std::path::PathBuf;
809
810 fn scratch(name: &str) -> PathBuf {
811 let dir = std::env::temp_dir().join(format!("zond-lock-{}-{name}", std::process::id()));
812 std::fs::create_dir_all(&dir).expect("scratch directory");
813 dir
814 }
815
816 /// Taking, holding and releasing, and what each looks like from outside.
817 #[test]
818 fn a_lock_is_visible_while_held_and_gone_once_released() {
819 let dir = scratch("lifecycle");
820 let path = dir.join("LOCK");
821
822 assert_eq!(inspect(&path), LockState::Free, "nothing has it yet");
823
824 let lock = Lock::acquire(&path).expect("takes a free lock");
825 assert_eq!(lock.record().pid, std::process::id());
826 assert!(
827 matches!(inspect(&path), LockState::Held { .. }),
828 "this process is alive and has just beaten"
829 );
830
831 lock.release().expect("releases");
832 assert_eq!(inspect(&path), LockState::Free);
833
834 std::fs::remove_dir_all(&dir).ok();
835 }
836
837 /// Two scans racing for one journal: the second is refused and told which process
838 /// has it.
839 #[test]
840 fn a_second_acquisition_is_refused_while_the_first_holds_it() {
841 let dir = scratch("contended");
842 let path = dir.join("LOCK");
843
844 let first = Lock::acquire(&path).expect("takes it");
845
846 match Lock::acquire(&path) {
847 Err(LockRefused::Held(state)) => {
848 assert!(!state.is_resumable());
849 assert!(
850 state
851 .refusal()
852 .is_some_and(|r| r.contains(&std::process::id().to_string())),
853 "a refusal has to name who holds it"
854 );
855 }
856 other => panic!("expected a refusal, got {other:?}"),
857 }
858
859 // Forcing gets past it.
860 let forced = Lock::force(&path).expect("force overrides");
861 drop(forced);
862 drop(first);
863
864 std::fs::remove_dir_all(&dir).ok();
865 }
866
867 /// A lock whose holder stopped beating can be taken over, and one whose holder is
868 /// beating cannot. This process stands in for a reissued pid.
869 #[test]
870 fn a_stale_lock_can_be_taken_over_and_a_beating_one_cannot() {
871 let dir = scratch("stale");
872 let path = dir.join("LOCK");
873
874 let stale = LockRecord {
875 pid: std::process::id(),
876 boot: boot_identity(),
877 started_at: SystemTime::now() - Duration::from_secs(3_600),
878 heartbeat: SystemTime::now() - Duration::from_secs(600),
879 };
880 std::fs::write(&path, serde_json::to_string(&stale).expect("json")).expect("writes");
881 assert!(matches!(inspect(&path), LockState::Stale { .. }));
882 assert!(
883 Lock::acquire(&path).is_err(),
884 "a stale lock is refused unasked"
885 );
886
887 let taken = Lock::take_over(&path).expect("a stale lock is taken over");
888 assert!(
889 matches!(
890 Lock::take_over(&path),
891 Err(LockRefused::Held(LockState::Held { .. }))
892 ),
893 "a lock beating now is not taken over"
894 );
895 drop(taken);
896
897 std::fs::remove_dir_all(&dir).ok();
898 }
899
900 /// A lock left half-written by a crash reads as free, so it does not lock the
901 /// journal forever.
902 #[test]
903 fn a_corrupt_lock_does_not_wedge_the_journal() {
904 let dir = scratch("corrupt");
905 let path = dir.join("LOCK");
906
907 std::fs::write(&path, "{\"pid\":42,\"bo").expect("writes a torn lock");
908 assert_eq!(inspect(&path), LockState::Free);
909
910 Lock::acquire(&path).expect("a torn lock can be taken over");
911
912 std::fs::remove_dir_all(&dir).ok();
913 }
914
915 /// A crashed writer's lock is taken without a force, and the new holder's heartbeat
916 /// replaces it.
917 #[test]
918 fn a_dead_writers_lock_is_taken_over() {
919 let dir = scratch("crashed");
920 let path = dir.join("LOCK");
921
922 // pid 0 is never a signalable process, so it stands in for a writer that is gone.
923 let dead = LockRecord {
924 pid: 0,
925 boot: boot_identity(),
926 started_at: SystemTime::now(),
927 heartbeat: SystemTime::now(),
928 };
929 std::fs::write(&path, serde_json::to_string(&dead).expect("json")).expect("writes");
930
931 assert_eq!(inspect(&path), LockState::Crashed { pid: 0 });
932
933 let lock = Lock::acquire(&path).expect("a crashed lock needs no force");
934 assert_eq!(lock.record().pid, std::process::id());
935
936 std::fs::remove_dir_all(&dir).ok();
937 }
938
939 /// Several racers finding the same crashed journal: exactly one takes it.
940 ///
941 /// The racers are threads in one process, so this also covers the staged-name counter in
942 /// `create_exclusively`. Without the guard or the counter the count came out at two about
943 /// one run in twenty, so the racers start on a barrier and the race runs in rounds.
944 #[test]
945 fn only_one_of_several_racers_breaks_a_crashed_lock() {
946 for round in 0..16 {
947 let dir = scratch(&format!("break-race-{round}"));
948 let path = dir.join("LOCK");
949
950 // A lock from before a reboot, resumable whatever its pid is doing. A test
951 // cannot name a pid it is certain nothing holds.
952 let stale = LockRecord {
953 pid: std::process::id(),
954 boot: "a boot that is over".to_string(),
955 started_at: SystemTime::UNIX_EPOCH,
956 heartbeat: SystemTime::UNIX_EPOCH,
957 };
958 std::fs::write(&path, serde_json::to_string(&stale).expect("json")).expect("writes");
959 assert!(matches!(inspect(&path), LockState::RebootedUnder { .. }));
960
961 let taken = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
962 // Racers start together; without the barrier the first thread usually finished
963 // before the last began.
964 let start = std::sync::Arc::new(std::sync::Barrier::new(8));
965 std::thread::scope(|scope| {
966 for _ in 0..8 {
967 let path = path.clone();
968 let taken = std::sync::Arc::clone(&taken);
969 let start = std::sync::Arc::clone(&start);
970 scope.spawn(move || {
971 start.wait();
972 if let Ok(lock) = Lock::acquire(&path) {
973 taken.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
974 // Held for the rest of the scope, so a later racer is refused.
975 std::thread::sleep(std::time::Duration::from_millis(20));
976 std::mem::forget(lock);
977 }
978 });
979 }
980 });
981
982 assert_eq!(
983 taken.load(std::sync::atomic::Ordering::Relaxed),
984 1,
985 "round {round}: this many racers believed they held the journal"
986 );
987
988 std::fs::remove_dir_all(&dir).ok();
989 }
990 }
991
992 /// The heartbeat moves, and leaves no temporary file behind.
993 #[test]
994 fn beating_moves_the_heartbeat_forward() {
995 let dir = scratch("heartbeat");
996 let path = dir.join("LOCK");
997
998 let mut lock = Lock::acquire(&path).expect("takes it");
999 let before = lock.record().heartbeat;
1000
1001 std::thread::sleep(Duration::from_millis(5));
1002 lock.beat().expect("beats");
1003
1004 assert!(lock.record().heartbeat > before);
1005 assert!(
1006 !dir.join("LOCK.lock-tmp").exists(),
1007 "the temporary must not survive the rename"
1008 );
1009
1010 std::fs::remove_dir_all(&dir).ok();
1011 }
1012}