Skip to main content

zond_engine/journal/
lock.rs

1// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
2//
3// This file is part of Zond Engine, licensed under the GNU Affero General
4// Public License, version 3 or later. See the LICENSE file for details, or
5// <https://www.gnu.org/licenses/agpl-3.0.html>.
6//
7// SPDX-License-Identifier: AGPL-3.0-or-later
8
9//! # Telling a running scan from a crashed one
10//!
11//! A journal that is being written to must not be resumed, and a journal whose writer
12//! died must not stay locked forever. Resuming a live scan corrupts both sittings' cursors;
13//! a lock nobody can clear makes a crashed scan impossible to continue.
14//!
15//! ## Three facts
16//!
17//! The lock file records a process id, a boot identity and a heartbeat, because each alone
18//! is wrong in some case:
19//!
20//! - **The pid alone is wrong after a reboot.** Process ids restart from a low number, so
21//!   the pid in a lock written before a crash-and-reboot often belongs to something live
22//!   and unrelated, such as `launchd` or a shell.
23//! - **The boot identity alone is wrong within one boot.** A pid freed by a crash can be
24//!   reissued to an unrelated process minutes later.
25//! - **The heartbeat alone is wrong about a slow scan.** A scan may legitimately do
26//!   nothing for a while, under a long silence tolerance or against a tarpit.
27//!
28//! The boot identity rules out everything before the last boot; within one boot the pid
29//! says whether something holds that number, and the heartbeat says whether it is still
30//! this scan.
31//!
32//! ## The policy is a pure function
33//!
34//! [`classify`] takes the record, the current boot identity, whether the pid is alive and
35//! the time, and returns a [`LockState`], so a reboot, a reused pid and a hung writer can be
36//! tested in CI. The syscalls live in [`inspect`], a thin wrapper over it.
37//!
38//! ## What is refused
39//!
40//! A scan that is or might be running is refused; one that certainly is not may be
41//! resumed. A live pid that has stopped touching the lock is refused, since it may be a hung
42//! writer that will wake up, and two writers on one journal is what this module prevents.
43//!
44//! Every refusal is overridable, so a stale lock left by an unforeseen defect cannot make a
45//! journal unusable forever. [`Lock::take_over`] overrides only that ambiguous case, where
46//! the user can judge whether the process holding the number is the scan.
47//! [`Lock::force`] also overrides a writer that is checkpointing now.
48
49use std::time::{Duration, SystemTime};
50
51use serde::{Deserialize, Serialize};
52
53/// How long a heartbeat may go untouched before the writer holding a lock is
54/// treated as no longer obviously alive.
55///
56/// A scan checkpoints every few seconds, so a minute of silence is many missed beats.
57/// Erring long costs an overridable refusal; erring short costs two writers on one
58/// journal.
59pub const HEARTBEAT_STALE_AFTER: Duration = Duration::from_secs(60);
60
61/// What a lock file says about the process that wrote it.
62#[non_exhaustive]
63#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
64pub struct LockRecord {
65    /// The process holding the journal.
66    pub pid: u32,
67    /// Which boot that pid belongs to. A pid is only meaningful within one.
68    pub boot: String,
69    /// When the scan started, so a listing can say how old each journal is.
70    pub started_at: SystemTime,
71    /// Last touched by the writer, once per checkpoint.
72    pub heartbeat: SystemTime,
73}
74
75impl LockRecord {
76    /// The record this process would write now.
77    pub fn current() -> Self {
78        let now = SystemTime::now();
79        Self {
80            pid: std::process::id(),
81            boot: boot_identity(),
82            started_at: now,
83            heartbeat: now,
84        }
85    }
86
87    /// The same record with its heartbeat moved to `now`.
88    pub fn beating_at(&self, now: SystemTime) -> Self {
89        Self {
90            heartbeat: now,
91            ..self.clone()
92        }
93    }
94}
95
96/// What is known about a journal's writer.
97#[non_exhaustive]
98#[derive(Debug, Clone, PartialEq, Eq)]
99pub enum LockState {
100    /// No lock file. The journal is free.
101    Free,
102
103    /// A live process is writing this journal now. Refuse.
104    Held {
105        /// The process holding it, so a refusal can name it.
106        pid: u32,
107        /// How long ago it last checkpointed.
108        last_beat: Duration,
109    },
110
111    /// The pid is gone and the writer died without releasing. Resume, noting that the
112    /// last checkpoint interval may be missing.
113    Crashed {
114        /// The process that held it, for the note.
115        pid: u32,
116    },
117
118    /// The machine rebooted while this journal was locked, so the pid in the file is not
119    /// the writer. Resume, and warn that a reboot loses more than a crash: anything the page
120    /// cache had not flushed is gone.
121    RebootedUnder {
122        /// The process that held it, before the reboot.
123        pid: u32,
124    },
125
126    /// Something holds that pid and it has stopped touching the lock: either the writer
127    /// is hung or it died and the number was reissued. Refused, overridably.
128    Stale {
129        /// The process that number now belongs to, whoever that is.
130        pid: u32,
131        /// How long the heartbeat has been untouched.
132        last_beat: Duration,
133    },
134}
135
136impl LockState {
137    /// Whether the journal may be resumed without the user overriding anything.
138    pub fn is_resumable(&self) -> bool {
139        matches!(
140            self,
141            LockState::Free | LockState::Crashed { .. } | LockState::RebootedUnder { .. }
142        )
143    }
144
145    /// Whether resuming loses more than one checkpoint interval.
146    ///
147    /// True only after a reboot: every other stop is a process death, and the page cache
148    /// outlives a process. See [`journal`](crate::journal) for the survival table.
149    pub fn may_have_lost_the_tail(&self) -> bool {
150        matches!(self, LockState::RebootedUnder { .. })
151    }
152
153    /// Why a resume was refused, phrased for a user deciding what to do about it. `None`
154    /// where nothing was refused.
155    pub fn refusal(&self) -> Option<String> {
156        match self {
157            LockState::Free | LockState::Crashed { .. } | LockState::RebootedUnder { .. } => None,
158            LockState::Held { pid, last_beat } => Some(format!(
159                "process {pid} is scanning this journal now (last checkpoint {}s ago); \
160                 wait for it to finish, or stop it first",
161                last_beat.as_secs()
162            )),
163            LockState::Stale { pid, last_beat } => Some(format!(
164                "this journal is locked by process {pid}, which has not checkpointed for {}s — \
165                 it is either hung or the number was reissued to something else. \
166                 Stop it, or take the journal over if you are sure it is not scanning",
167                last_beat.as_secs()
168            )),
169        }
170    }
171}
172
173/// Decides what a lock record means, given everything observed about it. Pure.
174///
175/// The boot identity is checked first: a pid from a previous boot says nothing, and asking
176/// whether it is alive first would leave every journal locked behind an unrelated process
177/// after a reboot.
178pub fn classify(
179    record: &LockRecord,
180    boot: &str,
181    pid_is_alive: bool,
182    now: SystemTime,
183    stale_after: Duration,
184) -> LockState {
185    if record.boot != boot {
186        return LockState::RebootedUnder { pid: record.pid };
187    }
188
189    if !pid_is_alive {
190        return LockState::Crashed { pid: record.pid };
191    }
192
193    // A clock that went backwards yields no elapsed time, read as a fresh beat: the safe
194    // reading of an unknowable age is that the writer is alive.
195    let last_beat = now.duration_since(record.heartbeat).unwrap_or_default();
196
197    if last_beat > stale_after {
198        LockState::Stale {
199            pid: record.pid,
200            last_beat,
201        }
202    } else {
203        LockState::Held {
204            pid: record.pid,
205            last_beat,
206        }
207    }
208}
209
210/// A value that changes every boot, so a pid from before one can be disregarded.
211///
212/// Linux has a boot id; macOS and the BSDs use the boot time. Where neither can be read the
213/// identity is empty, which differs from any recorded one and so reads as a reboot,
214/// releasing the lock.
215pub fn boot_identity() -> String {
216    imp::boot_identity()
217}
218
219/// Whether any process currently holds `pid`.
220///
221/// Says nothing about which process. Within one boot the heartbeat distinguishes this
222/// scan from a reused number.
223pub fn pid_is_alive(pid: u32) -> bool {
224    imp::pid_is_alive(pid)
225}
226
227#[cfg(unix)]
228mod imp {
229    pub fn boot_identity() -> String {
230        // Linux: a UUID minted per boot.
231        if let Ok(id) = std::fs::read_to_string("/proc/sys/kernel/random/boot_id") {
232            return id.trim().to_string();
233        }
234
235        // macOS and the BSDs: the boot instant from `KERN_BOOTTIME`.
236        #[cfg(any(target_os = "macos", target_os = "ios", target_vendor = "apple"))]
237        {
238            let mut boot = libc::timeval {
239                tv_sec: 0,
240                tv_usec: 0,
241            };
242            let mut size = std::mem::size_of::<libc::timeval>();
243            let mut mib = [libc::CTL_KERN, libc::KERN_BOOTTIME];
244
245            // SAFETY: `mib` is a live array of the length passed, `boot` is a
246            // live `timeval` and `size` names its exact size, which is what
247            // `KERN_BOOTTIME` writes. The call reads and writes only those.
248            let code = unsafe {
249                libc::sysctl(
250                    mib.as_mut_ptr(),
251                    mib.len() as libc::c_uint,
252                    (&raw mut boot).cast::<libc::c_void>(),
253                    &mut size,
254                    std::ptr::null_mut(),
255                    0,
256                )
257            };
258
259            if code == 0 {
260                return format!("{}.{}", boot.tv_sec, boot.tv_usec);
261            }
262        }
263
264        String::new()
265    }
266
267    pub fn pid_is_alive(pid: u32) -> bool {
268        // `pid_t` is signed and `kill` reads a negative argument as a process group:
269        // `u32::MAX` casts to `-1`, which asks about every process the caller may signal and is
270        // always answered yes. A lock naming it would then be refused forever.
271        let Ok(pid) = libc::pid_t::try_from(pid) else {
272            return false;
273        };
274        if pid <= 0 {
275            return false;
276        }
277
278        // SAFETY: `kill` with signal 0 sends nothing. It performs the existence
279        // and permission checks and returns, dereferencing nothing.
280        let code = unsafe { libc::kill(pid, 0) };
281        if code == 0 {
282            return true;
283        }
284
285        // `EPERM` means the process exists and belongs to somebody else, the ordinary case
286        // for a scan started under `sudo` and inspected without it.
287        std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
288    }
289}
290
291#[cfg(windows)]
292mod imp {
293    /// Empty on Windows, so the process id alone decides.
294    ///
295    /// Wall clock minus `GetTickCount64` is not constant within a boot: the two are read at
296    /// different instants, the tick counter has about 15 ms resolution, and it stops across
297    /// some suspend states. A value that moved within a boot would read every lock as
298    /// `RebootedUnder`, which is resumable.
299    ///
300    /// Two empty identities compare equal, so [`pid_is_alive`] decides every lock. The cost is
301    /// that a lock left by a crash before a reboot, whose number another process has taken
302    /// since, reads as held and is refused.
303    pub fn boot_identity() -> String {
304        String::new()
305    }
306
307    /// Whether a process with this id is running, as the unix arm asks it.
308    ///
309    /// Access denied means the process exists and belongs to somebody else, as `EPERM`
310    /// does on unix, so it reads as alive. A query that fails after the process was opened also
311    /// reads as alive: a refused resume only waits, while a resume under a live writer
312    /// corrupts its journal.
313    pub fn pid_is_alive(pid: u32) -> bool {
314        use windows_sys::Win32::Foundation::{
315            CloseHandle, ERROR_ACCESS_DENIED, GetLastError, STILL_ACTIVE,
316        };
317        use windows_sys::Win32::System::Threading::{
318            GetExitCodeProcess, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
319        };
320
321        // Zero is the idle process, which no scan runs as.
322        if pid == 0 {
323            return false;
324        }
325
326        // SAFETY: `OpenProcess` takes no pointers and returns a null handle
327        // on failure, which is checked before any use.
328        let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) };
329        if handle.is_null() {
330            // SAFETY: reads the calling thread's last-error value.
331            return unsafe { GetLastError() } == ERROR_ACCESS_DENIED;
332        }
333
334        let mut code = 0u32;
335        // SAFETY: `handle` was opened above with the right to query, and `code`
336        // is a live `u32` the call writes the exit code into.
337        let answered = unsafe { GetExitCodeProcess(handle, &mut code) } != 0;
338        // SAFETY: `handle` was opened above and is not used again.
339        unsafe { CloseHandle(handle) };
340
341        !answered || code == STILL_ACTIVE as u32
342    }
343}
344
345#[cfg(feature = "journal-format")]
346mod persistence {
347    use std::fs;
348    use std::io::Write;
349    use std::path::{Path, PathBuf};
350    use std::time::SystemTime;
351
352    use super::{HEARTBEAT_STALE_AFTER, LockRecord, LockState, boot_identity, classify};
353    use crate::journal::file::{link_new, open_or_create_private, open_to_read, remove, replace};
354    use crate::journal::format::JournalError;
355
356    /// Reads a lock file and says what it means.
357    ///
358    /// A missing file is [`LockState::Free`], and so is a file that cannot be parsed or a
359    /// link at the name, which is not followed: nothing that takes a lock leaves one there. A
360    /// truncated or corrupt lock was left by a writer that crashed mid-write, and refusing it
361    /// would make the crash unrecoverable. The next holder's heartbeat protects the journal.
362    pub fn inspect(path: &Path) -> LockState {
363        let text = open_to_read(path).and_then(|mut file| {
364            let mut text = String::new();
365            std::io::Read::read_to_string(&mut file, &mut text).map(|_| text)
366        });
367        let Ok(text) = text else {
368            return LockState::Free;
369        };
370
371        let Ok(record) = serde_json::from_str::<LockRecord>(&text) else {
372            return LockState::Free;
373        };
374
375        classify(
376            &record,
377            &boot_identity(),
378            super::pid_is_alive(record.pid),
379            SystemTime::now(),
380            HEARTBEAT_STALE_AFTER,
381        )
382    }
383
384    /// A held lock, which releases itself when dropped.
385    #[derive(Debug)]
386    pub struct Lock {
387        path: PathBuf,
388        record: LockRecord,
389        /// Set by [`Lock::release`] so the `Drop` path does not try again. A flag, since
390        /// `mem::forget` would leak the path and the record.
391        released: bool,
392    }
393
394    impl Lock {
395        /// Takes the lock, or explains why it could not be taken.
396        ///
397        /// Creating the file is the exclusion: it fails if anything is there, so two
398        /// processes racing for one journal cannot both succeed. The existing lock is inspected
399        /// only after that fails, to decide whether it may be broken.
400        pub fn acquire(path: &Path) -> Result<Self, LockRefused> {
401            Self::acquire_inner(path, Breaks::Dead)
402        }
403
404        /// [`acquire`](Self::acquire), breaking a [`Stale`](LockState::Stale) lock as well
405        /// as a dead one.
406        ///
407        /// For a caller who has judged that the process holding the number in a lock that stopped
408        /// beating is not the scan, usually because the number was reissued after a crash. A lock
409        /// whose writer is checkpointing now is still refused.
410        pub fn take_over(path: &Path) -> Result<Self, LockRefused> {
411            Self::acquire_inner(path, Breaks::Stale)
412        }
413
414        /// [`acquire`](Self::acquire), overriding any refusal.
415        ///
416        /// For a lock left by a defect nothing here anticipated, which would otherwise make the
417        /// journal unusable forever.
418        pub fn force(path: &Path) -> Result<Self, LockRefused> {
419            Self::acquire_inner(path, Breaks::Anything)
420        }
421
422        /// How many times a break is retried before giving up.
423        ///
424        /// A retry happens only when another process won the create between this one removing a
425        /// dead lock and replacing it; the winner's lock is live, so the next round refuses. More
426        /// rounds than that means two processes are breaking each other's locks in a loop.
427        const BREAK_ATTEMPTS: usize = 3;
428
429        fn acquire_inner(path: &Path, breaks: Breaks) -> Result<Self, LockRefused> {
430            // Built only after the create succeeds: `Lock` removes its file on drop, so one
431            // dropped after losing the create would delete the winner's lock.
432            let held = |record| Self {
433                path: path.to_path_buf(),
434                record,
435                released: false,
436            };
437            let mut refusal = None;
438
439            for _ in 0..Self::BREAK_ATTEMPTS {
440                let record = LockRecord::current();
441                match Self::create_exclusively(path, &record) {
442                    Ok(()) => return Ok(held(record)),
443                    Err(error) if error.kind() != std::io::ErrorKind::AlreadyExists => {
444                        return Err(LockRefused::Io(error.to_string()));
445                    }
446                    Err(_) => {}
447                }
448
449                // Deciding a lock is dead and replacing it must be one operation. `create_new`
450                // only excludes racers for a free journal. Replacing a dead lock by rename
451                // bypasses the create, and removing it first lets every racer remove whatever is
452                // at the name, including the lock the last winner just created: eight processes
453                // on one crashed journal yielded two to four holders.
454                //
455                // So the inspect, removal and create run under an advisory lock on a sibling
456                // file, held only for those three steps. The kernel drops it when its holder
457                // exits, so it cannot go stale and `force` never needs to clear it.
458                let _breaking = Breaking::take(path).map_err(|e| LockRefused::Io(e.to_string()))?;
459
460                // Asked again under the guard: the previous guard holder may have taken the
461                // journal in the meantime.
462                let state = inspect(path);
463                if !breaks.permits(&state) {
464                    return Err(LockRefused::Held(state));
465                }
466
467                match remove(path) {
468                    Ok(()) => {}
469                    // Removed between the two inspections; the create below decides either way.
470                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
471                    Err(error) => return Err(LockRefused::Io(error.to_string())),
472                }
473
474                let record = LockRecord::current();
475                match Self::create_exclusively(path, &record) {
476                    Ok(()) => return Ok(held(record)),
477                    Err(error) if error.kind() != std::io::ErrorKind::AlreadyExists => {
478                        return Err(LockRefused::Io(error.to_string()));
479                    }
480                    // A process arriving after this one removed the lock sees nothing to break,
481                    // takes no guard, and can win the create in that gap. The next round then
482                    // finds a beating lock and refuses by name.
483                    Err(_) => refusal = Some(state),
484                }
485            }
486
487            Err(match refusal {
488                Some(state) => LockRefused::Held(state),
489                None => LockRefused::Io("the lock could not be taken".to_string()),
490            })
491        }
492
493        /// Moves the heartbeat forward. Called once per checkpoint.
494        pub fn beat(&mut self) -> Result<(), JournalError> {
495            self.record = self.record.beating_at(SystemTime::now());
496            Self::write(&self.path, &self.record)?;
497            Ok(())
498        }
499
500        /// What this lock claims, as written.
501        pub fn record(&self) -> &LockRecord {
502            &self.record
503        }
504
505        /// Releases the lock, reporting a failure the `Drop` path would swallow.
506        pub fn release(mut self) -> Result<(), JournalError> {
507            self.released = true;
508            remove(&self.path)?;
509            Ok(())
510        }
511
512        /// Takes the lock, or fails with
513        /// [`AlreadyExists`](std::io::ErrorKind::AlreadyExists) if somebody has it.
514        ///
515        /// The name appears already holding the record. Creating then writing would let a racer
516        /// read an empty lock between the two steps, which [`inspect`] reports as `Free` (that is
517        /// what a writer killed mid-write leaves), and delete a lock taken a microsecond earlier.
518        /// So the record is written to a staged file and linked into place: `link` refuses an
519        /// existing name, as `create_new` does, over a file that already has its contents.
520        ///
521        /// Created like every other journal file, so under `sudo` it belongs to whoever invoked the
522        /// scan and they can release it.
523        ///
524        /// The staged name carries a counter as well as the pid, because this is a library and two
525        /// threads of one caller may take the same journal. With the pid alone they would share the
526        /// staged name, and since [`link_new`] removes a staged name it finds occupied, one thread
527        /// could delete the other's file (failing its link with `NotFound`) or link an empty file
528        /// that [`inspect`] reads as `Free`. Either way the lock would get two holders.
529        fn create_exclusively(path: &Path, record: &LockRecord) -> std::io::Result<()> {
530            static STAGING: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
531            let staged = path.with_extension(format!(
532                "lock-{}-{}",
533                std::process::id(),
534                STAGING.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
535            ));
536
537            let text = serde_json::to_string(record).map_err(std::io::Error::other)?;
538            // Staged with a retry, since a reused pid can find a staged file left by a run
539            // that died between the create and the link.
540            link_new(path, &staged, |mut file| file.write_all(text.as_bytes()))
541        }
542
543        /// Replaces the lock file in place, for the holder moving its own heartbeat forward.
544        /// Taking a lock goes through [`create_exclusively`](Self::create_exclusively).
545        fn write(path: &Path, record: &LockRecord) -> std::io::Result<()> {
546            let text = serde_json::to_string(record).map_err(std::io::Error::other)?;
547            // Private from creation like the original, so a heartbeat never widens the mode.
548            // The lock becomes the staged file's inode, ownership included.
549            replace(path, &path.with_extension("lock-tmp"), |mut file| {
550                file.write_all(text.as_bytes())
551            })
552        }
553    }
554
555    /// Which locks an attempt to take one may break.
556    #[derive(Debug, Clone, Copy)]
557    enum Breaks {
558        /// Only one nothing holds: free, crashed, or from before a reboot.
559        Dead,
560        /// Those, and one whose holder stopped checkpointing.
561        Stale,
562        /// Whatever is there.
563        Anything,
564    }
565
566    impl Breaks {
567        fn permits(self, state: &LockState) -> bool {
568            match self {
569                Breaks::Dead => state.is_resumable(),
570                Breaks::Stale => state.is_resumable() || matches!(state, LockState::Stale { .. }),
571                Breaks::Anything => true,
572            }
573        }
574    }
575
576    /// Serialises deciding that a lock is dead and replacing it.
577    ///
578    /// See the comment at the call site. Dropped as soon as the three steps are done.
579    struct Breaking {
580        /// Held only for the handle: the lock lives on the open file and is released when
581        /// it closes.
582        _file: fs::File,
583    }
584
585    impl Breaking {
586        fn take(lock: &Path) -> std::io::Result<Self> {
587            // A sibling file, because the lock file is about to be removed and an advisory
588            // lock follows the open file, not the name. Opened, not created, so every racer reaches
589            // the same file.
590            let file = open_or_create_private(&lock.with_extension("break"))?;
591
592            // Blocks for the exclusive lock: `flock` on unix, `LockFileEx` on Windows.
593            file.lock()?;
594            Ok(Self { _file: file })
595        }
596    }
597
598    impl Drop for Lock {
599        fn drop(&mut self) {
600            if self.released {
601                return;
602            }
603
604            // Best effort. A lock left by a failed removal reads as `Crashed` to the next
605            // reader, which is resumable.
606            let _ = remove(&self.path);
607        }
608    }
609
610    /// Why a lock could not be taken.
611    #[non_exhaustive]
612    #[derive(Debug, Clone, PartialEq, Eq)]
613    pub enum LockRefused {
614        /// Somebody else has it. Carries the state so a caller can render
615        /// [`LockState::refusal`].
616        Held(LockState),
617        /// The lock file could not be written.
618        Io(String),
619    }
620
621    impl std::fmt::Display for LockRefused {
622        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
623            match self {
624                LockRefused::Held(state) => match state.refusal() {
625                    Some(reason) => write!(f, "{reason}"),
626                    None => write!(f, "the journal is locked"),
627                },
628                LockRefused::Io(error) => write!(f, "could not write the lock: {error}"),
629            }
630        }
631    }
632
633    impl std::error::Error for LockRefused {}
634}
635
636#[cfg(feature = "journal-format")]
637pub use persistence::{Lock, LockRefused, inspect};
638
639// ╔════════════════════════════════════════════╗
640// ║ ████████╗███████╗███████╗████████╗███████╗ ║
641// ║ ╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝ ║
642// ║    ██║   █████╗  ███████╗   ██║   ███████╗ ║
643// ║    ██║   ██╔══╝  ╚════██║   ██║   ╚════██║ ║
644// ║    ██║   ███████╗███████║   ██║   ███████║ ║
645// ║    ╚═╝   ╚══════╝╚══════╝   ╚═╝   ╚══════╝ ║
646// ╚════════════════════════════════════════════╝
647
648#[cfg(test)]
649mod tests {
650    use super::*;
651
652    const BOOT: &str = "boot-a";
653    const OTHER_BOOT: &str = "boot-b";
654
655    fn at(seconds: u64) -> SystemTime {
656        SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
657    }
658
659    fn record(boot: &str, beat: u64) -> LockRecord {
660        LockRecord {
661            pid: 4_242,
662            boot: boot.to_string(),
663            started_at: at(0),
664            heartbeat: at(beat),
665        }
666    }
667
668    /// A live writer checkpointing normally is always refused.
669    #[test]
670    fn a_beating_lock_on_a_live_pid_is_held() {
671        let state = classify(
672            &record(BOOT, 100),
673            BOOT,
674            true,
675            at(105),
676            HEARTBEAT_STALE_AFTER,
677        );
678
679        assert!(matches!(state, LockState::Held { pid: 4_242, .. }));
680        assert!(!state.is_resumable());
681        assert!(state.refusal().is_some_and(|r| r.contains("4242")));
682    }
683
684    /// The ordinary crash: the process is gone and the journal is free.
685    #[test]
686    fn a_lock_whose_pid_is_gone_is_a_crash() {
687        let state = classify(
688            &record(BOOT, 100),
689            BOOT,
690            false,
691            at(105),
692            HEARTBEAT_STALE_AFTER,
693        );
694
695        assert_eq!(state, LockState::Crashed { pid: 4_242 });
696        assert!(state.is_resumable());
697        assert!(!state.may_have_lost_the_tail());
698        assert_eq!(state.refusal(), None);
699    }
700
701    /// The case the pid alone gets wrong. After a reboot low pids go to init and its
702    /// children, so the pid in an old lock is often alive and unrelated.
703    #[test]
704    fn a_lock_from_a_previous_boot_is_released_even_though_its_pid_is_alive() {
705        let state = classify(
706            &record(OTHER_BOOT, 100),
707            BOOT,
708            true, // alive, but not the same process
709            at(105),
710            HEARTBEAT_STALE_AFTER,
711        );
712
713        assert_eq!(state, LockState::RebootedUnder { pid: 4_242 });
714        assert!(state.is_resumable());
715        assert!(
716            state.may_have_lost_the_tail(),
717            "a reboot loses what the page cache had not flushed"
718        );
719    }
720
721    /// The case the boot identity alone gets wrong. Within one boot a freed pid can be
722    /// reissued, and only the heartbeat tells the two apart, so a stale beat is refused.
723    #[test]
724    fn a_live_pid_that_stopped_beating_is_stale_and_refused() {
725        let state = classify(
726            &record(BOOT, 100),
727            BOOT,
728            true,
729            at(100 + HEARTBEAT_STALE_AFTER.as_secs() + 1),
730            HEARTBEAT_STALE_AFTER,
731        );
732
733        assert!(matches!(state, LockState::Stale { pid: 4_242, .. }));
734        assert!(!state.is_resumable(), "hung and reused look the same here");
735        assert!(
736            state
737                .refusal()
738                .is_some_and(|r| r.contains("take the journal over"))
739        );
740    }
741
742    /// A beat exactly at the threshold is not yet stale. The boundary decides between
743    /// refusing a live scan and permitting a second writer.
744    #[test]
745    fn the_staleness_boundary_is_exclusive() {
746        let held = classify(
747            &record(BOOT, 100),
748            BOOT,
749            true,
750            at(100) + HEARTBEAT_STALE_AFTER,
751            HEARTBEAT_STALE_AFTER,
752        );
753        assert!(matches!(held, LockState::Held { .. }), "{held:?}");
754
755        let stale = classify(
756            &record(BOOT, 100),
757            BOOT,
758            true,
759            at(100) + HEARTBEAT_STALE_AFTER + Duration::from_nanos(1),
760            HEARTBEAT_STALE_AFTER,
761        );
762        assert!(matches!(stale, LockState::Stale { .. }), "{stale:?}");
763    }
764
765    /// A clock that moved backwards does not turn a live scan into a stale one.
766    #[test]
767    fn a_heartbeat_from_the_future_reads_as_fresh() {
768        let state = classify(
769            &record(BOOT, 500),
770            BOOT,
771            true,
772            at(100),
773            HEARTBEAT_STALE_AFTER,
774        );
775
776        assert!(matches!(state, LockState::Held { .. }), "{state:?}");
777    }
778
779    /// An unreadable boot identity compares unequal to any recorded one, so it releases
780    /// locks.
781    #[test]
782    fn an_unknown_boot_identity_releases_rather_than_holds() {
783        let state = classify(&record(BOOT, 100), "", true, at(105), HEARTBEAT_STALE_AFTER);
784
785        assert!(state.is_resumable(), "{state:?}");
786    }
787
788    /// This host reports its boot identity, and the same one twice.
789    #[test]
790    fn the_boot_identity_is_readable_and_stable() {
791        let first = boot_identity();
792        assert!(!first.is_empty(), "no boot identity on this platform");
793        assert_eq!(first, boot_identity(), "it must not change while running");
794    }
795
796    /// The liveness check is right about the one process a test can be sure of:
797    /// itself.
798    #[test]
799    fn this_process_is_alive() {
800        assert!(pid_is_alive(std::process::id()));
801        assert!(!pid_is_alive(0), "pid 0 is not a process this can signal");
802    }
803}
804
805#[cfg(all(test, feature = "journal-format"))]
806mod file_tests {
807    use super::*;
808    use std::path::PathBuf;
809
810    fn scratch(name: &str) -> PathBuf {
811        let dir = std::env::temp_dir().join(format!("zond-lock-{}-{name}", std::process::id()));
812        std::fs::create_dir_all(&dir).expect("scratch directory");
813        dir
814    }
815
816    /// Taking, holding and releasing, and what each looks like from outside.
817    #[test]
818    fn a_lock_is_visible_while_held_and_gone_once_released() {
819        let dir = scratch("lifecycle");
820        let path = dir.join("LOCK");
821
822        assert_eq!(inspect(&path), LockState::Free, "nothing has it yet");
823
824        let lock = Lock::acquire(&path).expect("takes a free lock");
825        assert_eq!(lock.record().pid, std::process::id());
826        assert!(
827            matches!(inspect(&path), LockState::Held { .. }),
828            "this process is alive and has just beaten"
829        );
830
831        lock.release().expect("releases");
832        assert_eq!(inspect(&path), LockState::Free);
833
834        std::fs::remove_dir_all(&dir).ok();
835    }
836
837    /// Two scans racing for one journal: the second is refused and told which process
838    /// has it.
839    #[test]
840    fn a_second_acquisition_is_refused_while_the_first_holds_it() {
841        let dir = scratch("contended");
842        let path = dir.join("LOCK");
843
844        let first = Lock::acquire(&path).expect("takes it");
845
846        match Lock::acquire(&path) {
847            Err(LockRefused::Held(state)) => {
848                assert!(!state.is_resumable());
849                assert!(
850                    state
851                        .refusal()
852                        .is_some_and(|r| r.contains(&std::process::id().to_string())),
853                    "a refusal has to name who holds it"
854                );
855            }
856            other => panic!("expected a refusal, got {other:?}"),
857        }
858
859        // Forcing gets past it.
860        let forced = Lock::force(&path).expect("force overrides");
861        drop(forced);
862        drop(first);
863
864        std::fs::remove_dir_all(&dir).ok();
865    }
866
867    /// A lock whose holder stopped beating can be taken over, and one whose holder is
868    /// beating cannot. This process stands in for a reissued pid.
869    #[test]
870    fn a_stale_lock_can_be_taken_over_and_a_beating_one_cannot() {
871        let dir = scratch("stale");
872        let path = dir.join("LOCK");
873
874        let stale = LockRecord {
875            pid: std::process::id(),
876            boot: boot_identity(),
877            started_at: SystemTime::now() - Duration::from_secs(3_600),
878            heartbeat: SystemTime::now() - Duration::from_secs(600),
879        };
880        std::fs::write(&path, serde_json::to_string(&stale).expect("json")).expect("writes");
881        assert!(matches!(inspect(&path), LockState::Stale { .. }));
882        assert!(
883            Lock::acquire(&path).is_err(),
884            "a stale lock is refused unasked"
885        );
886
887        let taken = Lock::take_over(&path).expect("a stale lock is taken over");
888        assert!(
889            matches!(
890                Lock::take_over(&path),
891                Err(LockRefused::Held(LockState::Held { .. }))
892            ),
893            "a lock beating now is not taken over"
894        );
895        drop(taken);
896
897        std::fs::remove_dir_all(&dir).ok();
898    }
899
900    /// A lock left half-written by a crash reads as free, so it does not lock the
901    /// journal forever.
902    #[test]
903    fn a_corrupt_lock_does_not_wedge_the_journal() {
904        let dir = scratch("corrupt");
905        let path = dir.join("LOCK");
906
907        std::fs::write(&path, "{\"pid\":42,\"bo").expect("writes a torn lock");
908        assert_eq!(inspect(&path), LockState::Free);
909
910        Lock::acquire(&path).expect("a torn lock can be taken over");
911
912        std::fs::remove_dir_all(&dir).ok();
913    }
914
915    /// A crashed writer's lock is taken without a force, and the new holder's heartbeat
916    /// replaces it.
917    #[test]
918    fn a_dead_writers_lock_is_taken_over() {
919        let dir = scratch("crashed");
920        let path = dir.join("LOCK");
921
922        // pid 0 is never a signalable process, so it stands in for a writer that is gone.
923        let dead = LockRecord {
924            pid: 0,
925            boot: boot_identity(),
926            started_at: SystemTime::now(),
927            heartbeat: SystemTime::now(),
928        };
929        std::fs::write(&path, serde_json::to_string(&dead).expect("json")).expect("writes");
930
931        assert_eq!(inspect(&path), LockState::Crashed { pid: 0 });
932
933        let lock = Lock::acquire(&path).expect("a crashed lock needs no force");
934        assert_eq!(lock.record().pid, std::process::id());
935
936        std::fs::remove_dir_all(&dir).ok();
937    }
938
939    /// Several racers finding the same crashed journal: exactly one takes it.
940    ///
941    /// The racers are threads in one process, so this also covers the staged-name counter in
942    /// `create_exclusively`. Without the guard or the counter the count came out at two about
943    /// one run in twenty, so the racers start on a barrier and the race runs in rounds.
944    #[test]
945    fn only_one_of_several_racers_breaks_a_crashed_lock() {
946        for round in 0..16 {
947            let dir = scratch(&format!("break-race-{round}"));
948            let path = dir.join("LOCK");
949
950            // A lock from before a reboot, resumable whatever its pid is doing. A test
951            // cannot name a pid it is certain nothing holds.
952            let stale = LockRecord {
953                pid: std::process::id(),
954                boot: "a boot that is over".to_string(),
955                started_at: SystemTime::UNIX_EPOCH,
956                heartbeat: SystemTime::UNIX_EPOCH,
957            };
958            std::fs::write(&path, serde_json::to_string(&stale).expect("json")).expect("writes");
959            assert!(matches!(inspect(&path), LockState::RebootedUnder { .. }));
960
961            let taken = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
962            // Racers start together; without the barrier the first thread usually finished
963            // before the last began.
964            let start = std::sync::Arc::new(std::sync::Barrier::new(8));
965            std::thread::scope(|scope| {
966                for _ in 0..8 {
967                    let path = path.clone();
968                    let taken = std::sync::Arc::clone(&taken);
969                    let start = std::sync::Arc::clone(&start);
970                    scope.spawn(move || {
971                        start.wait();
972                        if let Ok(lock) = Lock::acquire(&path) {
973                            taken.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
974                            // Held for the rest of the scope, so a later racer is refused.
975                            std::thread::sleep(std::time::Duration::from_millis(20));
976                            std::mem::forget(lock);
977                        }
978                    });
979                }
980            });
981
982            assert_eq!(
983                taken.load(std::sync::atomic::Ordering::Relaxed),
984                1,
985                "round {round}: this many racers believed they held the journal"
986            );
987
988            std::fs::remove_dir_all(&dir).ok();
989        }
990    }
991
992    /// The heartbeat moves, and leaves no temporary file behind.
993    #[test]
994    fn beating_moves_the_heartbeat_forward() {
995        let dir = scratch("heartbeat");
996        let path = dir.join("LOCK");
997
998        let mut lock = Lock::acquire(&path).expect("takes it");
999        let before = lock.record().heartbeat;
1000
1001        std::thread::sleep(Duration::from_millis(5));
1002        lock.beat().expect("beats");
1003
1004        assert!(lock.record().heartbeat > before);
1005        assert!(
1006            !dir.join("LOCK.lock-tmp").exists(),
1007            "the temporary must not survive the rename"
1008        );
1009
1010        std::fs::remove_dir_all(&dir).ok();
1011    }
1012}