use std::time::{Duration, SystemTime};
use serde::{Deserialize, Serialize};
pub const HEARTBEAT_STALE_AFTER: Duration = Duration::from_secs(60);
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LockRecord {
pub pid: u32,
pub boot: String,
pub started_at: SystemTime,
pub heartbeat: SystemTime,
}
impl LockRecord {
pub fn current() -> Self {
let now = SystemTime::now();
Self {
pid: std::process::id(),
boot: boot_identity(),
started_at: now,
heartbeat: now,
}
}
pub fn beating_at(&self, now: SystemTime) -> Self {
Self {
heartbeat: now,
..self.clone()
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LockState {
Free,
Held {
pid: u32,
last_beat: Duration,
},
Crashed {
pid: u32,
},
RebootedUnder {
pid: u32,
},
Stale {
pid: u32,
last_beat: Duration,
},
}
impl LockState {
pub fn is_resumable(&self) -> bool {
matches!(
self,
LockState::Free | LockState::Crashed { .. } | LockState::RebootedUnder { .. }
)
}
pub fn may_have_lost_the_tail(&self) -> bool {
matches!(self, LockState::RebootedUnder { .. })
}
pub fn refusal(&self) -> Option<String> {
match self {
LockState::Free | LockState::Crashed { .. } | LockState::RebootedUnder { .. } => None,
LockState::Held { pid, last_beat } => Some(format!(
"process {pid} is scanning this journal now (last checkpoint {}s ago); \
wait for it to finish, or stop it first",
last_beat.as_secs()
)),
LockState::Stale { pid, last_beat } => Some(format!(
"this journal is locked by process {pid}, which has not checkpointed for {}s — \
it is either hung or the number was reissued to something else. \
Stop it, or take the lock forcibly if you are sure it is not scanning",
last_beat.as_secs()
)),
}
}
}
pub fn classify(
record: &LockRecord,
boot: &str,
pid_is_alive: bool,
now: SystemTime,
stale_after: Duration,
) -> LockState {
if record.boot != boot {
return LockState::RebootedUnder { pid: record.pid };
}
if !pid_is_alive {
return LockState::Crashed { pid: record.pid };
}
let last_beat = now.duration_since(record.heartbeat).unwrap_or_default();
if last_beat > stale_after {
LockState::Stale {
pid: record.pid,
last_beat,
}
} else {
LockState::Held {
pid: record.pid,
last_beat,
}
}
}
pub fn boot_identity() -> String {
imp::boot_identity()
}
pub fn pid_is_alive(pid: u32) -> bool {
imp::pid_is_alive(pid)
}
#[cfg(unix)]
mod imp {
pub fn boot_identity() -> String {
if let Ok(id) = std::fs::read_to_string("/proc/sys/kernel/random/boot_id") {
return id.trim().to_string();
}
#[cfg(any(target_os = "macos", target_os = "ios", target_vendor = "apple"))]
{
let mut boot = libc::timeval {
tv_sec: 0,
tv_usec: 0,
};
let mut size = std::mem::size_of::<libc::timeval>();
let mut mib = [libc::CTL_KERN, libc::KERN_BOOTTIME];
let code = unsafe {
libc::sysctl(
mib.as_mut_ptr(),
mib.len() as libc::c_uint,
(&raw mut boot).cast::<libc::c_void>(),
&mut size,
std::ptr::null_mut(),
0,
)
};
if code == 0 {
return format!("{}.{}", boot.tv_sec, boot.tv_usec);
}
}
String::new()
}
pub fn pid_is_alive(pid: u32) -> bool {
let Ok(pid) = libc::pid_t::try_from(pid) else {
return false;
};
if pid <= 0 {
return false;
}
let code = unsafe { libc::kill(pid, 0) };
if code == 0 {
return true;
}
std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
}
}
#[cfg(windows)]
mod imp {
pub fn boot_identity() -> String {
String::new()
}
pub fn pid_is_alive(_pid: u32) -> bool {
true
}
}
#[cfg(feature = "journal-format")]
mod persistence {
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::SystemTime;
use super::{HEARTBEAT_STALE_AFTER, LockRecord, LockState, boot_identity, classify};
use crate::journal::file::{create_staged, open_or_create_private};
use crate::journal::format::JournalError;
pub fn inspect(path: &Path) -> LockState {
let Ok(text) = fs::read_to_string(path) else {
return LockState::Free;
};
let Ok(record) = serde_json::from_str::<LockRecord>(&text) else {
return LockState::Free;
};
classify(
&record,
&boot_identity(),
super::pid_is_alive(record.pid),
SystemTime::now(),
HEARTBEAT_STALE_AFTER,
)
}
#[derive(Debug)]
pub struct Lock {
path: PathBuf,
record: LockRecord,
released: bool,
}
impl Lock {
pub fn acquire(path: &Path) -> Result<Self, LockRefused> {
Self::acquire_inner(path, false)
}
pub fn force(path: &Path) -> Result<Self, LockRefused> {
Self::acquire_inner(path, true)
}
const BREAK_ATTEMPTS: usize = 3;
fn acquire_inner(path: &Path, force: bool) -> Result<Self, LockRefused> {
let held = |record| Self {
path: path.to_path_buf(),
record,
released: false,
};
let mut refusal = None;
for _ in 0..Self::BREAK_ATTEMPTS {
let record = LockRecord::current();
match Self::create_exclusively(path, &record) {
Ok(()) => return Ok(held(record)),
Err(error) if error.kind() != std::io::ErrorKind::AlreadyExists => {
return Err(LockRefused::Io(error.to_string()));
}
Err(_) => {}
}
let _breaking = Breaking::take(path).map_err(|e| LockRefused::Io(e.to_string()))?;
let state = inspect(path);
if !force && !state.is_resumable() {
return Err(LockRefused::Held(state));
}
match fs::remove_file(path) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(LockRefused::Io(error.to_string())),
}
let record = LockRecord::current();
match Self::create_exclusively(path, &record) {
Ok(()) => return Ok(held(record)),
Err(error) if error.kind() != std::io::ErrorKind::AlreadyExists => {
return Err(LockRefused::Io(error.to_string()));
}
Err(_) => refusal = Some(state),
}
}
Err(match refusal {
Some(state) => LockRefused::Held(state),
None => LockRefused::Io("the lock could not be taken".to_string()),
})
}
pub fn beat(&mut self) -> Result<(), JournalError> {
self.record = self.record.beating_at(SystemTime::now());
Self::write(&self.path, &self.record)?;
Ok(())
}
pub fn record(&self) -> &LockRecord {
&self.record
}
pub fn release(mut self) -> Result<(), JournalError> {
self.released = true;
fs::remove_file(&self.path)?;
Ok(())
}
fn create_exclusively(path: &Path, record: &LockRecord) -> std::io::Result<()> {
static STAGING: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let staged = path.with_extension(format!(
"lock-{}-{}",
std::process::id(),
STAGING.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
));
{
let mut file = create_staged(&staged)?;
file.write_all(
serde_json::to_string(record)
.map_err(std::io::Error::other)?
.as_bytes(),
)?;
}
let linked = fs::hard_link(&staged, path);
let _ = fs::remove_file(&staged);
linked
}
fn write(path: &Path, record: &LockRecord) -> std::io::Result<()> {
let temporary = path.with_extension("lock-tmp");
{
let mut file = create_staged(&temporary)?;
file.write_all(
serde_json::to_string(record)
.map_err(std::io::Error::other)?
.as_bytes(),
)?;
}
fs::rename(&temporary, path)?;
Ok(())
}
}
#[cfg(unix)]
struct Breaking {
_file: fs::File,
}
#[cfg(unix)]
impl Breaking {
fn take(lock: &Path) -> std::io::Result<Self> {
use std::os::unix::io::AsRawFd;
let file = open_or_create_private(&lock.with_extension("break"))?;
if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } != 0 {
return Err(std::io::Error::last_os_error());
}
Ok(Self { _file: file })
}
}
#[cfg(not(unix))]
struct Breaking;
#[cfg(not(unix))]
impl Breaking {
fn take(_lock: &Path) -> std::io::Result<Self> {
Ok(Self)
}
}
impl Drop for Lock {
fn drop(&mut self) {
if self.released {
return;
}
let _ = fs::remove_file(&self.path);
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LockRefused {
Held(LockState),
Io(String),
}
impl std::fmt::Display for LockRefused {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
LockRefused::Held(state) => match state.refusal() {
Some(reason) => write!(f, "{reason}"),
None => write!(f, "the journal is locked"),
},
LockRefused::Io(error) => write!(f, "could not write the lock: {error}"),
}
}
}
impl std::error::Error for LockRefused {}
}
#[cfg(feature = "journal-format")]
pub use persistence::{Lock, LockRefused, inspect};
#[cfg(test)]
mod tests {
use super::*;
const BOOT: &str = "boot-a";
const OTHER_BOOT: &str = "boot-b";
fn at(seconds: u64) -> SystemTime {
SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
}
fn record(boot: &str, beat: u64) -> LockRecord {
LockRecord {
pid: 4_242,
boot: boot.to_string(),
started_at: at(0),
heartbeat: at(beat),
}
}
#[test]
fn a_beating_lock_on_a_live_pid_is_held() {
let state = classify(
&record(BOOT, 100),
BOOT,
true,
at(105),
HEARTBEAT_STALE_AFTER,
);
assert!(matches!(state, LockState::Held { pid: 4_242, .. }));
assert!(!state.is_resumable());
assert!(state.refusal().is_some_and(|r| r.contains("4242")));
}
#[test]
fn a_lock_whose_pid_is_gone_is_a_crash() {
let state = classify(
&record(BOOT, 100),
BOOT,
false,
at(105),
HEARTBEAT_STALE_AFTER,
);
assert_eq!(state, LockState::Crashed { pid: 4_242 });
assert!(state.is_resumable());
assert!(!state.may_have_lost_the_tail());
assert_eq!(state.refusal(), None);
}
#[test]
fn a_lock_from_a_previous_boot_is_released_even_though_its_pid_is_alive() {
let state = classify(
&record(OTHER_BOOT, 100),
BOOT,
true, at(105),
HEARTBEAT_STALE_AFTER,
);
assert_eq!(state, LockState::RebootedUnder { pid: 4_242 });
assert!(state.is_resumable());
assert!(
state.may_have_lost_the_tail(),
"a reboot loses what the page cache had not flushed"
);
}
#[test]
fn a_live_pid_that_stopped_beating_is_stale_and_refused() {
let state = classify(
&record(BOOT, 100),
BOOT,
true,
at(100 + HEARTBEAT_STALE_AFTER.as_secs() + 1),
HEARTBEAT_STALE_AFTER,
);
assert!(matches!(state, LockState::Stale { pid: 4_242, .. }));
assert!(!state.is_resumable(), "hung and reused look the same here");
assert!(state.refusal().is_some_and(|r| r.contains("forcibly")));
}
#[test]
fn the_staleness_boundary_is_exclusive() {
let held = classify(
&record(BOOT, 100),
BOOT,
true,
at(100) + HEARTBEAT_STALE_AFTER,
HEARTBEAT_STALE_AFTER,
);
assert!(matches!(held, LockState::Held { .. }), "{held:?}");
let stale = classify(
&record(BOOT, 100),
BOOT,
true,
at(100) + HEARTBEAT_STALE_AFTER + Duration::from_nanos(1),
HEARTBEAT_STALE_AFTER,
);
assert!(matches!(stale, LockState::Stale { .. }), "{stale:?}");
}
#[test]
fn a_heartbeat_from_the_future_reads_as_fresh() {
let state = classify(
&record(BOOT, 500),
BOOT,
true,
at(100),
HEARTBEAT_STALE_AFTER,
);
assert!(matches!(state, LockState::Held { .. }), "{state:?}");
}
#[test]
fn an_unknown_boot_identity_releases_rather_than_holds() {
let state = classify(&record(BOOT, 100), "", true, at(105), HEARTBEAT_STALE_AFTER);
assert!(state.is_resumable(), "{state:?}");
}
#[test]
fn the_boot_identity_is_readable_and_stable() {
let first = boot_identity();
assert!(!first.is_empty(), "no boot identity on this platform");
assert_eq!(first, boot_identity(), "it must not change while running");
}
#[test]
fn this_process_is_alive() {
assert!(pid_is_alive(std::process::id()));
assert!(!pid_is_alive(0), "pid 0 is not a process this can signal");
}
}
#[cfg(all(test, feature = "journal-format"))]
mod file_tests {
use super::*;
use std::path::PathBuf;
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("zond-lock-{}-{name}", std::process::id()));
std::fs::create_dir_all(&dir).expect("scratch directory");
dir
}
#[test]
fn a_lock_is_visible_while_held_and_gone_once_released() {
let dir = scratch("lifecycle");
let path = dir.join("LOCK");
assert_eq!(inspect(&path), LockState::Free, "nothing has it yet");
let lock = Lock::acquire(&path).expect("takes a free lock");
assert_eq!(lock.record().pid, std::process::id());
assert!(
matches!(inspect(&path), LockState::Held { .. }),
"this process is alive and has just beaten"
);
lock.release().expect("releases");
assert_eq!(inspect(&path), LockState::Free);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_second_acquisition_is_refused_while_the_first_holds_it() {
let dir = scratch("contended");
let path = dir.join("LOCK");
let first = Lock::acquire(&path).expect("takes it");
match Lock::acquire(&path) {
Err(LockRefused::Held(state)) => {
assert!(!state.is_resumable());
assert!(
state
.refusal()
.is_some_and(|r| r.contains(&std::process::id().to_string())),
"a refusal has to name who holds it"
);
}
other => panic!("expected a refusal, got {other:?}"),
}
let forced = Lock::force(&path).expect("force overrides");
drop(forced);
drop(first);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_corrupt_lock_does_not_wedge_the_journal() {
let dir = scratch("corrupt");
let path = dir.join("LOCK");
std::fs::write(&path, "{\"pid\":42,\"bo").expect("writes a torn lock");
assert_eq!(inspect(&path), LockState::Free);
Lock::acquire(&path).expect("a torn lock can be taken over");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_dead_writers_lock_is_taken_over() {
let dir = scratch("crashed");
let path = dir.join("LOCK");
let dead = LockRecord {
pid: 0,
boot: boot_identity(),
started_at: SystemTime::now(),
heartbeat: SystemTime::now(),
};
std::fs::write(&path, serde_json::to_string(&dead).expect("json")).expect("writes");
assert_eq!(inspect(&path), LockState::Crashed { pid: 0 });
let lock = Lock::acquire(&path).expect("a crashed lock needs no force");
assert_eq!(lock.record().pid, std::process::id());
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn only_one_of_several_racers_breaks_a_crashed_lock() {
for round in 0..16 {
let dir = scratch(&format!("break-race-{round}"));
let path = dir.join("LOCK");
let stale = LockRecord {
pid: std::process::id(),
boot: "a boot that is over".to_string(),
started_at: SystemTime::UNIX_EPOCH,
heartbeat: SystemTime::UNIX_EPOCH,
};
std::fs::write(&path, serde_json::to_string(&stale).expect("json")).expect("writes");
assert!(matches!(inspect(&path), LockState::RebootedUnder { .. }));
let taken = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let start = std::sync::Arc::new(std::sync::Barrier::new(8));
std::thread::scope(|scope| {
for _ in 0..8 {
let path = path.clone();
let taken = std::sync::Arc::clone(&taken);
let start = std::sync::Arc::clone(&start);
scope.spawn(move || {
start.wait();
if let Ok(lock) = Lock::acquire(&path) {
taken.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
std::thread::sleep(std::time::Duration::from_millis(20));
std::mem::forget(lock);
}
});
}
});
assert_eq!(
taken.load(std::sync::atomic::Ordering::Relaxed),
1,
"round {round}: this many racers believed they held the journal"
);
std::fs::remove_dir_all(&dir).ok();
}
}
#[test]
fn beating_moves_the_heartbeat_forward() {
let dir = scratch("heartbeat");
let path = dir.join("LOCK");
let mut lock = Lock::acquire(&path).expect("takes it");
let before = lock.record().heartbeat;
std::thread::sleep(Duration::from_millis(5));
lock.beat().expect("beats");
assert!(lock.record().heartbeat > before);
assert!(
!dir.join("LOCK.lock-tmp").exists(),
"the temporary must not survive the rename"
);
std::fs::remove_dir_all(&dir).ok();
}
}