zc2 0.0.52

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! The admission lock (ZC-0018 ยง2, zc#578): "sum what runs, then start" must
//! be one step across every process on the node. The agent (Atelier jobs,
//! share workers) and the broker (Spaces leases, deployments) are separate
//! processes sharing one budget; an in-process mutex alone lets both pass
//! admission against the same free share.
//!
//! Two layers: a process-wide mutex (cheap, and what keeps two threads of one
//! process from racing on one file descriptor) and an OS `flock` on a file
//! under the zakuro home (`sandbox/admission.lock`), released by the kernel
//! when the holder exits, so a crashed process never wedges the node.
//! Windows has no `flock` here: the file layer is a no-op there (the broker
//! and the agent do not share a budget on a desktop).

use std::path::{Path, PathBuf};
use std::sync::MutexGuard;
use std::time::Duration;

/// How long [`admission_lock`] waits for another process before it carries on
/// with the in-process lock alone (and says so).
pub const FILE_LOCK_WAIT: Duration = Duration::from_secs(120);

/// Held from "read what runs" to "the workload exists". Dropping it releases
/// both layers.
pub struct AdmissionGuard {
    _process: MutexGuard<'static, ()>,
    #[cfg(unix)]
    _file: Option<std::fs::File>,
}

/// The default lock file: `<zakuro home>/sandbox/admission.lock`.
pub fn default_lock_path() -> Option<PathBuf> {
    // Machine-wide: every node on this machine starts into the same engine.
    crate::credentials::machine_dir().map(|d| d.join("sandbox").join("admission.lock"))
}

fn process_lock() -> MutexGuard<'static, ()> {
    static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
    LOCK.lock().unwrap_or_else(|p| p.into_inner())
}

/// A second, independent lock for creating the sandbox and its golden image
/// (minutes, not milliseconds). Separate from the admission lock because
/// provisioning happens inside a start that already holds admission.
fn provision_process_lock() -> MutexGuard<'static, ()> {
    static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
    LOCK.lock().unwrap_or_else(|p| p.into_inner())
}

/// Held while the sandbox instance or the golden image is created or rebuilt,
/// so the agent and the broker do not both do it.
pub fn provision_lock(wait: Duration) -> Result<AdmissionGuard, String> {
    let path = crate::credentials::dir().map(|d| d.join("sandbox").join("provision.lock"));
    lock_with(provision_process_lock(), path.as_deref(), wait)
}

/// Take the lock at `path` (`None`: in-process only), waiting up to `wait`
/// for another process. `Err` says who is holding it up; nothing is held then.
pub fn lock_at(path: Option<&Path>, wait: Duration) -> Result<AdmissionGuard, String> {
    lock_with(process_lock(), path, wait)
}

fn lock_with(
    process: MutexGuard<'static, ()>,
    path: Option<&Path>,
    wait: Duration,
) -> Result<AdmissionGuard, String> {
    #[cfg(unix)]
    {
        let file = match path {
            None => None,
            Some(p) => Some(flock_exclusive(p, wait)?),
        };
        Ok(AdmissionGuard {
            _process: process,
            _file: file,
        })
    }
    #[cfg(not(unix))]
    {
        let _ = (path, wait);
        Ok(AdmissionGuard { _process: process })
    }
}

#[cfg(unix)]
fn flock_exclusive(path: &Path, wait: Duration) -> Result<std::fs::File, String> {
    use std::os::unix::io::AsRawFd;
    if let Some(dir) = path.parent() {
        std::fs::create_dir_all(dir).map_err(|e| format!("admission lock dir {dir:?}: {e}"))?;
    }
    let file = std::fs::OpenOptions::new()
        .create(true)
        .truncate(false)
        .write(true)
        .open(path)
        .map_err(|e| format!("admission lock {path:?}: {e}"))?;
    let deadline = std::time::Instant::now() + wait;
    loop {
        // SAFETY: `fd` is a valid open descriptor for the life of `file`.
        let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
        if rc == 0 {
            return Ok(file);
        }
        let err = std::io::Error::last_os_error();
        if err.raw_os_error() != Some(libc::EWOULDBLOCK) && err.raw_os_error() != Some(libc::EINTR)
        {
            return Err(format!("admission lock {path:?}: {err}"));
        }
        if std::time::Instant::now() >= deadline {
            return Err(format!(
                "admission lock {path:?} is held by another zc process (waited {}s)",
                wait.as_secs()
            ));
        }
        std::thread::sleep(Duration::from_millis(20));
    }
}

/// Serialises "sum what runs, then start" across this node: every zc process
/// that shares the budget takes the same file lock. When the file cannot be
/// taken (unwritable home, a holder that never lets go) the start still goes
/// ahead under the in-process lock, with a warning: refusing every start on a
/// node because of a lock file would be worse than the race it closes.
pub fn admission_lock() -> AdmissionGuard {
    match lock_at(default_lock_path().as_deref(), FILE_LOCK_WAIT) {
        Ok(g) => g,
        Err(e) => {
            eprintln!("  [SANDBOX] {e}; admitting under the in-process lock only");
            AdmissionGuard {
                _process: process_lock(),
                #[cfg(unix)]
                _file: None,
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn scratch(name: &str) -> PathBuf {
        let d = std::env::temp_dir().join(format!("zc-lock-{name}-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&d);
        d
    }

    #[cfg(unix)]
    #[test]
    fn the_file_lock_excludes_a_second_holder_until_it_is_dropped() {
        let dir = scratch("excl");
        let path = dir.join("sandbox").join("admission.lock");
        // `flock` locks belong to the open file description, so a second
        // descriptor in this process stands in for a second process. The
        // process mutex is bypassed on purpose: it would serialise first.
        let first = flock_exclusive(&path, Duration::from_millis(50)).unwrap();
        let e = flock_exclusive(&path, Duration::from_millis(60)).unwrap_err();
        assert!(e.contains("held by another zc process"), "{e}");
        drop(first);
        assert!(flock_exclusive(&path, Duration::from_millis(50)).is_ok());
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[cfg(unix)]
    #[test]
    fn the_lock_path_is_injectable_and_created() {
        let dir = scratch("inject");
        let path = dir.join("a").join("b.lock");
        let g = lock_at(Some(&path), Duration::from_millis(50)).unwrap();
        assert!(path.exists());
        drop(g);
        // No path: in-process only, always available.
        assert!(lock_at(None, Duration::from_millis(1)).is_ok());
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn an_unusable_lock_path_is_an_error_not_a_silent_skip() {
        let dir = scratch("bad");
        std::fs::create_dir_all(&dir).unwrap();
        // The lock file's "parent" is a regular file.
        let blocker = dir.join("file");
        std::fs::write(&blocker, "x").unwrap();
        let e = lock_at(
            Some(&blocker.join("admission.lock")),
            Duration::from_millis(1),
        );
        #[cfg(unix)]
        assert!(e.is_err());
        #[cfg(not(unix))]
        assert!(e.is_ok());
        let _ = std::fs::remove_dir_all(&dir);
    }
}