link-assistant-router 1.4.6

//! Owner-only, crash-durable file replacement and inter-process locking.

use std::fs::{self, OpenOptions, TryLockError};
use std::io::{self, Write};
use std::path::Path;

const ROLLBACK_SUFFIX: &str = ".router-rollback";
const COMMIT_SUFFIX: &str = ".router-commit";

fn sibling_with_suffix(path: &Path, suffix: &str) -> io::Result<std::path::PathBuf> {
    let name = path
        .file_name()
        .and_then(|name| name.to_str())
        .ok_or_else(|| io::Error::other("durable file name is not valid UTF-8"))?;
    Ok(path.with_file_name(format!(".{name}{suffix}")))
}

#[cfg(test)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum FaultPoint {
    AfterRename,
    RemoveRollback,
    SyncAfterRollbackRemoval,
    RemoveCommit,
    SyncAfterCommitRemoval,
    Unlock,
}

#[cfg(test)]
#[derive(Debug)]
struct InjectedFault {
    path: std::path::PathBuf,
    point: FaultPoint,
}

#[cfg(test)]
fn fault_slot() -> &'static std::sync::Mutex<Option<InjectedFault>> {
    static SLOT: std::sync::OnceLock<std::sync::Mutex<Option<InjectedFault>>> =
        std::sync::OnceLock::new();
    SLOT.get_or_init(|| std::sync::Mutex::new(None))
}

#[cfg(test)]
fn fault_serial() -> &'static std::sync::Mutex<()> {
    static SERIAL: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
    SERIAL.get_or_init(|| std::sync::Mutex::new(()))
}

#[cfg(test)]
pub(crate) struct FaultGuard {
    _serial: std::sync::MutexGuard<'static, ()>,
}

#[cfg(test)]
impl Drop for FaultGuard {
    fn drop(&mut self) {
        *fault_slot()
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
    }
}

#[cfg(test)]
pub(crate) fn inject_fault(path: &Path, point: FaultPoint) -> FaultGuard {
    let serial = fault_serial()
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    *fault_slot()
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(InjectedFault {
        path: path.to_path_buf(),
        point,
    });
    FaultGuard { _serial: serial }
}

#[cfg(test)]
fn fail_if_injected(path: &Path, point: FaultPoint) -> io::Result<()> {
    let mut slot = fault_slot()
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    let injected = slot
        .as_ref()
        .is_some_and(|fault| fault.path == path && fault.point == point);
    if injected {
        *slot = None;
    }
    drop(slot);
    if injected {
        return Err(io::Error::other(format!(
            "injected durable-file failure at {point:?}"
        )));
    }
    Ok(())
}

/// Describe a credential-write failure in terms an operator can act on.
///
/// A read-only mount is the common case — the deployment docs tell you to mount
/// the credential directory `:ro` — and it otherwise surfaces as a bare
/// `Read-only file system (os error 30)`, which does not say what to change
/// (issue #205).
#[must_use]
pub fn describe_write_failure(path: &Path, error: &io::Error) -> String {
    if error.kind() == io::ErrorKind::ReadOnlyFilesystem {
        return format!(
            "cannot write {}: the credential directory is mounted read-only. \
             Re-run without `:ro` to authorize, then restore it — serving and \
             token renewal do not need write access.",
            path.display()
        );
    }
    format!("could not create {}: {error}", path.display())
}

/// Replace `path` atomically, syncing both the file and its containing
/// directory so the rename survives power loss.
pub fn atomic_write_owner_only(path: &Path, contents: &[u8]) -> io::Result<()> {
    let parent = path
        .parent()
        .ok_or_else(|| io::Error::other("durable path has no parent directory"))?;
    fs::create_dir_all(parent)?;
    let name = path
        .file_name()
        .and_then(|name| name.to_str())
        .ok_or_else(|| io::Error::other("durable file name is not valid UTF-8"))?;
    let temporary = parent.join(format!(
        ".{name}.{}.{}.tmp",
        std::process::id(),
        uuid::Uuid::new_v4()
    ));
    let result = (|| {
        let mut options = OpenOptions::new();
        options.write(true).create_new(true);
        #[cfg(unix)]
        {
            use std::os::unix::fs::OpenOptionsExt as _;
            options.mode(0o600);
        }
        let mut file = options.open(&temporary)?;
        file.write_all(contents)?;
        file.sync_all()?;
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt as _;
            file.set_permissions(fs::Permissions::from_mode(0o600))?;
        }
        drop(file);
        fs::rename(&temporary, path)?;
        #[cfg(test)]
        fail_if_injected(path, FaultPoint::AfterRename)?;
        sync_directory(parent)
    })();
    if result.is_err() {
        let _ = fs::remove_file(&temporary);
    }
    result
}

/// Recover an interrupted transactional replacement of `path`.
///
/// A surviving commit marker makes the current primary authoritative. A
/// rollback document without that marker means the replacement had not
/// committed, so the prior bytes are restored before callers read the file.
pub fn recover_transactional_write(path: &Path) -> io::Result<()> {
    let rollback = sibling_with_suffix(path, ROLLBACK_SUFFIX)?;
    let commit = sibling_with_suffix(path, COMMIT_SUFFIX)?;
    if commit.exists() {
        return cleanup_committed_transaction(path, &rollback, &commit);
    }
    let rollback_document = match fs::read(&rollback) {
        Ok(prior) => prior,
        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
        Err(error) => return Err(error),
    };
    match rollback_document.split_first() {
        Some((0, _)) => match fs::remove_file(path) {
            Ok(()) => {}
            Err(error) if error.kind() == io::ErrorKind::NotFound => {}
            Err(error) => return Err(error),
        },
        Some((1, prior)) => atomic_write_owner_only(path, prior)?,
        _ => return Err(io::Error::other("invalid transactional rollback document")),
    }
    fs::remove_file(&rollback)?;
    if let Some(parent) = path.parent() {
        sync_directory(parent)?;
    }
    Ok(())
}

fn remove_file_if_present(path: &Path) -> io::Result<()> {
    match fs::remove_file(path) {
        Ok(()) => Ok(()),
        Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
        Err(error) => Err(error),
    }
}

/// Remove committed transaction sidecars without ever making the rollback
/// document authoritative again.
///
/// The rollback is removed and that removal is made durable before the commit
/// marker can be removed. Every failure before that point therefore leaves the
/// commit marker in place, so another recovery pass still keeps the new
/// primary. A failure syncing the final marker removal is safe too: either the
/// marker reappears after a crash and cleanup repeats, or both sidecars stay
/// absent and the primary remains authoritative.
fn cleanup_committed_transaction(path: &Path, rollback: &Path, commit: &Path) -> io::Result<()> {
    #[cfg(test)]
    fail_if_injected(path, FaultPoint::RemoveRollback)?;
    remove_file_if_present(rollback)?;
    if let Some(parent) = path.parent() {
        #[cfg(test)]
        fail_if_injected(path, FaultPoint::SyncAfterRollbackRemoval)?;
        sync_directory(parent)?;
    }

    #[cfg(test)]
    fail_if_injected(path, FaultPoint::RemoveCommit)?;
    remove_file_if_present(commit)?;
    if let Some(parent) = path.parent() {
        #[cfg(test)]
        fail_if_injected(path, FaultPoint::SyncAfterCommitRemoval)?;
        sync_directory(parent)?;
    }
    Ok(())
}

/// Replace `path` as a recoverable transaction.
///
/// The prior bytes are made durable before the new primary is written. A
/// durable commit marker is then the boundary between rollback and commit, so
/// a crash or a late directory-sync error never leaves callers guessing which
/// document is authoritative.
pub fn transactional_write_owner_only(path: &Path, contents: &[u8]) -> io::Result<()> {
    recover_transactional_write(path)?;
    let rollback = sibling_with_suffix(path, ROLLBACK_SUFFIX)?;
    let commit = sibling_with_suffix(path, COMMIT_SUFFIX)?;
    let rollback_document = match fs::read(path) {
        Ok(prior) => {
            let mut rollback = Vec::with_capacity(prior.len() + 1);
            rollback.push(1);
            rollback.extend(prior);
            rollback
        }
        Err(error) if error.kind() == io::ErrorKind::NotFound => vec![0],
        Err(error) => return Err(error),
    };
    atomic_write_owner_only(&rollback, &rollback_document)?;
    if let Err(error) = atomic_write_owner_only(path, contents) {
        return match recover_transactional_write(path) {
            Ok(()) => Err(error),
            Err(recovery) => Err(io::Error::other(format!(
                "replacement failed ({error}); rollback remains recoverable but immediate restore failed ({recovery})"
            ))),
        };
    }
    if let Err(error) = atomic_write_owner_only(&commit, b"committed\n") {
        // The marker may already have been renamed before its directory sync
        // failed. It did not cross the durable commit boundary, so remove it
        // before invoking ordinary rollback recovery.
        let _ = fs::remove_file(&commit);
        if let Some(parent) = path.parent() {
            let _ = sync_directory(parent);
        }
        return match recover_transactional_write(path) {
            Ok(()) => Err(error),
            Err(recovery) => Err(io::Error::other(format!(
                "commit failed ({error}); rollback remains recoverable but immediate restore failed ({recovery})"
            ))),
        };
    }

    // Once the marker is durable, cleanup is not part of the commit result.
    // A restart seeing it keeps the new primary and finishes the same cleanup.
    let _ = cleanup_committed_transaction(path, &rollback, &commit);
    Ok(())
}

/// Execute a state mutation while holding an owner-only advisory lock shared
/// by every router process using the same data directory.
pub fn with_exclusive_lock<T, E>(
    path: &Path,
    operation: impl FnOnce() -> Result<T, E>,
) -> Result<T, E>
where
    E: From<io::Error>,
{
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent).map_err(E::from)?;
    }
    let mut options = OpenOptions::new();
    options.read(true).write(true).create(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt as _;
        options.mode(0o600);
    }
    let lock = options.open(path).map_err(E::from)?;
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt as _;
        lock.set_permissions(fs::Permissions::from_mode(0o600))
            .map_err(E::from)?;
    }
    lock.lock().map_err(E::from)?;
    let result = operation();
    let unlock = lock.unlock();
    #[cfg(test)]
    let unlock = unlock.and_then(|()| fail_if_injected(path, FaultPoint::Unlock));
    match (result, unlock) {
        (Err(error), _) => Err(error),
        // Closing the descriptor releases the lock as well. A late explicit
        // unlock failure cannot undo a completed durable operation and must
        // not turn its public result into an ambiguous failure.
        (Ok(value), _) => Ok(value),
    }
}

/// Execute a read while holding a *shared* advisory lock on the same file.
///
/// A listing is a read, and a read that takes the exclusive lock serialises
/// itself against the request path: `try_consume_request` runs per proxied
/// request and wants the same lock, so one slow listing queues live traffic
/// behind it (issue #351). A shared lock lets concurrent readers proceed
/// together and still excludes writers.
pub fn with_shared_lock<T, E>(path: &Path, operation: impl FnOnce() -> Result<T, E>) -> Result<T, E>
where
    E: From<io::Error>,
{
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent).map_err(E::from)?;
    }
    let mut options = OpenOptions::new();
    options.read(true).write(true).create(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt as _;
        options.mode(0o600);
    }
    let lock = options.open(path).map_err(E::from)?;
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt as _;
        lock.set_permissions(fs::Permissions::from_mode(0o600))
            .map_err(E::from)?;
    }
    lock.lock_shared().map_err(E::from)?;
    let result = operation();
    let unlock = lock.unlock();
    match (result, unlock) {
        (Err(error), _) => Err(error),
        (Ok(_), Err(error)) => Err(E::from(error)),
        (Ok(value), Ok(())) => Ok(value),
    }
}

/// An exclusive advisory lock held for as long as the guard lives.
///
/// Returned by [`lock_exclusive_async`] so an `async` critical section — a
/// token exchange over the network — can serialise against other holders
/// without blocking a runtime worker on `flock`.
#[derive(Debug)]
pub struct FileLockGuard {
    file: fs::File,
    path: std::path::PathBuf,
}

impl FileLockGuard {
    /// Path of the lock file this guard holds.
    #[must_use]
    pub fn path(&self) -> &Path {
        &self.path
    }
}

impl Drop for FileLockGuard {
    fn drop(&mut self) {
        // Best effort: the lock is released by closing the descriptor anyway.
        let _ = self.file.unlock();
    }
}

/// How often a contended lock is re-tried while waiting.
const LOCK_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(20);

/// Acquire an exclusive advisory lock on `path`, waiting up to `timeout`.
///
/// `flock` has no async form, so the lock is polled rather than waited on: a
/// blocking [`std::fs::File::lock`] inside an `async fn` would park a runtime
/// worker for as long as another process holds it, which for a credential
/// refresh can be a full network round trip.
///
/// Contention is [`TryLockError::WouldBlock`], a variant of its own rather than
/// a platform errno to classify: `EWOULDBLOCK` maps to
/// [`io::ErrorKind::WouldBlock`] on unix, but Windows answers
/// `ERROR_LOCK_VIOLATION`, which maps to nothing in particular. Reading
/// contention as a broken lock would make the waiter proceed *unlocked*, and
/// two holders of one credential would then spend the same refresh token twice
/// (issue #239).
///
/// # Errors
///
/// Returns [`io::ErrorKind::WouldBlock`] when the lock is still held after
/// `timeout`, or the underlying error when the lock file cannot be opened.
pub async fn lock_exclusive_async(
    path: &Path,
    timeout: std::time::Duration,
) -> io::Result<FileLockGuard> {
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    let mut options = OpenOptions::new();
    options.read(true).write(true).create(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt as _;
        options.mode(0o600);
    }
    let file = options.open(path)?;
    let mut waited = std::time::Duration::ZERO;
    loop {
        match file.try_lock() {
            Ok(()) => {
                return Ok(FileLockGuard {
                    file,
                    path: path.to_path_buf(),
                });
            }
            Err(TryLockError::WouldBlock) => {
                if waited >= timeout {
                    return Err(io::Error::new(
                        io::ErrorKind::WouldBlock,
                        format!("timed out waiting for the lock on {}", path.display()),
                    ));
                }
                tokio::time::sleep(LOCK_POLL_INTERVAL).await;
                waited = waited.saturating_add(LOCK_POLL_INTERVAL);
            }
            Err(TryLockError::Error(error)) => return Err(error),
        }
    }
}

/// Sync a directory entry update on platforms that support directory fsync.
pub fn sync_directory(path: &Path) -> io::Result<()> {
    #[cfg(unix)]
    {
        fs::File::open(path)?.sync_all()
    }
    #[cfg(not(unix))]
    {
        let _ = path;
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn durable_write_is_owner_only_and_leaves_no_temporary_file() {
        let directory = tempfile::tempdir().unwrap();
        let path = directory.path().join("state.json");
        atomic_write_owner_only(&path, b"one").unwrap();
        atomic_write_owner_only(&path, b"two").unwrap();
        assert_eq!(fs::read(&path).unwrap(), b"two");
        assert_eq!(fs::read_dir(directory.path()).unwrap().count(), 1);
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt as _;
            assert_eq!(
                fs::metadata(path).unwrap().permissions().mode() & 0o777,
                0o600
            );
        }
    }

    #[test]
    fn transactional_failure_after_primary_rename_restores_previous_bytes() {
        let directory = tempfile::tempdir().unwrap();
        let path = directory.path().join("state.json");
        atomic_write_owner_only(&path, b"old").unwrap();
        let _fault = inject_fault(&path, FaultPoint::AfterRename);

        transactional_write_owner_only(&path, b"new").expect_err("late write must fail");

        assert_eq!(fs::read(&path).unwrap(), b"old");
        assert!(
            !sibling_with_suffix(&path, ROLLBACK_SUFFIX)
                .unwrap()
                .exists()
        );
        assert!(!sibling_with_suffix(&path, COMMIT_SUFFIX).unwrap().exists());
    }

    #[test]
    fn transactional_failure_while_committing_restores_previous_bytes() {
        let directory = tempfile::tempdir().unwrap();
        let path = directory.path().join("state.json");
        atomic_write_owner_only(&path, b"old").unwrap();
        let commit = sibling_with_suffix(&path, COMMIT_SUFFIX).unwrap();
        let _fault = inject_fault(&commit, FaultPoint::AfterRename);

        transactional_write_owner_only(&path, b"new").expect_err("commit must fail");

        assert_eq!(fs::read(&path).unwrap(), b"old");
        assert!(
            !sibling_with_suffix(&path, ROLLBACK_SUFFIX)
                .unwrap()
                .exists()
        );
        assert!(!commit.exists());
    }

    #[test]
    fn restart_recovery_obeys_the_durable_commit_marker() {
        for committed in [false, true] {
            let directory = tempfile::tempdir().unwrap();
            let path = directory.path().join("state.json");
            let rollback = sibling_with_suffix(&path, ROLLBACK_SUFFIX).unwrap();
            let commit = sibling_with_suffix(&path, COMMIT_SUFFIX).unwrap();
            atomic_write_owner_only(&path, b"new").unwrap();
            atomic_write_owner_only(&rollback, b"\x01old").unwrap();
            if committed {
                atomic_write_owner_only(&commit, b"committed\n").unwrap();
            }

            recover_transactional_write(&path).unwrap();

            assert_eq!(
                fs::read(&path).unwrap(),
                if committed { b"new" } else { b"old" }
            );
            assert!(!rollback.exists());
            assert!(!commit.exists());
        }
    }

    #[test]
    fn committed_cleanup_failures_never_make_rollback_authoritative() {
        for point in [
            FaultPoint::RemoveRollback,
            FaultPoint::SyncAfterRollbackRemoval,
            FaultPoint::RemoveCommit,
            FaultPoint::SyncAfterCommitRemoval,
        ] {
            let directory = tempfile::tempdir().unwrap();
            let path = directory.path().join("state.json");
            let rollback = sibling_with_suffix(&path, ROLLBACK_SUFFIX).unwrap();
            let commit = sibling_with_suffix(&path, COMMIT_SUFFIX).unwrap();
            atomic_write_owner_only(&path, b"new").unwrap();
            atomic_write_owner_only(&rollback, b"\x01old").unwrap();
            atomic_write_owner_only(&commit, b"committed\n").unwrap();
            let fault = inject_fault(&path, point);

            recover_transactional_write(&path).expect_err("cleanup fault must be reported");
            assert_eq!(fs::read(&path).unwrap(), b"new", "fault at {point:?}");
            if matches!(
                point,
                FaultPoint::RemoveRollback
                    | FaultPoint::SyncAfterRollbackRemoval
                    | FaultPoint::RemoveCommit
            ) {
                assert!(commit.exists(), "fault at {point:?}");
            }
            drop(fault);

            recover_transactional_write(&path).unwrap();
            recover_transactional_write(&path).unwrap();
            assert_eq!(fs::read(&path).unwrap(), b"new", "fault at {point:?}");
            assert!(!rollback.exists(), "fault at {point:?}");
            assert!(!commit.exists(), "fault at {point:?}");
        }
    }

    #[test]
    fn late_unlock_failure_does_not_reclassify_a_completed_operation() {
        let directory = tempfile::tempdir().unwrap();
        let lock_path = directory.path().join("state.lock");
        let _fault = inject_fault(&lock_path, FaultPoint::Unlock);

        let result = with_exclusive_lock::<_, io::Error>(&lock_path, || Ok(7)).unwrap();

        assert_eq!(result, 7);
    }

    /// Contention has to be recognised on every platform, not only where it
    /// happens to map onto `WouldBlock`.
    ///
    /// Windows answers a contended `LockFileEx` with `ERROR_LOCK_VIOLATION`,
    /// which `io::ErrorKind` does not classify; reading that as a broken lock
    /// makes the waiter proceed *unlocked*, and two holders of one credential
    /// then spend the same refresh token twice — exactly the race the lock
    /// exists to prevent (issue #239). The standard library answers with
    /// [`TryLockError::WouldBlock`] on every platform, so this asserts the
    /// variant rather than an errno.
    ///
    /// Advisory locks belong to the *open file description*, so two separate
    /// handles inside one process contend exactly as two processes do — which
    /// is what lets this run without spawning one.
    #[tokio::test]
    async fn contention_is_told_apart_from_a_lock_that_cannot_work() {
        let directory = tempfile::tempdir().unwrap();
        let path = directory.path().join("credential.lock");

        let holder = OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .truncate(false)
            .open(&path)
            .unwrap();
        holder.lock().unwrap();

        let waiter = OpenOptions::new()
            .read(true)
            .write(true)
            .open(&path)
            .unwrap();
        assert!(
            matches!(waiter.try_lock(), Err(TryLockError::WouldBlock)),
            "a contended lock must report WouldBlock, not a platform errno"
        );

        // And the polling waiter must read that as "held", not as "broken".
        let refused = lock_exclusive_async(&path, std::time::Duration::from_millis(60)).await;
        let error = refused.expect_err("the lock was held");
        assert_eq!(error.kind(), io::ErrorKind::WouldBlock);

        holder.unlock().unwrap();
        assert!(
            lock_exclusive_async(&path, std::time::Duration::from_millis(60))
                .await
                .is_ok(),
            "the lock must be available once the holder releases it"
        );
    }

    #[tokio::test]
    async fn lock_open_errors_are_returned_to_the_caller() {
        let directory = tempfile::tempdir().unwrap();
        let blocking_file = directory.path().join("not-a-directory");
        fs::write(&blocking_file, b"occupied").unwrap();

        let error = lock_exclusive_async(
            &blocking_file.join("credential.lock"),
            std::time::Duration::from_millis(60),
        )
        .await
        .expect_err("a lock below a regular file cannot be opened");

        assert_ne!(error.kind(), io::ErrorKind::WouldBlock);
        assert!(error.raw_os_error().is_some(), "{error}");
    }

    /// Two holders of one credential must serialise, and a holder that cannot
    /// get in must give up rather than wait forever: a stale lock must never be
    /// able to wedge token renewal (issue #239).
    ///
    /// Linux-only because the contending holder is `flock(1)`, which macOS does
    /// not ship; the code under test is the same on both.
    #[cfg(target_os = "linux")]
    #[tokio::test]
    async fn an_exclusive_lock_excludes_and_then_gives_up() {
        use std::os::unix::fs::PermissionsExt as _;

        let directory = tempfile::tempdir().unwrap();
        let path = directory.path().join("nested").join("credential.lock");
        let taken = directory.path().join("taken");
        {
            let guard = lock_exclusive_async(&path, std::time::Duration::from_secs(1))
                .await
                .expect("first holder");
            assert_eq!(guard.path(), path);
            assert_eq!(
                fs::metadata(&path).unwrap().permissions().mode() & 0o777,
                0o600
            );
        }

        // Contention is exercised from another process: two lock attempts on
        // the same descriptor within one process would not exclude each other.
        let mut holder = std::process::Command::new("sh")
            .arg("-c")
            .arg(format!(
                "exec 9>>'{}'; flock 9 && touch '{}' && sleep 5",
                path.display(),
                taken.display()
            ))
            .spawn()
            .expect("spawn the competing holder");
        for _ in 0..200 {
            if taken.exists() {
                break;
            }
            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
        }
        assert!(taken.exists(), "the competing holder never took the lock");

        let refused = lock_exclusive_async(&path, std::time::Duration::from_millis(60)).await;
        let error = refused.expect_err("the lock was held by another process");
        assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
        assert!(error.to_string().contains("credential.lock"), "{error}");

        let _ = holder.kill();
        let _ = holder.wait();
    }

    /// A read-only mount is the common cause of a failed credential write, and
    /// the bare `errno` does not say what to change (issue #205).
    #[test]
    fn a_read_only_mount_is_named_as_the_cause() {
        let message = describe_write_failure(
            Path::new("/data/claude/.credentials.json"),
            &io::Error::from(io::ErrorKind::ReadOnlyFilesystem),
        );
        assert!(
            message.contains("/data/claude/.credentials.json"),
            "{message}"
        );
        assert!(message.contains("read-only"), "{message}");
        // The remedy must be actionable, and say the cost of applying it.
        assert!(message.contains(":ro"), "{message}");
        assert!(
            message.contains("token renewal do not need write"),
            "{message}"
        );
    }

    #[test]
    fn other_write_failures_keep_the_underlying_error() {
        let message = describe_write_failure(
            Path::new("/data/x.json"),
            &io::Error::from(io::ErrorKind::PermissionDenied),
        );
        assert!(message.contains("/data/x.json"), "{message}");
        assert!(!message.contains("read-only"), "{message}");
    }
}