car-registry 0.55.0

//! Crash-durable, prove-first reboot cutover transaction.
//!
//! This module does not install, load, unload, or signal a launch agent. It
//! only replaces one caller-supplied file path and retains its original bytes
//! beside it. The transaction internals compile only on Unix; other targets
//! retain the public surface and return
//! [`CutoverError::UnsupportedPlatform`](crate::reboot_cutover::CutoverError::UnsupportedPlatform)
//! before creating transaction state. A future installer must call
//! [`RebootCutover::recover_boot`](crate::reboot_cutover::RebootCutover::recover_boot)
//! before it lets the operating system consume the canonical path.

use serde::{Deserialize, Serialize};
#[cfg(unix)]
use sha2::{Digest, Sha256};
#[cfg(unix)]
use std::fs::{self, File, OpenOptions};
use std::io;
#[cfg(unix)]
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
#[cfg(unix)]
use std::time::{SystemTime, UNIX_EPOCH};
#[cfg(unix)]
use uuid::Uuid;

#[cfg(unix)]
const JOURNAL_LIMIT: u64 = 64 * 1024;
#[cfg(unix)]
const JOURNAL_VERSION: u8 = 1;
#[cfg(unix)]
const JOURNAL_TEMP: &str = "journal.tmp";
#[cfg(unix)]
const JOURNAL_ZERO: &str = "journal.0";
#[cfg(unix)]
const JOURNAL_ONE: &str = "journal.1";
#[cfg(unix)]
const BACKUP_TEMP: &str = "original.tmp";
#[cfg(unix)]
const CANDIDATE: &str = "candidate";
#[cfg(unix)]
const ROLLBACK_TEMP: &str = "rollback.tmp";

/// Ownership which every transaction artifact and published generation must
/// retain. The mode contains only the Unix permission bits.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct CutoverOwnership {
    pub uid: u32,
    pub gid: u32,
    pub mode: u32,
}

impl CutoverOwnership {
    /// Read the ownership contract from an existing regular, single-link file.
    #[cfg(unix)]
    pub fn from_path(path: &Path) -> Result<Self, CutoverError> {
        use std::os::unix::fs::MetadataExt;

        let metadata = fs::symlink_metadata(path)?;
        if !metadata.file_type().is_file() || metadata.nlink() != 1 {
            return Err(CutoverError::UnsafeArtifact(path.to_path_buf()));
        }
        Ok(Self {
            uid: metadata.uid(),
            gid: metadata.gid(),
            mode: metadata.mode() & 0o7777,
        })
    }

    #[cfg(not(unix))]
    pub fn from_path(_path: &Path) -> Result<Self, CutoverError> {
        Err(CutoverError::UnsupportedPlatform)
    }
}

/// Durable transaction state. Each state owns the artifacts named in the
/// transaction document; no unjournaled artifact is ever adopted.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CutoverState {
    Started,
    Prepared,
    Published,
    Committed,
    RollbackPrepared,
    RolledBack,
}

/// Result of boot recovery, apply, or rollback.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CutoverOutcome {
    NoTransaction,
    Committed,
    RolledBack,
}

/// Every mutation boundary exposed to the crash-simulation harness.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CutoverBoundary {
    OwnerLeaseWritten,
    OwnerLeaseSynced,
    OwnerLeaseDirectorySynced,
    TransactionDirectoryCreated,
    TransactionDirectorySynced,
    TransactionParentDirectorySynced,
    JournalTempWritten(CutoverState),
    JournalTempSynced(CutoverState),
    JournalPublished(CutoverState),
    JournalDirectorySynced(CutoverState),
    BackupTempWritten,
    BackupTempSynced,
    BackupTempDirectorySynced,
    BackupPublished,
    BackupParentDirectorySynced,
    BackupTempRemoved,
    BackupCleanupDirectorySynced,
    CandidateWritten,
    CandidateSynced,
    CandidateDirectorySynced,
    CanonicalPublished,
    CanonicalSourceDirectorySynced,
    CanonicalParentDirectorySynced,
    RollbackTempWritten,
    RollbackTempSynced,
    RollbackTempDirectorySynced,
    RollbackCanonicalPublished,
    RollbackSourceDirectorySynced,
    RollbackParentDirectorySynced,
}

impl CutoverBoundary {
    pub fn name(self) -> String {
        let state = |state: CutoverState| match state {
            CutoverState::Started => "started",
            CutoverState::Prepared => "prepared",
            CutoverState::Published => "published",
            CutoverState::Committed => "committed",
            CutoverState::RollbackPrepared => "rollback_prepared",
            CutoverState::RolledBack => "rolled_back",
        };
        match self {
            Self::OwnerLeaseWritten => "owner_lease_written".into(),
            Self::OwnerLeaseSynced => "owner_lease_synced".into(),
            Self::OwnerLeaseDirectorySynced => "owner_lease_directory_synced".into(),
            Self::TransactionDirectoryCreated => "transaction_directory_created".into(),
            Self::TransactionDirectorySynced => "transaction_directory_synced".into(),
            Self::TransactionParentDirectorySynced => "transaction_parent_directory_synced".into(),
            Self::JournalTempWritten(s) => format!("journal_{}_temp_written", state(s)),
            Self::JournalTempSynced(s) => format!("journal_{}_temp_synced", state(s)),
            Self::JournalPublished(s) => format!("journal_{}_published", state(s)),
            Self::JournalDirectorySynced(s) => {
                format!("journal_{}_directory_synced", state(s))
            }
            Self::BackupTempWritten => "backup_temp_written".into(),
            Self::BackupTempSynced => "backup_temp_synced".into(),
            Self::BackupTempDirectorySynced => "backup_temp_directory_synced".into(),
            Self::BackupPublished => "backup_published".into(),
            Self::BackupParentDirectorySynced => "backup_parent_directory_synced".into(),
            Self::BackupTempRemoved => "backup_temp_removed".into(),
            Self::BackupCleanupDirectorySynced => "backup_cleanup_directory_synced".into(),
            Self::CandidateWritten => "candidate_written".into(),
            Self::CandidateSynced => "candidate_synced".into(),
            Self::CandidateDirectorySynced => "candidate_directory_synced".into(),
            Self::CanonicalPublished => "canonical_published".into(),
            Self::CanonicalSourceDirectorySynced => "canonical_source_directory_synced".into(),
            Self::CanonicalParentDirectorySynced => "canonical_parent_directory_synced".into(),
            Self::RollbackTempWritten => "rollback_temp_written".into(),
            Self::RollbackTempSynced => "rollback_temp_synced".into(),
            Self::RollbackTempDirectorySynced => "rollback_temp_directory_synced".into(),
            Self::RollbackCanonicalPublished => "rollback_canonical_published".into(),
            Self::RollbackSourceDirectorySynced => "rollback_source_directory_synced".into(),
            Self::RollbackParentDirectorySynced => "rollback_parent_directory_synced".into(),
        }
    }

    #[cfg(all(test, unix))]
    fn closes_durability_epoch(self) -> bool {
        matches!(
            self,
            Self::OwnerLeaseDirectorySynced
                | Self::TransactionParentDirectorySynced
                | Self::JournalDirectorySynced(_)
                | Self::BackupTempDirectorySynced
                | Self::BackupParentDirectorySynced
                | Self::BackupCleanupDirectorySynced
                | Self::CandidateDirectorySynced
                | Self::CanonicalParentDirectorySynced
                | Self::RollbackTempDirectorySynced
                | Self::RollbackParentDirectorySynced
        )
    }
}

#[derive(Debug, thiserror::Error)]
pub enum CutoverError {
    #[error("reboot cutover is only supported on Unix hosts")]
    UnsupportedPlatform,
    #[error("cutover I/O error: {0}")]
    Io(#[from] io::Error),
    #[error("another cutover owner holds the transaction lease")]
    ActiveOwner,
    #[error("rollback requires an owned cutover transaction")]
    NoTransactionForRollback,
    #[error("cutover artifact is not a regular, private, single-link file: {0}")]
    UnsafeArtifact(PathBuf),
    #[error("cutover artifact ownership does not match the transaction: {0}")]
    OwnershipMismatch(PathBuf),
    #[error("an unowned backup already occupies the cutover backup path: {0}")]
    StaleBackupCollision(PathBuf),
    #[error("the owned backup diverges from the journaled original: {0}")]
    OwnedBackupCollision(PathBuf),
    #[error("transaction journal is torn and has no valid durable slot")]
    TornJournal,
    #[error("transaction journal slots disagree about ownership")]
    JournalOwnershipConflict,
    #[error("the candidate supplied for recovery differs from the journaled candidate")]
    CandidateChanged,
    #[error("the canonical file changed outside the owned transaction: {0}")]
    CanonicalCollision(PathBuf),
    #[error("transaction directory contains unjournaled artifacts")]
    UnjournaledArtifacts,
    #[error("invalid transaction state transition from {from:?} to {to:?}")]
    InvalidTransition {
        from: CutoverState,
        to: CutoverState,
    },
    #[error("cutover observer failed: {0}")]
    Observer(String),
}

#[cfg(unix)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct JournalRecord {
    version: u8,
    transaction_id: String,
    sequence: u64,
    state: CutoverState,
    ownership: CutoverOwnership,
    original_sha256: String,
    candidate_sha256: String,
}

#[cfg(unix)]
#[derive(Debug, Serialize, Deserialize)]
struct JournalEnvelope {
    record: JournalRecord,
    checksum_sha256: String,
}

#[cfg(unix)]
#[derive(Debug)]
struct Lease(File);

#[cfg(unix)]
impl Drop for Lease {
    fn drop(&mut self) {
        let _ = self.0.unlock();
    }
}

#[cfg(unix)]
#[derive(Debug)]
struct Paths {
    canonical: PathBuf,
    parent: PathBuf,
    backup: PathBuf,
    transaction: PathBuf,
    lease: PathBuf,
}

#[cfg(unix)]
impl Paths {
    fn new(canonical: &Path) -> Result<Self, CutoverError> {
        let parent = canonical
            .parent()
            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing parent"))?
            .to_path_buf();
        let name = canonical
            .file_name()
            .and_then(|name| name.to_str())
            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid file name"))?;
        Ok(Self {
            canonical: canonical.to_path_buf(),
            parent: parent.clone(),
            backup: parent.join(format!("{name}.car-reboot-cutover.original")),
            transaction: parent.join(format!(".{name}.car-reboot-cutover.txn")),
            lease: parent.join(format!(".{name}.car-reboot-cutover.lock")),
        })
    }

    fn transaction_path(&self, name: &str) -> PathBuf {
        self.transaction.join(name)
    }
}

/// A cutover transaction for one canonical LaunchAgent path.
///
/// The caller supplies the exact candidate bytes and the expected owner. The
/// transaction derives every sidecar path from the canonical file name, never
/// touches launchd, and never traverses outside the canonical file's parent.
pub struct RebootCutover {
    #[cfg(unix)]
    paths: Paths,
    #[cfg(unix)]
    candidate: Vec<u8>,
    #[cfg(unix)]
    ownership: CutoverOwnership,
    // Keep the public type opaque on unsupported targets without compiling the
    // Unix transaction implementation there.
    #[cfg(not(unix))]
    _unsupported: (),
}

impl RebootCutover {
    pub fn new(
        canonical: impl AsRef<Path>,
        candidate: impl Into<Vec<u8>>,
        ownership: CutoverOwnership,
    ) -> Result<Self, CutoverError> {
        #[cfg(not(unix))]
        {
            let _ = (canonical, candidate, ownership);
            Err(CutoverError::UnsupportedPlatform)
        }

        #[cfg(unix)]
        {
            Ok(Self {
                paths: Paths::new(canonical.as_ref())?,
                candidate: candidate.into(),
                ownership,
            })
        }
    }

    /// Recover a durable transaction before boot consumes the canonical path.
    /// No journal means no transaction; the method never adopts a backup that
    /// lacks a valid journal owner.
    pub fn recover_boot(&self) -> Result<CutoverOutcome, CutoverError> {
        self.run(Operation::Recover, &mut |_| Ok(()))
    }

    /// Start a new cutover or recover the existing owned transaction first.
    pub fn apply(&self) -> Result<CutoverOutcome, CutoverError> {
        self.run(Operation::Apply, &mut |_| Ok(()))
    }

    /// Restore the exact journaled original bytes. Repeating rollback returns
    /// the same terminal result without replacing the canonical file again.
    pub fn rollback(&self) -> Result<CutoverOutcome, CutoverError> {
        self.run(Operation::Rollback, &mut |_| Ok(()))
    }

    fn run(
        &self,
        operation: Operation,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<CutoverOutcome, CutoverError> {
        #[cfg(not(unix))]
        {
            let _ = (operation, observer);
            Err(CutoverError::UnsupportedPlatform)
        }

        #[cfg(unix)]
        {
            fs::create_dir_all(&self.paths.parent)?;
            let _lease = self.acquire_lease(observer)?;
            let loaded = self.load_journal()?;
            let mut record = match loaded {
                Some(record) => {
                    if record.ownership != self.ownership {
                        return Err(CutoverError::JournalOwnershipConflict);
                    }
                    if record.candidate_sha256 != digest(&self.candidate) {
                        return Err(CutoverError::CandidateChanged);
                    }
                    record
                }
                None => {
                    let no_transaction = self.no_transaction_outcome()?;
                    match operation {
                        Operation::Recover => return Ok(no_transaction),
                        Operation::Apply => self.start_transaction(observer)?,
                        Operation::Rollback => return Err(CutoverError::NoTransactionForRollback),
                    }
                }
            };

            match operation {
                Operation::Recover | Operation::Apply => self.drive_apply(&mut record, observer),
                Operation::Rollback => {
                    if !matches!(
                        record.state,
                        CutoverState::Committed | CutoverState::RolledBack
                    ) {
                        self.drive_apply(&mut record, observer)?;
                    }
                    self.drive_rollback(&mut record, observer)
                }
            }
        }
    }
}

#[cfg(unix)]
impl RebootCutover {
    fn no_transaction_outcome(&self) -> Result<CutoverOutcome, CutoverError> {
        if self.paths.backup.exists() {
            return Err(CutoverError::StaleBackupCollision(
                self.paths.backup.clone(),
            ));
        }
        if !self.paths.transaction.exists() {
            return Ok(CutoverOutcome::NoTransaction);
        }
        let mut entries = fs::read_dir(&self.paths.transaction)?;
        while let Some(entry) = entries.next().transpose()? {
            let name = entry.file_name();
            if name != JOURNAL_TEMP {
                return Err(CutoverError::UnjournaledArtifacts);
            }
        }
        Ok(CutoverOutcome::NoTransaction)
    }

    fn acquire_lease(
        &self,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<Lease, CutoverError> {
        let existed = self.paths.lease.exists();
        let mut options = OpenOptions::new();
        options.read(true).write(true).create(true).truncate(false);
        #[cfg(unix)]
        {
            use std::os::unix::fs::OpenOptionsExt;
            options.custom_flags(libc::O_NOFOLLOW);
        }
        let mut lease = options.open(&self.paths.lease)?;
        if existed {
            validate_artifact(&self.paths.lease, self.ownership, false)?;
        } else {
            apply_ownership(&lease, self.ownership)?;
        }
        validate_open_file(&lease, &self.paths.lease, self.ownership, 1)?;
        match lease.try_lock() {
            Ok(()) => {}
            Err(fs::TryLockError::WouldBlock) => return Err(CutoverError::ActiveOwner),
            Err(fs::TryLockError::Error(error)) => return Err(error.into()),
        }
        lease.set_len(0)?;
        lease.seek(SeekFrom::Start(0))?;
        let acquired = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap_or_default()
            .as_secs();
        write!(
            lease,
            "version={JOURNAL_VERSION}\npid={}\nacquired_unix={acquired}\n",
            std::process::id()
        )?;
        observer(CutoverBoundary::OwnerLeaseWritten)?;
        full_sync(&lease)?;
        observer(CutoverBoundary::OwnerLeaseSynced)?;
        sync_directory(&self.paths.parent)?;
        observer(CutoverBoundary::OwnerLeaseDirectorySynced)?;
        Ok(Lease(lease))
    }

    fn start_transaction(
        &self,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<JournalRecord, CutoverError> {
        if self.paths.backup.exists() {
            return Err(CutoverError::StaleBackupCollision(
                self.paths.backup.clone(),
            ));
        }
        self.ensure_transaction_directory(observer)?;
        let canonical = read_owned(&self.paths.canonical, self.ownership)?;
        let record = JournalRecord {
            version: JOURNAL_VERSION,
            transaction_id: Uuid::new_v4().to_string(),
            sequence: 0,
            state: CutoverState::Started,
            ownership: self.ownership,
            original_sha256: digest(&canonical),
            candidate_sha256: digest(&self.candidate),
        };
        self.persist_record(&record, observer)?;
        Ok(record)
    }

    fn ensure_transaction_directory(
        &self,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        if self.paths.transaction.exists() {
            validate_directory(&self.paths.transaction, self.ownership)?;
            // A prior process may have died after mkdir but before either
            // directory sync. Stabilize both names before publishing state.
            sync_directory(&self.paths.transaction)?;
            sync_directory(&self.paths.parent)?;
            return Ok(());
        }
        fs::create_dir(&self.paths.transaction)?;
        set_directory_ownership(&self.paths.transaction, self.ownership)?;
        observer(CutoverBoundary::TransactionDirectoryCreated)?;
        sync_directory(&self.paths.transaction)?;
        observer(CutoverBoundary::TransactionDirectorySynced)?;
        sync_directory(&self.paths.parent)?;
        observer(CutoverBoundary::TransactionParentDirectorySynced)?;
        Ok(())
    }

    fn load_journal(&self) -> Result<Option<JournalRecord>, CutoverError> {
        if !self.paths.transaction.exists() {
            if self.paths.backup.exists() {
                return Err(CutoverError::StaleBackupCollision(
                    self.paths.backup.clone(),
                ));
            }
            return Ok(None);
        }
        validate_directory(&self.paths.transaction, self.ownership)?;
        let mut valid = Vec::new();
        let mut slots_present = false;
        for (index, name) in [(0_u64, JOURNAL_ZERO), (1, JOURNAL_ONE)] {
            let path = self.paths.transaction_path(name);
            if !path_present(&path)? {
                continue;
            }
            slots_present = true;
            match self.read_journal_slot(&path) {
                Ok(record) if record.sequence % 2 == index => valid.push(record),
                Ok(_) | Err(CutoverError::TornJournal) => {}
                Err(error) => return Err(error),
            }
        }
        if valid.is_empty() {
            if slots_present {
                return Err(CutoverError::TornJournal);
            }
            return Ok(None);
        }
        valid.sort_by_key(|record| record.sequence);
        let newest = valid.pop().expect("valid journal is non-empty");
        if valid.iter().any(|record| {
            record.transaction_id != newest.transaction_id
                || record.ownership != newest.ownership
                || record.original_sha256 != newest.original_sha256
                || record.candidate_sha256 != newest.candidate_sha256
        }) {
            return Err(CutoverError::JournalOwnershipConflict);
        }
        Ok(Some(newest))
    }

    fn read_journal_slot(&self, path: &Path) -> Result<JournalRecord, CutoverError> {
        let bytes = read_owned(path, self.ownership)?;
        if bytes.len() as u64 > JOURNAL_LIMIT {
            return Err(CutoverError::TornJournal);
        }
        let envelope: JournalEnvelope =
            serde_json::from_slice(&bytes).map_err(|_| CutoverError::TornJournal)?;
        let payload =
            serde_json::to_vec(&envelope.record).map_err(|_| CutoverError::TornJournal)?;
        if envelope.record.version != JOURNAL_VERSION
            || envelope.checksum_sha256 != digest(&payload)
        {
            return Err(CutoverError::TornJournal);
        }
        Ok(envelope.record)
    }

    fn persist_record(
        &self,
        record: &JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        let payload = serde_json::to_vec(record).map_err(io::Error::other)?;
        let envelope = JournalEnvelope {
            checksum_sha256: digest(&payload),
            record: record.clone(),
        };
        let bytes = serde_json::to_vec(&envelope).map_err(io::Error::other)?;
        let temp = self.paths.transaction_path(JOURNAL_TEMP);
        let mut file = open_owned_replace(&temp, self.ownership)?;
        file.write_all(&bytes)?;
        observer(CutoverBoundary::JournalTempWritten(record.state))?;
        full_sync(&file)?;
        observer(CutoverBoundary::JournalTempSynced(record.state))?;
        drop(file);
        let slot = if record.sequence.is_multiple_of(2) {
            JOURNAL_ZERO
        } else {
            JOURNAL_ONE
        };
        fs::rename(&temp, self.paths.transaction_path(slot))?;
        observer(CutoverBoundary::JournalPublished(record.state))?;
        sync_directory(&self.paths.transaction)?;
        observer(CutoverBoundary::JournalDirectorySynced(record.state))?;
        Ok(())
    }

    fn transition(
        &self,
        record: &mut JournalRecord,
        next: CutoverState,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        let allowed = matches!(
            (record.state, next),
            (CutoverState::Started, CutoverState::Prepared)
                | (CutoverState::Prepared, CutoverState::Published)
                | (CutoverState::Published, CutoverState::Committed)
                | (CutoverState::Committed, CutoverState::RollbackPrepared)
                | (CutoverState::RollbackPrepared, CutoverState::RolledBack)
        );
        if !allowed {
            return Err(CutoverError::InvalidTransition {
                from: record.state,
                to: next,
            });
        }
        let next_record = JournalRecord {
            sequence: record.sequence + 1,
            state: next,
            ..record.clone()
        };
        self.persist_record(&next_record, observer)?;
        *record = next_record;
        Ok(())
    }

    fn drive_apply(
        &self,
        record: &mut JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<CutoverOutcome, CutoverError> {
        loop {
            match record.state {
                CutoverState::Started => {
                    self.ensure_backup(record, observer)?;
                    self.ensure_candidate(record, observer)?;
                    self.transition(record, CutoverState::Prepared, observer)?;
                }
                CutoverState::Prepared => {
                    self.publish_candidate(record, observer)?;
                    self.transition(record, CutoverState::Published, observer)?;
                }
                CutoverState::Published => {
                    self.validate_committed_layout(record)?;
                    self.transition(record, CutoverState::Committed, observer)?;
                }
                CutoverState::Committed => {
                    self.validate_committed_layout(record)?;
                    return Ok(CutoverOutcome::Committed);
                }
                CutoverState::RollbackPrepared => {
                    self.publish_rollback(record, observer)?;
                    self.transition(record, CutoverState::RolledBack, observer)?;
                }
                CutoverState::RolledBack => {
                    self.validate_rolled_back_layout(record)?;
                    return Ok(CutoverOutcome::RolledBack);
                }
            }
        }
    }

    fn ensure_backup(
        &self,
        record: &JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        let temp = self.paths.transaction_path(BACKUP_TEMP);
        if self.paths.backup.exists() {
            self.validate_backup(record)?;
        } else {
            let original = read_owned(&self.paths.canonical, record.ownership)?;
            if digest(&original) != record.original_sha256 {
                return Err(CutoverError::CanonicalCollision(
                    self.paths.canonical.clone(),
                ));
            }
            let mut file = open_owned_replace(&temp, record.ownership)?;
            file.write_all(&original)?;
            observer(CutoverBoundary::BackupTempWritten)?;
            full_sync(&file)?;
            observer(CutoverBoundary::BackupTempSynced)?;
            drop(file);
            sync_directory(&self.paths.transaction)?;
            observer(CutoverBoundary::BackupTempDirectorySynced)?;
            match fs::hard_link(&temp, &self.paths.backup) {
                Ok(()) => observer(CutoverBoundary::BackupPublished)?,
                Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
                    self.validate_backup(record)?;
                }
                Err(error) => return Err(error.into()),
            }
        }
        // This is also required when recovery observes a link published by a
        // process killed before it synced the parent directory.
        sync_directory(&self.paths.parent)?;
        observer(CutoverBoundary::BackupParentDirectorySynced)?;
        self.validate_backup(record)?;
        if temp.exists() {
            validate_artifact(&temp, record.ownership, true)?;
            if digest(&fs::read(&temp)?) != record.original_sha256 {
                return Err(CutoverError::OwnedBackupCollision(temp));
            }
            fs::remove_file(&temp)?;
            observer(CutoverBoundary::BackupTempRemoved)?;
            sync_directory(&self.paths.transaction)?;
            observer(CutoverBoundary::BackupCleanupDirectorySynced)?;
        }
        Ok(())
    }

    fn validate_backup(&self, record: &JournalRecord) -> Result<(), CutoverError> {
        let temp = self.paths.transaction_path(BACKUP_TEMP);
        let temp_exists = temp.exists();
        let bytes = read_owned_allow_links(
            &self.paths.backup,
            record.ownership,
            if temp_exists { 2 } else { 1 },
        )?;
        if digest(&bytes) != record.original_sha256 {
            return Err(CutoverError::OwnedBackupCollision(
                self.paths.backup.clone(),
            ));
        }
        if temp_exists && !same_inode(&self.paths.backup, &temp)? {
            return Err(CutoverError::OwnedBackupCollision(temp));
        }
        Ok(())
    }

    fn ensure_candidate(
        &self,
        record: &JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        let path = self.paths.transaction_path(CANDIDATE);
        let mut file = open_owned_replace(&path, record.ownership)?;
        file.write_all(&self.candidate)?;
        observer(CutoverBoundary::CandidateWritten)?;
        full_sync(&file)?;
        observer(CutoverBoundary::CandidateSynced)?;
        drop(file);
        sync_directory(&self.paths.transaction)?;
        observer(CutoverBoundary::CandidateDirectorySynced)?;
        let actual = read_owned(&path, record.ownership)?;
        if digest(&actual) != record.candidate_sha256 {
            return Err(CutoverError::CandidateChanged);
        }
        Ok(())
    }

    fn publish_candidate(
        &self,
        record: &JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        self.validate_backup(record)?;
        let canonical = read_owned(&self.paths.canonical, record.ownership)?;
        let canonical_sha = digest(&canonical);
        let candidate = self.paths.transaction_path(CANDIDATE);
        if canonical_sha == record.original_sha256 {
            if !candidate.exists() {
                self.ensure_candidate(record, observer)?;
            }
            let bytes = read_owned(&candidate, record.ownership)?;
            if digest(&bytes) != record.candidate_sha256 {
                return Err(CutoverError::CandidateChanged);
            }
            fs::rename(&candidate, &self.paths.canonical)?;
            observer(CutoverBoundary::CanonicalPublished)?;
            sync_directory(&self.paths.transaction)?;
            observer(CutoverBoundary::CanonicalSourceDirectorySynced)?;
            sync_directory(&self.paths.parent)?;
            observer(CutoverBoundary::CanonicalParentDirectorySynced)?;
        } else if canonical_sha != record.candidate_sha256 || candidate.exists() {
            return Err(CutoverError::CanonicalCollision(
                self.paths.canonical.clone(),
            ));
        } else {
            // The rename reached the canonical name but the state record did
            // not. Sync both directories before recording Published.
            sync_directory(&self.paths.transaction)?;
            sync_directory(&self.paths.parent)?;
        }
        self.validate_committed_layout(record)
    }

    fn validate_committed_layout(&self, record: &JournalRecord) -> Result<(), CutoverError> {
        self.validate_backup(record)?;
        let canonical = read_owned(&self.paths.canonical, record.ownership)?;
        if digest(&canonical) != record.candidate_sha256 {
            return Err(CutoverError::CanonicalCollision(
                self.paths.canonical.clone(),
            ));
        }
        Ok(())
    }

    fn drive_rollback(
        &self,
        record: &mut JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<CutoverOutcome, CutoverError> {
        match record.state {
            CutoverState::Committed => {
                self.prepare_rollback(record, observer)?;
                self.transition(record, CutoverState::RollbackPrepared, observer)?;
                self.publish_rollback(record, observer)?;
                self.transition(record, CutoverState::RolledBack, observer)?;
            }
            CutoverState::RollbackPrepared => {
                self.publish_rollback(record, observer)?;
                self.transition(record, CutoverState::RolledBack, observer)?;
            }
            CutoverState::RolledBack => {}
            state => {
                return Err(CutoverError::InvalidTransition {
                    from: state,
                    to: CutoverState::RollbackPrepared,
                })
            }
        }
        self.validate_rolled_back_layout(record)?;
        Ok(CutoverOutcome::RolledBack)
    }

    fn prepare_rollback(
        &self,
        record: &JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        self.validate_committed_layout(record)?;
        let backup = read_owned_allow_links(&self.paths.backup, record.ownership, 2)?;
        let rollback = self.paths.transaction_path(ROLLBACK_TEMP);
        let mut file = open_owned_replace(&rollback, record.ownership)?;
        file.write_all(&backup)?;
        observer(CutoverBoundary::RollbackTempWritten)?;
        full_sync(&file)?;
        observer(CutoverBoundary::RollbackTempSynced)?;
        drop(file);
        sync_directory(&self.paths.transaction)?;
        observer(CutoverBoundary::RollbackTempDirectorySynced)?;
        Ok(())
    }

    fn publish_rollback(
        &self,
        record: &JournalRecord,
        observer: &mut dyn FnMut(CutoverBoundary) -> Result<(), CutoverError>,
    ) -> Result<(), CutoverError> {
        self.validate_backup(record)?;
        let canonical = read_owned(&self.paths.canonical, record.ownership)?;
        let canonical_sha = digest(&canonical);
        let rollback = self.paths.transaction_path(ROLLBACK_TEMP);
        if canonical_sha == record.candidate_sha256 {
            if !rollback.exists() {
                self.prepare_rollback(record, observer)?;
            }
            let bytes = read_owned(&rollback, record.ownership)?;
            if digest(&bytes) != record.original_sha256 {
                return Err(CutoverError::OwnedBackupCollision(rollback));
            }
            fs::rename(&rollback, &self.paths.canonical)?;
            observer(CutoverBoundary::RollbackCanonicalPublished)?;
            sync_directory(&self.paths.transaction)?;
            observer(CutoverBoundary::RollbackSourceDirectorySynced)?;
            sync_directory(&self.paths.parent)?;
            observer(CutoverBoundary::RollbackParentDirectorySynced)?;
        } else if canonical_sha != record.original_sha256 || rollback.exists() {
            return Err(CutoverError::CanonicalCollision(
                self.paths.canonical.clone(),
            ));
        } else {
            sync_directory(&self.paths.transaction)?;
            sync_directory(&self.paths.parent)?;
        }
        Ok(())
    }

    fn validate_rolled_back_layout(&self, record: &JournalRecord) -> Result<(), CutoverError> {
        self.validate_backup(record)?;
        let canonical = read_owned(&self.paths.canonical, record.ownership)?;
        if digest(&canonical) != record.original_sha256 {
            return Err(CutoverError::CanonicalCollision(
                self.paths.canonical.clone(),
            ));
        }
        Ok(())
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Operation {
    Recover,
    Apply,
    Rollback,
}

#[cfg(unix)]
fn digest(bytes: &[u8]) -> String {
    format!("{:x}", Sha256::digest(bytes))
}

#[cfg(unix)]
fn path_present(path: &Path) -> Result<bool, CutoverError> {
    match fs::symlink_metadata(path) {
        Ok(_) => Ok(true),
        Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false),
        Err(error) => Err(error.into()),
    }
}

#[cfg(unix)]
fn apply_ownership(file: &File, ownership: CutoverOwnership) -> Result<(), CutoverError> {
    use std::os::fd::AsRawFd;

    // SAFETY: fchown and fchmod receive a live descriptor owned by `file` and
    // plain integer ownership values. Neither call retains the descriptor.
    let result = unsafe {
        if libc::fchown(file.as_raw_fd(), ownership.uid, ownership.gid) != 0 {
            -1
        } else {
            libc::fchmod(file.as_raw_fd(), ownership.mode as libc::mode_t)
        }
    };
    if result != 0 {
        return Err(io::Error::last_os_error().into());
    }
    Ok(())
}

#[cfg(unix)]
fn open_owned_replace(path: &Path, ownership: CutoverOwnership) -> Result<File, CutoverError> {
    if let Ok(metadata) = fs::symlink_metadata(path) {
        if !metadata.file_type().is_file() {
            return Err(CutoverError::UnsafeArtifact(path.to_path_buf()));
        }
        #[cfg(unix)]
        {
            use std::os::unix::fs::MetadataExt;
            if metadata.nlink() != 1 {
                return Err(CutoverError::UnsafeArtifact(path.to_path_buf()));
            }
        }
    }
    let mut options = OpenOptions::new();
    options.read(true).write(true).create(true).truncate(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        options.custom_flags(libc::O_NOFOLLOW);
    }
    let file = options.open(path)?;
    apply_ownership(&file, ownership)?;
    validate_open_file(&file, path, ownership, 1)?;
    Ok(file)
}

#[cfg(unix)]
fn validate_artifact(
    path: &Path,
    ownership: CutoverOwnership,
    allow_second_link: bool,
) -> Result<(), CutoverError> {
    use std::os::unix::fs::MetadataExt;

    let metadata = fs::symlink_metadata(path)?;
    let links_ok = metadata.nlink() == 1 || (allow_second_link && metadata.nlink() == 2);
    if !metadata.file_type().is_file() || !links_ok {
        return Err(CutoverError::UnsafeArtifact(path.to_path_buf()));
    }
    if metadata.uid() != ownership.uid
        || metadata.gid() != ownership.gid
        || metadata.mode() & 0o7777 != ownership.mode
    {
        return Err(CutoverError::OwnershipMismatch(path.to_path_buf()));
    }
    Ok(())
}

#[cfg(unix)]
fn read_owned(path: &Path, ownership: CutoverOwnership) -> Result<Vec<u8>, CutoverError> {
    read_owned_allow_links(path, ownership, 1)
}

#[cfg(unix)]
fn read_owned_allow_links(
    path: &Path,
    ownership: CutoverOwnership,
    max_links: u64,
) -> Result<Vec<u8>, CutoverError> {
    let mut options = OpenOptions::new();
    options.read(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        options.custom_flags(libc::O_NOFOLLOW);
    }
    let mut file = options.open(path)?;
    validate_open_file(&file, path, ownership, max_links)?;
    let mut bytes = Vec::new();
    file.read_to_end(&mut bytes)?;
    Ok(bytes)
}

#[cfg(unix)]
fn validate_open_file(
    file: &File,
    path: &Path,
    ownership: CutoverOwnership,
    max_links: u64,
) -> Result<(), CutoverError> {
    use std::os::unix::fs::MetadataExt;

    let metadata = file.metadata()?;
    if !metadata.file_type().is_file() || metadata.nlink() == 0 || metadata.nlink() > max_links {
        return Err(CutoverError::UnsafeArtifact(path.to_path_buf()));
    }
    if metadata.uid() != ownership.uid
        || metadata.gid() != ownership.gid
        || metadata.mode() & 0o7777 != ownership.mode
    {
        return Err(CutoverError::OwnershipMismatch(path.to_path_buf()));
    }
    Ok(())
}

#[cfg(unix)]
fn same_inode(left: &Path, right: &Path) -> Result<bool, CutoverError> {
    use std::os::unix::fs::MetadataExt;

    let left = fs::symlink_metadata(left)?;
    let right = fs::symlink_metadata(right)?;
    Ok(left.dev() == right.dev() && left.ino() == right.ino())
}

#[cfg(unix)]
fn validate_directory(path: &Path, ownership: CutoverOwnership) -> Result<(), CutoverError> {
    use std::os::unix::fs::MetadataExt;

    let metadata = fs::symlink_metadata(path)?;
    if !metadata.file_type().is_dir()
        || metadata.uid() != ownership.uid
        || metadata.gid() != ownership.gid
        || metadata.mode() & 0o7777 != 0o700
    {
        return Err(CutoverError::OwnershipMismatch(path.to_path_buf()));
    }
    Ok(())
}

#[cfg(unix)]
fn set_directory_ownership(path: &Path, ownership: CutoverOwnership) -> Result<(), CutoverError> {
    use std::os::unix::fs::PermissionsExt;

    fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;
    let directory = File::open(path)?;
    let directory_owner = CutoverOwnership {
        mode: 0o700,
        ..ownership
    };
    apply_ownership(&directory, directory_owner)
}

#[cfg(target_os = "macos")]
fn full_sync(file: &File) -> Result<(), CutoverError> {
    use std::os::fd::AsRawFd;

    // SAFETY: fcntl receives a live descriptor and the parameter-free
    // F_FULLFSYNC command. It does not retain the descriptor.
    if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_FULLFSYNC) } == -1 {
        return Err(io::Error::last_os_error().into());
    }
    Ok(())
}

#[cfg(all(unix, not(target_os = "macos")))]
fn full_sync(file: &File) -> Result<(), CutoverError> {
    file.sync_all()?;
    Ok(())
}

#[cfg(unix)]
fn sync_directory(path: &Path) -> Result<(), CutoverError> {
    let directory = File::open(path)?;
    full_sync(&directory)
}

#[cfg(all(test, not(unix)))]
mod unsupported_tests {
    use super::*;

    #[test]
    fn non_unix_constructor_fails_before_creating_a_transaction() {
        let result = RebootCutover::new(
            "unsupported-launch-agent",
            b"candidate".to_vec(),
            CutoverOwnership {
                uid: 0,
                gid: 0,
                mode: 0o600,
            },
        );
        assert!(matches!(result, Err(CutoverError::UnsupportedPlatform)));
    }
}

#[cfg(all(test, unix))]
mod tests {
    use super::*;
    use std::collections::HashMap;
    use std::os::unix::fs::MetadataExt;
    use std::os::unix::process::ExitStatusExt;
    use std::process::{Child, Command, Stdio};
    use std::thread;
    use std::time::{Duration, Instant};
    use tempfile::TempDir;

    const ORIGINAL: &[u8] = b"<?xml version=\"1.0\"?><plist><string>original</string></plist>\n";
    const CANDIDATE_BYTES: &[u8] =
        b"<?xml version=\"1.0\"?><plist><string>candidate</string></plist>\n";

    fn apply_boundaries() -> Vec<CutoverBoundary> {
        let mut result = vec![
            CutoverBoundary::OwnerLeaseWritten,
            CutoverBoundary::OwnerLeaseSynced,
            CutoverBoundary::OwnerLeaseDirectorySynced,
            CutoverBoundary::TransactionDirectoryCreated,
            CutoverBoundary::TransactionDirectorySynced,
            CutoverBoundary::TransactionParentDirectorySynced,
        ];
        result.extend(journal_boundaries(CutoverState::Started));
        result.extend([
            CutoverBoundary::BackupTempWritten,
            CutoverBoundary::BackupTempSynced,
            CutoverBoundary::BackupTempDirectorySynced,
            CutoverBoundary::BackupPublished,
            CutoverBoundary::BackupParentDirectorySynced,
            CutoverBoundary::BackupTempRemoved,
            CutoverBoundary::BackupCleanupDirectorySynced,
            CutoverBoundary::CandidateWritten,
            CutoverBoundary::CandidateSynced,
            CutoverBoundary::CandidateDirectorySynced,
        ]);
        result.extend(journal_boundaries(CutoverState::Prepared));
        result.extend([
            CutoverBoundary::CanonicalPublished,
            CutoverBoundary::CanonicalSourceDirectorySynced,
            CutoverBoundary::CanonicalParentDirectorySynced,
        ]);
        result.extend(journal_boundaries(CutoverState::Published));
        result.extend(journal_boundaries(CutoverState::Committed));
        result
    }

    fn rollback_boundaries() -> Vec<CutoverBoundary> {
        let mut result = vec![
            CutoverBoundary::OwnerLeaseWritten,
            CutoverBoundary::OwnerLeaseSynced,
            CutoverBoundary::OwnerLeaseDirectorySynced,
            CutoverBoundary::RollbackTempWritten,
            CutoverBoundary::RollbackTempSynced,
            CutoverBoundary::RollbackTempDirectorySynced,
        ];
        result.extend(journal_boundaries(CutoverState::RollbackPrepared));
        result.extend([
            CutoverBoundary::RollbackCanonicalPublished,
            CutoverBoundary::RollbackSourceDirectorySynced,
            CutoverBoundary::RollbackParentDirectorySynced,
        ]);
        result.extend(journal_boundaries(CutoverState::RolledBack));
        result
    }

    fn journal_boundaries(state: CutoverState) -> [CutoverBoundary; 4] {
        [
            CutoverBoundary::JournalTempWritten(state),
            CutoverBoundary::JournalTempSynced(state),
            CutoverBoundary::JournalPublished(state),
            CutoverBoundary::JournalDirectorySynced(state),
        ]
    }

    struct Fixture {
        _temp: TempDir,
        car_home: PathBuf,
        launch_agents: PathBuf,
        canonical: PathBuf,
        ownership: CutoverOwnership,
    }

    impl Fixture {
        fn new() -> Self {
            let temp = tempfile::Builder::new()
                .prefix("car-14b42-cutover.")
                .tempdir()
                .expect("scratch root");
            let car_home = temp.path().join("car-home");
            let launch_agents = temp.path().join("home/Library/LaunchAgents");
            fs::create_dir_all(&car_home).expect("scratch CAR_HOME");
            fs::create_dir_all(&launch_agents).expect("scratch LaunchAgents");
            let canonical = launch_agents.join("ai.parslee.car.daemon.plist");
            fs::write(&canonical, ORIGINAL).expect("write original");
            #[cfg(unix)]
            {
                use std::os::unix::fs::PermissionsExt;
                fs::set_permissions(&canonical, fs::Permissions::from_mode(0o600))
                    .expect("private original");
            }
            let ownership = CutoverOwnership::from_path(&canonical).expect("ownership");
            Self {
                _temp: temp,
                car_home,
                launch_agents,
                canonical,
                ownership,
            }
        }

        fn transaction(&self) -> RebootCutover {
            RebootCutover::new(&self.canonical, CANDIDATE_BYTES, self.ownership)
                .expect("transaction")
        }

        fn backup(&self) -> PathBuf {
            Paths::new(&self.canonical).expect("paths").backup
        }

        fn journal_slots(&self) -> [PathBuf; 2] {
            let paths = Paths::new(&self.canonical).expect("paths");
            [
                paths.transaction_path(JOURNAL_ZERO),
                paths.transaction_path(JOURNAL_ONE),
            ]
        }
    }

    fn copy_tree(source: &Path, destination: &Path) {
        if destination.exists() {
            fs::remove_dir_all(destination).expect("remove prior durability snapshot");
        }
        let mut hard_links = HashMap::new();
        copy_tree_inner(source, destination, &mut hard_links);
    }

    fn copy_tree_inner(
        source: &Path,
        destination: &Path,
        hard_links: &mut HashMap<(u64, u64), PathBuf>,
    ) {
        fs::create_dir(destination).expect("create durability snapshot");
        fs::set_permissions(
            destination,
            fs::metadata(source)
                .expect("snapshot root metadata")
                .permissions(),
        )
        .expect("copy snapshot directory permissions");
        for entry in fs::read_dir(source).expect("read snapshot source") {
            let entry = entry.expect("snapshot entry");
            let source_path = entry.path();
            let destination_path = destination.join(entry.file_name());
            let metadata = entry.metadata().expect("snapshot metadata");
            if metadata.is_dir() {
                copy_tree_inner(&source_path, &destination_path, hard_links);
            } else {
                let inode = (metadata.dev(), metadata.ino());
                if metadata.nlink() > 1 {
                    if let Some(first) = hard_links.get(&inode) {
                        fs::hard_link(first, &destination_path).expect("copy hard link");
                    } else {
                        fs::copy(&source_path, &destination_path).expect("copy snapshot file");
                        hard_links.insert(inode, destination_path.clone());
                    }
                } else {
                    fs::copy(&source_path, &destination_path).expect("copy snapshot file");
                }
                fs::set_permissions(&destination_path, metadata.permissions())
                    .expect("copy permissions");
            }
        }
    }

    fn spawn_child(
        fixture: &Fixture,
        operation: &str,
        boundary: CutoverBoundary,
        marker: &Path,
        durable: &Path,
    ) -> Child {
        Command::new(std::env::current_exe().expect("test executable"))
            .arg("--exact")
            .arg("reboot_cutover::tests::kill_child")
            .arg("--nocapture")
            .env("CAR_HOME", &fixture.car_home)
            .env("CAR_CUTOVER_CHILD", operation)
            .env("CAR_CUTOVER_CANONICAL", &fixture.canonical)
            .env("CAR_CUTOVER_UID", fixture.ownership.uid.to_string())
            .env("CAR_CUTOVER_GID", fixture.ownership.gid.to_string())
            .env("CAR_CUTOVER_MODE", fixture.ownership.mode.to_string())
            .env("CAR_CUTOVER_BOUNDARY", boundary.name())
            .env("CAR_CUTOVER_MARKER", marker)
            .env("CAR_CUTOVER_DURABLE", durable)
            .stdin(Stdio::null())
            .stdout(Stdio::null())
            .stderr(Stdio::null())
            .spawn()
            .expect("spawn cutover child")
    }

    fn wait_for_marker(marker: &Path, child: &mut Child) {
        let deadline = Instant::now() + Duration::from_secs(10);
        while Instant::now() < deadline {
            if marker.exists() {
                return;
            }
            if let Some(status) = child.try_wait().expect("child status") {
                panic!("child exited before boundary marker: {status}");
            }
            thread::sleep(Duration::from_millis(5));
        }
        panic!("timed out waiting for {}", marker.display());
    }

    fn kill_at_boundary(operation: &str, boundary: CutoverBoundary, power_loss: bool) {
        let fixture = Fixture::new();
        if operation == "rollback" {
            assert_eq!(
                fixture
                    .transaction()
                    .apply()
                    .expect("prepare committed cutover"),
                CutoverOutcome::Committed
            );
        }
        let durable = fixture._temp.path().join("durable");
        copy_tree(&fixture.launch_agents, &durable);
        let marker = fixture._temp.path().join("boundary.ready");
        let mut child = spawn_child(&fixture, operation, boundary, &marker, &durable);
        wait_for_marker(&marker, &mut child);
        let kill = Command::new("/bin/kill")
            .args(["-9", &child.id().to_string()])
            .status()
            .expect("invoke kill");
        assert!(kill.success(), "SIGKILL command failed");
        let status = child.wait().expect("reap killed child");
        assert_eq!(status.signal(), Some(libc::SIGKILL));

        if power_loss {
            fs::remove_dir_all(&fixture.launch_agents).expect("discard volatile namespace");
            copy_tree(&durable, &fixture.launch_agents);
        }

        let transaction = fixture.transaction();
        let outcome = match operation {
            "apply" => match transaction.recover_boot().unwrap_or_else(|error| {
                panic!("boot-first recovery at {}: {error}", boundary.name())
            }) {
                CutoverOutcome::NoTransaction => transaction.apply().expect("restart cutover"),
                outcome => outcome,
            },
            "rollback" => transaction.rollback().expect("recover rollback"),
            _ => unreachable!(),
        };
        let expected = if operation == "apply" {
            CutoverOutcome::Committed
        } else {
            CutoverOutcome::RolledBack
        };
        assert_eq!(outcome, expected, "boundary {}", boundary.name());
        let second = if operation == "apply" {
            transaction.recover_boot().unwrap_or_else(|error| {
                panic!("idempotent boot recovery at {}: {error}", boundary.name())
            })
        } else {
            transaction.rollback().expect("idempotent rollback")
        };
        assert_eq!(second, expected, "idempotence at {}", boundary.name());
        assert_eq!(
            fs::read(fixture.backup()).expect("original backup"),
            ORIGINAL
        );
        let expected_canonical = if operation == "apply" {
            CANDIDATE_BYTES
        } else {
            ORIGINAL
        };
        assert_eq!(
            fs::read(&fixture.canonical).expect("canonical generation"),
            expected_canonical,
            "canonical bytes at {}",
            boundary.name()
        );
    }

    #[test]
    fn kill_child() {
        let Ok(operation) = std::env::var("CAR_CUTOVER_CHILD") else {
            return;
        };
        let canonical = PathBuf::from(std::env::var_os("CAR_CUTOVER_CANONICAL").unwrap());
        let ownership = CutoverOwnership {
            uid: std::env::var("CAR_CUTOVER_UID").unwrap().parse().unwrap(),
            gid: std::env::var("CAR_CUTOVER_GID").unwrap().parse().unwrap(),
            mode: std::env::var("CAR_CUTOVER_MODE").unwrap().parse().unwrap(),
        };
        let target = std::env::var("CAR_CUTOVER_BOUNDARY").unwrap();
        let marker = PathBuf::from(std::env::var_os("CAR_CUTOVER_MARKER").unwrap());
        let durable = PathBuf::from(std::env::var_os("CAR_CUTOVER_DURABLE").unwrap());
        let launch_agents = canonical.parent().unwrap().to_path_buf();
        let transaction = RebootCutover::new(canonical, CANDIDATE_BYTES, ownership).unwrap();
        let mut observer = |boundary: CutoverBoundary| {
            if boundary.closes_durability_epoch() {
                copy_tree(&launch_agents, &durable);
            }
            if boundary.name() == target {
                fs::write(&marker, boundary.name()).map_err(CutoverError::Io)?;
                File::open(&marker)
                    .and_then(|file| file.sync_all())
                    .map_err(CutoverError::Io)?;
                loop {
                    thread::sleep(Duration::from_secs(60));
                }
            }
            Ok(())
        };
        match operation.as_str() {
            "apply" => {
                transaction.run(Operation::Apply, &mut observer).unwrap();
            }
            "rollback" => {
                transaction.run(Operation::Rollback, &mut observer).unwrap();
            }
            _ => panic!("unknown child operation"),
        }
    }

    #[test]
    fn sigkill_and_power_loss_recover_at_every_mutation_boundary() {
        let apply = apply_boundaries();
        let rollback = rollback_boundaries();
        assert_eq!(apply.len(), 35);
        assert_eq!(rollback.len(), 17);
        let mut names = apply
            .iter()
            .chain(&rollback)
            .map(|boundary| boundary.name())
            .collect::<Vec<_>>();
        names.sort();
        let total = names.len();
        names.dedup();
        // Lease checkpoints intentionally occur once in apply and once in
        // rollback; every other operation boundary is unique.
        assert_eq!(total - names.len(), 3);
        let coverage = Fixture::new();
        let transaction = coverage.transaction();
        let mut observed_apply = Vec::new();
        transaction
            .run(Operation::Apply, &mut |boundary| {
                observed_apply.push(boundary);
                Ok(())
            })
            .expect("observe apply boundary sequence");
        assert_eq!(observed_apply, apply);
        let mut observed_rollback = Vec::new();
        transaction
            .run(Operation::Rollback, &mut |boundary| {
                observed_rollback.push(boundary);
                Ok(())
            })
            .expect("observe rollback boundary sequence");
        assert_eq!(observed_rollback, rollback);

        eprintln!(
            "cutover boundary matrix: apply={} rollback={} boundaries={} interruption_cases={}",
            apply.len(),
            rollback.len(),
            total,
            total * 2
        );
        for boundary in apply {
            kill_at_boundary("apply", boundary, false);
            kill_at_boundary("apply", boundary, true);
        }
        for boundary in rollback {
            kill_at_boundary("rollback", boundary, false);
            kill_at_boundary("rollback", boundary, true);
        }
    }

    #[test]
    fn active_stale_and_owned_backup_collisions_fail_closed() {
        let fixture = Fixture::new();
        let durable = fixture._temp.path().join("durable");
        copy_tree(&fixture.launch_agents, &durable);
        let marker = fixture._temp.path().join("active.ready");
        let boundary = CutoverBoundary::BackupParentDirectorySynced;
        let mut child = spawn_child(&fixture, "apply", boundary, &marker, &durable);
        wait_for_marker(&marker, &mut child);
        assert!(matches!(
            fixture.transaction().apply(),
            Err(CutoverError::ActiveOwner)
        ));
        assert_eq!(fs::read(fixture.backup()).unwrap(), ORIGINAL);
        Command::new("/bin/kill")
            .args(["-9", &child.id().to_string()])
            .status()
            .unwrap();
        child.wait().unwrap();
        assert_eq!(
            fixture.transaction().apply().unwrap(),
            CutoverOutcome::Committed
        );

        let stale = Fixture::new();
        fs::write(stale.backup(), b"unowned-backup").unwrap();
        assert!(matches!(
            stale.transaction().apply(),
            Err(CutoverError::StaleBackupCollision(_))
        ));
        assert_eq!(fs::read(&stale.canonical).unwrap(), ORIGINAL);
        assert_eq!(fs::read(stale.backup()).unwrap(), b"unowned-backup");

        let owned = Fixture::new();
        assert_eq!(
            owned.transaction().apply().unwrap(),
            CutoverOutcome::Committed
        );
        fs::write(owned.backup(), b"diverged-owned-backup").unwrap();
        assert!(matches!(
            owned.transaction().recover_boot(),
            Err(CutoverError::OwnedBackupCollision(_))
        ));
        assert_eq!(fs::read(&owned.canonical).unwrap(), CANDIDATE_BYTES);
        assert_eq!(fs::read(owned.backup()).unwrap(), b"diverged-owned-backup");
    }

    #[test]
    fn newest_torn_slot_recovers_and_two_torn_slots_fail_closed() {
        let fixture = Fixture::new();
        assert_eq!(
            fixture.transaction().apply().unwrap(),
            CutoverOutcome::Committed
        );
        let slots = fixture.journal_slots();
        fs::write(&slots[1], b"{torn").unwrap();
        assert_eq!(
            fixture.transaction().recover_boot().unwrap(),
            CutoverOutcome::Committed
        );
        assert_eq!(fs::read(fixture.backup()).unwrap(), ORIGINAL);
        assert_eq!(fs::read(&fixture.canonical).unwrap(), CANDIDATE_BYTES);

        fs::write(&slots[0], b"{also-torn").unwrap();
        fs::write(&slots[1], b"{torn-again").unwrap();
        assert!(matches!(
            fixture.transaction().recover_boot(),
            Err(CutoverError::TornJournal)
        ));
        assert_eq!(fs::read(fixture.backup()).unwrap(), ORIGINAL);
        assert_eq!(fs::read(&fixture.canonical).unwrap(), CANDIDATE_BYTES);
    }

    #[test]
    fn unsafe_journal_slot_is_not_downgraded_to_a_torn_tail() {
        use std::os::unix::fs::symlink;

        let fixture = Fixture::new();
        assert_eq!(
            fixture.transaction().apply().unwrap(),
            CutoverOutcome::Committed
        );
        let slots = fixture.journal_slots();
        fs::remove_file(&slots[1]).unwrap();
        symlink(&slots[0], &slots[1]).unwrap();
        assert!(fixture.transaction().recover_boot().is_err());
        fs::remove_file(&slots[1]).unwrap();
        symlink("missing-journal-target", &slots[1]).unwrap();
        assert!(fixture.transaction().recover_boot().is_err());
        fs::remove_file(&slots[1]).unwrap();
        fs::hard_link(&slots[0], &slots[1]).unwrap();
        assert!(matches!(
            fixture.transaction().recover_boot(),
            Err(CutoverError::UnsafeArtifact(_))
        ));
    }

    #[test]
    fn apply_and_rollback_are_idempotent_without_republishing() {
        let untouched = Fixture::new();
        assert!(matches!(
            untouched.transaction().rollback(),
            Err(CutoverError::NoTransactionForRollback)
        ));
        assert_eq!(fs::read(&untouched.canonical).unwrap(), ORIGINAL);
        assert!(!untouched.backup().exists());

        let fixture = Fixture::new();
        let transaction = fixture.transaction();
        assert_eq!(
            transaction.recover_boot().unwrap(),
            CutoverOutcome::NoTransaction
        );
        assert_eq!(transaction.apply().unwrap(), CutoverOutcome::Committed);
        let committed_inode = fs::metadata(&fixture.canonical).unwrap().ino();
        assert_eq!(transaction.apply().unwrap(), CutoverOutcome::Committed);
        assert_eq!(
            fs::metadata(&fixture.canonical).unwrap().ino(),
            committed_inode
        );
        assert_eq!(transaction.rollback().unwrap(), CutoverOutcome::RolledBack);
        let rolled_back_inode = fs::metadata(&fixture.canonical).unwrap().ino();
        assert_eq!(transaction.rollback().unwrap(), CutoverOutcome::RolledBack);
        assert_eq!(
            fs::metadata(&fixture.canonical).unwrap().ino(),
            rolled_back_inode
        );
        assert_eq!(fs::read(&fixture.canonical).unwrap(), ORIGINAL);
        assert_eq!(fs::read(fixture.backup()).unwrap(), ORIGINAL);
    }
}