aion-server 0.29.0

Aion workflow server library: HTTP, gRPC, WebSocket, and worker endpoints. Run it with the `aion` binary from the aion-cli crate.
Documentation
//! Red-first specimens for the birth claim's decision table.
//!
//! Every "live" case is proven against THIS process, whose pid and start
//! instant are a genuinely live incarnation by the same instrument the
//! production probe uses — so these exercise the real verification path.

use super::claim_at_birth;
use crate::control::pid_file::{IncarnationState, StaleReconciliation, read};
use crate::control::test_records::{
    GRPC, HTTP, TestResult, dead_record, intended, other_intended, own_birth_record,
    own_serving_record, plant,
};
use crate::error::ServerError;

/// The ordinary boot: no file, the claim writes one in BOOTING with no
/// addresses, and the guard's drop removes it.
#[test]
fn an_empty_home_is_claimed_at_birth() -> TestResult {
    let home = tempfile::tempdir()?;
    let record = own_birth_record()?;
    let guard = claim_at_birth(home.path(), &record, intended()?)?;
    assert_eq!(guard.reconciliation(), &StaleReconciliation::NonePresent);
    assert!(guard.holds_claim());

    let written = read(home.path())?.ok_or("the claim must write a record")?;
    assert_eq!(written.state, IncarnationState::Booting);
    assert_eq!(
        (written.http_address, written.grpc_address),
        (None, None),
        "a birth record promises no address it has not bound"
    );
    drop(guard);
    assert_eq!(read(home.path())?, None, "the guard removes its own record");
    Ok(())
}

/// A stale file whose pid is DEAD is reconciled and replaced — never
/// trusted, never an error.
///
/// The reaped pid COULD be recycled between the fixture's wait and the
/// claim's probe; the probe then honestly sees a stranger and reconciles the
/// same file as the reused-pid face. Both are honest stale-file answers and
/// both replace the record, so the specimen refuses only a claim that TRUSTS
/// the stale file or fails to replace it.
#[test]
fn a_dead_incarnations_record_is_reconciled_and_replaced() -> TestResult {
    let home = tempfile::tempdir()?;
    let stale = dead_record()?;
    plant(home.path(), &stale)?;

    let record = own_birth_record()?;
    let guard = claim_at_birth(home.path(), &record, intended()?)?;
    match guard.reconciliation() {
        StaleReconciliation::DeadIncarnation(named) | StaleReconciliation::ReusedPid(named) => {
            assert_eq!(named, &stale, "the stale predecessor must be named");
        }
        other => {
            return Err(
                format!("a dead record must reconcile as dead or reused: {other:?}").into(),
            );
        }
    }
    assert_eq!(
        read(home.path())?.map(|written| written.state),
        Some(IncarnationState::Booting),
        "the new incarnation's booting record must replace the stale one"
    );
    Ok(())
}

/// A file naming a LIVE pid with the wrong start instant is the reused-pid
/// face: reconciled as such, the stranger left untouched.
#[test]
fn a_reused_pids_record_is_reconciled_as_reused() -> TestResult {
    let home = tempfile::tempdir()?;
    let mut stale = own_serving_record(None, None)?;
    stale.started_at_unix_secs = stale.started_at_unix_secs.wrapping_add(31);
    plant(home.path(), &stale)?;

    let guard = claim_at_birth(home.path(), &own_birth_record()?, intended()?)?;
    assert_eq!(
        guard.reconciliation(),
        &StaleReconciliation::ReusedPid(stale),
        "a live pid with a foreign start instant is a reused pid"
    );
    Ok(())
}

/// An unparseable file (the pre-verb hand-written ritual wrote bare pids) is
/// reconciled as unreadable and replaced.
#[test]
fn a_hand_written_pid_file_is_reconciled_as_unreadable() -> TestResult {
    let home = tempfile::tempdir()?;
    let path = crate::control::pid_file::pid_file_path(home.path());
    std::fs::create_dir_all(path.parent().ok_or("pid path must have a parent")?)?;
    std::fs::write(&path, "93400\n")?;

    let guard = claim_at_birth(home.path(), &own_birth_record()?, intended()?)?;
    assert!(
        matches!(
            guard.reconciliation(),
            StaleReconciliation::Unreadable { .. }
        ),
        "a bare-pid file must reconcile as unreadable, got {:?}",
        guard.reconciliation()
    );
    assert!(read(home.path())?.is_some());
    Ok(())
}

/// 🔴 The stacking killer, as a specimen: a live BOOTING sibling refuses the
/// boot outright.
///
/// This is the exact 2026-08-26 shape. The sibling has bound nothing — it is
/// minutes into WAL recovery — so there is no port to probe and, before the
/// birth claim, nothing else to find either. A booting record with no
/// addresses is ALWAYS a collision, and the refusal must name the holder well
/// enough for the operator to act without further archaeology.
#[test]
fn a_live_booting_sibling_refuses_the_boot() -> TestResult {
    let home = tempfile::tempdir()?;
    let mut sibling = own_birth_record()?;
    sibling.stage = Some("wal-recovery".to_owned());
    sibling.stage_detail = Some("materializing shard 17 of 64".to_owned());
    sibling.stage_seq = 17;
    sibling.stage_updated_at_unix_secs = crate::control::pid_file::now_unix_secs();
    plant(home.path(), &sibling)?;

    // This sibling's record carries NO addresses at all — bound or intended —
    // the shape a pre-0.26 build wrote. Nothing is comparable, so the
    // collision presumption stands even against addresses it never named.
    // A record WITH intended addresses is decidable instead; the test below
    // proves both faces of that.
    let error = match claim_at_birth(home.path(), &own_birth_record()?, other_intended()?) {
        Err(error) => error,
        Ok(_guard) => {
            return Err("a live booting sibling must refuse the boot".into());
        }
    };
    let ServerError::HomeAlreadyClaimed { refusal } = &error else {
        return Err(
            format!("the refusal must be typed as HomeAlreadyClaimed, got {error:?}").into(),
        );
    };
    assert_eq!(refusal.holder.pid, sibling.pid);
    let message = error.to_string();
    for expected in [
        "BOOTING",
        "wal-recovery",
        "materializing shard 17 of 64",
        "aion server status",
        "aion server stop",
        "AION_HOME",
    ] {
        assert!(
            message.contains(expected),
            "the refusal must carry {expected:?}: {message}"
        );
    }
    assert!(
        error.is_config(),
        "a refusal an operator can act on is exit 2, not a crash"
    );
    assert_eq!(
        read(home.path())?.as_ref(),
        Some(&sibling),
        "a refused boot must leave the holder's record untouched"
    );
    Ok(())
}

/// A live BOOTING sibling that RECORDED its intended doors is decidable
/// mid-boot: different doors boot unclaimed beside it, the same doors
/// refuse. Measured before this existed (battery run 0b066959): two e2e
/// servers with distinct stores and distinct ports on one shared home, and
/// the second was refused mid-`store-open` because the holder's record
/// carried no addresses to compare.
#[test]
fn a_booting_siblings_intended_doors_decide_the_claim() -> TestResult {
    // Different intended doors: the boot proceeds UNCLAIMED, exactly like
    // meeting a bound server on other addresses.
    let home = tempfile::tempdir()?;
    let mut sibling = own_birth_record()?;
    sibling.intended_http_address = Some(HTTP.parse()?);
    sibling.intended_grpc_address = Some(GRPC.parse()?);
    plant(home.path(), &sibling)?;
    let guard = claim_at_birth(home.path(), &own_birth_record()?, other_intended()?)?;
    assert!(
        matches!(
            guard.reconciliation(),
            StaleReconciliation::LiveIncarnationElsewhere(_)
        ),
        "distinct intended doors must boot unclaimed, got {:?}",
        guard.reconciliation()
    );
    assert!(!guard.holds_claim());
    assert_eq!(
        read(home.path())?.as_ref(),
        Some(&sibling),
        "the booting holder's record must survive the second boot"
    );
    drop(guard);

    // The SAME intended doors: the refusal stands — this genuinely is one
    // configuration booting twice.
    let home = tempfile::tempdir()?;
    plant(home.path(), &sibling)?;
    match claim_at_birth(home.path(), &own_birth_record()?, intended()?) {
        Err(ServerError::HomeAlreadyClaimed { refusal }) => {
            assert_eq!(refusal.holder.pid, sibling.pid);
        }
        Err(error) => {
            return Err(
                format!("the refusal must be typed as HomeAlreadyClaimed, got {error:?}").into(),
            );
        }
        Ok(_guard) => {
            return Err("a booting sibling intending the SAME doors must refuse the boot".into());
        }
    }
    Ok(())
}

/// A live SERVING incarnation on the addresses this boot intends refuses it
/// too: the second server would only lose the bind race, after blocking on
/// the store's writer lock for however long the first one lives.
#[test]
fn a_live_serving_incarnation_on_the_same_addresses_refuses_the_boot() -> TestResult {
    let home = tempfile::tempdir()?;
    let holder = own_serving_record(None, None)?;
    plant(home.path(), &holder)?;

    let error = match claim_at_birth(home.path(), &own_birth_record()?, intended()?) {
        Err(error) => error,
        Ok(_guard) => return Err("a colliding live server must refuse the boot".into()),
    };
    assert!(
        matches!(error, ServerError::HomeAlreadyClaimed { .. }),
        "got {error:?}"
    );
    assert!(error.to_string().contains("SERVING"));
    assert_eq!(read(home.path())?.as_ref(), Some(&holder));
    Ok(())
}

/// A live SERVING incarnation on DIFFERENT addresses is not in this boot's
/// way: the multi-server home stays legal, the first claimant keeps the
/// record and the verbs, and the second boots UNCLAIMED — writing nothing,
/// removing nothing.
#[test]
fn a_live_incarnation_on_different_addresses_keeps_its_claim() -> TestResult {
    let home = tempfile::tempdir()?;
    let holder = own_serving_record(
        Some("127.0.0.1:19090".parse()?),
        Some("127.0.0.1:19091".parse()?),
    )?;
    plant(home.path(), &holder)?;

    let guard = claim_at_birth(home.path(), &own_birth_record()?, intended()?)?;
    assert!(
        matches!(
            guard.reconciliation(),
            StaleReconciliation::LiveIncarnationElsewhere(_)
        ),
        "the unclaimed boot must wear its OWN face: {:?}",
        guard.reconciliation()
    );
    assert!(!guard.holds_claim());
    assert_eq!(
        read(home.path())?.as_ref(),
        Some(&holder),
        "the first claimant's record must survive the second boot"
    );
    // An unclaimed guard writes nothing while it lives...
    guard
        .stage_reporter()
        .report("store-open", "opening".to_owned());
    assert_eq!(read(home.path())?.as_ref(), Some(&holder));
    // ...and removes nothing when it exits.
    drop(guard);
    assert_eq!(read(home.path())?.as_ref(), Some(&holder));
    Ok(())
}

/// A live DRAINING incarnation is SUCCEEDED: the record is replaced, and the
/// handover is the reconciliation's own face so a caller can tell it from
/// claiming over debris.
#[test]
fn a_draining_incarnation_is_succeeded() -> TestResult {
    let home = tempfile::tempdir()?;
    let mut drainer = own_serving_record(None, None)?;
    drainer.state = IncarnationState::Draining;
    plant(home.path(), &drainer)?;

    let successor = own_birth_record()?;
    let guard = claim_at_birth(home.path(), &successor, intended()?)?;
    assert!(
        matches!(
            guard.reconciliation(),
            StaleReconciliation::SucceededDrainer(_)
        ),
        "succeeding a drainer must be its own reported face: {:?}",
        guard.reconciliation()
    );
    assert!(guard.holds_claim());
    assert_eq!(
        read(home.path())?.map(|record| record.state),
        Some(IncarnationState::Booting),
        "the successor's booting record must replace the drainer's"
    );
    Ok(())
}

/// Two successors racing one drainer: the mutation lock serializes them, so
/// the second one's reconcile sees the FIRST successor's live booting record
/// and is refused. Exactly one server ever comes out of a handover.
#[test]
fn a_second_successor_is_refused_by_the_first() -> TestResult {
    let home = tempfile::tempdir()?;
    let mut drainer = own_serving_record(None, None)?;
    drainer.state = IncarnationState::Draining;
    plant(home.path(), &drainer)?;

    let first = claim_at_birth(home.path(), &own_birth_record()?, intended()?)?;
    assert!(first.holds_claim());

    let error = match claim_at_birth(home.path(), &own_birth_record()?, intended()?) {
        Err(error) => error,
        Ok(_second) => {
            return Err("the second successor must be refused by the first".into());
        }
    };
    assert!(
        matches!(error, ServerError::HomeAlreadyClaimed { .. }),
        "got {error:?}"
    );
    Ok(())
}