aion-server 0.27.1

//! Red-first specimens for the outcome record and the death-note reader.

use std::path::Path;

use super::{NoteFate, OutcomeRecord, ParkedWorkerRecord, read_fate, render_entry};
use crate::shutdown::ShutdownOutcome;

type TestResult = Result<(), Box<dyn std::error::Error>>;

fn parked_record(pid: u32) -> OutcomeRecord {
    OutcomeRecord {
        pid,
        outcome: ShutdownOutcome::Parked,
        drain_timeout_seconds: 30,
        delivered_drain_requests: 2,
        parked_declared_commands: vec![
            "11111111-1111-1111-1111-111111111111/activity:0#1".to_owned(),
        ],
        parked: vec![ParkedWorkerRecord {
            worker: "worker-7".to_owned(),
            queue: Some("fleet_dev".to_owned()),
            tasks: vec!["wf-1/act-3#1".to_owned(), "wf-2/act-1#4".to_owned()],
        }],
        managed_workers_stopped: vec!["managed-a".to_owned()],
        managed_workers_unstopped: Vec::new(),
    }
}

fn write_note(home: &Path, lines: &[String]) -> std::io::Result<()> {
    let logs = home.join("logs");
    std::fs::create_dir_all(&logs)?;
    std::fs::write(logs.join("aion-server.death.log"), lines.join("\n") + "\n")
}

/// One note line as the writer emits it: timestamp, pid frame, entry.
fn stamped(pid: u32, body: &str) -> String {
    format!("2026-08-24T08:00:00+00:00 pid={pid} {body}")
}

/// A note line in the UN-FRAMED format that preceded per-pid framing. Used
/// only to prove the reader refuses to attribute one.
fn untagged(body: &str) -> String {
    format!("2026-08-24T08:00:00+00:00 {body}")
}

/// The record survives its trip through the note entry byte-exactly.
#[test]
fn outcome_record_roundtrips_through_its_entry() -> TestResult {
    let record = parked_record(4242);
    let entry = render_entry(&record)?;
    let json = entry
        .strip_prefix("OUTCOME ")
        .ok_or("entry must carry the OUTCOME kind")?;
    let read_back: OutcomeRecord = serde_json::from_str(json)?;
    assert_eq!(read_back, record, "the record must roundtrip unchanged");
    Ok(())
}

/// A record written before the declared-command census existed still reads:
/// the death note is shared with older builds, and their records carry no
/// `parked_declared_commands` field. The honest reading of that absence is
/// "none named" — never a parse refusal that erases the rest of the record.
#[test]
fn a_record_without_the_declared_field_reads_as_none_named() -> TestResult {
    let json = r#"{"pid":7,"outcome":"Clean","drain_timeout_seconds":30,"delivered_drain_requests":0,"parked":[],"managed_workers_stopped":[],"managed_workers_unstopped":[]}"#;
    let record: OutcomeRecord = serde_json::from_str(json)?;
    assert_eq!(record.pid, 7);
    assert!(
        record.parked_declared_commands.is_empty(),
        "an absent field must read as an empty census, not an error"
    );
    Ok(())
}

/// The ordinary shutdown shape: ARMED, OUTCOME, DISARMED — read back as
/// Disarmed with the record and the reason.
#[test]
fn a_disarmed_bracket_with_a_record_reads_whole() -> TestResult {
    let home = tempfile::tempdir()?;
    let record = parked_record(100);
    write_note(
        home.path(),
        &[
            stamped(100, "ARMED version=0.23.0 build=test"),
            stamped(100, &render_entry(&record)?),
            stamped(100, "DISARMED clean run-loop exit: shutdown outcome Parked"),
        ],
    )?;
    match read_fate(home.path(), 100)? {
        NoteFate::Disarmed {
            outcome,
            outcome_unreadable,
            reason,
        } => {
            assert_eq!(
                outcome.as_ref(),
                Some(&record),
                "the record must be read back"
            );
            assert_eq!(outcome_unreadable, None, "a read-back record is readable");
            assert!(
                reason.contains("Parked"),
                "the DISARMED reason must be carried: {reason}"
            );
        }
        other => return Err(format!("expected Disarmed, got {other:?}").into()),
    }
    Ok(())
}

/// Red first — the kill -9 honesty face: an ARMED bracket with no OUTCOME
/// and no DISARMED reads as `ArmedNotDisarmed` with NO record, never a
/// fabricated summary.
#[test]
fn a_killed_server_reads_as_armed_not_disarmed_with_no_record() -> TestResult {
    let home = tempfile::tempdir()?;
    write_note(
        home.path(),
        &[stamped(200, "ARMED version=0.23.0 build=test")],
    )?;
    match read_fate(home.path(), 200)? {
        NoteFate::ArmedNotDisarmed {
            outcome,
            outcome_unreadable,
        } => {
            assert_eq!(outcome, None, "no record must mean NO record");
            assert_eq!(
                outcome_unreadable, None,
                "an absent record is an absence, not an unreadable presence"
            );
        }
        other => return Err(format!("expected ArmedNotDisarmed, got {other:?}").into()),
    }
    Ok(())
}

/// The reader binds to the LAST bracket for the pid: an earlier clean
/// shutdown's record must not answer for the CURRENT incarnation.
#[test]
fn the_last_bracket_for_the_pid_wins() -> TestResult {
    let home = tempfile::tempdir()?;
    let old_record = parked_record(300);
    write_note(
        home.path(),
        &[
            stamped(300, "ARMED version=0.23.0 build=test"),
            stamped(300, &render_entry(&old_record)?),
            stamped(300, "DISARMED clean run-loop exit: shutdown outcome Parked"),
            stamped(300, "ARMED version=0.23.0 build=test"),
        ],
    )?;
    match read_fate(home.path(), 300)? {
        NoteFate::ArmedNotDisarmed {
            outcome,
            outcome_unreadable,
        } => {
            assert_eq!(
                outcome, None,
                "the previous bracket's record must not leak into the new bracket"
            );
            assert_eq!(outcome_unreadable, None, "nor may its unreadable face");
        }
        other => return Err(format!("expected ArmedNotDisarmed, got {other:?}").into()),
    }
    Ok(())
}

/// A bracket that closed without a record reads as Disarmed with an honest
/// absence — the pre-record-binary and error-return shapes.
#[test]
fn a_disarmed_bracket_without_a_record_reports_the_absence() -> TestResult {
    let home = tempfile::tempdir()?;
    write_note(
        home.path(),
        &[
            stamped(400, "ARMED version=0.22.0 build=test"),
            stamped(400, "DISARMED run scope exited without an explicit disarm"),
        ],
    )?;
    match read_fate(home.path(), 400)? {
        NoteFate::Disarmed { outcome, .. } => {
            assert_eq!(outcome, None, "absence must be reported as absence");
        }
        other => return Err(format!("expected Disarmed, got {other:?}").into()),
    }
    Ok(())
}

/// No note and no bracket are DISTINCT states, and another pid's bracket is
/// not ours.
#[test]
fn absence_states_are_distinct() -> TestResult {
    let home = tempfile::tempdir()?;
    assert_eq!(read_fate(home.path(), 1)?, NoteFate::NoNote);
    write_note(
        home.path(),
        &[stamped(555, "ARMED version=0.23.0 build=test")],
    )?;
    assert_eq!(
        read_fate(home.path(), 1)?,
        NoteFate::NoBracketForPid,
        "another pid's bracket must not answer for ours"
    );
    Ok(())
}

/// A torn OUTCOME line (a crash mid-write) never becomes a fabricated
/// record — and it is reported as an unreadable PRESENCE, not silently
/// collapsed into "the drain never wrote one".
#[test]
fn a_torn_outcome_line_reads_as_an_unreadable_presence() -> TestResult {
    let home = tempfile::tempdir()?;
    write_note(
        home.path(),
        &[
            stamped(600, "ARMED version=0.23.0 build=test"),
            stamped(600, "OUTCOME {\"pid\":600,\"outcome\":\"Par"),
        ],
    )?;
    match read_fate(home.path(), 600)? {
        NoteFate::ArmedNotDisarmed {
            outcome,
            outcome_unreadable,
        } => {
            assert_eq!(
                outcome, None,
                "a torn record must never be guessed into a summary"
            );
            assert!(
                outcome_unreadable.is_some(),
                "a torn OUTCOME line is an UNREADABLE PRESENCE, distinct from the \
                 honest absence: the drain wrote a record even if this binary \
                 cannot read it"
            );
        }
        other => return Err(format!("expected ArmedNotDisarmed, got {other:?}").into()),
    }
    Ok(())
}

/// Both mixed orders keep BOTH facts. The renderers' rule — "a readable
/// record does not un-happen the torn line beside it" — is enforced at the
/// reader: whichever order the lines arrived in, a torn line and a readable
/// record inside one bracket are both carried.
#[test]
fn a_torn_line_survives_beside_a_readable_record_in_both_orders() -> TestResult {
    let record = parked_record(600);
    let good = stamped(600, &render_entry(&record)?);
    let torn = stamped(600, "OUTCOME {\"pid\":600,\"outcome\":\"Par");

    for (label, lines) in [
        ("good-then-torn", [good.clone(), torn.clone()]),
        ("torn-then-good", [torn, good]),
    ] {
        let home = tempfile::tempdir()?;
        let mut note = vec![stamped(600, "ARMED version=0.23.0 build=test")];
        note.extend(lines);
        write_note(home.path(), &note)?;
        match read_fate(home.path(), 600)? {
            NoteFate::ArmedNotDisarmed {
                outcome,
                outcome_unreadable,
            } => {
                assert_eq!(
                    outcome.as_ref(),
                    Some(&record),
                    "{label}: the readable record must be carried"
                );
                assert!(
                    outcome_unreadable.is_some(),
                    "{label}: the torn line must be carried BESIDE the record, \
                     never erased by it"
                );
            }
            other => {
                return Err(format!("{label}: expected ArmedNotDisarmed, got {other:?}").into());
            }
        }
    }
    Ok(())
}

/// 🔴 THE INTERLEAVED-BRACKET REGRESSION, in the shape it was driven in.
///
/// Measured 2026-08-26 at the coordinator's hands: boot a server (A) to
/// SERVING, attempt a second `aion server` (B), which arms its own note and is
/// then REFUSED the home by the birth claim, then `aion stop` A. A drained
/// cleanly, with its `OUTCOME` and `DISARMED` on disk — and the stop verb
/// reported "the death note's bracket for pid A never closed … the `kill -9`
/// shape", because B's `ARMED` line landed inside A's bracket and the reader
/// scanned only as far as the next `ARMED`.
///
/// The birth claim did not create this defect; it changed its FREQUENCY
/// CLASS. Every double-launch attempt now arms-and-abandons, so interleaved
/// brackets are ordinary, and the stop verb would routinely accuse an
/// operator's clean stop of being a kill -9.
///
/// The line ORDER here is the measured one, not a convenient one — B's whole
/// life sits between A's `SIGNAL` and A's `OUTCOME`.
#[test]
fn a_refused_boots_bracket_inside_a_live_one_attributes_to_neither_wrongly() -> TestResult {
    const SERVING: u32 = 12845;
    const REFUSED: u32 = 12903;

    let home = tempfile::tempdir()?;
    let clean = OutcomeRecord {
        pid: SERVING,
        outcome: ShutdownOutcome::Clean,
        drain_timeout_seconds: 30,
        delivered_drain_requests: 0,
        parked: Vec::new(),
        parked_declared_commands: Vec::new(),
        managed_workers_stopped: Vec::new(),
        managed_workers_unstopped: Vec::new(),
    };
    write_note(
        home.path(),
        &[
            stamped(SERVING, "ARMED version=0.26.0 build=test"),
            // B arms, is refused the home by the birth claim, and abandons —
            // its whole life inside A's bracket.
            stamped(REFUSED, "ARMED version=0.26.0 build=test"),
            stamped(
                REFUSED,
                "DISARMED run scope exited without an explicit disarm (an error return, \
                 or an unwind — see any PANIC entry directly above)",
            ),
            // A is signalled and drains cleanly, AFTER B's bracket closed.
            stamped(
                SERVING,
                "SIGNAL SIGTERM observed; the graceful drain owns the response",
            ),
            stamped(SERVING, &render_entry(&clean)?),
            stamped(
                SERVING,
                "DISARMED clean run-loop exit: shutdown outcome Clean",
            ),
        ],
    )?;

    // A: the clean exit it actually had — outcome and reason both.
    match read_fate(home.path(), SERVING)? {
        NoteFate::Disarmed {
            outcome,
            outcome_unreadable,
            reason,
        } => {
            assert_eq!(
                outcome.as_ref(),
                Some(&clean),
                "the serving incarnation's own OUTCOME must be read back, however \
                 many other brackets opened and closed across it"
            );
            assert_eq!(outcome_unreadable, None);
            assert!(
                reason.contains("clean run-loop exit"),
                "A's DISARMED reason must be A's, got: {reason}"
            );
        }
        other => {
            return Err(format!(
                "expected A's clean Disarmed — this is the regression: got {other:?}"
            )
            .into());
        }
    }

    // B: its own abandonment, and none of A's story.
    match read_fate(home.path(), REFUSED)? {
        NoteFate::Disarmed {
            outcome,
            outcome_unreadable,
            reason,
        } => {
            assert_eq!(
                outcome, None,
                "the refused boot wrote no drain outcome — and must not inherit the \
                 live server's"
            );
            assert_eq!(outcome_unreadable, None);
            assert!(
                reason.contains("without an explicit disarm"),
                "B's DISARMED reason must be B's, got: {reason}"
            );
        }
        other => return Err(format!("expected B's Disarmed, got {other:?}").into()),
    }
    Ok(())
}

/// The same shape with the SECOND boot abandoned by a mid-boot termination
/// signal rather than by the claim refusal — the other way a bracket lands
/// inside a live one now that `aion server stop` addresses a booting server.
#[test]
fn a_mid_boot_abandonment_inside_a_live_bracket_attributes_correctly() -> TestResult {
    const SERVING: u32 = 700;
    const BOOTING: u32 = 701;

    let home = tempfile::tempdir()?;
    write_note(
        home.path(),
        &[
            stamped(SERVING, "ARMED version=0.26.0 build=test"),
            stamped(BOOTING, "ARMED version=0.26.0 build=test"),
            stamped(
                BOOTING,
                "SIGNAL SIGTERM received during boot; no listener is bound and nothing \
                 is draining, so the boot is abandoned",
            ),
            stamped(
                BOOTING,
                "DISARMED SIGTERM received before the doors opened; the boot was \
                 abandoned with nothing serving and nothing draining",
            ),
        ],
    )?;
    // The live server is still running: its bracket is genuinely open, and
    // that must NOT be confused with the abandoned boot's closed one.
    assert!(
        matches!(
            read_fate(home.path(), SERVING)?,
            NoteFate::ArmedNotDisarmed {
                outcome: None,
                outcome_unreadable: None
            }
        ),
        "a live server's open bracket must not be closed by another pid's DISARMED"
    );
    match read_fate(home.path(), BOOTING)? {
        NoteFate::Disarmed { reason, .. } => assert!(
            reason.contains("before the doors opened"),
            "the abandoned boot must report its own reason, got: {reason}"
        ),
        other => {
            return Err(format!("expected the abandoned boot's Disarmed, got {other:?}").into());
        }
    }
    Ok(())
}

/// An entry with no `pid=` frame is UNATTRIBUTABLE, never guessed onto a pid.
///
/// This is the no-backwards-compatibility ruling made observable: a note
/// written by a server older than the framing is reported as unreadable-by-
/// attribution rather than scanned for a bracket. Guessing is what produced
/// the interleaving defect in the first place.
#[test]
fn an_unframed_note_is_reported_as_unattributable() -> TestResult {
    let home = tempfile::tempdir()?;
    write_note(
        home.path(),
        &[
            untagged("ARMED pid=900 version=0.25.1 build=test"),
            untagged("DISARMED clean run-loop exit: shutdown outcome Clean"),
        ],
    )?;
    assert_eq!(
        read_fate(home.path(), 900)?,
        NoteFate::Unattributable {
            untagged_entries: 2
        },
        "an old-format note must be reported, not read"
    );
    Ok(())
}

/// A note holding BOTH — an old server's unframed lines and a new server's
/// framed ones — answers the framed pid from its own entries, and answers the
/// unframed pid with the unattributable face. Neither leaks into the other.
#[test]
fn framed_and_unframed_entries_do_not_contaminate_each_other() -> TestResult {
    let home = tempfile::tempdir()?;
    write_note(
        home.path(),
        &[
            untagged("ARMED pid=900 version=0.25.1 build=test"),
            untagged("DISARMED clean run-loop exit: shutdown outcome Clean"),
            stamped(901, "ARMED version=0.26.0 build=test"),
            stamped(901, "DISARMED clean run-loop exit: shutdown outcome Clean"),
        ],
    )?;
    match read_fate(home.path(), 901)? {
        NoteFate::Disarmed { reason, .. } => assert!(reason.contains("clean run-loop exit")),
        other => return Err(format!("expected the framed pid's Disarmed, got {other:?}").into()),
    }
    assert_eq!(
        read_fate(home.path(), 900)?,
        NoteFate::Unattributable {
            untagged_entries: 2
        },
        "the unframed entries stay unattributable even beside framed ones"
    );
    Ok(())
}

/// A PANIC entry embeds a multi-line backtrace. Its continuation lines carry
/// no timestamp and no frame — and they must count as neither entries nor
/// evidence of an old format, or every panicking server would read as
/// unattributable.
#[test]
fn a_panic_entrys_backtrace_lines_are_not_mistaken_for_unframed_entries() -> TestResult {
    let home = tempfile::tempdir()?;
    write_note(
        home.path(),
        &[
            stamped(800, "ARMED version=0.26.0 build=test"),
            stamped(800, "PANIC thread=main location=src/x.rs:1 payload=boom"),
            "   0: <std::backtrace::Backtrace>::create".to_owned(),
            "   1: aion_server::death_note::panic_entry".to_owned(),
            stamped(800, "DISARMED run scope exited without an explicit disarm"),
        ],
    )?;
    match read_fate(home.path(), 800)? {
        NoteFate::Disarmed { reason, .. } => {
            assert!(reason.contains("without an explicit disarm"), "{reason}");
        }
        other => {
            return Err(format!(
                "backtrace continuation lines must not disturb the read, got {other:?}"
            )
            .into());
        }
    }
    Ok(())
}