gunnar_server/

narrate.rs

//! **The app's half of the boot narration: stages, ticks and fault codes.**
//!
//! # The measurement this exists for
//!
//! 2026-09-23, the live estate. The twin was installed with the narration
//! channel ON (`korp.log_udp=10.13.2.127:5140`) and it delivered **66
//! datagrams**: three application lines and sixty-three kernel records.
//! Network up at 6.34 s, xfs recovered, `end of hand-off` at 6.35 s — and then
//! silence, for ever. The twin then read its config, bound its listeners, took
//! its role, held its certificate, started following its primary, and panicked
//! (`Cannot drop a runtime in a context where blocking is not allowed`). gunnar
//! is PID 1, so the kernel panicked with it and the box stopped.
//!
//! **Everything that mattered happened after the last datagram.** The rung
//! waiting on that box timed out at 600 s with an empty status, and the only
//! account anybody got was a VNC screenshot taken by hand.
//!
//! `boot::ship_boot_narration` was never meant to be the whole story: it is the
//! hand-off, the moment the DHCP lease makes the wire usable, and it says so —
//! `end of hand-off`. Nothing said anything after it. This module is what
//! speaks from there on.
//!
//! # What it does
//!
//! * **[`at`]** — one line at every boundary the app crosses, with a monotonic
//!   stamp: config read, every listener BOUND with its address, the role, a
//!   twin's primary and certificate, serving.
//! * **[`tick`](Narrator::tick)** — one short line a minute with uptime and the
//!   current stage, so silence means dead rather than idle, and the ear can say
//!   "last tick N s ago" instead of "nothing has ever arrived" (which cannot
//!   tell "never started" from "died at minute 12").
//! * **[`fault`]** — a short STABLE code, the stage, one line of reason and the
//!   crash-record path; re-sent because this is UDP; acknowledged by the ear;
//!   then a CLEAN POWER-OFF. No kernel panic, no reboot loop. The rung refuses
//!   by that code.
//!
//! # Three readers, one story
//!
//! Every line goes to the console, to `BOOTLOG.TXT` on the ESP (which
//! `korp-start` left mounted read-only at `/boot`, flipped read-write for one
//! append at a time — the same flip the rollback floor makes) and to
//! `/data/log/boot.log`. A fault also writes `/data/crash/fault.txt`. A box
//! that dies before its DHCP lease has said nothing on the wire and everything
//! on the other two.
//!
//! # The off switch
//!
//! `korp.log_udp=` on the kernel command line, and nothing else. Unset ⇒
//! [`armed`] is false, no socket is bound, no tick thread exists and not one
//! datagram is sent; the ESP and `/data` lines cost nothing anybody can hear
//! and are kept, because a box that powers itself off has to leave the reason
//! somewhere. The estate's steady state ships media with no `log_udp` param, so
//! a box that narrates for ever cannot be produced by forgetting to turn
//! something off — only by asking for it.
//!
//! # One vocabulary, three spellings
//!
//! `korp-installer::narrate` (korp-start), this module and the ear
//! (`private-gunnar-ops/xtask/src/bootnarr.rs`) each spell these constants.
//! There is no dependency edge between the three repositories, which is the
//! same arrangement the secret-class door already lives under and for the same
//! reason. What keeps them honest is that the wire format is a TOKEN and not
//! prose: a spelling that drifts stops matching, and the ear says so loudly
//! rather than quietly judging something else.

use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};

// ── the vocabulary ───────────────────────────────────────────────────────────

/// A boundary in the boot, named. The first eight are `korp-start`'s, before
/// the exec; this process emits the last eight. One enum because it is one
/// story on one wire.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Stage {
    /// PID 1's first instruction.
    Entered,
    /// `devtmpfs` is on `/dev`.
    Devtmpfs,
    /// fds 0-2 take a byte.
    Console,
    /// The kernel command line, as parsed.
    Cmdline,
    /// The disks seen, and which was chosen.
    Disks,
    /// The data filesystem is mounted.
    DataFs,
    /// The app archive is open and its digest is known.
    Archive,
    /// The exec: which binary, which version, which args.
    Exec,
    /// This process read its configuration.
    Config,
    /// The network link is up.
    Network,
    /// A DHCP lease, with the address. The first stage the wire can carry live.
    Lease,
    /// The narration sink is resolved and the backlog has been flushed.
    Sink,
    /// One listener is BOUND, with its address. Once per listener.
    Listener,
    /// The role this box took.
    Role,
    /// A twin: the primary it follows and the certificate it holds.
    Follow,
    /// Serving. The end of the boot story and the beginning of the ticks.
    Serving,
}

impl Stage {
    /// The wire token: one lower-case word, matched rather than read.
    pub const fn token(self) -> &'static str {
        match self {
            Stage::Entered => "entered",
            Stage::Devtmpfs => "devtmpfs",
            Stage::Console => "console",
            Stage::Cmdline => "cmdline",
            Stage::Disks => "disks",
            Stage::DataFs => "datafs",
            Stage::Archive => "archive",
            Stage::Exec => "exec",
            Stage::Config => "config",
            Stage::Network => "network",
            Stage::Lease => "lease",
            Stage::Sink => "sink",
            Stage::Listener => "listener",
            Stage::Role => "role",
            Stage::Follow => "follow",
            Stage::Serving => "serving",
        }
    }
}

/// Every stage, in the order a healthy boot crosses them. Index = discriminant,
/// which is what lets the tick thread carry the stage in one atomic byte.
pub const ORDER: [Stage; 16] = [
    Stage::Entered,
    Stage::Devtmpfs,
    Stage::Console,
    Stage::Cmdline,
    Stage::Disks,
    Stage::DataFs,
    Stage::Archive,
    Stage::Exec,
    Stage::Config,
    Stage::Network,
    Stage::Lease,
    Stage::Sink,
    Stage::Listener,
    Stage::Role,
    Stage::Follow,
    Stage::Serving,
];

/// The marker a stage line begins with.
pub const STAGE_MARKER: &str = "STAGE";
/// The marker a tick line begins with.
pub const TICK_MARKER: &str = "TICK";
/// The marker a fault line begins with.
pub const FAULT_MARKER: &str = "FAULT";
/// What the ear sends back.
pub const ACK_MARKER: &str = "ACK";

/// One line a minute. Short on purpose: it is sent for the life of the box.
pub const TICK_EVERY: Duration = Duration::from_secs(60);

/// How many times a fault datagram is sent. One is a coin flip on UDP from a
/// box that is about to power itself off.
pub const FAULT_RESENDS: usize = 3;
/// The gap between the re-sends.
pub const FAULT_RESEND_EVERY: Duration = Duration::from_millis(400);

/// **How long a faulting box waits to be heard, and no longer: five seconds.**
///
/// Chosen and stated. Long enough for three datagrams and a reply to cross a
/// utility network (sub-millisecond) with four orders of magnitude of slack;
/// short enough that a box with no ear waits about as long as a human blinks
/// before it powers off. Unheard is NOT a reason to stay up — the whole point
/// is that the box stops rather than loops.
pub const ACK_BUDGET: Duration = Duration::from_secs(5);

/// The stable fault codes this process can emit. `KS-*` are `korp-start`'s and
/// live in `korp_installer::narrate`; these are the app's. Add codes, never
/// renumber one — a rung refuses BY the code.
pub const CODES: [(&str, &str); 6] = [
    ("GN-01", "the app could not read its configuration"),
    ("GN-02", "a declared listener could not be bound"),
    (
        "GN-03",
        "the role this box was given is not one it can take",
    ),
    ("GN-04", "a twin could not follow its primary"),
    ("GN-05", "the app stopped serving with an error"),
    ("GN-99", "the app panicked"),
];

/// **The code an ordinary `Err` out of the serve path carries**, read off the
/// last boundary the narrator crossed.
///
/// Until 2026-09-23 only the panic hook sent a code (`GN-99`). An `Err` out of
/// `serve` went `std::process::exit(1)`, and as PID 1 that is the kernel's
/// `Attempted to kill init!` — no code, no power-off, the tick simply stops.
/// `GN-01..04` were declared and never called. This is what calls them: the
/// stage names the boundary that WAS crossed, so the failure lies in the step
/// after it.
///
/// * nothing past `exec` — the configuration was never read: `GN-01`;
/// * `config`, `network`, `lease`, `sink` — the next boundary is a bound
///   listener, so this is the bind: `GN-02`;
/// * `listener` — the next boundary is the role: `GN-03`;
/// * `role`, `follow` — a twin that could not follow (a primary crosses from
///   `role` straight to `serving` with nothing fallible between): `GN-04`;
/// * `serving` — the box was up and its serve loop returned an error: `GN-05`.
pub fn code_after(stage: Stage) -> &'static str {
    match stage {
        Stage::Entered
        | Stage::Devtmpfs
        | Stage::Console
        | Stage::Cmdline
        | Stage::Disks
        | Stage::DataFs
        | Stage::Archive
        | Stage::Exec => "GN-01",
        Stage::Config | Stage::Network | Stage::Lease | Stage::Sink => "GN-02",
        Stage::Listener => "GN-03",
        Stage::Role | Stage::Follow => "GN-04",
        Stage::Serving => "GN-05",
    }
}

/// What a code means, in one clause.
pub fn code_meaning(code: &str) -> Option<&'static str> {
    CODES.iter().find(|(c, _)| *c == code).map(|(_, m)| *m)
}

/// The acknowledgement for `code`.
pub fn ack_line(code: &str) -> String {
    format!("{ACK_MARKER} {code}")
}

/// Does this datagram acknowledge `code`? Tolerant of an ear that says more
/// after the code; strict about the code itself.
pub fn ack_matches(datagram: &[u8], code: &str) -> bool {
    let text = String::from_utf8_lossy(datagram);
    let mut parts = text.split_whitespace();
    parts.next() == Some(ACK_MARKER) && parts.next() == Some(code)
}

/// `STAGE <token> +<ms>ms <detail>`.
pub fn stage_line(stage: Stage, monotonic_ms: u128, detail: &str) -> String {
    let detail = detail.trim();
    if detail.is_empty() {
        format!("{STAGE_MARKER} {} +{monotonic_ms}ms", stage.token())
    } else {
        format!(
            "{STAGE_MARKER} {} +{monotonic_ms}ms {detail}",
            stage.token()
        )
    }
}

/// `TICK stage=<token> up=<s>s`.
pub fn tick_line(stage: Stage, uptime_s: u64) -> String {
    format!("{TICK_MARKER} stage={} up={uptime_s}s", stage.token())
}

/// `FAULT <code> stage=<token> +<ms>ms reason=<one line> crash=<path>`.
///
/// The reason is flattened to one line: a newline would forge a second line at
/// every reader here, and all three are line-oriented.
pub fn fault_line(
    code: &str,
    stage: Stage,
    monotonic_ms: u128,
    reason: &str,
    crash: Option<&Path>,
) -> String {
    let reason = reason.replace(['\r', '\n'], " ");
    let reason = reason.trim();
    let crash = crash
        .map(|p| p.display().to_string())
        .unwrap_or_else(|| "-".to_string());
    format!(
        "{FAULT_MARKER} {code} stage={} +{monotonic_ms}ms reason={reason} crash={crash}",
        stage.token()
    )
}

/// Where the fault record goes on the data disk. `crate::slow::CRASH_DIR` is
/// the same directory; this is the fixed file inside it, one per boot, so a
/// reboot loop cannot fill `/data` with a file per attempt.
pub const CRASH_RECORD: &str = "fault.txt";

// ── the narrator ─────────────────────────────────────────────────────────────

/// The file `korp-start` appends this boot's narration to on the ESP.
const ESP_LOG: &str = "/boot/BOOTLOG.TXT";
/// The mountpoint that file lives on, flipped read-write for one append.
const ESP_MOUNT: &str = "/boot";
/// The durable log `korp-start` opened on the data home, which this process
/// continues rather than starting a second one beside.
const DATA_LOG: &str = "log/boot.log";

/// The one narrator, installed once by [`install`].
static NARRATOR: OnceLock<Narrator> = OnceLock::new();

/// **The app's narrator.** One per process, installed at the top of the serve
/// path and read from everywhere afterwards — including the panic hook, which
/// takes no arguments and is why this is a process global rather than a value
/// threaded through the call graph.
pub struct Narrator {
    /// The boot's monotonic zero for this process. The kernel's uptime is the
    /// number on the tick; this is the number on the stage lines, so they are
    /// stamped from the exec rather than from the power-on.
    t0: Instant,
    /// The last boundary crossed. An atomic because the tick thread reads it.
    stage: AtomicU8,
    /// The data home, when there is one — where the crash record goes.
    data: Mutex<Option<PathBuf>>,
    /// Everything narrated so far, for the crash record. A boot narrates a few
    /// KiB; unbounded is fine and bounded would lose the beginning, which is
    /// the part a fault at minute twelve needs.
    memory: Mutex<Vec<u8>>,
}

/// **Install the narrator.** Since 2026-09-24 there is no CLEAR-TEXT UDP wire:
/// the boot narration over `korp.log_udp` is RETIRED, because the installer
/// collects each VM's whole boot log over its own mutual TLS
/// (`crate::installer_listener`). Every line goes to the console, the ESP's
/// `BOOTLOG.TXT`, `/data`'s boot log and the log the installer reads — and on a
/// DEV build (feature `dev-narrate`, 2026-09-30) that same log also leaves
/// SEALED to the installer's key over udp/51822 from the moment the NIC has an
/// address, before the installer's link exists (`crate::dev_narrate`).
///
/// Idempotent: the first caller wins.
pub fn install() {
    let _ = NARRATOR.set(Narrator {
        t0: Instant::now(),
        // The exec happened, so this process starts where korp-start stopped.
        stage: AtomicU8::new(Stage::Exec as u8),
        data: Mutex::new(None),
        memory: Mutex::new(Vec::new()),
    });
}

/// The installed narrator, if there is one. Everything here is a no-op before
/// [`install`] — a unit test, `gunnar-solo` on a laptop, `serve --check`.
pub fn narrator() -> Option<&'static Narrator> {
    NARRATOR.get()
}

/// **Everything narrated so far** (empty before [`install`]): what a panic's
/// crash record holds under its fault line (`boot::install_panic_fault` names
/// it to edda's `fault::narration_from`).
pub fn memory() -> Vec<u8> {
    NARRATOR.get().map(Narrator::memory).unwrap_or_default()
}

/// **Cross a boundary, and say so on every reader.** The free function every
/// call site uses; a no-op before [`install`].
pub fn at(stage: Stage, detail: &str) {
    if let Some(n) = NARRATOR.get() {
        n.at(stage, detail);
    }
}

/// Tell the narrator where the data home is, so a fault can leave a crash
/// record on it. Called once the stores are open.
pub fn data_home(root: &Path) {
    if let Some(n) = NARRATOR.get() {
        *n.data.lock().expect("the narrator's data home") = Some(root.to_path_buf());
    }
}

impl Narrator {
    /// Milliseconds since this process started narrating.
    pub fn monotonic_ms(&self) -> u128 {
        self.t0.elapsed().as_millis()
    }

    /// Everything this narrator said so far, as rendered (`gunnar: …\r\n`).
    pub fn memory(&self) -> Vec<u8> {
        self.memory.lock().map(|m| m.clone()).unwrap_or_default()
    }

    /// The last boundary crossed.
    pub fn stage(&self) -> Stage {
        let raw = self.stage.load(Ordering::Relaxed) as usize;
        ORDER.get(raw).copied().unwrap_or(Stage::Exec)
    }

    /// Cross a boundary: remember it, then say it on all three readers.
    pub fn at(&self, stage: Stage, detail: &str) {
        self.stage.store(stage as u8, Ordering::Relaxed);
        // And to edda's PID 1, which a panic (`GN-99`, `boot::install_panic_fault`)
        // reads its stage and its restart-or-power-off off.
        if stage == Stage::Serving {
            appliance_pid1::fault::serving_now();
        } else {
            appliance_pid1::fault::reached(stage.token());
        }
        let line = stage_line(stage, self.monotonic_ms(), detail);
        self.say(&line);
    }

    /// **One line, on every reader.** The console (which on the appliance is
    /// the serial line teed with the VGA screen), the wire, `BOOTLOG.TXT` on
    /// the ESP and `/data`'s boot log. Every failure is swallowed: this is
    /// diagnostics, and a narration that could refuse a boot would be worse
    /// than no narration.
    pub fn say(&self, line: &str) {
        let rendered = format!("gunnar: {line}\r\n");
        if let Ok(mut m) = self.memory.lock() {
            m.extend_from_slice(rendered.as_bytes());
        }
        // The console first: it is the reader that needs nothing to exist.
        appliance_pid1::console::mirror_to_gfx_console(&rendered);
        eprint!("{rendered}");
        gunnar_common::logqueue::tap_line(rendered.trim_end());
        append_to_esp(rendered.as_bytes());
        if let Some(root) = self.data.lock().ok().and_then(|d| d.clone()) {
            append_file(&root.join(DATA_LOG), rendered.as_bytes());
        }
    }
}

/// **A fatal fault: the code, the record, the ear's word, then a clean stop.**
///
/// In this order, and the order is the point:
///
/// 1. the crash record on `/data/crash/fault.txt`, holding the whole narration
///    so far under the fault line — so the fault line can NAME it;
/// 2. the fault line on every reader, including the two that need no network;
/// 3. [`FAULT_RESENDS`] datagrams, spaced, because this is UDP;
/// 4. a bounded wait for `ACK <code>` ([`ACK_BUDGET`], five seconds);
/// 5. `sync` and power off — or, for a box that was serving, restart ([`end_after`]).
///
/// **Not a panic, and a reboot only once it serves.** gunnar is PID 1: a panic is
/// the kernel's `Attempted to kill init!`, which names an exit code and nothing
/// else, and a reboot into the same fault is a loop, which is how a fault
/// becomes a mystery — so a fault during the boot powers off. A SERVING box
/// restarts unless its last fault was recent ([`end_after`]). With the narration off it still writes the code to the two file
/// readers and still powers off cleanly.
///
/// Never returns.
pub fn fault(code: &'static str, stage: Stage, reason: &str) -> ! {
    let Some(n) = NARRATOR.get() else {
        // Nothing was installed: a unit test, or a fault before the serve path
        // began. Say it where a process without a narrator can say it, and
        // stop — but by exiting, not by powering off a developer's laptop.
        eprintln!("gunnar: {}", fault_line(code, stage, 0, reason, None));
        std::process::exit(1);
    };
    n.stage.store(stage as u8, Ordering::Relaxed);
    let ms = n.monotonic_ms();
    let root = n.data.lock().ok().and_then(|d| d.clone());
    // The previous fault's age, read BEFORE this one overwrites its record.
    let since_last = root
        .as_ref()
        .and_then(|r| std::fs::metadata(r.join(crate::slow::CRASH_DIR).join(CRASH_RECORD)).ok())
        .and_then(|m| m.modified().ok())
        .and_then(|t| t.elapsed().ok());
    let crash = root.and_then(|root| write_crash_record(&root, code, stage, ms, reason, n));
    let line = fault_line(code, stage, ms, reason, crash.as_deref());
    n.say(&line);
    if let Some(meaning) = code_meaning(code) {
        n.say(&format!("{code}: {meaning}"));
    }
    match end_after(stage, since_last) {
        End::Restart => {
            n.say(&format!("{code}: the box was SERVING and its last fault is older than {} min — restarting, not powering off", RESTART_LOOP_WINDOW.as_secs() / 60));
            restart_now()
        }
        End::PowerOff => power_off_now(),
    }
}

/// How recent the previous fault must be for a fault on a SERVING box to count
/// as a loop (and power off instead of restarting).
pub const RESTART_LOOP_WINDOW: Duration = Duration::from_secs(30 * 60);

/// How a fault ends the box.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum End {
    /// `sync` and reboot: the box comes back by itself.
    Restart,
    /// `sync` and power off: a person decides (see [`fault`]).
    PowerOff,
}

/// **How a fault at `stage` ends, given how long ago the previous one was.**
///
/// During the boot a reboot is a reboot into the same fault — a loop that turns
/// a fault into a mystery — so it powers off, as it always did. Once the box
/// SERVES, a fault is one request task's panic or one listener's error, and
/// powering off leaves the forge down until a person starts it, with nothing
/// alerting anybody (finalization 2026-09-28, row h-panic-poweroff; UpCloud
/// leaves a powered-off server stopped). So a serving box restarts — unless its
/// previous fault was within [`RESTART_LOOP_WINDOW`], which is a loop again.
/// `since_last_fault` is read off the crash record's mtime on `/data` (it
/// survives the restart); `None` = no earlier record.
pub fn end_after(stage: Stage, since_last_fault: Option<Duration>) -> End {
    match (stage, since_last_fault) {
        (Stage::Serving, None) => End::Restart,
        (Stage::Serving, Some(d)) if d >= RESTART_LOOP_WINDOW => End::Restart,
        _ => End::PowerOff,
    }
}

/// `sync(2)` then `reboot(RB_AUTOBOOT)`, and park if the kernel refuses.
pub fn restart_now() -> ! {
    #[cfg(target_os = "linux")]
    {
        rustix::fs::sync();
        let _ = rustix::system::reboot(rustix::system::RebootCommand::Restart);
    }
    loop {
        std::thread::sleep(Duration::from_secs(3600));
    }
}

/// `sync(2)` then `reboot(RB_POWER_OFF)`, and park if the kernel refuses.
///
/// A PID 1 that returns from `main` is `Attempted to kill init!` all over
/// again, so the park is the last resort rather than the plan.
pub fn power_off_now() -> ! {
    #[cfg(target_os = "linux")]
    {
        // `sync(2)` first: the crash record and the two log appends above were
        // fsync'd by their writers, but the box is about to lose power and
        // anything else in flight has one chance.
        rustix::fs::sync();
        let _ = rustix::system::reboot(rustix::system::RebootCommand::PowerOff);
    }
    loop {
        std::thread::sleep(Duration::from_secs(3600));
    }
}

/// `sync(2)` then `reboot(RB_AUTOBOOT)`, and park if the kernel refuses — the
/// restart a box restored from its twin owes (`twin::restart_after_restore`).
pub fn reboot_now() -> ! {
    #[cfg(target_os = "linux")]
    {
        rustix::fs::sync();
        let _ = rustix::system::reboot(rustix::system::RebootCommand::Restart);
    }
    loop {
        std::thread::sleep(Duration::from_secs(3600));
    }
}

/// Write `<data>/crash/fault.txt`: the fault line, what the code means, and the
/// narration that led to it. Best effort — the caller is on its way to a
/// power-off and a failed write must not change that.
fn write_crash_record(
    root: &Path,
    code: &str,
    stage: Stage,
    ms: u128,
    reason: &str,
    n: &Narrator,
) -> Option<PathBuf> {
    use std::io::Write as _;
    let path = root.join(crate::slow::CRASH_DIR).join(CRASH_RECORD);
    std::fs::create_dir_all(path.parent()?).ok()?;
    let mut f = std::fs::File::create(&path).ok()?;
    writeln!(f, "---- {} ----", fault_line(code, stage, ms, reason, None)).ok()?;
    if let Some(meaning) = code_meaning(code) {
        writeln!(f, "{code}: {meaning}").ok()?;
    }
    writeln!(f, "---- the boot narration up to this fault ----").ok()?;
    if let Ok(m) = n.memory.lock() {
        f.write_all(&m).ok()?;
    }
    f.sync_all().ok()?;
    Some(path)
}

/// Append to `BOOTLOG.TXT` on the ESP, flipping `/boot` read-write for exactly
/// one append and straight back — the same flip `korp-start` and the rollback
/// floor make, and for the same reason: nothing may hold a writer open across
/// a remount to read-only, which answers `EBUSY`.
///
/// Every failure is swallowed. A box with no ESP mounted (a container, a
/// laptop, `gunnar-solo`) simply has no such reader.
fn append_to_esp(bytes: &[u8]) {
    if !Path::new(ESP_LOG).exists() {
        return;
    }
    let flipped = remount(ESP_MOUNT, false).is_ok();
    append_file(Path::new(ESP_LOG), bytes);
    if flipped {
        let _ = remount(ESP_MOUNT, true);
    }
}

/// **Write one small file on the ESP** (a marker), `/boot` flipped read-write
/// for exactly that write, the way [`append_to_esp`] flips it. `false` when it
/// could not be written — the caller decides what that means.
pub fn write_esp_file(path: &Path, bytes: &[u8]) -> bool {
    use std::io::Write as _;
    let flipped = remount(ESP_MOUNT, false).is_ok();
    let written = std::fs::File::create(path)
        .and_then(|mut f| f.write_all(bytes).and_then(|_| f.sync_all()))
        .is_ok();
    if flipped {
        let _ = remount(ESP_MOUNT, true);
    }
    written
}

/// **Remove one small file from the ESP**, `/boot` flipped read-write for exactly that
/// removal (the next boot's cleanup of a record [`write_esp_file`] left).
pub fn remove_esp_file(path: &Path) -> bool {
    let flipped = remount(ESP_MOUNT, false).is_ok();
    let removed = std::fs::remove_file(path).is_ok();
    if flipped {
        let _ = remount(ESP_MOUNT, true);
    }
    removed
}

/// Open-append-fsync-close. No descriptor is held between lines, so the mount
/// under it can go back to read-only immediately.
fn append_file(path: &Path, bytes: &[u8]) {
    use std::io::Write as _;
    if let Ok(mut f) = std::fs::OpenOptions::new()
        .append(true)
        .create(true)
        .open(path)
    {
        let _ = f.write_all(bytes);
        let _ = f.sync_all();
    }
}

/// Flip a mountpoint between read-only and read-write in place.
#[cfg(target_os = "linux")]
fn remount(mountpoint: &str, readonly: bool) -> std::io::Result<()> {
    use rustix::mount::{mount_remount, MountFlags};
    let mut flags = MountFlags::NOSUID | MountFlags::NODEV | MountFlags::NOEXEC;
    if readonly {
        flags |= MountFlags::RDONLY;
    }
    mount_remount(mountpoint, flags, "").map_err(std::io::Error::from)
}

#[cfg(not(target_os = "linux"))]
fn remount(_mountpoint: &str, _readonly: bool) -> std::io::Result<()> {
    Ok(())
}

/// Seconds since the kernel started — the one clock `korp-start` and this
/// process share, so their ticks and stamps sit on the same number line.
pub fn uptime_s() -> u64 {
    std::fs::read_to_string("/proc/uptime")
        .ok()
        .and_then(|s| s.split_whitespace().next()?.parse::<f64>().ok())
        .map(|f| f as u64)
        .unwrap_or_default()
}

#[cfg(test)]
mod tests {
    use super::*;

    /// ★ **A panic's crash record holds Gunnar's narration again** (lane L10,
    /// appliance-pid1 0.1.5): edda's panic hook wrote `crash/fault.txt` with an
    /// empty narration; `boot::install_panic_fault` now names the narrator's
    /// memory to `fault::narration_from`, so the record carries every line the
    /// narrator said, under the fault line, as `narrate::fault`'s own record does.
    #[test]
    fn a_panics_crash_record_holds_the_narrators_memory() {
        static N: OnceLock<Narrator> = OnceLock::new();
        let n = N.get_or_init(|| Narrator {
            t0: Instant::now(),
            stage: AtomicU8::new(Stage::Exec as u8),
            data: Mutex::new(None),
            memory: Mutex::new(Vec::new()),
        });
        n.say("STAGE serving +812ms l10-crash-narration");
        assert_eq!(
            n.memory(),
            b"gunnar: STAGE serving +812ms l10-crash-narration\r\n"
        );
        // Not installed in a test process: the global is empty, never a panic.
        if NARRATOR.get().is_none() {
            assert!(memory().is_empty());
        }
        // Gunnar's wiring: the hook's installer names `narrate::memory` …
        let boot = include_str!("boot.rs");
        let hook = &boot[boot.find("pub fn install_panic_fault").unwrap()..];
        let hook = &hook[..hook.find("\n}\n").unwrap()];
        assert!(
            hook.contains("appliance_pid1::fault::narration_from(crate::narrate::memory)"),
            "{hook}"
        );
        // … and what such a source answers is what the panic's record holds.
        appliance_pid1::fault::narration_from(|| N.get().map(Narrator::memory).unwrap_or_default());
        let d = tempfile::tempdir().unwrap();
        let p = appliance_pid1::fault::PanicFault {
            code: "GN-99",
            meaning: "the app panicked",
            data: Some(d.path().to_path_buf()),
        };
        appliance_pid1::fault::panic_ending(
            &p,
            "panicked at src/x.rs:1: boom",
            ("serving", true),
            9,
        );
        let text = std::fs::read_to_string(
            d.path()
                .join(appliance_pid1::fault::CRASH_DIR)
                .join(appliance_pid1::fault::CRASH_RECORD),
        )
        .unwrap();
        assert!(
            text.contains("gunnar: STAGE serving +812ms l10-crash-narration"),
            "{text}"
        );
        assert!(text.contains("GN-99: the app panicked"), "{text}");
        appliance_pid1::fault::narration_from(memory);
    }

    /// **A serving box that faults comes back by itself; a loop still stops.**
    /// (finalization 2026-09-28, row h-panic-poweroff.) Until now every fault
    /// powered the box off — right during the boot, where a reboot is a loop,
    /// and wrong once it SERVES: one panic in one request task left the forge
    /// down until a person started it, and nothing alerted. A second fault
    /// within [`RESTART_LOOP_WINDOW`] of the last one is a loop and powers off,
    /// exactly as before.
    #[test]
    fn a_serving_fault_restarts_the_box_unless_it_is_a_loop() {
        assert_eq!(
            end_after(Stage::Serving, None),
            End::Restart,
            "the first fault of a serving box: back by itself"
        );
        assert_eq!(
            end_after(Stage::Serving, Some(RESTART_LOOP_WINDOW)),
            End::Restart,
            "the last fault is old: not a loop"
        );
        assert_eq!(
            end_after(Stage::Serving, Some(Duration::from_secs(60))),
            End::PowerOff,
            "a fault a minute after the last: a loop, stop"
        );
        for s in ORDER.iter().copied().filter(|s| *s != Stage::Serving) {
            assert_eq!(
                end_after(s, None),
                End::PowerOff,
                "{s:?}: a fault during the boot reboots into the same fault — power off"
            );
        }
    }

    /// The tokens are a closed set, unique, and one word each — the ear
    /// matches on them, so a token with a space in it is a token that splits.
    #[test]
    fn the_stage_tokens_are_unique_single_words() {
        let mut seen: Vec<&str> = Vec::new();
        for (i, s) in ORDER.iter().enumerate() {
            assert_eq!(
                *s as usize, i,
                "the discriminant IS the index: the tick carries it as one byte"
            );
            assert!(!s.token().contains(' '), "{:?}", s.token());
            assert!(!seen.contains(&s.token()), "two stages spell {}", s.token());
            seen.push(s.token());
        }
    }

    /// **The vocabulary parity check.** These spellings exist three times, in
    /// three repositories with no dependency edge between them. If one drifts,
    /// the ear stops matching and says nothing useful — which is the failure
    /// this whole module exists to end. So the exact wire shapes are asserted
    /// here, as literals, rather than built from the constants that would drift
    /// with them.
    #[test]
    fn the_wire_shapes_are_exactly_what_the_ear_matches() {
        assert_eq!(
            stage_line(Stage::Lease, 6_340, "eth1 10.13.9.24/22"),
            "STAGE lease +6340ms eth1 10.13.9.24/22"
        );
        assert_eq!(tick_line(Stage::Serving, 720), "TICK stage=serving up=720s");
        assert_eq!(
            fault_line("GN-04", Stage::Follow, 12_345, "the primary refused", Some(Path::new("/data/crash/fault.txt"))),
            "FAULT GN-04 stage=follow +12345ms reason=the primary refused crash=/data/crash/fault.txt"
        );
        assert_eq!(ack_line("GN-99"), "ACK GN-99");
    }

    /// A fault is ONE line whatever the reason was — the twin's panic message
    /// has three lines in it, and a fault that forged two more lines at every
    /// reader would be a fault the ear judged as something else.
    #[test]
    fn a_fault_is_one_line_even_when_the_reason_is_not() {
        let l = fault_line(
            "GN-99",
            Stage::Follow,
            1,
            "panicked at\nsrc/runtime/shutdown.rs:51:\nCannot drop a runtime",
            None,
        );
        assert_eq!(l.lines().count(), 1, "{l}");
        assert!(l.contains("Cannot drop a runtime"), "{l}");
        // The field is stated as absent rather than omitted: a field that
        // sometimes disappears is a field the ear has to guess at.
        assert!(l.ends_with(" crash=-"), "{l}");
    }

    /// **Every death path has a code.** An `Err` at any stage maps to one of
    /// the declared codes — never `GN-99`, which is the panic's and would make
    /// a refused configuration read as a crash — and the four boot codes are
    /// each reachable from the boundary they sit behind.
    #[test]
    fn an_err_at_every_stage_carries_a_declared_code_that_is_not_the_panics() {
        for s in ORDER {
            let code = code_after(s);
            assert!(
                code_meaning(code).is_some(),
                "{code} for {:?} is not declared",
                s
            );
            assert_ne!(code, "GN-99", "an ordinary Err at {:?} is not a panic", s);
        }
        assert_eq!(
            code_after(Stage::Exec),
            "GN-01",
            "nothing read yet: the configuration"
        );
        assert_eq!(
            code_after(Stage::Config),
            "GN-02",
            "config read; the next boundary is a bound listener"
        );
        assert_eq!(code_after(Stage::Sink), "GN-02");
        assert_eq!(
            code_after(Stage::Listener),
            "GN-03",
            "bound; the next boundary is the role"
        );
        assert_eq!(
            code_after(Stage::Role),
            "GN-04",
            "role taken; a twin follows next"
        );
        assert_eq!(code_after(Stage::Follow), "GN-04");
        assert_eq!(
            code_after(Stage::Serving),
            "GN-05",
            "up, and the serve loop returned an error"
        );
        assert_eq!(
            [
                code_after(Stage::Exec),
                code_after(Stage::Config),
                code_after(Stage::Listener),
                code_after(Stage::Role)
            ],
            ["GN-01", "GN-02", "GN-03", "GN-04"],
            "the four boot codes are all reachable"
        );
    }

    /// The acknowledgement matches its own code and nothing else's.
    #[test]
    fn an_ack_matches_its_code_and_only_its_code() {
        assert!(ack_matches(b"ACK GN-99", "GN-99"));
        assert!(
            ack_matches(b"ACK GN-99 heard\n", "GN-99"),
            "an ear may say more"
        );
        assert!(!ack_matches(b"ACK GN-01", "GN-99"));
        assert!(!ack_matches(b"GN-99", "GN-99"), "a bare code is not an ack");
        assert!(!ack_matches(b"NACK GN-99", "GN-99"));
    }

    /// Every code is unique, shaped the way a matcher expects, and means
    /// something. A rung refuses BY these, so they are a contract.
    #[test]
    fn the_fault_codes_are_unique_and_stable_shaped() {
        let mut seen: Vec<&str> = Vec::new();
        for (c, meaning) in CODES {
            assert!(
                c.len() == 5 && c.as_bytes()[2] == b'-',
                "a code is XX-NN: {c}"
            );
            assert!(!seen.contains(&c), "two faults spell {c}");
            assert!(!meaning.is_empty());
            assert_eq!(code_meaning(c), Some(meaning));
            seen.push(c);
        }
        assert_eq!(code_meaning("ZZ-99"), None);
    }
}