aion_server/

death_note.rs

//! The server's death note: a durable record of what is killing the process
//! on paths that never reach the graceful-shutdown log.
//!
//! Two production servers died on 2026-08-16 (pids 52844 and 2681) with
//! identical faces: the main log ends mid-flight on routine lines — no drain,
//! no panic output, no OS crash report. This module exists so the third such
//! death is a diagnosis instead of a mystery. It arms three recorders around
//! the run loop, writing to `<AION_HOME>/logs/aion-server.death.log`:
//!
//! - an **armed/disarmed bracket**: one line when the server boots, one line
//!   when the run scope exits by ordinary control flow (clean shutdown or an
//!   error return, written by [`DeathNote`]'s `Drop`). A note that is armed
//!   but never disarmed, with no other entry, means the process was destroyed
//!   without the run loop seeing it — the `SIGKILL` / raw-`_exit` class no
//!   in-process handler can observe.
//! - a **panic hook**, chained in front of the previously installed hook,
//!   that records every panic's thread, location, payload, and backtrace. A
//!   panic in a spawned task can be survivable; the entry says so. Death by
//!   panic reads as a `PANIC` entry followed by the drop-written `DISARMED`
//!   line as the unwind leaves the run scope.
//! - a **termination-signal watcher** for the catchable signals whose default
//!   action kills the process silently (`SIGHUP`, `SIGQUIT`, `SIGUSR1`,
//!   `SIGUSR2`): the note records the signal, then re-applies the signal's
//!   default action so process behaviour is unchanged — the death is noted,
//!   not prevented. `SIGTERM` and `SIGINT` are recorded as observations only:
//!   the graceful drain in [`crate::run`] owns those two, and both watchers
//!   coexist because tokio and this module register through the same
//!   `signal-hook-registry` chain.
//! - **breadcrumbs**: work sites call [`breadcrumb`] as they begin a unit of
//!   in-flight work (today: every declared action body the server executes),
//!   so a corpse's LAST breadcrumb names what was running when the process
//!   died. The third 2026-08-16 death (pid 25680, ~16:27Z) happened
//!   mid-declared-action with only an ordinary INFO log line to say so; this
//!   makes that fact a note entry the corpse reader sees first.
//!
//! # Every entry carries its pid
//!
//! One home's note is shared by every server that ever ran against it, and
//! those lives INTERLEAVE: an unclaimed multi-server boot, a restart's
//! succession handover, and — routinely, since the birth claim landed — a
//! second `aion server` that arms, is refused the home, and abandons. So the
//! frame of every line is `<rfc3339> pid=<pid> <ENTRY>`, and the reader
//! ([`crate::control::outcome::read_fate`]) SELECTS by pid rather than
//! scanning a bracket until the next `ARMED`.
//!
//! 🔴 That bracket scan was a real defect, measured 2026-08-26: a refused
//! boot's `ARMED` line landed between a live server's `SIGNAL` and its
//! `OUTCOME`, the scan stopped there, and `aion server stop` reported "the
//! bracket never closed … the `kill -9` shape" about a server that had just
//! drained cleanly with its outcome on disk two lines below. Interleaving is
//! ORDINARY now, so the framing has to be right rather than the interleaving
//! rare.
//!
//! There is deliberately no reading of the un-framed format that preceded
//! this one. A line with no `pid=` tag is reported as UNATTRIBUTABLE — the
//! note is on disk and readable by eye — never guessed onto a pid.
//!
//! NOT covered, stated plainly: `SIGKILL`/`SIGSTOP` (uncatchable by kernel
//! contract); the fatal-fault signals (`SIGSEGV`/`SIGBUS`/`SIGILL`/`SIGFPE`)
//! and raw `abort()`, whose handlers must run in async-signal context and
//! therefore require `unsafe` this workspace denies (the OS crash reporter
//! remains the observer for that class — both 2026-08-16 corpses left no
//! crash report, which is itself evidence against that class); and a direct
//! `exit()` that bypasses the run scope. With the note armed, each of those
//! reads as ARMED-without-DISARMED plus the absence of every entry above — a
//! narrow, named remainder instead of an anonymous death.

use std::fs::{File, OpenOptions};
use std::io::Write;
use std::panic::PanicHookInfo;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, PoisonError};

use tracing::{error, info, warn};

use crate::ServerError;

/// One death note per process: the panic hook and signal registrations are
/// process-global, so a second armed instance would double-write every entry.
static ARMED_ONCE: AtomicBool = AtomicBool::new(false);

/// File name of the death note inside the Aion home's `logs/` directory.
const NOTE_FILE_NAME: &str = "aion-server.death.log";

/// The death note's path under `home` — the ONE derivation, shared with the
/// reader ([`crate::control::outcome::read_fate`]) so the writer and the
/// reader cannot drift apart: a drifted reader would report "the exit left no
/// recorded account" over a perfectly good note, silently.
#[must_use]
pub fn note_path(home: &std::path::Path) -> std::path::PathBuf {
    home.join("logs").join(NOTE_FILE_NAME)
}

/// The armed note's breadcrumb route. Work sites reach the note through this
/// process-global slot because the note's other recorders (panic hook, signal
/// watcher) are process-global by nature and [`ARMED_ONCE`] already enforces
/// a single armed note per process. Set by [`DeathNote::arm`], cleared on
/// disarm; [`breadcrumb`] is a silent no-op in between.
static BREADCRUMB_SLOT: Mutex<Option<BreadcrumbWriter>> = Mutex::new(None);

/// The pieces [`breadcrumb`] needs: the shared note file and the armed flag
/// that gates every recorder off after disarm.
struct BreadcrumbWriter {
    note: Arc<NoteFile>,
    armed: Arc<AtomicBool>,
}

/// Append a `BREADCRUMB` entry to the armed death note, naming work now in
/// flight so a corpse's last breadcrumb identifies the site that never
/// finished. A silent no-op when no note is armed (tests, tools, the CLI).
///
/// Every note entry is written through to disk (`sync_all`), so call this at
/// work-start cadence — one entry per declared action body — never inside a
/// hot loop.
pub fn breadcrumb(entry: &str) {
    let slot = BREADCRUMB_SLOT
        .lock()
        .unwrap_or_else(PoisonError::into_inner);
    if let Some(writer) = slot.as_ref()
        && writer.armed.load(Ordering::SeqCst)
    {
        writer.note.write_entry(&format!("BREADCRUMB {entry}"));
    }
}

/// What the signal watcher does after noting a signal whose default action
/// terminates the process. Production re-applies the default action (the
/// process dies exactly as it would have, but on the record); tests inject a
/// recorder so the suite is not killed by its own probe.
type FatalAction = Box<dyn Fn(i32) + Send + Sync>;

/// The armed death note. Constructed by [`DeathNote::arm`] early in the
/// server run loop; its `Drop` writes the `DISARMED` line, so any ordinary
/// exit from the run scope — clean shutdown, error return, or a panic
/// unwinding through it — closes the bracket on the record.
pub struct DeathNote {
    note: Arc<NoteFile>,
    armed: Arc<AtomicBool>,
    /// Whether the run loop's graceful drain is watching for a termination
    /// signal yet. See [`DeathNote::drain_owns_termination`].
    drain_owns_termination: Arc<AtomicBool>,
    #[cfg(unix)]
    signals_handle: signal_hook::iterator::Handle,
    disarm_reason: Option<String>,
}

impl std::fmt::Debug for DeathNote {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("DeathNote")
            .field("path", &self.note.path)
            .field("armed", &self.armed.load(Ordering::SeqCst))
            .finish_non_exhaustive()
    }
}

impl DeathNote {
    /// Arm the death note under the resolved Aion home: create
    /// `<home>/logs/`, open the note file append-only, write the `ARMED`
    /// line, install the panic hook, and start the signal watcher.
    ///
    /// # Errors
    ///
    /// Returns [`ServerError::DeathNote`] when a note is already armed in
    /// this process, when the logs directory or note file cannot be created,
    /// or when the signal watcher cannot be installed. Arming failures are
    /// fatal to boot by design: a server that cannot record its own death is
    /// exactly the server this module exists for.
    pub fn arm(home: &Path) -> Result<Self, ServerError> {
        Self::arm_with_fatal_action(home, None)
    }

    /// [`DeathNote::arm`] with an injectable fatal action. `None` selects the
    /// production action (re-apply the signal's default, terminating the
    /// process); tests inject a recorder so the probe signal does not kill
    /// the test binary. The seam sits on the production path — the injected
    /// closure replaces only the final kill, never the note write.
    fn arm_with_fatal_action(
        home: &Path,
        fatal_action: Option<FatalAction>,
    ) -> Result<Self, ServerError> {
        if ARMED_ONCE
            .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
            .is_err()
        {
            return Err(death_note_error(
                "a death note is already armed in this process; the panic hook and \
                 signal registrations are process-global and must not be doubled",
            ));
        }
        let logs_dir = home.join("logs");
        std::fs::create_dir_all(&logs_dir).map_err(|source| {
            death_note_error(format!(
                "could not create logs directory `{}`: {source}",
                logs_dir.display()
            ))
        })?;
        let path = note_path(home);
        let file = OpenOptions::new()
            .create(true)
            .append(true)
            .open(&path)
            .map_err(|source| {
                death_note_error(format!(
                    "could not open death note `{}`: {source}",
                    path.display()
                ))
            })?;
        let note = Arc::new(NoteFile {
            path,
            file: Mutex::new(file),
            pid: std::process::id(),
        });
        let build = crate::build_identity::BuildIdentity::current();
        // No `pid=` in the body: the frame every entry carries already has
        // it, and two spellings of one fact is how a reader ends up trusting
        // the wrong one.
        note.write_entry(&format!(
            "ARMED version={} build={}",
            env!("CARGO_PKG_VERSION"),
            build.line(),
        ));
        let armed = Arc::new(AtomicBool::new(true));
        // False until the run loop is actually watching for a termination
        // signal — see `drain_owns_termination` for why the boot window is
        // not the drain's to own.
        let drain_owns_termination = Arc::new(AtomicBool::new(false));
        *BREADCRUMB_SLOT
            .lock()
            .unwrap_or_else(PoisonError::into_inner) = Some(BreadcrumbWriter {
            note: Arc::clone(&note),
            armed: Arc::clone(&armed),
        });
        install_panic_hook(Arc::clone(&note), Arc::clone(&armed));
        #[cfg(unix)]
        let signals_handle = {
            let action =
                fatal_action.unwrap_or_else(|| terminate_with_default_action(Arc::clone(&note)));
            spawn_signal_watcher(
                Arc::clone(&note),
                Arc::clone(&armed),
                Arc::clone(&drain_owns_termination),
                action,
            )?
        };
        #[cfg(not(unix))]
        drop(fatal_action);
        info!(path = %note.path.display(), "death note armed");
        Ok(Self {
            note,
            armed,
            drain_owns_termination,
            #[cfg(unix)]
            signals_handle,
            disarm_reason: None,
        })
    }

    /// Declare that the run loop's graceful drain is now watching for a
    /// termination signal, so the watcher must only OBSERVE one.
    ///
    /// 🔴 Before this existed, the watcher observed `SIGTERM`/`SIGINT` from
    /// the instant the note was armed — which is the instant the home is
    /// claimed, minutes before the doors open on a large store. Registering a
    /// handler MASKS the default action, so during the whole boot a
    /// termination signal was caught, written down, and answered by nobody:
    /// the drain that "owns the response" was not listening yet. The process
    /// was un-terminable by `SIGTERM` for the length of its own recovery, and
    /// then went on to serve as though nothing had been asked of it. Measured
    /// 2026-08-26: `aion server stop` against a booting server reported
    /// `still draining` at patience against a server that was not draining,
    /// and the server came up serving thirty seconds later.
    ///
    /// While this is false, a termination signal ABANDONS THE BOOT. That is
    /// the honest response: no listener is bound, no work has been accepted,
    /// and there is nothing whatsoever to drain — so the only thing a drain
    /// could add is delay. The exit is recorded first, and the record the
    /// process leaves behind is reconciled as a dead incarnation by the next
    /// boot and removed by `aion server stop`'s own bookkeeping.
    pub fn drain_owns_termination(&self) {
        self.drain_owns_termination.store(true, Ordering::SeqCst);
    }

    /// Path of the note file, for the startup banner and operator docs.
    #[must_use]
    pub fn path(&self) -> &Path {
        &self.note.path
    }

    /// Close the bracket explicitly with the run loop's own account of how it
    /// ended. Consumes the note; the `DISARMED` line is written by `Drop`.
    pub fn disarm(mut self, reason: &str) {
        self.disarm_reason = Some(reason.to_owned());
    }

    /// Append the shutdown outcome record as an `OUTCOME` entry, written
    /// BEFORE disarm so `aion server stop` can read the drain's full result
    /// after the process is gone ([`crate::control::outcome`] is the
    /// reader). The death note already owns the "what ended this process"
    /// seam; extending it with one entry kind keeps a single file and a
    /// single writer — no parallel outcome channel.
    ///
    /// A record that cannot be serialized is reported and dropped: the
    /// shutdown must not fail over its own receipt, and the reader treats
    /// the record's absence as a state it names honestly.
    pub fn record_outcome(&self, record: &crate::control::outcome::OutcomeRecord) {
        match crate::control::outcome::render_entry(record) {
            Ok(entry) => self.note.write_entry(&entry),
            Err(error) => {
                error!(
                    %error,
                    "could not write the shutdown outcome record into the death note; \
                     `aion server stop` will report the record as absent"
                );
            }
        }
    }
}

impl Drop for DeathNote {
    fn drop(&mut self) {
        self.armed.store(false, Ordering::SeqCst);
        #[cfg(unix)]
        self.signals_handle.close();
        let reason = self.disarm_reason.as_deref().unwrap_or(
            "run scope exited without an explicit disarm (an error return, or an unwind — \
             see any PANIC entry directly above)",
        );
        self.note.write_entry(&format!("DISARMED {reason}"));
        *BREADCRUMB_SLOT
            .lock()
            .unwrap_or_else(PoisonError::into_inner) = None;
        info!(path = %self.note.path.display(), reason, "death note disarmed");
    }
}

/// The note file plus its path, shared by the run scope, the panic hook, and
/// the signal watcher thread.
struct NoteFile {
    path: PathBuf,
    file: Mutex<File>,
    /// This process's id, stamped onto EVERY entry. One note per process
    /// (`ARMED_ONCE`), so one pid per writer — the frame is a property of the
    /// file handle, not of each call site, which is what makes it impossible
    /// for a new entry kind to be added without it.
    pid: u32,
}

impl NoteFile {
    /// Append one timestamped, pid-framed entry and force it to disk. A
    /// poisoned lock is recovered and written through — the writer that
    /// poisoned it was a panicking thread, which is precisely when this file
    /// must still accept entries. A write or sync failure falls back to
    /// stderr so the entry is never silently lost while the process can still
    /// say anything at all.
    fn write_entry(&self, entry: &str) {
        let line = format!(
            "{} pid={} {entry}\n",
            chrono::Utc::now().to_rfc3339(),
            self.pid
        );
        let mut file = self.file.lock().unwrap_or_else(PoisonError::into_inner);
        let written = file
            .write_all(line.as_bytes())
            .and_then(|()| file.flush())
            .and_then(|()| file.sync_all());
        if let Err(io_error) = written {
            eprintln!(
                "death note write to `{}` failed ({io_error}); the entry was: {line}",
                self.path.display()
            );
        }
    }
}

/// Build the [`ServerError`] for an arming failure.
fn death_note_error(message: impl Into<String>) -> ServerError {
    ServerError::DeathNote {
        message: message.into(),
    }
}

/// Install the panic hook, chained IN FRONT of the previously installed hook
/// so default stderr reporting (and anything a test harness installed) still
/// runs after the note is on disk.
fn install_panic_hook(note: Arc<NoteFile>, armed: Arc<AtomicBool>) {
    let previous = std::panic::take_hook();
    std::panic::set_hook(Box::new(move |panic_info: &PanicHookInfo<'_>| {
        if armed.load(Ordering::SeqCst) {
            note.write_entry(&panic_entry(panic_info));
        }
        previous(panic_info);
    }));
}

/// Render a panic into one note entry: thread, location, payload, and the
/// full backtrace (force-captured — by the time this runs, cost is moot).
fn panic_entry(panic_info: &PanicHookInfo<'_>) -> String {
    let thread = std::thread::current();
    let thread_name = thread.name().unwrap_or("<unnamed>").to_owned();
    let location = panic_info
        .location()
        .map_or_else(|| "<unknown location>".to_owned(), ToString::to_string);
    let payload: &str = panic_info
        .payload()
        .downcast_ref::<&str>()
        .copied()
        .or_else(|| {
            panic_info
                .payload()
                .downcast_ref::<String>()
                .map(String::as_str)
        })
        .unwrap_or("<non-string panic payload>");
    let backtrace = std::backtrace::Backtrace::force_capture();
    format!(
        "PANIC thread={thread_name} location={location} payload={payload} \
         (a panic in a spawned task can be survivable; death by panic is this entry \
         followed by a DISARMED line as the unwind leaves the run scope)\n{backtrace}"
    )
}

/// The production fatal action: re-apply the signal's default disposition so
/// the process terminates exactly as it would have without the watcher. If
/// the default action somehow fails to terminate, that failure is itself
/// noted and the process exits loudly with the conventional `128 + signal`
/// code rather than continuing in an undefined disposition.
#[cfg(unix)]
fn terminate_with_default_action(note: Arc<NoteFile>) -> FatalAction {
    Box::new(move |signal| {
        if let Err(io_error) = signal_hook::low_level::emulate_default_handler(signal) {
            note.write_entry(&format!(
                "SIGNAL-DEFAULT-FAILED signal={} error={io_error}; exiting {} instead",
                signal_label(signal),
                128 + signal,
            ));
            std::process::exit(128 + signal);
        }
    })
}

/// Spawn the signal watcher thread over the catchable termination set.
///
/// `SIGTERM`/`SIGINT` entries are observations ONCE the run loop's graceful
/// drain is watching (`drain_owns_termination`); before that — the whole boot
/// window, which is minutes on a large store — they ABANDON THE BOOT, because
/// registering this watcher masks the default action and nothing else is
/// listening yet. `SIGHUP`/`SIGQUIT`/`SIGUSR1`/`SIGUSR2` had a default
/// disposition of silent process death, so those are noted, synced, and then
/// handed to `fatal_action`. Registration goes through `signal-hook`'s
/// registry, which chains with tokio's own listeners rather than replacing
/// them.
#[cfg(unix)]
fn spawn_signal_watcher(
    note: Arc<NoteFile>,
    armed: Arc<AtomicBool>,
    drain_owns_termination: Arc<AtomicBool>,
    fatal_action: FatalAction,
) -> Result<signal_hook::iterator::Handle, ServerError> {
    use signal_hook::consts::{SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGUSR2};

    let mut signals =
        signal_hook::iterator::Signals::new([SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGUSR2])
            .map_err(|source| {
            death_note_error(format!("could not register the signal watcher: {source}"))
        })?;
    let handle = signals.handle();
    std::thread::Builder::new()
        .name("aion-death-note".to_owned())
        .spawn(move || {
            for signal in &mut signals {
                if !armed.load(Ordering::SeqCst) {
                    continue;
                }
                let label = signal_label(signal);
                if signal == SIGTERM || signal == SIGINT {
                    if drain_owns_termination.load(Ordering::SeqCst) {
                        note.write_entry(&format!(
                            "SIGNAL {label} observed; the graceful drain owns the response"
                        ));
                        warn!(signal = label, "death note observed a termination signal");
                        continue;
                    }
                    // No drain is listening yet: this is the BOOT window, and
                    // registering this handler is what masked the default
                    // action. Answer it here rather than leaving the signal
                    // caught and unanswered.
                    note.write_entry(&format!(
                        "SIGNAL {label} received during boot; no listener is bound and \
                         nothing is draining, so the boot is abandoned"
                    ));
                    // Close the bracket HERE, before the default action ends
                    // the process. `Drop` never runs on this path, so without
                    // this the note would show an ARMED bracket that never
                    // closed — which every reader is entitled to read as the
                    // `kill -9` shape, and `aion server stop` duly reported
                    // exactly that about a stop the operator had just asked
                    // for. An exit this deliberate is owed a recorded reason.
                    //
                    // `armed` is deliberately NOT cleared: the default action
                    // ends the process on the next line, and clearing the flag
                    // would only silence entries from the microseconds in
                    // between — including a PANIC one, which is precisely what
                    // a reader would most want to see.
                    note.write_entry(&format!(
                        "DISARMED {label} received before the doors opened; the boot \
                         was abandoned with nothing serving and nothing draining"
                    ));
                    warn!(
                        signal = label,
                        "death note received a termination signal before the doors \
                         opened; abandoning the boot (nothing is serving and nothing \
                         is draining, so there is nothing a drain could do)"
                    );
                    fatal_action(signal);
                } else {
                    note.write_entry(&format!(
                        "SIGNAL {label} received; terminating with the signal's default action"
                    ));
                    error!(
                        signal = label,
                        "death note recorded a fatal signal; applying its default action"
                    );
                    fatal_action(signal);
                }
            }
        })
        .map_err(|source| {
            death_note_error(format!(
                "could not spawn the signal watcher thread: {source}"
            ))
        })?;
    Ok(handle)
}

/// Human-readable signal name (`SIGHUP`), falling back to the raw number for
/// anything the platform table does not name.
#[cfg(unix)]
fn signal_label(signal: i32) -> String {
    signal_hook::low_level::signal_name(signal)
        .map_or_else(|| format!("signal {signal}"), ToOwned::to_owned)
}

#[cfg(test)]
#[path = "death_note_tests.rs"]
mod tests;