aion-cli 0.31.0

//! Narrating someone else's boot from the outside.
//!
//! A server that spends minutes replaying its WAL used to look, from a
//! launcher's seat, exactly like a server that had wedged: the port did not
//! answer, and nothing else was readable. The pid record now carries the boot
//! stage and a sequence number that advances on every report, which is enough
//! to tell the two apart — and this module is the shared reader that turns
//! that into lines an operator can watch.
//!
//! Two callers narrate the same way: the bare `aion` launcher waiting on the
//! server it just spawned, and `aion server restart` waiting on the successor
//! it just started.
//!
//! # On patience
//!
//! There is deliberately NO stall cutoff here. A boot whose `stage_seq` has
//! not moved for two minutes may be a store on a slow disk, a writer-lock
//! wait behind a draining predecessor, or a genuine wedge — and this process
//! cannot tell which. Inventing a deadline would mean choosing, on the
//! operator's behalf, the moment at which a healthy recovery gets abandoned.
//! So the narrator says what it sees, keeps waiting, and names the two
//! commands that end the wait either way. The only thing that ends it by
//! itself is the process actually dying.

use std::path::Path;
use std::time::{Duration, Instant};

use aion_server::control::{IncarnationProbe, IncarnationState, PidRecord, incarnation, pid_file};

/// How long a stalled `stage_seq` must persist before the narrator says so.
/// This is a REPORTING cadence, not a patience: nothing gives up when it
/// elapses. Derived from the pid record's one-second stage timestamp
/// resolution — reporting faster than the record can change would print the
/// same sentence twice.
const STALL_REPORT_INTERVAL: Duration = Duration::from_secs(10);

/// A reader that watches one home's pid record and prints what changes.
pub struct BootNarrator {
    /// The last stage line printed, so an unchanged stage prints once.
    last_stage_line: Option<String>,
    /// The last `stage_seq` observed, and when it was first observed at that
    /// value — the pair that separates "progressing" from "stuck".
    last_seq: Option<(u64, Instant)>,
    /// When the last stall report was printed, so the message repeats at a
    /// readable cadence rather than every poll.
    last_stall_report: Option<Instant>,
    /// Whether any record has been seen at all. A launcher whose home never
    /// grows a record is in a different world (a legacy server, an unclaimed
    /// boot) and follows a different patience rule.
    seen_record: bool,
}

/// What one poll of the record found.
#[derive(Clone, Debug)]
pub enum BootObservation {
    /// The home has no pid record at all.
    NoRecord,
    /// The record exists but its process is not a verified live incarnation:
    /// it died, or its pid was recycled onto a stranger.
    NotLive(Box<PidRecord>),
    /// A live verified incarnation, in whatever state it recorded.
    Live(Box<PidRecord>),
    /// The record could not be read. Carried rather than swallowed: a home
    /// whose record is unreadable is a home whose verbs cannot address it,
    /// and the operator needs to know that even mid-wait.
    Unreadable(String),
}

impl Default for BootNarrator {
    fn default() -> Self {
        Self::new()
    }
}

impl BootNarrator {
    /// A narrator that has seen nothing yet.
    #[must_use]
    pub const fn new() -> Self {
        Self {
            last_stage_line: None,
            last_seq: None,
            last_stall_report: None,
            seen_record: false,
        }
    }

    /// Whether a live verified record has ever been observed. While this is
    /// false the caller is watching a home that may never grow a record (a
    /// server predating the birth claim, or one that booted unclaimed), and
    /// a bounded wait is the only honest one.
    #[must_use]
    pub const fn has_seen_record(&self) -> bool {
        self.seen_record
    }

    /// Read the record once and print whatever changed since the last read.
    pub fn observe(&mut self, home: &Path) -> BootObservation {
        let observation = read_observation(home);
        if let BootObservation::Live(record) = &observation {
            self.seen_record = true;
            self.narrate_live(record);
        }
        observation
    }

    /// Print the stage line when it changed, and the no-progress line when
    /// the sequence has not advanced for a while.
    fn narrate_live(&mut self, record: &PidRecord) {
        let now = Instant::now();
        match self.last_seq {
            Some((seq, _)) if seq == record.stage_seq => {}
            _ => {
                self.last_seq = Some((record.stage_seq, now));
                self.last_stall_report = None;
            }
        }
        if let Some(line) = stage_line(record)
            && self.last_stage_line.as_deref() != Some(line.as_str())
        {
            println!("{line}");
            self.last_stage_line = Some(line);
        }
        let Some((_, since)) = self.last_seq else {
            return;
        };
        let stalled_for = now.saturating_duration_since(since);
        if stalled_for < STALL_REPORT_INTERVAL {
            return;
        }
        let due = self
            .last_stall_report
            .is_none_or(|at| now.saturating_duration_since(at) >= STALL_REPORT_INTERVAL);
        if !due {
            return;
        }
        self.last_stall_report = Some(now);
        println!(
            "no progress reported for {}s — server {} is alive and still {}; waiting. \
             Ctrl-C leaves it running; `aion server stop` stops it",
            stalled_for.as_secs(),
            record.pid,
            record.state.label().to_lowercase()
        );
    }
}

/// One line describing where a live incarnation is, for the narrator and for
/// anything else that wants the same sentence.
#[must_use]
pub fn stage_line(record: &PidRecord) -> Option<String> {
    match record.state {
        IncarnationState::Booting => Some(match record.stage_line() {
            Some(stage) => format!("server {} booting — {stage}", record.pid),
            None => format!("server {} booting", record.pid),
        }),
        IncarnationState::Draining => Some(format!(
            "server {} (version {}) is draining",
            record.pid, record.version
        )),
        // A serving incarnation is not mid-anything; its caller announces the
        // arrival in its own words.
        IncarnationState::Serving => None,
    }
}

/// Read the home's record and classify it, without printing anything.
#[must_use]
pub fn read_observation(home: &Path) -> BootObservation {
    match pid_file::read(home) {
        Err(error) => BootObservation::Unreadable(error.to_string()),
        Ok(None) => BootObservation::NoRecord,
        Ok(Some(record)) => match incarnation::probe(&record) {
            IncarnationProbe::Verified { .. } => BootObservation::Live(Box::new(record)),
            IncarnationProbe::ProcessGone | IncarnationProbe::DifferentIncarnation { .. } => {
                BootObservation::NotLive(Box::new(record))
            }
        },
    }
}

#[cfg(test)]
mod tests {
    use super::{BootNarrator, BootObservation, read_observation, stage_line};
    use aion_server::control::{IncarnationState, PidRecord};

    type TestResult = Result<(), Box<dyn std::error::Error>>;

    fn booting_record(stage: &str, detail: &str, seq: u64) -> PidRecord {
        PidRecord {
            pid: 4242,
            started_at_unix_secs: 1,
            binary_sha256: "0".repeat(64),
            version: "0.0.0-test".to_owned(),
            commit: "test".to_owned(),
            state: IncarnationState::Booting,
            http_address: None,
            grpc_address: None,
            intended_http_address: None,
            intended_grpc_address: None,
            stage: Some(stage.to_owned()),
            stage_detail: Some(detail.to_owned()),
            stage_seq: seq,
            stage_updated_at_unix_secs: 1,
            drain_timeout_seconds: 0,
        }
    }

    /// The narration names the pid, the stage token, and the human detail —
    /// the three things an operator needs to decide whether to wait.
    #[test]
    fn a_booting_record_narrates_its_stage() {
        let line = stage_line(&booting_record(
            "wal-recovery",
            "materializing shard 17 of 64",
            3,
        ))
        .unwrap_or_default();
        assert!(line.contains("4242"), "the pid must be named: {line}");
        assert!(line.contains("wal-recovery"), "the stage token: {line}");
        assert!(line.contains("shard 17 of 64"), "the detail: {line}");
    }

    /// A SERVING record is not mid-anything: the narrator says nothing and
    /// lets its caller announce the arrival in its own words.
    #[test]
    fn a_serving_record_has_no_stage_line() {
        let mut record = booting_record("binding", "x", 1);
        record.state = IncarnationState::Serving;
        assert!(stage_line(&record).is_none());
    }

    /// An empty home reads as `NoRecord`, and the narrator knows it has not
    /// seen one — which is what keeps the launcher's bounded legacy patience
    /// from being applied to a boot it CAN watch.
    #[test]
    fn an_empty_home_reads_as_no_record() -> TestResult {
        let home = tempfile::tempdir()?;
        assert!(matches!(
            read_observation(home.path()),
            BootObservation::NoRecord
        ));
        let mut narrator = BootNarrator::new();
        assert!(matches!(
            narrator.observe(home.path()),
            BootObservation::NoRecord
        ));
        assert!(
            !narrator.has_seen_record(),
            "an empty home must not count as a watchable boot"
        );
        Ok(())
    }

    /// A record naming a DEAD process is `NotLive` — the face that ends a
    /// launcher's wait honestly instead of waiting out a corpse.
    #[test]
    fn a_dead_records_home_reads_as_not_live() -> TestResult {
        let home = tempfile::tempdir()?;
        let run_dir = home.path().join("run");
        std::fs::create_dir_all(&run_dir)?;
        let mut child = std::process::Command::new("true").spawn()?;
        let dead_pid = child.id();
        child.wait()?;
        let mut record = booting_record("store-open", "opening", 1);
        record.pid = dead_pid;
        std::fs::write(
            run_dir.join("aion-server.pid"),
            serde_json::to_string(&record)?,
        )?;
        assert!(
            matches!(
                read_observation(home.path()),
                BootObservation::NotLive(_) | BootObservation::NoRecord
            ),
            "a reaped pid must never read as a live boot"
        );
        Ok(())
    }
}