aion_server/control/stage.rs
1//! Boot-stage reporting: the running narration of a boot that takes minutes.
2//!
3//! A big store spends minutes in WAL recovery, and until the record carried a
4//! stage there was nothing to read but silence: `status` could say a server
5//! existed but not what it was doing, and an operator watching a home could
6//! not tell a boot that was PROGRESSING from one that was STUCK.
7//!
8//! A [`StageReporter`] is a cheap, cloneable handle on the incarnation's own
9//! pid record. It is safe to hand into a blocking closure or another thread —
10//! the store build runs on the blocking pool and reports from there — and
11//! every report does two things at once: it writes the stage into the record
12//! (where `aion server status` and the launcher read it) and emits a
13//! `tracing::info!` line (where the log reader sees it). One story, told in
14//! both places, from one call.
15//!
16//! Every report bumps `stage_seq`. That counter is the difference between
17//! "this boot is slow" and "this boot is wedged": a reader that sees the
18//! sequence advance knows work is happening even when the stage token has not
19//! changed for minutes.
20
21use std::path::PathBuf;
22use std::sync::{Arc, Mutex};
23
24use tracing::{info, warn};
25
26use super::guard::{RecordUpdate, update_record};
27use super::pid_file::{PidRecord, now_unix_secs};
28
29/// The machine token for the stage that resolves configuration — the first
30/// thing a boot finishes, and the one that names the home every later stage
31/// happens under.
32pub const STAGE_CONFIG: &str = "config";
33
34/// The store is being opened: directories checked, the database handed to
35/// haematite.
36pub const STAGE_STORE_OPEN: &str = "store-open";
37
38/// The open is blocked on the data directory's writer lock, which a live
39/// process holds — a draining predecessor mid-handover, or another server on
40/// the same data directory.
41pub const STAGE_WRITER_LOCK_WAIT: &str = "writer-lock-wait";
42
43/// The store's shards are being materialized, which is where WAL recovery
44/// replays. This is the stage that takes minutes on a big store.
45pub const STAGE_WAL_RECOVERY: &str = "wal-recovery";
46
47/// The engine is recovering resident workflows from durable state.
48pub const STAGE_ENGINE_RECOVERY: &str = "engine-recovery";
49
50/// The transport listeners are being bound. The last stage before serving.
51pub const STAGE_BINDING: &str = "binding";
52
53/// A cloneable handle for writing this incarnation's boot stages.
54///
55/// Cloning is cheap (a path, an `Arc`, and a flag) and every clone writes the
56/// SAME record, so a stage reported from the store's blocking thread is
57/// immediately visible through the run loop's guard.
58#[derive(Clone, Debug)]
59pub struct StageReporter {
60 path: PathBuf,
61 record: Arc<Mutex<PidRecord>>,
62 holds_claim: bool,
63}
64
65impl StageReporter {
66 /// Build a reporter over a guard's shared record. Created only by
67 /// [`PidFileGuard::stage_reporter`](super::guard::PidFileGuard::stage_reporter).
68 pub(super) fn new(path: PathBuf, record: Arc<Mutex<PidRecord>>, holds_claim: bool) -> Self {
69 Self {
70 path,
71 record,
72 holds_claim,
73 }
74 }
75
76 /// A reporter that writes nothing, for a caller with no claim to report
77 /// against: an embedder building [`ServerState`](crate::ServerState)
78 /// directly, or a test.
79 ///
80 /// It is deliberately a real reporter rather than an `Option`: the boot
81 /// path reports at half a dozen seams, and threading an `Option` through
82 /// all of them would make "no reporter" a case each seam had to handle.
83 /// Its reports are logged (at `debug`) and dropped.
84 #[must_use]
85 pub fn detached() -> Self {
86 Self {
87 path: PathBuf::new(),
88 record: Arc::new(Mutex::new(detached_record())),
89 holds_claim: false,
90 }
91 }
92
93 /// Report that this incarnation has reached `stage`, with one human
94 /// sentence of `detail`.
95 ///
96 /// Infallible by design: a stage report is narration, and a boot must
97 /// never fail because it could not describe itself. A write that cannot
98 /// happen is reported through `tracing` at `warn` — the same channel the
99 /// stage line itself goes to, so the loss is never silent.
100 pub fn report(&self, stage: &str, detail: String) {
101 info!(
102 boot_stage = stage,
103 boot_stage_detail = %detail,
104 "boot stage"
105 );
106 let stage_owned = stage.to_owned();
107 let update = update_record(&self.path, &self.record, self.holds_claim, move |record| {
108 record.stage = Some(stage_owned);
109 record.stage_detail = Some(detail);
110 record.stage_seq = record.stage_seq.saturating_add(1);
111 record.stage_updated_at_unix_secs = now_unix_secs();
112 });
113 match update {
114 Ok(RecordUpdate::Written | RecordUpdate::Unclaimed | RecordUpdate::NotOurs) => {}
115 Err(error) => warn!(
116 %error,
117 boot_stage = stage,
118 "could not write the boot stage into this home's pid record; the \
119 stage is in the log above, but `aion server status` and the \
120 launcher will not see it"
121 ),
122 }
123 }
124}
125
126/// The placeholder record a [`StageReporter::detached`] handle carries. It is
127/// never written anywhere — `holds_claim` is false, so every update short-
128/// circuits before touching it — and exists only so the handle has the same
129/// shape as a real one.
130fn detached_record() -> PidRecord {
131 PidRecord {
132 pid: 0,
133 started_at_unix_secs: 0,
134 binary_sha256: String::new(),
135 version: String::new(),
136 commit: String::new(),
137 state: super::pid_file::IncarnationState::Booting,
138 http_address: None,
139 grpc_address: None,
140 intended_http_address: None,
141 intended_grpc_address: None,
142 stage: None,
143 stage_detail: None,
144 stage_seq: 0,
145 stage_updated_at_unix_secs: 0,
146 drain_timeout_seconds: 0,
147 }
148}
149
150#[cfg(test)]
151#[path = "stage_tests.rs"]
152mod tests;