aion_server/worker/supervisor/status.rs
1//! What the server will say about a managed worker, and what it refuses to say.
2//!
3//! The status is a JOIN of two different kinds of truth: the durable record's
4//! desired state, and the live instance's actual state. Reporting one without
5//! the other is how "it says stopped" comes to mean "nobody looked". Every
6//! actual state below is written by the instance loop only after the fact it
7//! names has been established — `Running` after a spawn returned a pid,
8//! `Stopped` only after the process group was probed empty.
9
10use aion_store::{DeployedBinaryIdentity, DesiredState};
11use serde::{Deserialize, Serialize};
12
13/// What a managed worker instance is actually doing right now.
14#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
15#[serde(rename_all = "kebab-case")]
16pub enum ManagedWorkerState {
17 /// A spawn is in progress; no pid has been observed yet.
18 Starting,
19 /// A contained child is alive and leading its process group.
20 Running,
21 /// The child exited and a restart is waiting out the configured backoff.
22 Backoff,
23 /// No process is running and none is wanted: either the deployment desires
24 /// `Stopped`, or a stop confirmed the process group empty.
25 Stopped,
26 /// Supervision gave up: the restart budget for the window was exhausted, or
27 /// stopping could not be confirmed. Terminal until an operator acts.
28 Failed,
29 /// The deployment desires `Running` but this server has no supervision
30 /// policy, so nothing is supervising it. The remedy travels with the report.
31 Uncommissioned,
32}
33
34impl ManagedWorkerState {
35 /// Stable lowercase token used on the wire and in status history.
36 #[must_use]
37 pub const fn token(self) -> &'static str {
38 match self {
39 Self::Starting => "starting",
40 Self::Running => "running",
41 Self::Backoff => "backoff",
42 Self::Stopped => "stopped",
43 Self::Failed => "failed",
44 Self::Uncommissioned => "uncommissioned",
45 }
46 }
47
48 /// Whether the supervisor has stopped acting on this instance.
49 #[must_use]
50 pub const fn is_terminal(self) -> bool {
51 matches!(self, Self::Stopped | Self::Failed)
52 }
53}
54
55/// Identity of the executable one spawn actually ran.
56///
57/// The content hash is taken from the bytes on disk AT SPAWN, which is the
58/// whole point: a restart that picks up an upgraded binary changes this hash
59/// while the deployment record's deploy-time hash stays put, so the swap is a
60/// visible, reportable fact instead of an invisible one.
61#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
62pub struct SpawnedBinary {
63 /// Path of the executable that was launched.
64 pub path: String,
65 /// Lowercase hexadecimal SHA-256 digest of the executable bytes at spawn.
66 pub content_hash: String,
67 /// Package version stamped into the binary. Present only when the spawned
68 /// executable is this server's own binary, which is the only executable
69 /// whose build stamp this process can honestly claim to know.
70 pub version: Option<String>,
71 /// Source commit stamped into the binary, under the same condition.
72 pub commit: Option<String>,
73 /// Dirty-state token stamped into the binary, under the same condition.
74 pub dirty: Option<String>,
75}
76
77/// How a managed worker process last ended.
78#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
79pub struct ManagedWorkerExit {
80 /// When the exit was observed.
81 pub at: chrono::DateTime<chrono::Utc>,
82 /// Exit code, when the process exited normally.
83 pub code: Option<i32>,
84 /// Terminating signal number, when a signal ended the process.
85 pub signal: Option<i32>,
86 /// Whether the supervisor itself asked for this ending.
87 pub requested: bool,
88}
89
90/// One managed worker: what was asked for, and what is true.
91#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
92pub struct ManagedWorkerStatus {
93 /// Deployment primary key.
94 pub name: String,
95 /// Task queue the deployment binds.
96 pub task_queue: String,
97 /// Durable operator intent.
98 pub desired: DesiredState,
99 /// Live supervision state.
100 pub state: ManagedWorkerState,
101 /// Process id of the running child, while one is running.
102 pub pid: Option<u32>,
103 /// Process group id the child leads, while one is running.
104 pub process_group: Option<i32>,
105 /// Restarts performed since this instance was started.
106 pub restarts: u32,
107 /// How the process last ended, when it has ended at least once.
108 pub last_exit: Option<ManagedWorkerExit>,
109 /// The last failure worth an operator's attention, verbatim.
110 pub last_error: Option<String>,
111 /// Identity of the binary the deployment named AT DEPLOY TIME.
112 pub deployed_binary: DeployedBinaryIdentity,
113 /// Identity of the binary the CURRENT (or most recent) spawn actually ran.
114 ///
115 /// A restart across an upgrade is meant to be a visible fact rather than a
116 /// silent swap, so this is captured per spawn and reported beside the
117 /// deploy-time identity: the two disagreeing is the upgrade, stated.
118 pub spawn_binary: Option<SpawnedBinary>,
119}
120
121/// The whole managed-worker surface for this server.
122#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
123pub struct ManagedWorkerReport {
124 /// Whether a supervision policy is configured on this server.
125 pub commissioned: bool,
126 /// The remedy for an uncommissioned server, present exactly when
127 /// `commissioned` is false. Absent when there is nothing to remedy.
128 pub remedy: Option<String>,
129 /// One entry per durable deployment, ordered by name.
130 pub workers: Vec<ManagedWorkerStatus>,
131 /// Names of durable rows that are PRESENT but could not be decoded, and are
132 /// therefore supervised by nothing. Reported rather than dropped: a row the
133 /// server cannot read is exactly the row an operator needs told about.
134 pub undecodable: Vec<String>,
135 /// The latest auto-provision outcome for every task queue this server has
136 /// decided about, ordered by queue.
137 ///
138 /// This is where a queue that has NO worker says why. A deploy answers its
139 /// own caller, but a boot-time provisioning refusal — a dark outbox on the
140 /// assistant's queue, say — has no caller to answer, and an operator asking
141 /// "why is nothing serving this" hours later needs the reason to still be
142 /// here. Latest-per-queue, so it is bounded by what has been deployed
143 /// rather than by how often.
144 pub auto_provision: Vec<crate::worker::auto_provision::AutoWorkerOutcome>,
145}
146
147#[cfg(test)]
148mod tests {
149 use super::ManagedWorkerState;
150
151 #[test]
152 fn tokens_are_distinct_and_terminality_matches_them() {
153 let states = [
154 ManagedWorkerState::Starting,
155 ManagedWorkerState::Running,
156 ManagedWorkerState::Backoff,
157 ManagedWorkerState::Stopped,
158 ManagedWorkerState::Failed,
159 ManagedWorkerState::Uncommissioned,
160 ];
161 let mut tokens = states.map(ManagedWorkerState::token).to_vec();
162 tokens.sort_unstable();
163 tokens.dedup();
164 assert_eq!(tokens.len(), states.len());
165
166 assert!(ManagedWorkerState::Stopped.is_terminal());
167 assert!(ManagedWorkerState::Failed.is_terminal());
168 assert!(!ManagedWorkerState::Running.is_terminal());
169 assert!(!ManagedWorkerState::Backoff.is_terminal());
170 assert!(!ManagedWorkerState::Starting.is_terminal());
171 // Uncommissioned is not terminal: commissioning the server and starting
172 // the deployment is exactly the state it is waiting for.
173 assert!(!ManagedWorkerState::Uncommissioned.is_terminal());
174 }
175}