aion_server/worker/supervisor/
status.rs

1//! What the server will say about a managed worker, and what it refuses to say.
2//!
3//! The status is a JOIN of two different kinds of truth: the durable record's
4//! desired state, and the live instance's actual state. Reporting one without
5//! the other is how "it says stopped" comes to mean "nobody looked". Every
6//! actual state below is written by the instance loop only after the fact it
7//! names has been established — `Running` after a spawn returned a pid,
8//! `Stopped` only after the process group was probed empty.
9
10use aion_store::{DeployedBinaryIdentity, DesiredState};
11use serde::{Deserialize, Serialize};
12
13/// What a managed worker instance is actually doing right now.
14#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
15#[serde(rename_all = "kebab-case")]
16pub enum ManagedWorkerState {
17    /// A spawn is in progress; no pid has been observed yet.
18    Starting,
19    /// A contained child is alive and leading its process group.
20    Running,
21    /// The child exited and a restart is waiting out the configured backoff.
22    Backoff,
23    /// No process is running and none is wanted: either the deployment desires
24    /// `Stopped`, or a stop confirmed the process group empty.
25    Stopped,
26    /// Supervision gave up: the restart budget for the window was exhausted, or
27    /// stopping could not be confirmed. Terminal until an operator acts.
28    Failed,
29    /// The deployment desires `Running` but this server has no supervision
30    /// policy, so nothing is supervising it. The remedy travels with the report.
31    Uncommissioned,
32}
33
34impl ManagedWorkerState {
35    /// Stable lowercase token used on the wire and in status history.
36    #[must_use]
37    pub const fn token(self) -> &'static str {
38        match self {
39            Self::Starting => "starting",
40            Self::Running => "running",
41            Self::Backoff => "backoff",
42            Self::Stopped => "stopped",
43            Self::Failed => "failed",
44            Self::Uncommissioned => "uncommissioned",
45        }
46    }
47
48    /// Whether the supervisor has stopped acting on this instance.
49    #[must_use]
50    pub const fn is_terminal(self) -> bool {
51        matches!(self, Self::Stopped | Self::Failed)
52    }
53}
54
55/// Identity of the executable one spawn actually ran.
56///
57/// The content hash is taken from the bytes on disk AT SPAWN, which is the
58/// whole point: a restart that picks up an upgraded binary changes this hash
59/// while the deployment record's deploy-time hash stays put, so the swap is a
60/// visible, reportable fact instead of an invisible one.
61#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
62pub struct SpawnedBinary {
63    /// Path of the executable that was launched.
64    pub path: String,
65    /// Lowercase hexadecimal SHA-256 digest of the executable bytes at spawn.
66    pub content_hash: String,
67    /// Package version stamped into the binary. Present only when the spawned
68    /// executable is this server's own binary, which is the only executable
69    /// whose build stamp this process can honestly claim to know.
70    pub version: Option<String>,
71    /// Source commit stamped into the binary, under the same condition.
72    pub commit: Option<String>,
73    /// Dirty-state token stamped into the binary, under the same condition.
74    pub dirty: Option<String>,
75}
76
77/// How a managed worker process last ended.
78#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
79pub struct ManagedWorkerExit {
80    /// When the exit was observed.
81    pub at: chrono::DateTime<chrono::Utc>,
82    /// Exit code, when the process exited normally.
83    pub code: Option<i32>,
84    /// Terminating signal number, when a signal ended the process.
85    pub signal: Option<i32>,
86    /// Whether the supervisor itself asked for this ending.
87    pub requested: bool,
88}
89
90/// One managed worker: what was asked for, and what is true.
91#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
92pub struct ManagedWorkerStatus {
93    /// Deployment primary key.
94    pub name: String,
95    /// Task queue the deployment binds.
96    pub task_queue: String,
97    /// Durable operator intent.
98    pub desired: DesiredState,
99    /// Live supervision state.
100    pub state: ManagedWorkerState,
101    /// Process id of the running child, while one is running.
102    pub pid: Option<u32>,
103    /// Process group id the child leads, while one is running.
104    pub process_group: Option<i32>,
105    /// Restarts performed since this instance was started.
106    pub restarts: u32,
107    /// How the process last ended, when it has ended at least once.
108    pub last_exit: Option<ManagedWorkerExit>,
109    /// The last failure worth an operator's attention, verbatim.
110    pub last_error: Option<String>,
111    /// Identity of the binary the deployment named AT DEPLOY TIME.
112    pub deployed_binary: DeployedBinaryIdentity,
113    /// Identity of the binary the CURRENT (or most recent) spawn actually ran.
114    ///
115    /// A restart across an upgrade is meant to be a visible fact rather than a
116    /// silent swap, so this is captured per spawn and reported beside the
117    /// deploy-time identity: the two disagreeing is the upgrade, stated.
118    pub spawn_binary: Option<SpawnedBinary>,
119}
120
121/// The whole managed-worker surface for this server.
122#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
123pub struct ManagedWorkerReport {
124    /// Whether a supervision policy is configured on this server.
125    pub commissioned: bool,
126    /// The remedy for an uncommissioned server, present exactly when
127    /// `commissioned` is false. Absent when there is nothing to remedy.
128    pub remedy: Option<String>,
129    /// One entry per durable deployment, ordered by name.
130    pub workers: Vec<ManagedWorkerStatus>,
131    /// Names of durable rows that are PRESENT but could not be decoded, and are
132    /// therefore supervised by nothing. Reported rather than dropped: a row the
133    /// server cannot read is exactly the row an operator needs told about.
134    pub undecodable: Vec<String>,
135}
136
137#[cfg(test)]
138mod tests {
139    use super::ManagedWorkerState;
140
141    #[test]
142    fn tokens_are_distinct_and_terminality_matches_them() {
143        let states = [
144            ManagedWorkerState::Starting,
145            ManagedWorkerState::Running,
146            ManagedWorkerState::Backoff,
147            ManagedWorkerState::Stopped,
148            ManagedWorkerState::Failed,
149            ManagedWorkerState::Uncommissioned,
150        ];
151        let mut tokens = states.map(ManagedWorkerState::token).to_vec();
152        tokens.sort_unstable();
153        tokens.dedup();
154        assert_eq!(tokens.len(), states.len());
155
156        assert!(ManagedWorkerState::Stopped.is_terminal());
157        assert!(ManagedWorkerState::Failed.is_terminal());
158        assert!(!ManagedWorkerState::Running.is_terminal());
159        assert!(!ManagedWorkerState::Backoff.is_terminal());
160        assert!(!ManagedWorkerState::Starting.is_terminal());
161        // Uncommissioned is not terminal: commissioning the server and starting
162        // the deployment is exactly the state it is waiting for.
163        assert!(!ManagedWorkerState::Uncommissioned.is_terminal());
164    }
165}
aion_server/worker/supervisor/status.rs

aion_server/worker/supervisor/
status.rs
