aion_server/worker/supervisor/
error.rs

1//! Typed managed-worker supervision failures.
2//!
3//! Every variant names the thing that went wrong AND, where an operator can
4//! act, what to do about it. A refusal that does not name its remedy is a
5//! refusal the operator has to guess at, and guessing is how a fleet ends up
6//! hand-started again.
7
8use thiserror::Error;
9
10use super::policy::UNCOMMISSIONED_REMEDY;
11
12/// A managed-worker lifecycle operation that could not be carried out.
13#[derive(Debug, Error)]
14pub enum SupervisionError {
15    /// No supervision policy is configured, so nothing may be started.
16    #[error("{UNCOMMISSIONED_REMEDY}")]
17    NotCommissioned,
18    /// The named deployment does not exist.
19    #[error(
20        "worker deployment `{name}` was not found; create it first with a PUT to /worker-deployments/{name}"
21    )]
22    UnknownDeployment {
23        /// Deployment primary key the caller asked for.
24        name: String,
25    },
26    /// The durable deployment store refused or failed.
27    #[error("worker deployment store failed: {source}")]
28    Store {
29        /// Underlying store failure.
30        #[source]
31        source: aion_store::StoreError,
32    },
33    /// This server's own executable could not be identified.
34    #[error("could not resolve the running server executable: {detail}")]
35    ExecutableUnresolved {
36        /// Operating-system failure text.
37        detail: String,
38    },
39    /// The executable bytes could not be read for identity capture.
40    #[error("could not read managed worker executable `{path}`: {detail}")]
41    ExecutableUnreadable {
42        /// Path that could not be read.
43        path: String,
44        /// Operating-system failure text.
45        detail: String,
46    },
47    /// The contained child could not be spawned or observed.
48    #[error("managed worker `{name}` process failed: {source}")]
49    Process {
50        /// Deployment primary key.
51        name: String,
52        /// Underlying containment failure.
53        #[source]
54        source: aion_worker::ProcessGroupError,
55    },
56    /// A stop was asked for and could not be PROVEN complete.
57    ///
58    /// This exists so that "stopped" is never reported on the strength of
59    /// bookkeeping: the only path to a `Stopped` status runs through a process
60    /// group that a signal-zero probe found empty, and every other outcome
61    /// arrives here instead.
62    #[error("managed worker `{name}` could not be confirmed stopped: {detail}")]
63    StopIncomplete {
64        /// Deployment primary key.
65        name: String,
66        /// What was observed instead of an empty process group.
67        detail: String,
68    },
69    /// A supervision task ended abnormally (panicked or was aborted).
70    #[error("managed worker `{name}` supervision task ended abnormally: {detail}")]
71    TaskLost {
72        /// Deployment primary key.
73        name: String,
74        /// Join failure text.
75        detail: String,
76    },
77    /// A supervisor lock was poisoned by a panicking holder.
78    #[error("managed worker supervision state is poisoned: {detail}")]
79    StatePoisoned {
80        /// Poison report.
81        detail: String,
82    },
83}
84
85impl SupervisionError {
86    /// The operator remedy carried by this failure, when it has one.
87    #[must_use]
88    pub const fn remedy(&self) -> Option<&'static str> {
89        match self {
90            Self::NotCommissioned => Some(UNCOMMISSIONED_REMEDY),
91            _ => None,
92        }
93    }
94}
95
96#[cfg(test)]
97mod tests {
98    use super::SupervisionError;
99
100    /// The uncommissioned refusal is the one an operator meets first, and it is
101    /// useless unless it names the section AND every key that section needs.
102    #[test]
103    fn the_uncommissioned_refusal_names_the_section_and_every_required_key() {
104        let message = SupervisionError::NotCommissioned.to_string();
105        for expected in [
106            "[worker_supervision]",
107            "restart_backoff_initial_ms",
108            "restart_backoff_max_ms",
109            "restart_backoff_multiplier",
110            "restart_window_ms",
111            "max_restarts_per_window",
112            "stop_grace_ms",
113        ] {
114            assert!(
115                message.contains(expected),
116                "the refusal must name `{expected}`: {message}"
117            );
118        }
119        assert_eq!(
120            SupervisionError::NotCommissioned.remedy(),
121            Some(super::UNCOMMISSIONED_REMEDY)
122        );
123    }
124
125    #[test]
126    fn an_unknown_deployment_refusal_names_the_deployment_and_the_way_to_create_it() {
127        let message = SupervisionError::UnknownDeployment {
128            name: "shells".to_owned(),
129        }
130        .to_string();
131        assert!(message.contains("shells"));
132        assert!(message.contains("/worker-deployments/shells"));
133    }
134}
aion_server/worker/supervisor/error.rs

aion_server/worker/supervisor/
error.rs
