saddle_runtime/
recorded_lifecycle.rs

1use saddle_core::{ContextLabel, DiagnosticOccurrence, DiagnosticStage, ErrorKind};
2use saddle_observability::{SourceOutput, root_diagnostic::{
3    ComponentSourceKind, LifecycleSourceFailure, RecordedComponentLifecycle, RecordedSaddleError,
4}};
5use std::time::Duration;
6
7/// The mandatory-output consumer for a formal component. The Core lifecycle
8/// trait and its freely constructed SaddleError cannot enter this boundary.
9///
10/// ```compile_fail
11/// use saddle_core::{ErrorKind, SaddleError};
12/// use saddle_observability::root_diagnostic::LifecycleSourceFailure;
13/// fn fake() -> LifecycleSourceFailure {
14///     SaddleError::new(ErrorKind::Internal, "fake", "no original").into()
15/// }
16/// ```
17///
18/// ```compile_fail
19/// use saddle_core::{ErrorKind, SaddleError};
20/// use saddle_observability::root_diagnostic::LifecycleSourceFailure;
21/// fn fake() -> LifecycleSourceFailure {
22///     let light = SaddleError::new(ErrorKind::Internal, "fake", "safe")
23///         .with_source_receipt(7_u8);
24///     light.into()
25/// }
26/// ```
27///
28/// ```compile_fail
29/// use saddle_observability::root_diagnostic::LifecycleSourceFailure;
30/// fn forge() -> LifecycleSourceFailure { LifecycleSourceFailure(()) }
31/// ```
32///
33/// ```compile_fail
34/// use saddle_runtime::RecordedComponentBoundary;
35/// use saddle_core::ContextLabel;
36/// fn missing_output(app: ContextLabel) -> RecordedComponentBoundary {
37///     RecordedComponentBoundary::new(app, None)
38/// }
39/// ```
40pub struct RecordedComponentBoundary {
41    application: ContextLabel,
42    output: SourceOutput,
43}
44
45impl RecordedComponentBoundary {
46    pub fn new(application: ContextLabel, output: SourceOutput) -> Self {
47        Self { application, output }
48    }
49
50    pub(crate) fn timeout_failure(
51        &self, elapsed: tokio::time::error::Elapsed, kind: ComponentSourceKind,
52        primary: Option<DiagnosticOccurrence>,
53    ) -> LifecycleSourceFailure {
54        let (stage, code, message) = match kind {
55            ComponentSourceKind::Start => (ComponentSourceKind::Start,
56                "runtime.lifecycle_timeout.component_start",
57                "component startup exceeded its deadline"),
58            ComponentSourceKind::Cleanup => (ComponentSourceKind::Cleanup,
59                "runtime.lifecycle_timeout.component_shutdown",
60                "component cleanup exceeded its deadline"),
61        };
62        RecordedSaddleError::component_result::<(), _>(Err(elapsed), &self.application,
63            &self.output, stage, primary, ErrorKind::Infrastructure, code, message)
64            .expect_err("timeout is always a component failure")
65    }
66
67    pub async fn start(
68        &self, component: &dyn RecordedComponentLifecycle,
69    ) -> Result<(), LifecycleSourceFailure> {
70        component.start(&self.application, &self.output).await
71            .map_err(|failure| failure.require_component_kind(ComponentSourceKind::Start))
72    }
73
74    pub async fn shutdown(
75        &self,
76        component: &dyn RecordedComponentLifecycle,
77        primary: Option<DiagnosticOccurrence>,
78    ) -> Result<(), LifecycleSourceFailure> {
79        component.shutdown_with_primary(&self.application, &self.output, primary).await
80            .map_err(|failure| failure.require_component_kind(ComponentSourceKind::Cleanup))
81    }
82
83    /// The task envelope preserves the sealed failure. Its own timeout has a
84    /// distinct original, recorded before the cancelled task is reported.
85    pub async fn start_with_timeout(
86        &self, component: &dyn RecordedComponentLifecycle, timeout: Duration,
87    ) -> Result<(), LifecycleSourceFailure> {
88        match tokio::time::timeout(timeout, crate::diagnostics::task(
89            self.start(component), DiagnosticStage::StartupListener,
90            "runtime.component_start",
91        )).await {
92            Ok(result) => result,
93            Err(elapsed) => RecordedSaddleError::component_result::<(), _>(
94                Err(elapsed), &self.application, &self.output,
95                ComponentSourceKind::Start, None, ErrorKind::Infrastructure,
96                "runtime.lifecycle_timeout.component_start",
97                "component startup exceeded its deadline",
98            ),
99        }
100    }
101
102    pub async fn shutdown_with_timeout(
103        &self, component: &dyn RecordedComponentLifecycle,
104        primary: Option<DiagnosticOccurrence>, timeout: Duration,
105    ) -> Result<(), LifecycleSourceFailure> {
106        match tokio::time::timeout(timeout, crate::diagnostics::task(
107            self.shutdown(component, primary), DiagnosticStage::ShutdownComponent,
108            "runtime.component_shutdown",
109        )).await {
110            Ok(result) => result,
111            Err(elapsed) => RecordedSaddleError::component_result::<(), _>(
112                Err(elapsed), &self.application, &self.output,
113                ComponentSourceKind::Cleanup, primary, ErrorKind::Infrastructure,
114                "runtime.lifecycle_timeout.component_shutdown",
115                "component cleanup exceeded its deadline",
116            ),
117        }
118    }
119}
120
121#[cfg(test)]
122mod tests {
123    use super::*;
124    use std::{error::Error, fmt};
125    use saddle_core::ErrorKind;
126    use saddle_observability::{DiagnosticShutdown, EmergencyDiagnostics,
127        FileLoggingConfig, Rotation, root_diagnostic::{ComponentSourceKind,
128            RecordedLifecycleFuture, RecordedSaddleError}};
129
130    struct FailingComponent;
131    #[derive(Debug)]
132    struct CauseChain(&'static str, Option<Box<CauseChain>>);
133    impl fmt::Display for CauseChain {
134        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.0) }
135    }
136    impl Error for CauseChain {
137        fn source(&self) -> Option<&(dyn Error + 'static)> {
138            self.1.as_deref().map(|source| source as &(dyn Error + 'static))
139        }
140    }
141    impl RecordedComponentLifecycle for FailingComponent {
142        fn name(&self) -> &'static str { "recorded-failure" }
143
144        fn start<'a>(&'a self, application: &'a ContextLabel,
145            output: &'a SourceOutput) -> RecordedLifecycleFuture<'a> {
146            Box::pin(async move {
147                RecordedSaddleError::component_result::<(), _>(
148                    Err(CauseChain("actual component cause 6293", Some(Box::new(
149                        CauseChain("inner cause 7429", Some(Box::new(
150                            CauseChain("root cause 9531", None),
151                        ))),
152                    )))),
153                    application, output, ComponentSourceKind::Start, None,
154                    ErrorKind::Infrastructure, "recorded.boundary",
155                    "safe component failure",
156                )
157            })
158        }
159
160        fn shutdown<'a>(&'a self, _application: &'a ContextLabel,
161            _output: &'a SourceOutput) -> RecordedLifecycleFuture<'a> {
162            Box::pin(async { Ok(()) })
163        }
164
165        fn shutdown_with_primary<'a>(&'a self, application: &'a ContextLabel,
166            output: &'a SourceOutput,
167            _primary: Option<DiagnosticOccurrence>) -> RecordedLifecycleFuture<'a> {
168            self.shutdown(application, output)
169        }
170    }
171
172    struct WrongPhaseComponent;
173    struct PendingComponent;
174    impl RecordedComponentLifecycle for PendingComponent {
175        fn name(&self) -> &'static str { "pending-component" }
176        fn start<'a>(&'a self, _application: &'a ContextLabel,
177            _output: &'a SourceOutput) -> RecordedLifecycleFuture<'a> {
178            Box::pin(std::future::pending())
179        }
180        fn shutdown<'a>(&'a self, _application: &'a ContextLabel,
181            _output: &'a SourceOutput) -> RecordedLifecycleFuture<'a> {
182            Box::pin(std::future::pending())
183        }
184        fn shutdown_with_primary<'a>(&'a self, application: &'a ContextLabel,
185            output: &'a SourceOutput, _primary: Option<DiagnosticOccurrence>)
186            -> RecordedLifecycleFuture<'a> {
187            self.shutdown(application, output)
188        }
189    }
190
191    #[tokio::test]
192    async fn task_timeout_records_its_own_original_and_cleanup_parent() {
193        let directory = std::env::temp_dir().join(format!(
194            "saddle-recorded-timeout-{}-{}", std::process::id(),
195            std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
196                .unwrap().as_nanos()));
197        std::fs::create_dir(&directory).unwrap();
198        let mut owner = EmergencyDiagnostics::start_checked(
199            &FileLoggingConfig::new(directory.clone(), Rotation::Daily)).unwrap();
200        let target = owner.target().to_owned();
201        let boundary = RecordedComponentBoundary::new(
202            ContextLabel::checked("timeout-component").unwrap(),
203            owner.source_output().unwrap());
204        let start = boundary.start_with_timeout(&PendingComponent,
205            Duration::from_millis(1)).await.err().unwrap();
206        assert_eq!(start.safe().code(), "runtime.lifecycle_timeout.component_start");
207        assert!(start.original_if_unconfirmed().is_none());
208        let original = std::fs::read_to_string(&target).unwrap();
209        assert!(original.contains("deadline has elapsed"));
210        assert!(original.contains(&start.safe().diagnostic().unwrap().id().to_string()));
211        let cleanup = boundary.shutdown_with_timeout(&PendingComponent,
212            start.safe().diagnostic().map(|d| d.occurrence()),
213            Duration::from_millis(1)).await.err().unwrap();
214        assert_eq!(cleanup.safe().code(), "runtime.lifecycle_timeout.component_shutdown");
215        assert!(cleanup.original_if_unconfirmed().is_none());
216        assert_eq!(serde_json::to_value(cleanup.safe().diagnostic().unwrap()).unwrap()
217            ["primary_diagnostic_id"], start.safe().diagnostic().unwrap().id());
218        assert!(std::fs::read_to_string(&target).unwrap().contains(
219            &cleanup.safe().diagnostic().unwrap().id().to_string()));
220        drop(cleanup);
221        drop(start);
222        drop(boundary);
223        let deadline = std::time::Instant::now() + Duration::from_secs(5);
224        while owner.shutdown() == DiagnosticShutdown::Pending
225            && std::time::Instant::now() < deadline { tokio::task::yield_now().await; }
226        assert_eq!(owner.shutdown(), DiagnosticShutdown::Finished);
227        drop(owner);
228        std::fs::remove_dir_all(directory).unwrap();
229    }
230    impl RecordedComponentLifecycle for WrongPhaseComponent {
231        fn name(&self) -> &'static str { "wrong-phase" }
232        fn start<'a>(&'a self, application: &'a ContextLabel,
233            output: &'a SourceOutput) -> RecordedLifecycleFuture<'a> {
234            Box::pin(async move {
235                RecordedSaddleError::component_result::<(), _>(
236                    Err(std::io::Error::other("actual start source")),
237                    application, output, ComponentSourceKind::Cleanup, None,
238                    ErrorKind::Infrastructure, "wrong.phase", "safe",
239                )
240            })
241        }
242        fn shutdown<'a>(&'a self, _application: &'a ContextLabel,
243            _output: &'a SourceOutput) -> RecordedLifecycleFuture<'a> {
244            Box::pin(async { Ok(()) })
245        }
246        fn shutdown_with_primary<'a>(&'a self, application: &'a ContextLabel,
247            output: &'a SourceOutput,
248            _primary: Option<DiagnosticOccurrence>) -> RecordedLifecycleFuture<'a> {
249            self.shutdown(application, output)
250        }
251    }
252
253    #[tokio::test]
254    async fn start_rejects_cleanup_credential_without_claiming_source() {
255        let directory = std::env::temp_dir().join(format!(
256            "saddle-wrong-phase-{}-{}", std::process::id(),
257            std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
258                .unwrap().as_nanos()));
259        std::fs::create_dir(&directory).unwrap();
260        let mut output = EmergencyDiagnostics::start_checked(
261            &FileLoggingConfig::new(directory.clone(), Rotation::Daily)).unwrap();
262        let target = output.target().to_owned();
263        let boundary = RecordedComponentBoundary::new(
264            ContextLabel::checked("wrong-phase-test").unwrap(),
265            output.source_output().unwrap());
266        let failure = boundary.start(&WrongPhaseComponent).await.err().unwrap();
267        assert_eq!(failure.safe().code(), "component.source_phase_mismatch");
268        assert!(failure.safe().source_unavailable());
269        assert!(std::fs::read_to_string(&target).unwrap().contains("actual start source"));
270        drop(boundary);
271        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
272        while output.shutdown() == DiagnosticShutdown::Pending
273            && std::time::Instant::now() < deadline { std::thread::yield_now(); }
274        assert_eq!(output.shutdown(), DiagnosticShutdown::Finished);
275        drop(output);
276        std::fs::remove_dir_all(directory).unwrap();
277    }
278
279    #[tokio::test]
280    async fn cross_crate_boundary_requires_original_before_safe_result() {
281        let directory = std::env::temp_dir().join(format!(
282            "saddle-recorded-boundary-{}-{}", std::process::id(),
283            std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
284                .unwrap().as_nanos()));
285        std::fs::create_dir(&directory).unwrap();
286        let mut output = EmergencyDiagnostics::start_checked(
287            &FileLoggingConfig::new(directory.clone(), Rotation::Daily)).unwrap();
288        let target = output.target().to_owned();
289        let boundary = RecordedComponentBoundary::new(
290            ContextLabel::checked("recorded-test").unwrap(),
291            output.source_output().unwrap());
292        let failure = boundary.start(&FailingComponent).await.err().unwrap();
293        assert_eq!(failure.safe().code(), "recorded.boundary");
294        assert!(failure.original_if_unconfirmed().is_none());
295        let record = std::fs::read_to_string(&target).unwrap();
296        assert!(record.contains("actual component cause 6293"));
297        assert!(record.contains("inner cause 7429"));
298        assert!(record.contains("root cause 9531"));
299        assert!(record.contains(&failure.safe().diagnostic().unwrap().id().to_string()));
300        // The caller may recover after a confirmed original; the historical
301        // source remains in the controlled file and is never relabeled.
302        let recovered: Result<(), LifecycleSourceFailure> =
303            if failure.safe().code() == "recorded.boundary" { Ok(()) }
304            else { Err(failure) };
305        assert!(recovered.is_ok());
306        drop(boundary);
307        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
308        while output.shutdown() == DiagnosticShutdown::Pending
309            && std::time::Instant::now() < deadline { std::thread::yield_now(); }
310        assert_eq!(output.shutdown(), DiagnosticShutdown::Finished);
311        drop(output);
312        std::fs::remove_dir_all(directory).unwrap();
313    }
314}
saddle_runtime/recorded_lifecycle.rs

saddle_runtime/
recorded_lifecycle.rs
