Skip to main content

canic_core/api/runtime/
mod.rs

1pub mod install;
2
3use crate::{
4    cdk::types::Principal,
5    domain::runtime::{
6        FailureSeverity, RuntimeCheckStatus, RuntimeDiagnosticSeverity, RuntimeFieldVisibility,
7        RuntimeStateDomainStatus,
8    },
9    dto::{
10        error::Error,
11        runtime::{
12            CanicHealthStatus, CanicReadinessStatus, CanicRuntimeStatus, CanicTimerStatus,
13            RUNTIME_INTROSPECTION_SCHEMA_VERSION, ReadinessStatus, RuntimeAuthStatusSummary,
14            RuntimeBlobStorageStatusSummary, RuntimeBuildInfo, RuntimeCheck, RuntimeDiagnostic,
15            RuntimeFeatureStatus, RuntimeStateDomainSummary, RuntimeStateSummary, RuntimeStatus,
16            RuntimeTopologyStatus, RuntimeVisibilityEntry, TimerStatus,
17        },
18    },
19    ops::{
20        ic::IcOps,
21        runtime::{
22            env::EnvOps,
23            memory::MemoryRegistryOps,
24            metrics::timer::TimerMetrics,
25            ready::ReadyOps,
26            recent_failure::{RecentFailureInput, RecentFailureOps},
27        },
28    },
29    state_contract::{StateStorage, canic_state_manifest_for_role},
30};
31
32const MAX_TIMER_SUBSYSTEM_BYTES: usize = 64;
33const MAX_TIMER_NAME_BYTES: usize = 96;
34const RUNTIME_FEATURE_SOURCE: &str = "compile_feature";
35const RUNTIME_FEATURE_FLAGS: [(&str, bool); 10] = [
36    (
37        "auth-chain-key-ecdsa",
38        cfg!(feature = "auth-chain-key-ecdsa"),
39    ),
40    (
41        "auth-chain-key-root-sign",
42        cfg!(feature = "auth-chain-key-root-sign"),
43    ),
44    (
45        "auth-delegated-token-verify",
46        cfg!(feature = "auth-delegated-token-verify"),
47    ),
48    (
49        "auth-issuer-canister-sig-create",
50        cfg!(feature = "auth-issuer-canister-sig-create"),
51    ),
52    (
53        "auth-issuer-canister-sig-verify",
54        cfg!(feature = "auth-issuer-canister-sig-verify"),
55    ),
56    (
57        "auth-root-canister-sig-create",
58        cfg!(feature = "auth-root-canister-sig-create"),
59    ),
60    (
61        "auth-root-canister-sig-verify",
62        cfg!(feature = "auth-root-canister-sig-verify"),
63    ),
64    ("blob-storage", cfg!(feature = "blob-storage")),
65    (
66        "blob-storage-billing",
67        cfg!(feature = "blob-storage-billing"),
68    ),
69    ("sharding", cfg!(feature = "sharding")),
70];
71
72///
73/// MemoryRuntimeApi
74///
75
76pub struct MemoryRuntimeApi;
77
78impl MemoryRuntimeApi {
79    /// Bootstrap Canic's stable-memory declaration snapshot.
80    pub fn bootstrap_registry() -> Result<(), Error> {
81        MemoryRegistryOps::bootstrap_registry().map_err(Error::from)?;
82
83        Ok(())
84    }
85}
86
87///
88/// RuntimeIntrospectionApi
89///
90
91pub struct RuntimeIntrospectionApi;
92
93impl RuntimeIntrospectionApi {
94    /// Record one heap-only recent-failure summary for guarded runtime status.
95    pub fn record_recent_failure(
96        occurred_at_ns: u64,
97        subsystem: impl Into<String>,
98        code: impl Into<String>,
99        severity: FailureSeverity,
100        summary: impl Into<String>,
101        correlation_id: Option<String>,
102    ) {
103        RecentFailureOps::record(RecentFailureInput {
104            occurred_at_ns,
105            subsystem: subsystem.into(),
106            code: code.into(),
107            severity,
108            summary: summary.into(),
109            correlation_id,
110        });
111    }
112
113    /// Return the minimal health status for a canister that answered the query.
114    #[must_use]
115    pub fn health(observed_at_ns: Option<u64>) -> CanicHealthStatus {
116        CanicHealthStatus {
117            schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
118            status: crate::dto::runtime::HealthStatus::Healthy,
119            observed_at_ns,
120            checks: vec![RuntimeCheck {
121                category: "health".to_string(),
122                code: "canister_responsive".to_string(),
123                status: RuntimeCheckStatus::Pass,
124                subject: "canister".to_string(),
125                detail: "canister returned a health response".to_string(),
126                next: None,
127                source: "runtime_observed".to_string(),
128            }],
129        }
130    }
131
132    /// Return guarded readiness status for the local Canic role.
133    #[must_use]
134    pub fn readiness(observed_at_ns: u64) -> CanicReadinessStatus {
135        let ready = ReadyOps::is_ready();
136        let role = EnvOps::canister_role()
137            .ok()
138            .map(crate::ids::CanisterRole::into_string);
139
140        let (status, check_status, detail, next) = if ready {
141            (
142                ReadinessStatus::Ready,
143                RuntimeCheckStatus::Pass,
144                "runtime readiness barrier is marked ready",
145                None,
146            )
147        } else {
148            (
149                ReadinessStatus::NotReady,
150                RuntimeCheckStatus::Fail,
151                "runtime readiness barrier is not ready",
152                Some("wait for bootstrap to complete or inspect canic_bootstrap_status"),
153            )
154        };
155
156        let readiness_check = RuntimeCheck {
157            category: "readiness".to_string(),
158            code: "runtime_ready_barrier".to_string(),
159            status: check_status,
160            subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
161            detail: detail.to_string(),
162            next: next.map(str::to_string),
163            source: "runtime_observed".to_string(),
164        };
165
166        let blockers = if ready {
167            Vec::new()
168        } else {
169            vec![RuntimeDiagnostic {
170                category: "readiness".to_string(),
171                code: "runtime_not_ready".to_string(),
172                severity: RuntimeDiagnosticSeverity::Blocked,
173                subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
174                detail: "runtime readiness barrier has not completed".to_string(),
175                next: Some(
176                    "inspect bootstrap status before treating the role as ready".to_string(),
177                ),
178                source: "runtime_observed".to_string(),
179            }]
180        };
181
182        CanicReadinessStatus {
183            schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
184            role,
185            status,
186            observed_at_ns,
187            checks: vec![readiness_check],
188            blockers,
189            warnings: Vec::new(),
190        }
191    }
192
193    /// Return guarded runtime status for the local Canic role.
194    #[must_use]
195    pub fn runtime_status_for(
196        canister_id: Principal,
197        observed_at_ns: u64,
198        package_name: &str,
199        package_version: &str,
200        canic_version: &str,
201        canister_version: u64,
202    ) -> CanicRuntimeStatus {
203        let readiness = Self::readiness(observed_at_ns);
204        let role = readiness.role.clone();
205        let state = state_summary(role.as_deref());
206        let root = EnvOps::root_pid().ok();
207        let parent = EnvOps::parent_pid().ok();
208        let subnet = EnvOps::subnet_pid().ok();
209        let status = match readiness.status {
210            ReadinessStatus::Ready => RuntimeStatus::Ok,
211            ReadinessStatus::Degraded | ReadinessStatus::NotEvaluated => RuntimeStatus::Degraded,
212            ReadinessStatus::NotReady => RuntimeStatus::Failing,
213        };
214
215        CanicRuntimeStatus {
216            schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
217            observed_at_ns,
218            canister_id,
219            role,
220            root,
221            network: None,
222            build: RuntimeBuildInfo {
223                package_name: package_name.to_string(),
224                package_version: package_version.to_string(),
225                canic_version: canic_version.to_string(),
226                canister_version,
227            },
228            features: runtime_features(),
229            topology: Some(RuntimeTopologyStatus {
230                root,
231                parent,
232                subnet,
233                source: "runtime_observed".to_string(),
234            }),
235            timers: timer_statuses(),
236            state,
237            auth: Some(runtime_auth_status()),
238            blob_storage: runtime_blob_storage_status(),
239            recent_failures: RecentFailureOps::snapshot(),
240            visibility: runtime_visibility(),
241            readiness,
242            status,
243        }
244    }
245
246    /// Return guarded runtime status using ambient IC runtime values.
247    #[must_use]
248    pub fn runtime_status(
249        observed_at_ns: u64,
250        package_name: &str,
251        package_version: &str,
252        canic_version: &str,
253        canister_version: u64,
254    ) -> CanicRuntimeStatus {
255        Self::runtime_status_for(
256            IcOps::canister_self(),
257            observed_at_ns,
258            package_name,
259            package_version,
260            canic_version,
261            canister_version,
262        )
263    }
264}
265
266fn runtime_features() -> Vec<RuntimeFeatureStatus> {
267    RUNTIME_FEATURE_FLAGS
268        .into_iter()
269        .map(|(name, enabled)| runtime_feature_status(name, enabled))
270        .collect()
271}
272
273fn runtime_feature_status(name: &str, enabled: bool) -> RuntimeFeatureStatus {
274    RuntimeFeatureStatus {
275        name: name.to_string(),
276        enabled,
277        visibility: RuntimeFieldVisibility::OperatorOnly,
278        source: RUNTIME_FEATURE_SOURCE.to_string(),
279    }
280}
281
282fn runtime_auth_status() -> RuntimeAuthStatusSummary {
283    RuntimeAuthStatusSummary {
284        auth_features: RUNTIME_FEATURE_FLAGS
285            .into_iter()
286            .filter(|(name, _)| name.starts_with("auth-"))
287            .map(|(name, enabled)| runtime_feature_status(name, enabled))
288            .collect(),
289    }
290}
291
292fn runtime_blob_storage_status() -> Option<RuntimeBlobStorageStatusSummary> {
293    let blob_storage_enabled = cfg!(feature = "blob-storage");
294    let billing_enabled = cfg!(feature = "blob-storage-billing");
295
296    (blob_storage_enabled || billing_enabled).then(|| RuntimeBlobStorageStatusSummary {
297        blob_storage_features: [
298            ("blob-storage", blob_storage_enabled),
299            ("blob-storage-billing", billing_enabled),
300        ]
301        .into_iter()
302        .map(|(name, enabled)| runtime_feature_status(name, enabled))
303        .collect(),
304    })
305}
306
307fn timer_statuses() -> Vec<CanicTimerStatus> {
308    let mut timers = TimerMetrics::snapshot()
309        .entries
310        .into_iter()
311        .map(|(key, ticks)| {
312            let (subsystem, name) = split_timer_label(&key.label);
313            CanicTimerStatus {
314                name,
315                subsystem,
316                status: if ticks > 0 {
317                    TimerStatus::Healthy
318                } else {
319                    TimerStatus::Unknown
320                },
321                enabled: true,
322                registered: true,
323                last_success_at_ns: None,
324                last_failure_at_ns: None,
325                next_due_at_ns: None,
326                consecutive_failures: 0,
327                last_error_code: None,
328                last_error_summary: None,
329            }
330        })
331        .collect::<Vec<_>>();
332    timers.sort_by(|left, right| {
333        left.subsystem
334            .cmp(&right.subsystem)
335            .then_with(|| left.name.cmp(&right.name))
336    });
337    timers
338}
339
340fn state_summary(role: Option<&str>) -> Option<RuntimeStateSummary> {
341    let role = role?;
342    let manifest = canic_state_manifest_for_role(Some(role));
343    let domains = manifest
344        .roles
345        .into_iter()
346        .flat_map(|role| role.state)
347        .map(|domain| RuntimeStateDomainSummary {
348            domain: domain.domain,
349            version: domain.version,
350            storage: state_storage_name(domain.storage).to_string(),
351            memory_id: domain.memory_id,
352            status: RuntimeStateDomainStatus::Ok,
353        })
354        .collect::<Vec<_>>();
355
356    if domains.is_empty() {
357        return None;
358    }
359
360    Some(RuntimeStateSummary {
361        manifest_schema_version: u32::from(manifest.schema_version),
362        domains,
363        total_stable_memory_pages: None,
364    })
365}
366
367const fn state_storage_name(storage: StateStorage) -> &'static str {
368    match storage {
369        StateStorage::StableMemory => "stable_memory",
370        StateStorage::HeapOnly => "heap_only",
371        StateStorage::NotApplicable => "not_applicable",
372    }
373}
374
375fn split_timer_label(label: &str) -> (String, String) {
376    label.split_once(':').map_or_else(
377        || {
378            (
379                "runtime".to_string(),
380                bounded_runtime_text(label, MAX_TIMER_NAME_BYTES),
381            )
382        },
383        |(subsystem, name)| {
384            (
385                bounded_runtime_text(subsystem, MAX_TIMER_SUBSYSTEM_BYTES),
386                bounded_runtime_text(name, MAX_TIMER_NAME_BYTES),
387            )
388        },
389    )
390}
391
392fn bounded_runtime_text(value: &str, max_bytes: usize) -> String {
393    let sanitized = value
394        .chars()
395        .map(|character| {
396            if character.is_control() {
397                ' '
398            } else {
399                character
400            }
401        })
402        .collect::<String>();
403
404    if sanitized.len() <= max_bytes {
405        return sanitized;
406    }
407
408    let mut end = 0;
409    for (index, character) in sanitized.char_indices() {
410        let next = index + character.len_utf8();
411        if next > max_bytes {
412            break;
413        }
414        end = next;
415    }
416
417    sanitized[..end].to_string()
418}
419
420fn runtime_visibility() -> Vec<RuntimeVisibilityEntry> {
421    [
422        ("schema_version", RuntimeFieldVisibility::PublicSafe),
423        ("observed_at_ns", RuntimeFieldVisibility::PublicSafe),
424        ("canister_id", RuntimeFieldVisibility::OperatorOnly),
425        ("role", RuntimeFieldVisibility::OperatorOnly),
426        ("root", RuntimeFieldVisibility::OperatorOnly),
427        ("network", RuntimeFieldVisibility::OperatorOnly),
428        ("build", RuntimeFieldVisibility::OperatorOnly),
429        ("features", RuntimeFieldVisibility::OperatorOnly),
430        ("topology", RuntimeFieldVisibility::ControllerOnly),
431        ("timers", RuntimeFieldVisibility::OperatorOnly),
432        ("state", RuntimeFieldVisibility::OperatorOnly),
433        ("auth", RuntimeFieldVisibility::OperatorOnly),
434        ("blob_storage", RuntimeFieldVisibility::FeatureGated),
435        ("recent_failures", RuntimeFieldVisibility::OperatorOnly),
436        ("readiness", RuntimeFieldVisibility::OperatorOnly),
437        ("status", RuntimeFieldVisibility::OperatorOnly),
438        ("visibility", RuntimeFieldVisibility::OperatorOnly),
439    ]
440    .into_iter()
441    .map(|(field, visibility)| RuntimeVisibilityEntry {
442        field: field.to_string(),
443        visibility,
444    })
445    .collect()
446}
447
448#[cfg(test)]
449mod tests {
450    use super::*;
451    use crate::ops::runtime::bootstrap::BootstrapStatusOps;
452    use crate::ops::runtime::metrics::timer::{TimerMetrics, TimerMode};
453    use crate::ops::runtime::recent_failure::RecentFailureOps;
454    use std::time::Duration;
455
456    #[test]
457    fn health_is_minimal_and_schema_versioned() {
458        let health = RuntimeIntrospectionApi::health(Some(42));
459
460        assert_eq!(health.schema_version, RUNTIME_INTROSPECTION_SCHEMA_VERSION);
461        assert_eq!(health.status, crate::dto::runtime::HealthStatus::Healthy);
462        assert_eq!(health.observed_at_ns, Some(42));
463        assert_eq!(health.checks.len(), 1);
464        assert_eq!(health.checks[0].code, "canister_responsive");
465    }
466
467    #[test]
468    fn runtime_status_embeds_guarded_readiness_and_build_info() {
469        let status = RuntimeIntrospectionApi::runtime_status_for(
470            Principal::anonymous(),
471            100,
472            "test-canister",
473            "1.2.3",
474            "0.81.0",
475            7,
476        );
477
478        assert_eq!(status.schema_version, RUNTIME_INTROSPECTION_SCHEMA_VERSION);
479        assert_eq!(status.observed_at_ns, 100);
480        assert_eq!(status.canister_id, Principal::anonymous());
481        assert_eq!(status.build.package_name, "test-canister");
482        assert_eq!(status.build.package_version, "1.2.3");
483        assert_eq!(status.build.canic_version, "0.81.0");
484        assert_eq!(status.build.canister_version, 7);
485        assert_eq!(status.readiness.observed_at_ns, 100);
486        assert!(
487            status
488                .visibility
489                .iter()
490                .any(|entry| entry.field == "topology"
491                    && entry.visibility == RuntimeFieldVisibility::ControllerOnly)
492        );
493    }
494
495    #[test]
496    fn runtime_status_classifies_each_top_level_field_visibility() {
497        let status = RuntimeIntrospectionApi::runtime_status_for(
498            Principal::anonymous(),
499            100,
500            "test-canister",
501            "1.2.3",
502            "0.81.0",
503            7,
504        );
505        let expected = [
506            ("schema_version", RuntimeFieldVisibility::PublicSafe),
507            ("observed_at_ns", RuntimeFieldVisibility::PublicSafe),
508            ("canister_id", RuntimeFieldVisibility::OperatorOnly),
509            ("role", RuntimeFieldVisibility::OperatorOnly),
510            ("root", RuntimeFieldVisibility::OperatorOnly),
511            ("network", RuntimeFieldVisibility::OperatorOnly),
512            ("build", RuntimeFieldVisibility::OperatorOnly),
513            ("features", RuntimeFieldVisibility::OperatorOnly),
514            ("topology", RuntimeFieldVisibility::ControllerOnly),
515            ("timers", RuntimeFieldVisibility::OperatorOnly),
516            ("state", RuntimeFieldVisibility::OperatorOnly),
517            ("auth", RuntimeFieldVisibility::OperatorOnly),
518            ("blob_storage", RuntimeFieldVisibility::FeatureGated),
519            ("recent_failures", RuntimeFieldVisibility::OperatorOnly),
520            ("readiness", RuntimeFieldVisibility::OperatorOnly),
521            ("status", RuntimeFieldVisibility::OperatorOnly),
522            ("visibility", RuntimeFieldVisibility::OperatorOnly),
523        ];
524
525        assert_eq!(status.visibility.len(), expected.len());
526        for (index, (field, visibility)) in expected.into_iter().enumerate() {
527            assert_eq!(status.visibility[index].field, field);
528            assert_eq!(status.visibility[index].visibility, visibility);
529        }
530    }
531
532    #[test]
533    fn runtime_status_reports_compile_features_deterministically() {
534        let status = RuntimeIntrospectionApi::runtime_status_for(
535            Principal::anonymous(),
536            100,
537            "test-canister",
538            "1.2.3",
539            "0.81.0",
540            7,
541        );
542        assert_eq!(status.features.len(), RUNTIME_FEATURE_FLAGS.len());
543        for (index, (name, enabled)) in RUNTIME_FEATURE_FLAGS.into_iter().enumerate() {
544            assert_eq!(status.features[index].name, name);
545            assert_eq!(status.features[index].enabled, enabled);
546            assert_eq!(
547                status.features[index].visibility,
548                RuntimeFieldVisibility::OperatorOnly
549            );
550            assert_eq!(status.features[index].source, RUNTIME_FEATURE_SOURCE);
551        }
552    }
553
554    #[test]
555    fn runtime_status_reports_auth_and_blob_storage_feature_summaries() {
556        let status = RuntimeIntrospectionApi::runtime_status_for(
557            Principal::anonymous(),
558            100,
559            "test-canister",
560            "1.2.3",
561            "0.81.0",
562            7,
563        );
564
565        let auth = status.auth.expect("auth feature summary");
566        assert!(
567            auth.auth_features
568                .windows(2)
569                .all(|features| features[0].name <= features[1].name)
570        );
571        assert_runtime_feature(
572            &auth.auth_features,
573            "auth-chain-key-ecdsa",
574            cfg!(feature = "auth-chain-key-ecdsa"),
575        );
576        assert_runtime_feature(
577            &auth.auth_features,
578            "auth-delegated-token-verify",
579            cfg!(feature = "auth-delegated-token-verify"),
580        );
581        assert_runtime_feature(
582            &auth.auth_features,
583            "auth-issuer-canister-sig-create",
584            cfg!(feature = "auth-issuer-canister-sig-create"),
585        );
586
587        if cfg!(any(
588            feature = "blob-storage",
589            feature = "blob-storage-billing"
590        )) {
591            let blob_storage = status.blob_storage.expect("blob-storage feature summary");
592            assert_runtime_feature(
593                &blob_storage.blob_storage_features,
594                "blob-storage",
595                cfg!(feature = "blob-storage"),
596            );
597            assert_runtime_feature(
598                &blob_storage.blob_storage_features,
599                "blob-storage-billing",
600                cfg!(feature = "blob-storage-billing"),
601            );
602        } else {
603            assert!(status.blob_storage.is_none());
604        }
605    }
606
607    fn assert_runtime_feature(
608        features: &[RuntimeFeatureStatus],
609        name: &str,
610        expected_enabled: bool,
611    ) {
612        let feature = features
613            .iter()
614            .find(|feature| feature.name == name)
615            .unwrap_or_else(|| panic!("expected runtime feature {name}"));
616
617        assert_eq!(feature.enabled, expected_enabled);
618        assert_eq!(feature.visibility, RuntimeFieldVisibility::OperatorOnly);
619        assert_eq!(feature.source, RUNTIME_FEATURE_SOURCE);
620    }
621
622    #[test]
623    fn runtime_status_projects_registered_timer_metrics() {
624        TimerMetrics::reset();
625        TimerMetrics::record_timer_scheduled(
626            TimerMode::Interval,
627            Duration::from_mins(1),
628            "cycles:interval",
629        );
630        TimerMetrics::record_timer_scheduled(
631            TimerMode::Once,
632            Duration::from_secs(1),
633            "auth_renewal:init",
634        );
635        TimerMetrics::record_timer_tick(
636            TimerMode::Once,
637            Duration::from_secs(1),
638            "auth_renewal:init",
639        );
640
641        let status = RuntimeIntrospectionApi::runtime_status_for(
642            Principal::anonymous(),
643            100,
644            "test-canister",
645            "1.2.3",
646            "0.81.0",
647            7,
648        );
649
650        assert_eq!(status.timers.len(), 2);
651        assert_eq!(status.timers[0].subsystem, "auth_renewal");
652        assert_eq!(status.timers[0].name, "init");
653        assert_eq!(status.timers[0].status, TimerStatus::Healthy);
654        assert_eq!(status.timers[1].subsystem, "cycles");
655        assert_eq!(status.timers[1].name, "interval");
656        assert_eq!(status.timers[1].status, TimerStatus::Unknown);
657
658        TimerMetrics::reset();
659    }
660
661    #[test]
662    fn runtime_status_bounds_timer_labels() {
663        TimerMetrics::reset();
664
665        let label = format!("{}\n:{}\n", "subsystem".repeat(12), "timer_name".repeat(16));
666        TimerMetrics::record_timer_scheduled(TimerMode::Once, Duration::from_secs(1), &label);
667
668        let status = RuntimeIntrospectionApi::runtime_status_for(
669            Principal::anonymous(),
670            100,
671            "test-canister",
672            "1.2.3",
673            "0.81.0",
674            7,
675        );
676
677        assert_eq!(status.timers.len(), 1);
678        assert!(status.timers[0].subsystem.len() <= MAX_TIMER_SUBSYSTEM_BYTES);
679        assert!(status.timers[0].name.len() <= MAX_TIMER_NAME_BYTES);
680        assert!(!status.timers[0].subsystem.contains('\n'));
681        assert!(!status.timers[0].name.contains('\n'));
682
683        TimerMetrics::reset();
684    }
685
686    #[test]
687    fn state_summary_uses_declared_metadata_without_value_counts() {
688        let summary = state_summary(Some("root")).expect("root state declarations");
689
690        assert_eq!(
691            summary.manifest_schema_version,
692            u32::from(crate::state_contract::STATE_MANIFEST_SCHEMA_VERSION)
693        );
694        assert!(summary.total_stable_memory_pages.is_none());
695        assert!(summary.domains.iter().any(|domain| {
696            domain.domain == "env"
697                && domain.storage == "stable_memory"
698                && domain.status == RuntimeStateDomainStatus::Ok
699        }));
700        assert!(state_summary(Some("unknown_role")).is_none());
701        assert!(state_summary(None).is_none());
702    }
703
704    #[test]
705    fn runtime_status_includes_recent_failure_snapshot() {
706        RecentFailureOps::reset();
707        RuntimeIntrospectionApi::record_recent_failure(
708            77,
709            "runtime",
710            "readiness_failed",
711            FailureSeverity::Error,
712            "bounded failure summary",
713            Some("runtime-check".to_string()),
714        );
715
716        let status = RuntimeIntrospectionApi::runtime_status_for(
717            Principal::anonymous(),
718            100,
719            "test-canister",
720            "1.2.3",
721            "0.81.0",
722            7,
723        );
724
725        assert_eq!(status.recent_failures.len(), 1);
726        assert_eq!(status.recent_failures[0].occurred_at_ns, 77);
727        assert_eq!(status.recent_failures[0].subsystem, "runtime");
728        assert_eq!(status.recent_failures[0].code, "readiness_failed");
729
730        RecentFailureOps::reset();
731    }
732
733    #[test]
734    fn runtime_status_includes_bootstrap_failure_metadata() {
735        RecentFailureOps::reset();
736        BootstrapStatusOps::set_phase("root:init");
737        BootstrapStatusOps::mark_failed("raw bootstrap failure detail");
738
739        let status = RuntimeIntrospectionApi::runtime_status_for(
740            Principal::anonymous(),
741            100,
742            "test-canister",
743            "1.2.3",
744            "0.81.0",
745            7,
746        );
747
748        let failure = status
749            .recent_failures
750            .iter()
751            .find(|failure| failure.code == "bootstrap_failed")
752            .expect("bootstrap failure metadata");
753
754        assert_eq!(failure.subsystem, "runtime_bootstrap");
755        assert_eq!(failure.severity, FailureSeverity::Error);
756        assert_eq!(failure.correlation_id.as_deref(), Some("root:init"));
757        assert!(
758            !failure.summary.contains("raw bootstrap failure detail"),
759            "runtime status recent failures should not mirror raw bootstrap errors"
760        );
761
762        RecentFailureOps::reset();
763    }
764}