1pub mod install;
2
3use crate::{
4 cdk::types::Principal,
5 domain::runtime::{
6 FailureSeverity, RuntimeCheckStatus, RuntimeDiagnosticSeverity, RuntimeFieldVisibility,
7 RuntimeStateDomainStatus,
8 },
9 dto::{
10 error::Error,
11 runtime::{
12 CanicHealthStatus, CanicReadinessStatus, CanicRuntimeStatus, CanicTimerStatus,
13 RUNTIME_INTROSPECTION_SCHEMA_VERSION, ReadinessStatus, RuntimeAuthStatusSummary,
14 RuntimeBlobStorageStatusSummary, RuntimeBuildInfo, RuntimeCheck, RuntimeDiagnostic,
15 RuntimeFeatureStatus, RuntimeStateDomainSummary, RuntimeStateSummary, RuntimeStatus,
16 RuntimeTopologyStatus, RuntimeVisibilityEntry, TimerStatus,
17 },
18 },
19 ops::{
20 ic::IcOps,
21 runtime::{
22 env::EnvOps,
23 memory::MemoryRegistryOps,
24 metrics::timer::TimerMetrics,
25 ready::ReadyOps,
26 recent_failure::{RecentFailureInput, RecentFailureOps},
27 },
28 },
29 state_contract::{StateStorage, canic_state_manifest_for_role},
30};
31
32const MAX_TIMER_SUBSYSTEM_BYTES: usize = 64;
33const MAX_TIMER_NAME_BYTES: usize = 96;
34const RUNTIME_FEATURE_SOURCE: &str = "compile_feature";
35const RUNTIME_FEATURE_FLAGS: [(&str, bool); 10] = [
36 (
37 "auth-chain-key-ecdsa",
38 cfg!(feature = "auth-chain-key-ecdsa"),
39 ),
40 (
41 "auth-chain-key-root-sign",
42 cfg!(feature = "auth-chain-key-root-sign"),
43 ),
44 (
45 "auth-delegated-token-verify",
46 cfg!(feature = "auth-delegated-token-verify"),
47 ),
48 (
49 "auth-issuer-canister-sig-create",
50 cfg!(feature = "auth-issuer-canister-sig-create"),
51 ),
52 (
53 "auth-issuer-canister-sig-verify",
54 cfg!(feature = "auth-issuer-canister-sig-verify"),
55 ),
56 (
57 "auth-root-canister-sig-create",
58 cfg!(feature = "auth-root-canister-sig-create"),
59 ),
60 (
61 "auth-root-canister-sig-verify",
62 cfg!(feature = "auth-root-canister-sig-verify"),
63 ),
64 ("blob-storage", cfg!(feature = "blob-storage")),
65 (
66 "blob-storage-billing",
67 cfg!(feature = "blob-storage-billing"),
68 ),
69 ("sharding", cfg!(feature = "sharding")),
70];
71
72pub struct MemoryRuntimeApi;
77
78impl MemoryRuntimeApi {
79 pub fn bootstrap_registry() -> Result<(), Error> {
81 MemoryRegistryOps::bootstrap_registry().map_err(Error::from)?;
82
83 Ok(())
84 }
85}
86
87pub struct RuntimeIntrospectionApi;
92
93impl RuntimeIntrospectionApi {
94 pub fn record_recent_failure(
96 occurred_at_ns: u64,
97 subsystem: impl Into<String>,
98 code: impl Into<String>,
99 severity: FailureSeverity,
100 summary: impl Into<String>,
101 correlation_id: Option<String>,
102 ) {
103 RecentFailureOps::record(RecentFailureInput {
104 occurred_at_ns,
105 subsystem: subsystem.into(),
106 code: code.into(),
107 severity,
108 summary: summary.into(),
109 correlation_id,
110 });
111 }
112
113 #[must_use]
115 pub fn health(observed_at_ns: Option<u64>) -> CanicHealthStatus {
116 CanicHealthStatus {
117 schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
118 status: crate::dto::runtime::HealthStatus::Healthy,
119 observed_at_ns,
120 checks: vec![RuntimeCheck {
121 category: "health".to_string(),
122 code: "canister_responsive".to_string(),
123 status: RuntimeCheckStatus::Pass,
124 subject: "canister".to_string(),
125 detail: "canister returned a health response".to_string(),
126 next: None,
127 source: "runtime_observed".to_string(),
128 }],
129 }
130 }
131
132 #[must_use]
134 pub fn readiness(observed_at_ns: u64) -> CanicReadinessStatus {
135 let ready = ReadyOps::is_ready();
136 let role = EnvOps::canister_role()
137 .ok()
138 .map(crate::ids::CanisterRole::into_string);
139
140 let (status, check_status, detail, next) = if ready {
141 (
142 ReadinessStatus::Ready,
143 RuntimeCheckStatus::Pass,
144 "runtime readiness barrier is marked ready",
145 None,
146 )
147 } else {
148 (
149 ReadinessStatus::NotReady,
150 RuntimeCheckStatus::Fail,
151 "runtime readiness barrier is not ready",
152 Some("wait for bootstrap to complete or inspect canic_bootstrap_status"),
153 )
154 };
155
156 let readiness_check = RuntimeCheck {
157 category: "readiness".to_string(),
158 code: "runtime_ready_barrier".to_string(),
159 status: check_status,
160 subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
161 detail: detail.to_string(),
162 next: next.map(str::to_string),
163 source: "runtime_observed".to_string(),
164 };
165
166 let blockers = if ready {
167 Vec::new()
168 } else {
169 vec![RuntimeDiagnostic {
170 category: "readiness".to_string(),
171 code: "runtime_not_ready".to_string(),
172 severity: RuntimeDiagnosticSeverity::Blocked,
173 subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
174 detail: "runtime readiness barrier has not completed".to_string(),
175 next: Some(
176 "inspect bootstrap status before treating the role as ready".to_string(),
177 ),
178 source: "runtime_observed".to_string(),
179 }]
180 };
181
182 CanicReadinessStatus {
183 schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
184 role,
185 status,
186 observed_at_ns,
187 checks: vec![readiness_check],
188 blockers,
189 warnings: Vec::new(),
190 }
191 }
192
193 #[must_use]
195 pub fn runtime_status_for(
196 canister_id: Principal,
197 observed_at_ns: u64,
198 package_name: &str,
199 package_version: &str,
200 canic_version: &str,
201 canister_version: u64,
202 ) -> CanicRuntimeStatus {
203 let readiness = Self::readiness(observed_at_ns);
204 let role = readiness.role.clone();
205 let state = state_summary(role.as_deref());
206 let root = EnvOps::root_pid().ok();
207 let parent = EnvOps::parent_pid().ok();
208 let subnet = EnvOps::subnet_pid().ok();
209 let status = match readiness.status {
210 ReadinessStatus::Ready => RuntimeStatus::Ok,
211 ReadinessStatus::Degraded | ReadinessStatus::NotEvaluated => RuntimeStatus::Degraded,
212 ReadinessStatus::NotReady => RuntimeStatus::Failing,
213 };
214
215 CanicRuntimeStatus {
216 schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
217 observed_at_ns,
218 canister_id,
219 role,
220 root,
221 network: None,
222 build: RuntimeBuildInfo {
223 package_name: package_name.to_string(),
224 package_version: package_version.to_string(),
225 canic_version: canic_version.to_string(),
226 canister_version,
227 },
228 features: runtime_features(),
229 topology: Some(RuntimeTopologyStatus {
230 root,
231 parent,
232 subnet,
233 source: "runtime_observed".to_string(),
234 }),
235 timers: timer_statuses(),
236 state,
237 auth: Some(runtime_auth_status()),
238 blob_storage: runtime_blob_storage_status(),
239 recent_failures: RecentFailureOps::snapshot(),
240 visibility: runtime_visibility(),
241 readiness,
242 status,
243 }
244 }
245
246 #[must_use]
248 pub fn runtime_status(
249 observed_at_ns: u64,
250 package_name: &str,
251 package_version: &str,
252 canic_version: &str,
253 canister_version: u64,
254 ) -> CanicRuntimeStatus {
255 Self::runtime_status_for(
256 IcOps::canister_self(),
257 observed_at_ns,
258 package_name,
259 package_version,
260 canic_version,
261 canister_version,
262 )
263 }
264}
265
266fn runtime_features() -> Vec<RuntimeFeatureStatus> {
267 RUNTIME_FEATURE_FLAGS
268 .into_iter()
269 .map(|(name, enabled)| runtime_feature_status(name, enabled))
270 .collect()
271}
272
273fn runtime_feature_status(name: &str, enabled: bool) -> RuntimeFeatureStatus {
274 RuntimeFeatureStatus {
275 name: name.to_string(),
276 enabled,
277 visibility: RuntimeFieldVisibility::OperatorOnly,
278 source: RUNTIME_FEATURE_SOURCE.to_string(),
279 }
280}
281
282fn runtime_auth_status() -> RuntimeAuthStatusSummary {
283 RuntimeAuthStatusSummary {
284 auth_features: RUNTIME_FEATURE_FLAGS
285 .into_iter()
286 .filter(|(name, _)| name.starts_with("auth-"))
287 .map(|(name, enabled)| runtime_feature_status(name, enabled))
288 .collect(),
289 }
290}
291
292fn runtime_blob_storage_status() -> Option<RuntimeBlobStorageStatusSummary> {
293 let blob_storage_enabled = cfg!(feature = "blob-storage");
294 let billing_enabled = cfg!(feature = "blob-storage-billing");
295
296 (blob_storage_enabled || billing_enabled).then(|| RuntimeBlobStorageStatusSummary {
297 blob_storage_features: [
298 ("blob-storage", blob_storage_enabled),
299 ("blob-storage-billing", billing_enabled),
300 ]
301 .into_iter()
302 .map(|(name, enabled)| runtime_feature_status(name, enabled))
303 .collect(),
304 })
305}
306
307fn timer_statuses() -> Vec<CanicTimerStatus> {
308 let mut timers = TimerMetrics::snapshot()
309 .entries
310 .into_iter()
311 .map(|(key, ticks)| {
312 let (subsystem, name) = split_timer_label(&key.label);
313 CanicTimerStatus {
314 name,
315 subsystem,
316 status: if ticks > 0 {
317 TimerStatus::Healthy
318 } else {
319 TimerStatus::Unknown
320 },
321 enabled: true,
322 registered: true,
323 last_success_at_ns: None,
324 last_failure_at_ns: None,
325 next_due_at_ns: None,
326 consecutive_failures: 0,
327 last_error_code: None,
328 last_error_summary: None,
329 }
330 })
331 .collect::<Vec<_>>();
332 timers.sort_by(|left, right| {
333 left.subsystem
334 .cmp(&right.subsystem)
335 .then_with(|| left.name.cmp(&right.name))
336 });
337 timers
338}
339
340fn state_summary(role: Option<&str>) -> Option<RuntimeStateSummary> {
341 let role = role?;
342 let manifest = canic_state_manifest_for_role(Some(role));
343 let domains = manifest
344 .roles
345 .into_iter()
346 .flat_map(|role| role.state)
347 .map(|domain| RuntimeStateDomainSummary {
348 domain: domain.domain,
349 version: domain.version,
350 storage: state_storage_name(domain.storage).to_string(),
351 memory_id: domain.memory_id,
352 status: RuntimeStateDomainStatus::Ok,
353 })
354 .collect::<Vec<_>>();
355
356 if domains.is_empty() {
357 return None;
358 }
359
360 Some(RuntimeStateSummary {
361 manifest_schema_version: u32::from(manifest.schema_version),
362 domains,
363 total_stable_memory_pages: None,
364 })
365}
366
367const fn state_storage_name(storage: StateStorage) -> &'static str {
368 match storage {
369 StateStorage::StableMemory => "stable_memory",
370 StateStorage::HeapOnly => "heap_only",
371 StateStorage::NotApplicable => "not_applicable",
372 }
373}
374
375fn split_timer_label(label: &str) -> (String, String) {
376 label.split_once(':').map_or_else(
377 || {
378 (
379 "runtime".to_string(),
380 bounded_runtime_text(label, MAX_TIMER_NAME_BYTES),
381 )
382 },
383 |(subsystem, name)| {
384 (
385 bounded_runtime_text(subsystem, MAX_TIMER_SUBSYSTEM_BYTES),
386 bounded_runtime_text(name, MAX_TIMER_NAME_BYTES),
387 )
388 },
389 )
390}
391
392fn bounded_runtime_text(value: &str, max_bytes: usize) -> String {
393 let sanitized = value
394 .chars()
395 .map(|character| {
396 if character.is_control() {
397 ' '
398 } else {
399 character
400 }
401 })
402 .collect::<String>();
403
404 if sanitized.len() <= max_bytes {
405 return sanitized;
406 }
407
408 let mut end = 0;
409 for (index, character) in sanitized.char_indices() {
410 let next = index + character.len_utf8();
411 if next > max_bytes {
412 break;
413 }
414 end = next;
415 }
416
417 sanitized[..end].to_string()
418}
419
420fn runtime_visibility() -> Vec<RuntimeVisibilityEntry> {
421 [
422 ("schema_version", RuntimeFieldVisibility::PublicSafe),
423 ("observed_at_ns", RuntimeFieldVisibility::PublicSafe),
424 ("canister_id", RuntimeFieldVisibility::OperatorOnly),
425 ("role", RuntimeFieldVisibility::OperatorOnly),
426 ("root", RuntimeFieldVisibility::OperatorOnly),
427 ("network", RuntimeFieldVisibility::OperatorOnly),
428 ("build", RuntimeFieldVisibility::OperatorOnly),
429 ("features", RuntimeFieldVisibility::OperatorOnly),
430 ("topology", RuntimeFieldVisibility::ControllerOnly),
431 ("timers", RuntimeFieldVisibility::OperatorOnly),
432 ("state", RuntimeFieldVisibility::OperatorOnly),
433 ("auth", RuntimeFieldVisibility::OperatorOnly),
434 ("blob_storage", RuntimeFieldVisibility::FeatureGated),
435 ("recent_failures", RuntimeFieldVisibility::OperatorOnly),
436 ("readiness", RuntimeFieldVisibility::OperatorOnly),
437 ("status", RuntimeFieldVisibility::OperatorOnly),
438 ("visibility", RuntimeFieldVisibility::OperatorOnly),
439 ]
440 .into_iter()
441 .map(|(field, visibility)| RuntimeVisibilityEntry {
442 field: field.to_string(),
443 visibility,
444 })
445 .collect()
446}
447
448#[cfg(test)]
449mod tests {
450 use super::*;
451 use crate::ops::runtime::bootstrap::BootstrapStatusOps;
452 use crate::ops::runtime::metrics::timer::{TimerMetrics, TimerMode};
453 use crate::ops::runtime::recent_failure::RecentFailureOps;
454 use std::time::Duration;
455
456 #[test]
457 fn health_is_minimal_and_schema_versioned() {
458 let health = RuntimeIntrospectionApi::health(Some(42));
459
460 assert_eq!(health.schema_version, RUNTIME_INTROSPECTION_SCHEMA_VERSION);
461 assert_eq!(health.status, crate::dto::runtime::HealthStatus::Healthy);
462 assert_eq!(health.observed_at_ns, Some(42));
463 assert_eq!(health.checks.len(), 1);
464 assert_eq!(health.checks[0].code, "canister_responsive");
465 }
466
467 #[test]
468 fn runtime_status_embeds_guarded_readiness_and_build_info() {
469 let status = RuntimeIntrospectionApi::runtime_status_for(
470 Principal::anonymous(),
471 100,
472 "test-canister",
473 "1.2.3",
474 "0.81.0",
475 7,
476 );
477
478 assert_eq!(status.schema_version, RUNTIME_INTROSPECTION_SCHEMA_VERSION);
479 assert_eq!(status.observed_at_ns, 100);
480 assert_eq!(status.canister_id, Principal::anonymous());
481 assert_eq!(status.build.package_name, "test-canister");
482 assert_eq!(status.build.package_version, "1.2.3");
483 assert_eq!(status.build.canic_version, "0.81.0");
484 assert_eq!(status.build.canister_version, 7);
485 assert_eq!(status.readiness.observed_at_ns, 100);
486 assert!(
487 status
488 .visibility
489 .iter()
490 .any(|entry| entry.field == "topology"
491 && entry.visibility == RuntimeFieldVisibility::ControllerOnly)
492 );
493 }
494
495 #[test]
496 fn runtime_status_classifies_each_top_level_field_visibility() {
497 let status = RuntimeIntrospectionApi::runtime_status_for(
498 Principal::anonymous(),
499 100,
500 "test-canister",
501 "1.2.3",
502 "0.81.0",
503 7,
504 );
505 let expected = [
506 ("schema_version", RuntimeFieldVisibility::PublicSafe),
507 ("observed_at_ns", RuntimeFieldVisibility::PublicSafe),
508 ("canister_id", RuntimeFieldVisibility::OperatorOnly),
509 ("role", RuntimeFieldVisibility::OperatorOnly),
510 ("root", RuntimeFieldVisibility::OperatorOnly),
511 ("network", RuntimeFieldVisibility::OperatorOnly),
512 ("build", RuntimeFieldVisibility::OperatorOnly),
513 ("features", RuntimeFieldVisibility::OperatorOnly),
514 ("topology", RuntimeFieldVisibility::ControllerOnly),
515 ("timers", RuntimeFieldVisibility::OperatorOnly),
516 ("state", RuntimeFieldVisibility::OperatorOnly),
517 ("auth", RuntimeFieldVisibility::OperatorOnly),
518 ("blob_storage", RuntimeFieldVisibility::FeatureGated),
519 ("recent_failures", RuntimeFieldVisibility::OperatorOnly),
520 ("readiness", RuntimeFieldVisibility::OperatorOnly),
521 ("status", RuntimeFieldVisibility::OperatorOnly),
522 ("visibility", RuntimeFieldVisibility::OperatorOnly),
523 ];
524
525 assert_eq!(status.visibility.len(), expected.len());
526 for (index, (field, visibility)) in expected.into_iter().enumerate() {
527 assert_eq!(status.visibility[index].field, field);
528 assert_eq!(status.visibility[index].visibility, visibility);
529 }
530 }
531
532 #[test]
533 fn runtime_status_reports_compile_features_deterministically() {
534 let status = RuntimeIntrospectionApi::runtime_status_for(
535 Principal::anonymous(),
536 100,
537 "test-canister",
538 "1.2.3",
539 "0.81.0",
540 7,
541 );
542 assert_eq!(status.features.len(), RUNTIME_FEATURE_FLAGS.len());
543 for (index, (name, enabled)) in RUNTIME_FEATURE_FLAGS.into_iter().enumerate() {
544 assert_eq!(status.features[index].name, name);
545 assert_eq!(status.features[index].enabled, enabled);
546 assert_eq!(
547 status.features[index].visibility,
548 RuntimeFieldVisibility::OperatorOnly
549 );
550 assert_eq!(status.features[index].source, RUNTIME_FEATURE_SOURCE);
551 }
552 }
553
554 #[test]
555 fn runtime_status_reports_auth_and_blob_storage_feature_summaries() {
556 let status = RuntimeIntrospectionApi::runtime_status_for(
557 Principal::anonymous(),
558 100,
559 "test-canister",
560 "1.2.3",
561 "0.81.0",
562 7,
563 );
564
565 let auth = status.auth.expect("auth feature summary");
566 assert!(
567 auth.auth_features
568 .windows(2)
569 .all(|features| features[0].name <= features[1].name)
570 );
571 assert_runtime_feature(
572 &auth.auth_features,
573 "auth-chain-key-ecdsa",
574 cfg!(feature = "auth-chain-key-ecdsa"),
575 );
576 assert_runtime_feature(
577 &auth.auth_features,
578 "auth-delegated-token-verify",
579 cfg!(feature = "auth-delegated-token-verify"),
580 );
581 assert_runtime_feature(
582 &auth.auth_features,
583 "auth-issuer-canister-sig-create",
584 cfg!(feature = "auth-issuer-canister-sig-create"),
585 );
586
587 if cfg!(any(
588 feature = "blob-storage",
589 feature = "blob-storage-billing"
590 )) {
591 let blob_storage = status.blob_storage.expect("blob-storage feature summary");
592 assert_runtime_feature(
593 &blob_storage.blob_storage_features,
594 "blob-storage",
595 cfg!(feature = "blob-storage"),
596 );
597 assert_runtime_feature(
598 &blob_storage.blob_storage_features,
599 "blob-storage-billing",
600 cfg!(feature = "blob-storage-billing"),
601 );
602 } else {
603 assert!(status.blob_storage.is_none());
604 }
605 }
606
607 fn assert_runtime_feature(
608 features: &[RuntimeFeatureStatus],
609 name: &str,
610 expected_enabled: bool,
611 ) {
612 let feature = features
613 .iter()
614 .find(|feature| feature.name == name)
615 .unwrap_or_else(|| panic!("expected runtime feature {name}"));
616
617 assert_eq!(feature.enabled, expected_enabled);
618 assert_eq!(feature.visibility, RuntimeFieldVisibility::OperatorOnly);
619 assert_eq!(feature.source, RUNTIME_FEATURE_SOURCE);
620 }
621
622 #[test]
623 fn runtime_status_projects_registered_timer_metrics() {
624 TimerMetrics::reset();
625 TimerMetrics::record_timer_scheduled(
626 TimerMode::Interval,
627 Duration::from_mins(1),
628 "cycles:interval",
629 );
630 TimerMetrics::record_timer_scheduled(
631 TimerMode::Once,
632 Duration::from_secs(1),
633 "auth_renewal:init",
634 );
635 TimerMetrics::record_timer_tick(
636 TimerMode::Once,
637 Duration::from_secs(1),
638 "auth_renewal:init",
639 );
640
641 let status = RuntimeIntrospectionApi::runtime_status_for(
642 Principal::anonymous(),
643 100,
644 "test-canister",
645 "1.2.3",
646 "0.81.0",
647 7,
648 );
649
650 assert_eq!(status.timers.len(), 2);
651 assert_eq!(status.timers[0].subsystem, "auth_renewal");
652 assert_eq!(status.timers[0].name, "init");
653 assert_eq!(status.timers[0].status, TimerStatus::Healthy);
654 assert_eq!(status.timers[1].subsystem, "cycles");
655 assert_eq!(status.timers[1].name, "interval");
656 assert_eq!(status.timers[1].status, TimerStatus::Unknown);
657
658 TimerMetrics::reset();
659 }
660
661 #[test]
662 fn runtime_status_bounds_timer_labels() {
663 TimerMetrics::reset();
664
665 let label = format!("{}\n:{}\n", "subsystem".repeat(12), "timer_name".repeat(16));
666 TimerMetrics::record_timer_scheduled(TimerMode::Once, Duration::from_secs(1), &label);
667
668 let status = RuntimeIntrospectionApi::runtime_status_for(
669 Principal::anonymous(),
670 100,
671 "test-canister",
672 "1.2.3",
673 "0.81.0",
674 7,
675 );
676
677 assert_eq!(status.timers.len(), 1);
678 assert!(status.timers[0].subsystem.len() <= MAX_TIMER_SUBSYSTEM_BYTES);
679 assert!(status.timers[0].name.len() <= MAX_TIMER_NAME_BYTES);
680 assert!(!status.timers[0].subsystem.contains('\n'));
681 assert!(!status.timers[0].name.contains('\n'));
682
683 TimerMetrics::reset();
684 }
685
686 #[test]
687 fn state_summary_uses_declared_metadata_without_value_counts() {
688 let summary = state_summary(Some("root")).expect("root state declarations");
689
690 assert_eq!(
691 summary.manifest_schema_version,
692 u32::from(crate::state_contract::STATE_MANIFEST_SCHEMA_VERSION)
693 );
694 assert!(summary.total_stable_memory_pages.is_none());
695 assert!(summary.domains.iter().any(|domain| {
696 domain.domain == "env"
697 && domain.storage == "stable_memory"
698 && domain.status == RuntimeStateDomainStatus::Ok
699 }));
700 assert!(state_summary(Some("unknown_role")).is_none());
701 assert!(state_summary(None).is_none());
702 }
703
704 #[test]
705 fn runtime_status_includes_recent_failure_snapshot() {
706 RecentFailureOps::reset();
707 RuntimeIntrospectionApi::record_recent_failure(
708 77,
709 "runtime",
710 "readiness_failed",
711 FailureSeverity::Error,
712 "bounded failure summary",
713 Some("runtime-check".to_string()),
714 );
715
716 let status = RuntimeIntrospectionApi::runtime_status_for(
717 Principal::anonymous(),
718 100,
719 "test-canister",
720 "1.2.3",
721 "0.81.0",
722 7,
723 );
724
725 assert_eq!(status.recent_failures.len(), 1);
726 assert_eq!(status.recent_failures[0].occurred_at_ns, 77);
727 assert_eq!(status.recent_failures[0].subsystem, "runtime");
728 assert_eq!(status.recent_failures[0].code, "readiness_failed");
729
730 RecentFailureOps::reset();
731 }
732
733 #[test]
734 fn runtime_status_includes_bootstrap_failure_metadata() {
735 RecentFailureOps::reset();
736 BootstrapStatusOps::set_phase("root:init");
737 BootstrapStatusOps::mark_failed("raw bootstrap failure detail");
738
739 let status = RuntimeIntrospectionApi::runtime_status_for(
740 Principal::anonymous(),
741 100,
742 "test-canister",
743 "1.2.3",
744 "0.81.0",
745 7,
746 );
747
748 let failure = status
749 .recent_failures
750 .iter()
751 .find(|failure| failure.code == "bootstrap_failed")
752 .expect("bootstrap failure metadata");
753
754 assert_eq!(failure.subsystem, "runtime_bootstrap");
755 assert_eq!(failure.severity, FailureSeverity::Error);
756 assert_eq!(failure.correlation_id.as_deref(), Some("root:init"));
757 assert!(
758 !failure.summary.contains("raw bootstrap failure detail"),
759 "runtime status recent failures should not mirror raw bootstrap errors"
760 );
761
762 RecentFailureOps::reset();
763 }
764}