pub mod install;
pub mod root_funding;
use crate::{
InternalError,
cdk::types::Principal,
domain::runtime::{
FailureSeverity, HealthStatus, ReadinessStatus, RuntimeCheckStatus,
RuntimeDiagnosticSeverity, RuntimeFieldVisibility, RuntimeStateDomainStatus, RuntimeStatus,
},
dto::{
error::Error,
runtime::{
CanicHealthStatus, CanicReadinessStatus, CanicRuntimeStatus, CanisterTimerStatus,
RUNTIME_INTROSPECTION_SCHEMA_VERSION, RuntimeAuthStatusSummary,
RuntimeBlobStorageStatusSummary, RuntimeBuildInfo, RuntimeCheck, RuntimeDiagnostic,
RuntimeFeatureStatus, RuntimeReceiptCapacityStatus, RuntimeStateDomainSummary,
RuntimeStateSummary, RuntimeTopologyStatus, RuntimeVisibilityEntry,
TimerCallbackPerformanceStatus, TimerMemoryPageExtentStatus,
TimerMemoryPageSampleStatus,
},
},
ops::{
ic::{IcOps, build_network::BuildNetworkOps},
runtime::{
env::EnvOps,
memory::MemoryRegistryOps,
ready::ReadyOps,
recent_failure::{RecentFailureInput, RecentFailureOps},
},
storage::intent::{RECEIPT_CAPACITY_WARNING_HEADROOM_THRESHOLD, ReceiptBackedIntentOps},
},
state_contract::{STATE_MANIFEST_SCHEMA_VERSION, canic_state_descriptors},
workflow::runtime::timer::TimerAuthorityWorkflow,
};
const RUNTIME_FEATURE_SOURCE: &str = "compile_feature";
const RUNTIME_FEATURE_FLAGS: [(&str, bool); 11] = [
(
"auth-chain-key-ecdsa",
cfg!(feature = "auth-chain-key-ecdsa"),
),
(
"auth-chain-key-root-sign",
cfg!(feature = "auth-chain-key-root-sign"),
),
(
"auth-delegated-token-verify",
cfg!(feature = "auth-delegated-token-verify"),
),
(
"auth-issuer-canister-sig-create",
cfg!(feature = "auth-issuer-canister-sig-create"),
),
(
"auth-issuer-canister-sig-verify",
cfg!(feature = "auth-issuer-canister-sig-verify"),
),
(
"auth-local-application-authorization",
cfg!(feature = "auth-local-application-authorization"),
),
(
"auth-root-canister-sig-create",
cfg!(feature = "auth-root-canister-sig-create"),
),
(
"auth-root-canister-sig-verify",
cfg!(feature = "auth-root-canister-sig-verify"),
),
("blob-storage", cfg!(feature = "blob-storage")),
(
"blob-storage-billing",
cfg!(feature = "blob-storage-billing"),
),
("sharding", cfg!(feature = "sharding")),
];
pub struct MemoryRuntimeApi;
impl MemoryRuntimeApi {
pub fn bootstrap_registry() -> Result<(), Error> {
MemoryRegistryOps::bootstrap_registry().map_err(Error::from)?;
Ok(())
}
}
pub struct RuntimeIntrospectionApi;
impl RuntimeIntrospectionApi {
#[must_use]
pub fn health(observed_at_ns: Option<u64>) -> CanicHealthStatus {
CanicHealthStatus {
schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
status: HealthStatus::Healthy,
observed_at_ns,
checks: vec![RuntimeCheck {
category: "health".to_string(),
code: "canister_responsive".to_string(),
status: RuntimeCheckStatus::Pass,
subject: "canister".to_string(),
detail: "canister returned a health response".to_string(),
next: None,
source: "runtime_observed".to_string(),
}],
}
}
#[must_use]
pub fn readiness(observed_at_ns: u64) -> CanicReadinessStatus {
let ready = ReadyOps::is_ready();
let role = EnvOps::canister_role()
.ok()
.map(crate::ids::CanisterRole::into_string);
let (status, check_status, detail, next) = if ready {
(
ReadinessStatus::Ready,
RuntimeCheckStatus::Pass,
"runtime readiness barrier is marked ready",
None,
)
} else {
(
ReadinessStatus::NotReady,
RuntimeCheckStatus::Fail,
"runtime readiness barrier is not ready",
Some("wait for bootstrap to complete or inspect the role-owned Overview status"),
)
};
let readiness_check = RuntimeCheck {
category: "readiness".to_string(),
code: "runtime_ready_barrier".to_string(),
status: check_status,
subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
detail: detail.to_string(),
next: next.map(str::to_string),
source: "runtime_observed".to_string(),
};
let blockers = if ready {
Vec::new()
} else {
vec![RuntimeDiagnostic {
category: "readiness".to_string(),
code: "runtime_not_ready".to_string(),
severity: RuntimeDiagnosticSeverity::Blocked,
subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
detail: "runtime readiness barrier has not completed".to_string(),
next: Some(
"inspect bootstrap status before treating the role as ready".to_string(),
),
source: "runtime_observed".to_string(),
}]
};
let fixture = crate::workflow::fixture_provisioning::status();
let fixture_ready = crate::workflow::fixture_provisioning::is_ready(&fixture);
let mut checks = vec![readiness_check];
let mut blockers = blockers;
checks.push(RuntimeCheck {
category: "readiness".to_string(),
code: "fixture_receipt".to_string(),
status: if fixture_ready {
RuntimeCheckStatus::Pass
} else {
RuntimeCheckStatus::Fail
},
subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
detail: "application fixture receipt for the installed selection".to_string(),
next: None,
source: "application_durable_progress".to_string(),
});
if !fixture_ready {
blockers.push(RuntimeDiagnostic {
category: "readiness".to_string(),
code: "fixture_not_ready".to_string(),
severity: RuntimeDiagnosticSeverity::Blocked,
subject: role.clone().unwrap_or_else(|| "unknown_role".to_string()),
detail: "required fixture has no validated completion receipt".to_string(),
next: Some(
"inspect the protected fixture result for progress or failure".to_string(),
),
source: "application_durable_progress".to_string(),
});
}
CanicReadinessStatus {
fixture,
schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
role,
status: if fixture_ready {
status
} else {
ReadinessStatus::NotReady
},
observed_at_ns,
checks,
blockers,
warnings: Vec::new(),
}
}
#[must_use]
pub fn runtime_status_for(
canister_id: Principal,
observed_at_ns: u64,
package_name: &str,
package_version: &str,
canic_version: &str,
canister_version: u64,
) -> CanicRuntimeStatus {
let readiness = Self::readiness(observed_at_ns);
let role = readiness.role.clone();
let state = state_summary(role.as_deref());
let root = EnvOps::root_pid().ok();
let parent = EnvOps::parent_pid().ok();
let subnet = EnvOps::subnet_pid().ok();
let receipt_capacity_result = runtime_receipt_capacity();
let receipt_capacity_status = receipt_capacity_result
.as_ref()
.ok()
.map(|capacity| capacity.status);
let timer_observation =
timer_status_observation(TimerAuthorityWorkflow::statuses(), observed_at_ns);
let status = aggregate_runtime_status(
readiness.status,
receipt_capacity_status,
timer_observation.check.status,
);
let (receipt_capacity, receipt_failure) = match receipt_capacity_result {
Ok(capacity) => (Some(capacity), None),
Err(err) => {
let code = err.code();
(
None,
Some(RecentFailureInput {
occurred_at_ns: observed_at_ns,
subsystem: "intent_capacity".to_string(),
code: code.to_string(),
severity: FailureSeverity::Error,
summary: format!("diagnostic={code}"),
correlation_id: None,
}),
)
}
};
let recent_failures = RecentFailureOps::snapshot_with_many(
[timer_observation.failure, receipt_failure]
.into_iter()
.flatten(),
);
CanicRuntimeStatus {
schema_version: RUNTIME_INTROSPECTION_SCHEMA_VERSION,
observed_at_ns,
canister_id,
role,
root,
build_network: BuildNetworkOps::build_network(),
build: RuntimeBuildInfo {
package_name: package_name.to_string(),
package_version: package_version.to_string(),
canic_version: canic_version.to_string(),
canister_version,
},
features: runtime_features(),
topology: Some(RuntimeTopologyStatus {
root,
parent,
subnet,
source: "runtime_observed".to_string(),
}),
timers: timer_observation.timers,
timer_inventory: timer_observation.check,
state,
auth: Some(runtime_auth_status()),
blob_storage: runtime_blob_storage_status(),
receipt_capacity,
recent_failures,
visibility: runtime_visibility(),
readiness,
status,
}
}
#[must_use]
pub fn runtime_status(
observed_at_ns: u64,
package_name: &str,
package_version: &str,
canic_version: &str,
canister_version: u64,
) -> CanicRuntimeStatus {
Self::runtime_status_for(
IcOps::canister_self(),
observed_at_ns,
package_name,
package_version,
canic_version,
canister_version,
)
}
}
fn runtime_features() -> Vec<RuntimeFeatureStatus> {
RUNTIME_FEATURE_FLAGS
.into_iter()
.map(|(name, enabled)| runtime_feature_status(name, enabled))
.collect()
}
fn runtime_feature_status(name: &str, enabled: bool) -> RuntimeFeatureStatus {
RuntimeFeatureStatus {
name: name.to_string(),
enabled,
visibility: RuntimeFieldVisibility::OperatorOnly,
source: RUNTIME_FEATURE_SOURCE.to_string(),
}
}
fn runtime_auth_status() -> RuntimeAuthStatusSummary {
RuntimeAuthStatusSummary {
auth_features: RUNTIME_FEATURE_FLAGS
.into_iter()
.filter(|(name, _)| name.starts_with("auth-"))
.map(|(name, enabled)| runtime_feature_status(name, enabled))
.collect(),
}
}
fn runtime_receipt_capacity() -> Result<RuntimeReceiptCapacityStatus, InternalError> {
let capacity = ReceiptBackedIntentOps::receipt_capacity()?;
Ok(RuntimeReceiptCapacityStatus {
status: receipt_capacity_condition(
capacity.remaining_record_headroom,
capacity.remaining_resource_total_headroom,
),
receipt_records: capacity.total_records,
application_receipt_records: capacity.application_records,
canic_owned_receipt_records: capacity.canic_owned_records,
pending_application_receipt_records: capacity.pending_records,
terminal_application_receipt_records: capacity.terminal_records,
receipt_record_limit: capacity.record_limit,
remaining_receipt_record_headroom: capacity.remaining_record_headroom,
resource_total_records: capacity.resource_total_records,
resource_total_record_limit: capacity.resource_total_record_limit,
remaining_resource_total_headroom: capacity.remaining_resource_total_headroom,
warning_headroom_threshold: RECEIPT_CAPACITY_WARNING_HEADROOM_THRESHOLD,
reserved_terminal_slots: capacity.reserved_terminal_slots,
reserved_terminal_pages: capacity.reserved_terminal_pages,
next_terminal_eligibility_at_ns: capacity.next_eligibility_at_ns,
source: "intent_storage".to_string(),
})
}
const fn receipt_capacity_condition(
remaining_receipt_records: u64,
remaining_resource_totals: u64,
) -> RuntimeCheckStatus {
let minimum_headroom = if remaining_receipt_records < remaining_resource_totals {
remaining_receipt_records
} else {
remaining_resource_totals
};
if minimum_headroom == 0 {
RuntimeCheckStatus::Fail
} else if minimum_headroom <= RECEIPT_CAPACITY_WARNING_HEADROOM_THRESHOLD {
RuntimeCheckStatus::Warn
} else {
RuntimeCheckStatus::Pass
}
}
const fn aggregate_runtime_status(
readiness: ReadinessStatus,
receipt_capacity: Option<RuntimeCheckStatus>,
timer_inventory: RuntimeCheckStatus,
) -> RuntimeStatus {
if matches!(readiness, ReadinessStatus::NotReady)
|| matches!(
timer_inventory,
RuntimeCheckStatus::Fail | RuntimeCheckStatus::NotEvaluated
)
|| matches!(
receipt_capacity,
None | Some(RuntimeCheckStatus::Fail | RuntimeCheckStatus::NotEvaluated)
)
{
RuntimeStatus::Failing
} else if matches!(
readiness,
ReadinessStatus::Degraded | ReadinessStatus::NotEvaluated
) || matches!(receipt_capacity, Some(RuntimeCheckStatus::Warn))
|| matches!(timer_inventory, RuntimeCheckStatus::Warn)
{
RuntimeStatus::Degraded
} else {
RuntimeStatus::Ok
}
}
fn runtime_blob_storage_status() -> Option<RuntimeBlobStorageStatusSummary> {
let blob_storage_enabled = cfg!(feature = "blob-storage");
let billing_enabled = cfg!(feature = "blob-storage-billing");
(blob_storage_enabled || billing_enabled).then(|| RuntimeBlobStorageStatusSummary {
blob_storage_features: [
("blob-storage", blob_storage_enabled),
("blob-storage-billing", billing_enabled),
]
.into_iter()
.map(|(name, enabled)| runtime_feature_status(name, enabled))
.collect(),
})
}
struct TimerStatusObservation {
check: RuntimeCheck,
timers: Vec<CanisterTimerStatus>,
failure: Option<RecentFailureInput>,
}
fn timer_status_observation(
snapshots: Result<Vec<ic_timers::TimerSnapshot>, crate::api::timer::TimerError>,
observed_at_ns: u64,
) -> TimerStatusObservation {
match snapshots {
Ok(snapshots) => TimerStatusObservation {
check: RuntimeCheck {
category: "runtime".to_string(),
code: "timer_inventory_available".to_string(),
status: RuntimeCheckStatus::Pass,
subject: "shared_timer_registry".to_string(),
detail: "the complete canister-local timer inventory was observed".to_string(),
next: None,
source: "ic_timers".to_string(),
},
timers: timer_statuses_from(snapshots),
failure: None,
},
Err(error) => TimerStatusObservation {
check: RuntimeCheck {
category: "runtime".to_string(),
code: "timer_inventory_unavailable".to_string(),
status: RuntimeCheckStatus::Fail,
subject: "shared_timer_registry".to_string(),
detail: "the canister-local timer inventory could not be observed".to_string(),
next: Some(
"retry runtime observation; inspect recent failures if unavailable persists"
.to_string(),
),
source: "ic_timers".to_string(),
},
timers: Vec::new(),
failure: Some(RecentFailureInput {
occurred_at_ns: observed_at_ns,
subsystem: "timer_runtime".to_string(),
code: "timer_inventory_unavailable".to_string(),
severity: FailureSeverity::Error,
summary: error.to_string(),
correlation_id: None,
}),
},
}
}
fn timer_statuses_from(snapshots: Vec<ic_timers::TimerSnapshot>) -> Vec<CanisterTimerStatus> {
let mut timers = snapshots
.into_iter()
.map(|snapshot| {
let identity = snapshot.identity();
let observability = snapshot.observability();
let outcomes = observability.outcomes();
let counters = observability.counters();
let performance = observability.performance();
let condition = timer_process_condition(snapshot.process_condition());
CanisterTimerStatus {
name: identity.name().to_string(),
owner: identity.owner().to_string(),
subsystem: identity.subsystem().to_string(),
scheduling_mode: timer_scheduling_mode(snapshot.scheduling_mode()),
registration: timer_registration_status(snapshot.registration_status()),
condition,
enabled: condition != crate::domain::runtime::TimerProcessCondition::Disabled,
generation: snapshot.generation(),
next_due_at_ns: snapshot.next_deadline_ns(),
last_outcome: outcomes.last_outcome().map(timer_execution_outcome),
last_work_count: outcomes.last_work_count().unwrap_or_default(),
last_success_at_ns: outcomes.last_success_at_ns(),
last_failure_at_ns: outcomes.last_failure_at_ns(),
consecutive_expected_failures: outcomes.consecutive_expected_failures(),
schedules_since_runtime_start: counters.wakeups_armed(),
executions_since_runtime_start: counters.work_started(),
successes_since_runtime_start: counters
.succeeded()
.saturating_add(counters.no_work()),
expected_failures_since_runtime_start: counters.retryable_failure(),
invariant_failures_since_runtime_start: counters.invariant_failure(),
stale_callbacks_since_runtime_start: counters
.stale_wakeups()
.saturating_add(counters.stale_work()),
scheduler_performance: timer_callback_performance_status(
performance.scheduler_instructions(),
performance.scheduler_memory_pages(),
),
work_performance: timer_callback_performance_status(
performance.work_instructions(),
performance.work_memory_pages(),
),
}
})
.collect::<Vec<_>>();
timers.sort_by(|left, right| {
left.owner
.cmp(&right.owner)
.then_with(|| left.subsystem.cmp(&right.subsystem))
.then_with(|| left.name.cmp(&right.name))
});
timers
}
fn timer_callback_performance_status(
instructions: ic_timers::MeasurementSummary,
memory: ic_timers::MemoryPageSummary,
) -> TimerCallbackPerformanceStatus {
TimerCallbackPerformanceStatus {
instruction_samples_since_runtime_start: instructions.samples(),
instructions_latest: instructions.latest(),
instructions_maximum: instructions.maximum(),
instructions_total_since_runtime_start: instructions.total(),
memory_page_samples_since_runtime_start: memory.samples(),
memory_pages_latest: memory.latest().map(timer_memory_page_sample_status),
maximum_wasm_memory_growth_pages: memory.maximum_wasm_growth_pages(),
maximum_stable_memory_growth_pages: memory.maximum_stable_growth_pages(),
}
}
const fn timer_memory_page_sample_status(
sample: ic_timers::MemoryPageSample,
) -> TimerMemoryPageSampleStatus {
TimerMemoryPageSampleStatus {
start: timer_memory_page_extent_status(sample.start()),
end: timer_memory_page_extent_status(sample.end()),
}
}
const fn timer_memory_page_extent_status(
extent: ic_timers::MemoryPageExtent,
) -> TimerMemoryPageExtentStatus {
TimerMemoryPageExtentStatus {
wasm_pages: extent.wasm_pages(),
stable_pages: extent.stable_pages(),
}
}
const fn timer_scheduling_mode(
value: ic_timers::TimerSchedulingMode,
) -> crate::domain::runtime::TimerSchedulingMode {
use crate::domain::runtime::TimerSchedulingMode as Canic;
use ic_timers::TimerSchedulingMode as Shared;
match value {
Shared::Once => Canic::Once,
Shared::AfterCompletion => Canic::AfterCompletion,
Shared::Deadline => Canic::Deadline,
Shared::Retry => Canic::Retry,
Shared::Continuation => Canic::Continuation,
Shared::Watchdog => Canic::Watchdog,
}
}
const fn timer_registration_status(
value: ic_timers::TimerRegistrationStatus,
) -> crate::domain::runtime::TimerRegistrationStatus {
use crate::domain::runtime::TimerRegistrationStatus as Canic;
match value {
ic_timers::TimerRegistrationStatus::Unregistered => Canic::Unregistered,
ic_timers::TimerRegistrationStatus::Scheduled => Canic::Scheduled,
ic_timers::TimerRegistrationStatus::Running => Canic::Running,
}
}
const fn timer_process_condition(
value: ic_timers::TimerProcessCondition,
) -> crate::domain::runtime::TimerProcessCondition {
use crate::domain::runtime::TimerProcessCondition as Canic;
match value {
ic_timers::TimerProcessCondition::Disabled => Canic::Disabled,
ic_timers::TimerProcessCondition::Idle => Canic::Idle,
ic_timers::TimerProcessCondition::Active => Canic::Active,
ic_timers::TimerProcessCondition::Retrying => Canic::Retrying,
ic_timers::TimerProcessCondition::Failed => Canic::Failed,
}
}
const fn timer_execution_outcome(
value: ic_timers::TimerLastOutcome,
) -> crate::domain::runtime::TimerExecutionOutcome {
use crate::domain::runtime::TimerExecutionOutcome as Canic;
match value {
ic_timers::TimerLastOutcome::Completed(completion) => match completion {
ic_timers::TimerCompletionOutcome::Success => Canic::Success,
ic_timers::TimerCompletionOutcome::NoWork => Canic::NoWork,
ic_timers::TimerCompletionOutcome::RetryableFailure => Canic::RetryableFailure,
ic_timers::TimerCompletionOutcome::InvariantFailure => Canic::InvariantFailure,
},
ic_timers::TimerLastOutcome::Unacknowledged => Canic::Unacknowledged,
}
}
fn state_summary(role: Option<&str>) -> Option<RuntimeStateSummary> {
let memory_ids = MemoryRegistryOps::ledger_snapshot()
.ok()?
.memories
.into_iter()
.map(|memory| memory.memory_manager_id)
.collect::<std::collections::BTreeSet<_>>();
state_summary_for_memory_ids(role, &memory_ids)
}
fn state_summary_for_memory_ids(
role: Option<&str>,
memory_ids: &std::collections::BTreeSet<u8>,
) -> Option<RuntimeStateSummary> {
role?;
let mut domains = canic_state_descriptors()
.into_iter()
.flat_map(|descriptor| descriptor.state)
.filter(|domain| domain.memory_id.is_some_and(|id| memory_ids.contains(&id)))
.map(|domain| RuntimeStateDomainSummary {
domain: domain.domain,
version: domain.version,
storage: domain.storage.as_str().to_string(),
memory_id: domain.memory_id,
status: RuntimeStateDomainStatus::Ok,
})
.collect::<Vec<_>>();
domains.sort_by(|left, right| left.domain.cmp(&right.domain));
if domains.is_empty() {
return None;
}
Some(RuntimeStateSummary {
manifest_schema_version: u32::from(STATE_MANIFEST_SCHEMA_VERSION),
domains,
total_stable_memory_pages: None,
})
}
fn runtime_visibility() -> Vec<RuntimeVisibilityEntry> {
[
("schema_version", RuntimeFieldVisibility::PublicSafe),
("observed_at_ns", RuntimeFieldVisibility::PublicSafe),
("canister_id", RuntimeFieldVisibility::OperatorOnly),
("role", RuntimeFieldVisibility::OperatorOnly),
("root", RuntimeFieldVisibility::OperatorOnly),
("build_network", RuntimeFieldVisibility::OperatorOnly),
("build", RuntimeFieldVisibility::OperatorOnly),
("features", RuntimeFieldVisibility::OperatorOnly),
("topology", RuntimeFieldVisibility::ControllerOnly),
("timers", RuntimeFieldVisibility::OperatorOnly),
("timer_inventory", RuntimeFieldVisibility::OperatorOnly),
("state", RuntimeFieldVisibility::OperatorOnly),
("auth", RuntimeFieldVisibility::OperatorOnly),
("blob_storage", RuntimeFieldVisibility::FeatureGated),
("receipt_capacity", RuntimeFieldVisibility::OperatorOnly),
("recent_failures", RuntimeFieldVisibility::OperatorOnly),
("readiness", RuntimeFieldVisibility::OperatorOnly),
("status", RuntimeFieldVisibility::OperatorOnly),
("visibility", RuntimeFieldVisibility::OperatorOnly),
]
.into_iter()
.map(|(field, visibility)| RuntimeVisibilityEntry {
field: field.to_string(),
visibility,
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ids::IntentResourceKey;
use crate::ops::runtime::bootstrap::{BootstrapPhaseLabel, BootstrapStatusOps};
use crate::ops::runtime::recent_failure::RecentFailureOps;
use crate::ops::storage::intent::{
INTENT_RESOURCE_TOTAL_RECORD_LIMIT, IntentStoreOps, RECEIPT_BACKED_INTENT_RECORD_LIMIT,
};
use crate::storage::stable::intent::{IntentResourceTotalsRecord, IntentStore};
#[test]
fn health_is_minimal_and_schema_versioned() {
let health = RuntimeIntrospectionApi::health(Some(42));
assert_eq!(health.schema_version, RUNTIME_INTROSPECTION_SCHEMA_VERSION);
assert_eq!(health.status, HealthStatus::Healthy);
assert_eq!(health.observed_at_ns, Some(42));
assert_eq!(health.checks.len(), 1);
assert_eq!(health.checks[0].code, "canister_responsive");
}
#[test]
fn unavailable_timer_inventory_is_explicit_and_fails_runtime_status() {
let observation =
timer_status_observation(Err(crate::api::timer::TimerError::CustodyBusy), 101);
assert_eq!(observation.check.status, RuntimeCheckStatus::Fail);
assert!(observation.timers.is_empty());
assert_eq!(
observation
.failure
.as_ref()
.map(|failure| failure.code.as_str()),
Some("timer_inventory_unavailable")
);
assert_eq!(
aggregate_runtime_status(
ReadinessStatus::Ready,
Some(RuntimeCheckStatus::Pass),
observation.check.status,
),
RuntimeStatus::Failing
);
}
#[test]
fn runtime_status_embeds_guarded_readiness_and_build_info() {
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.81.0",
7,
);
assert_eq!(status.schema_version, RUNTIME_INTROSPECTION_SCHEMA_VERSION);
assert_eq!(status.observed_at_ns, 100);
assert_eq!(status.canister_id, Principal::anonymous());
assert_eq!(status.build_network, BuildNetworkOps::build_network());
assert_eq!(status.build.package_name, "test-canister");
assert_eq!(status.build.package_version, "1.2.3");
assert_eq!(status.build.canic_version, "0.81.0");
assert_eq!(status.build.canister_version, 7);
assert_eq!(status.readiness.observed_at_ns, 100);
assert!(
status
.visibility
.iter()
.any(|entry| entry.field == "topology"
&& entry.visibility == RuntimeFieldVisibility::ControllerOnly)
);
}
#[test]
fn runtime_status_classifies_each_top_level_field_visibility() {
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.81.0",
7,
);
let expected = [
("schema_version", RuntimeFieldVisibility::PublicSafe),
("observed_at_ns", RuntimeFieldVisibility::PublicSafe),
("canister_id", RuntimeFieldVisibility::OperatorOnly),
("role", RuntimeFieldVisibility::OperatorOnly),
("root", RuntimeFieldVisibility::OperatorOnly),
("build_network", RuntimeFieldVisibility::OperatorOnly),
("build", RuntimeFieldVisibility::OperatorOnly),
("features", RuntimeFieldVisibility::OperatorOnly),
("topology", RuntimeFieldVisibility::ControllerOnly),
("timers", RuntimeFieldVisibility::OperatorOnly),
("timer_inventory", RuntimeFieldVisibility::OperatorOnly),
("state", RuntimeFieldVisibility::OperatorOnly),
("auth", RuntimeFieldVisibility::OperatorOnly),
("blob_storage", RuntimeFieldVisibility::FeatureGated),
("receipt_capacity", RuntimeFieldVisibility::OperatorOnly),
("recent_failures", RuntimeFieldVisibility::OperatorOnly),
("readiness", RuntimeFieldVisibility::OperatorOnly),
("status", RuntimeFieldVisibility::OperatorOnly),
("visibility", RuntimeFieldVisibility::OperatorOnly),
];
assert_eq!(status.visibility.len(), expected.len());
for (index, (field, visibility)) in expected.into_iter().enumerate() {
assert_eq!(status.visibility[index].field, field);
assert_eq!(status.visibility[index].visibility, visibility);
}
}
#[test]
fn runtime_status_projects_empty_receipt_capacity() {
IntentStoreOps::reset_for_tests();
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.96.6",
7,
);
let capacity = status.receipt_capacity.expect("receipt capacity");
assert_eq!(capacity.status, RuntimeCheckStatus::Pass);
assert_eq!(capacity.receipt_records, 0);
assert_eq!(
capacity.receipt_record_limit,
RECEIPT_BACKED_INTENT_RECORD_LIMIT
);
assert_eq!(capacity.resource_total_records, 0);
assert_eq!(
capacity.resource_total_record_limit,
INTENT_RESOURCE_TOTAL_RECORD_LIMIT
);
assert_eq!(
capacity.warning_headroom_threshold,
RECEIPT_CAPACITY_WARNING_HEADROOM_THRESHOLD
);
assert_eq!(capacity.source, "intent_storage");
}
#[test]
fn receipt_capacity_condition_has_exact_warning_and_failure_boundaries() {
assert_eq!(
receipt_capacity_condition(RECEIPT_CAPACITY_WARNING_HEADROOM_THRESHOLD + 1, u64::MAX,),
RuntimeCheckStatus::Pass
);
assert_eq!(
receipt_capacity_condition(RECEIPT_CAPACITY_WARNING_HEADROOM_THRESHOLD, u64::MAX),
RuntimeCheckStatus::Warn
);
assert_eq!(
receipt_capacity_condition(u64::MAX, 1),
RuntimeCheckStatus::Warn
);
assert_eq!(
receipt_capacity_condition(u64::MAX, 0),
RuntimeCheckStatus::Fail
);
assert_eq!(
aggregate_runtime_status(
ReadinessStatus::Ready,
Some(RuntimeCheckStatus::Warn),
RuntimeCheckStatus::Pass,
),
RuntimeStatus::Degraded
);
assert_eq!(
aggregate_runtime_status(ReadinessStatus::Ready, None, RuntimeCheckStatus::Pass,),
RuntimeStatus::Failing
);
assert_eq!(
aggregate_runtime_status(
ReadinessStatus::Ready,
Some(RuntimeCheckStatus::Pass),
RuntimeCheckStatus::Fail,
),
RuntimeStatus::Failing
);
}
#[test]
fn runtime_status_fails_closed_with_typed_capacity_diagnostic() {
IntentStoreOps::reset_for_tests();
RecentFailureOps::reset();
for value in 0..=INTENT_RESOURCE_TOTAL_RECORD_LIMIT {
IntentStore::set_totals(
IntentResourceKey::new(format!("runtime-capacity:{value}")),
IntentResourceTotalsRecord {
reserved_qty: 0,
committed_qty: 1,
pending_count: 0,
},
);
}
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.96.6",
7,
);
assert_eq!(status.status, RuntimeStatus::Failing);
assert!(status.receipt_capacity.is_none());
let expected_code = crate::diagnostics::codes::CAPACITY_LIMIT.to_string();
let failure = status
.recent_failures
.iter()
.find(|failure| failure.code == expected_code)
.expect("current capacity failure diagnostic");
assert_eq!(failure.subsystem, "intent_capacity");
assert_eq!(failure.code, expected_code);
assert_eq!(failure.severity, FailureSeverity::Error);
assert_eq!(
failure.summary,
format!("diagnostic={}", crate::diagnostics::codes::CAPACITY_LIMIT)
);
assert!(RecentFailureOps::snapshot().is_empty());
IntentStoreOps::reset_for_tests();
RecentFailureOps::reset();
}
#[test]
fn runtime_status_reports_compile_features_deterministically() {
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.81.0",
7,
);
assert_eq!(status.features.len(), RUNTIME_FEATURE_FLAGS.len());
for (index, (name, enabled)) in RUNTIME_FEATURE_FLAGS.into_iter().enumerate() {
assert_eq!(status.features[index].name, name);
assert_eq!(status.features[index].enabled, enabled);
assert_eq!(
status.features[index].visibility,
RuntimeFieldVisibility::OperatorOnly
);
assert_eq!(status.features[index].source, RUNTIME_FEATURE_SOURCE);
}
}
#[test]
fn runtime_status_reports_auth_and_blob_storage_feature_summaries() {
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.81.0",
7,
);
let auth = status.auth.expect("auth feature summary");
assert!(
auth.auth_features
.windows(2)
.all(|features| features[0].name <= features[1].name)
);
assert_runtime_feature(
&auth.auth_features,
"auth-chain-key-ecdsa",
cfg!(feature = "auth-chain-key-ecdsa"),
);
assert_runtime_feature(
&auth.auth_features,
"auth-delegated-token-verify",
cfg!(feature = "auth-delegated-token-verify"),
);
assert_runtime_feature(
&auth.auth_features,
"auth-issuer-canister-sig-create",
cfg!(feature = "auth-issuer-canister-sig-create"),
);
if cfg!(any(
feature = "blob-storage",
feature = "blob-storage-billing"
)) {
let blob_storage = status.blob_storage.expect("blob-storage feature summary");
assert_runtime_feature(
&blob_storage.blob_storage_features,
"blob-storage",
cfg!(feature = "blob-storage"),
);
assert_runtime_feature(
&blob_storage.blob_storage_features,
"blob-storage-billing",
cfg!(feature = "blob-storage-billing"),
);
} else {
assert!(status.blob_storage.is_none());
}
}
fn assert_runtime_feature(
features: &[RuntimeFeatureStatus],
name: &str,
expected_enabled: bool,
) {
let feature = features
.iter()
.find(|feature| feature.name == name)
.unwrap_or_else(|| panic!("expected runtime feature {name}"));
assert_eq!(feature.enabled, expected_enabled);
assert_eq!(feature.visibility, RuntimeFieldVisibility::OperatorOnly);
assert_eq!(feature.source, RUNTIME_FEATURE_SOURCE);
}
#[test]
fn state_summary_joins_runtime_memory_ids_to_owner_metadata() {
let summary = state_summary_for_memory_ids(
Some("root"),
&std::collections::BTreeSet::from([
crate::role_contract::allocation::memory::runtime::RUNTIME_BINDINGS_ID,
]),
)
.expect("runtime state declarations");
assert_eq!(
summary.manifest_schema_version,
u32::from(crate::state_contract::STATE_MANIFEST_SCHEMA_VERSION)
);
assert!(summary.total_stable_memory_pages.is_none());
assert!(summary.domains.iter().any(|domain| {
domain.domain == "runtime_bindings"
&& domain.storage == "stable_memory"
&& domain.status == RuntimeStateDomainStatus::Ok
}));
assert!(state_summary_for_memory_ids(None, &std::collections::BTreeSet::new()).is_none());
}
#[test]
fn runtime_status_includes_recent_failure_snapshot() {
RecentFailureOps::reset();
RecentFailureOps::record(RecentFailureInput {
occurred_at_ns: 77,
subsystem: "runtime".to_string(),
code: "readiness_failed".to_string(),
severity: FailureSeverity::Error,
summary: "bounded failure summary".to_string(),
correlation_id: Some("runtime-check".to_string()),
});
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.81.0",
7,
);
let failure = status
.recent_failures
.iter()
.find(|failure| failure.code == "readiness_failed")
.expect("retained recent failure");
assert_eq!(failure.occurred_at_ns, 77);
assert_eq!(failure.subsystem, "runtime");
RecentFailureOps::reset();
}
#[test]
fn runtime_status_includes_bootstrap_failure_metadata() {
RecentFailureOps::reset();
BootstrapStatusOps::set_phase(BootstrapPhaseLabel::ROOT_INIT);
BootstrapStatusOps::mark_failed("raw bootstrap failure detail");
let status = RuntimeIntrospectionApi::runtime_status_for(
Principal::anonymous(),
100,
"test-canister",
"1.2.3",
"0.81.0",
7,
);
let failure = status
.recent_failures
.iter()
.find(|failure| failure.code == "bootstrap_failed")
.expect("bootstrap failure metadata");
assert_eq!(failure.subsystem, "runtime_bootstrap");
assert_eq!(failure.severity, FailureSeverity::Error);
assert_eq!(failure.correlation_id.as_deref(), Some("root:init"));
assert!(
!failure.summary.contains("raw bootstrap failure detail"),
"runtime status recent failures should not mirror raw bootstrap errors"
);
RecentFailureOps::reset();
}
}