canic_host/observability/
mod.rs

1//! Module: observability
2//!
3//! Responsibility: route exact Fleet telemetry through controller-authenticated Canic surfaces.
4//! Does not own: Fleet discovery, metric interpretation, or report rendering.
5//! Boundary: Root observations query the Root directly; descendants use its protected relay.
6
7use crate::{
8    CanisterProtocolError, call_canister_with_arg,
9    fleet_ensure::CurrentFleetResolution,
10    icp::IcpCli,
11    protocol_binding::{ProtocolBindingError, resolve_registry_protocol_binding},
12    query_canister_with_arg,
13    registry::RegistryEntry,
14};
15use candid::{CandidType, Deserialize, Principal, types::principal::PrincipalError};
16use canic_core::{
17    dto::{
18        canister::{CanisterInspectionRequest, CanisterStatusResponse},
19        observability::{
20            CanisterObservabilityRequest, CanisterObservabilityResponse,
21            FleetCanisterObservabilityRequest,
22        },
23        page::Page,
24    },
25    ids::CanisterRole,
26    protocol,
27};
28use std::{collections::BTreeSet, path::Path};
29use thiserror::Error as ThisError;
30
31#[derive(CandidType)]
32enum RootCommandFragment {
33    InspectCanister(CanisterInspectionRequest),
34    ObserveCanister(FleetCanisterObservabilityRequest),
35}
36
37#[derive(CandidType, Deserialize)]
38enum RootCommandResponseFragment {
39    InspectCanister(Box<CanisterStatusResponse>),
40    ObserveCanister(CanisterObservabilityResponse),
41}
42
43#[derive(CandidType)]
44enum RootStatusRequestFragment {
45    CycleBalance,
46    CycleHistory(canic_core::dto::page::PageRequest),
47    Metrics(canic_core::dto::role::MetricsStatusRequest),
48}
49
50#[derive(CandidType, Deserialize)]
51enum RootStatusResponseFragment {
52    CycleBalance(canic_core::dto::role::CycleBalanceStatusResponse),
53    CycleHistory(Page<canic_core::dto::cycles::CycleTrackerEntry>),
54    Metrics(Page<canic_core::dto::metrics::MetricEntry>),
55}
56
57#[derive(CandidType)]
58enum StoreStatusRequestFragment {
59    CycleHistory(canic_core::dto::page::PageRequest),
60}
61
62#[derive(CandidType, Deserialize)]
63enum StoreStatusResponseFragment {
64    CycleHistory(Page<canic_core::dto::cycles::CycleTrackerEntry>),
65}
66
67/// Failure to route one protected observation through exact current Fleet authority.
68#[derive(Debug, ThisError)]
69pub enum FleetObservabilityError {
70    #[error("Canister {canister} has no owning Fleet Subnet Root in current topology")]
71    MissingRoot { canister: String },
72
73    #[error("current Fleet observability topology contains a parent cycle at {canister}")]
74    ParentCycle { canister: String },
75
76    #[error("invalid {field} Principal {value}: {source}")]
77    Principal {
78        field: &'static str,
79        value: String,
80        #[source]
81        source: PrincipalError,
82    },
83
84    #[error(transparent)]
85    Protocol(#[from] CanisterProtocolError),
86
87    #[error(transparent)]
88    ProtocolBinding(#[from] ProtocolBindingError),
89
90    #[error("Fleet Subnet Root does not expose cycle top-up history")]
91    RootCycleTopupsUnsupported,
92
93    #[error("Wasm Store does not expose cycle top-up history")]
94    StoreCycleTopupsUnsupported,
95
96    #[error("Wasm Store does not expose runtime metrics")]
97    StoreMetricsUnsupported,
98
99    #[error("Fleet Subnet Root returned a different observability response variant")]
100    UnexpectedRootResponse,
101
102    #[error("Canister {canister} cycle balance exceeds u128")]
103    CycleBalanceOverflow { canister: String },
104}
105
106/// Observe one current Fleet canister without granting its human operator lifecycle control.
107pub fn observe_fleet_canister(
108    icp: &IcpCli,
109    icp_root: &Path,
110    environment: &str,
111    fleet: &CurrentFleetResolution,
112    entry: &RegistryEntry,
113    request: CanisterObservabilityRequest,
114) -> Result<CanisterObservabilityResponse, FleetObservabilityError> {
115    if entry.role.as_deref() == Some(CanisterRole::ROOT.as_str()) {
116        return observe_root(icp, icp_root, environment, entry, request);
117    }
118
119    let root = fleet_subnet_root_entry(&fleet.registry.entries, entry)?;
120    let binding = resolve_registry_protocol_binding(icp_root, environment, root)?;
121    let root_canister = parse_principal("Fleet Subnet Root", &root.pid)?;
122    let target = parse_principal("observability target", &entry.pid)?;
123    if matches!(&request, CanisterObservabilityRequest::CycleBalance) {
124        let response: RootCommandResponseFragment = call_canister_with_arg(
125            icp,
126            &binding,
127            root_canister,
128            protocol::CANIC_ROOT_COMMAND,
129            &RootCommandFragment::InspectCanister(CanisterInspectionRequest {
130                canister_id: target,
131            }),
132        )?;
133        let RootCommandResponseFragment::InspectCanister(response) = response else {
134            return Err(FleetObservabilityError::UnexpectedRootResponse);
135        };
136        let cycles = u128::try_from(response.cycles.0).map_err(|_| {
137            FleetObservabilityError::CycleBalanceOverflow {
138                canister: entry.pid.clone(),
139            }
140        })?;
141        return Ok(CanisterObservabilityResponse::CycleBalance(
142            canic_core::dto::role::CycleBalanceStatusResponse { cycles },
143        ));
144    }
145    if entry.role.as_deref() == Some(CanisterRole::WASM_STORE.as_str()) {
146        return observe_store(icp, icp_root, environment, entry, request);
147    }
148    let response: RootCommandResponseFragment = call_canister_with_arg(
149        icp,
150        &binding,
151        root_canister,
152        protocol::CANIC_ROOT_COMMAND,
153        &RootCommandFragment::ObserveCanister(FleetCanisterObservabilityRequest {
154            canister_id: target,
155            request,
156        }),
157    )?;
158    let RootCommandResponseFragment::ObserveCanister(response) = response else {
159        return Err(FleetObservabilityError::UnexpectedRootResponse);
160    };
161    Ok(response)
162}
163
164fn observe_store(
165    icp: &IcpCli,
166    icp_root: &Path,
167    environment: &str,
168    store: &RegistryEntry,
169    request: CanisterObservabilityRequest,
170) -> Result<CanisterObservabilityResponse, FleetObservabilityError> {
171    let request = match request {
172        CanisterObservabilityRequest::CycleHistory(page) => {
173            StoreStatusRequestFragment::CycleHistory(page)
174        }
175        CanisterObservabilityRequest::CycleTopups(_) => {
176            return Err(FleetObservabilityError::StoreCycleTopupsUnsupported);
177        }
178        CanisterObservabilityRequest::Metrics(_) => {
179            return Err(FleetObservabilityError::StoreMetricsUnsupported);
180        }
181        CanisterObservabilityRequest::CycleBalance => {
182            unreachable!("Store CycleBalance uses Root management inspection");
183        }
184    };
185    let binding = resolve_registry_protocol_binding(icp_root, environment, store)?;
186    let store_canister = parse_principal("Wasm Store", &store.pid)?;
187    let response: StoreStatusResponseFragment = query_canister_with_arg(
188        icp,
189        &binding,
190        store_canister,
191        protocol::CANIC_OBSERVABILITY,
192        &request,
193    )?;
194    let StoreStatusResponseFragment::CycleHistory(page) = response;
195    Ok(CanisterObservabilityResponse::CycleHistory(page))
196}
197
198fn observe_root(
199    icp: &IcpCli,
200    icp_root: &Path,
201    environment: &str,
202    root: &RegistryEntry,
203    request: CanisterObservabilityRequest,
204) -> Result<CanisterObservabilityResponse, FleetObservabilityError> {
205    let request = match request {
206        CanisterObservabilityRequest::CycleBalance => RootStatusRequestFragment::CycleBalance,
207        CanisterObservabilityRequest::CycleHistory(page) => {
208            RootStatusRequestFragment::CycleHistory(page)
209        }
210        CanisterObservabilityRequest::CycleTopups(_) => {
211            return Err(FleetObservabilityError::RootCycleTopupsUnsupported);
212        }
213        CanisterObservabilityRequest::Metrics(request) => {
214            RootStatusRequestFragment::Metrics(request)
215        }
216    };
217    let binding = resolve_registry_protocol_binding(icp_root, environment, root)?;
218    let root_canister = parse_principal("Fleet Subnet Root", &root.pid)?;
219    let response: RootStatusResponseFragment = query_canister_with_arg(
220        icp,
221        &binding,
222        root_canister,
223        protocol::CANIC_OBSERVABILITY,
224        &request,
225    )?;
226    Ok(match response {
227        RootStatusResponseFragment::CycleBalance(response) => {
228            CanisterObservabilityResponse::CycleBalance(response)
229        }
230        RootStatusResponseFragment::CycleHistory(response) => {
231            CanisterObservabilityResponse::CycleHistory(response)
232        }
233        RootStatusResponseFragment::Metrics(response) => {
234            CanisterObservabilityResponse::Metrics(response)
235        }
236    })
237}
238
239fn fleet_subnet_root_entry<'a>(
240    registry: &'a [RegistryEntry],
241    entry: &RegistryEntry,
242) -> Result<&'a RegistryEntry, FleetObservabilityError> {
243    let mut current = entry;
244    let mut visited = BTreeSet::new();
245    loop {
246        if !visited.insert(current.pid.as_str()) {
247            return Err(FleetObservabilityError::ParentCycle {
248                canister: current.pid.clone(),
249            });
250        }
251        if current.role.as_deref() == Some(CanisterRole::ROOT.as_str()) {
252            return registry
253                .iter()
254                .find(|candidate| candidate.pid == current.pid)
255                .ok_or_else(|| FleetObservabilityError::MissingRoot {
256                    canister: entry.pid.clone(),
257                });
258        }
259        let Some(parent) = current.parent_pid.as_deref() else {
260            return Err(FleetObservabilityError::MissingRoot {
261                canister: entry.pid.clone(),
262            });
263        };
264        current = registry
265            .iter()
266            .find(|candidate| candidate.pid == parent)
267            .ok_or_else(|| FleetObservabilityError::MissingRoot {
268                canister: entry.pid.clone(),
269            })?;
270    }
271}
272
273fn parse_principal(field: &'static str, value: &str) -> Result<Principal, FleetObservabilityError> {
274    Principal::from_text(value).map_err(|source| FleetObservabilityError::Principal {
275        field,
276        value: value.to_string(),
277        source,
278    })
279}
280
281#[cfg(test)]
282mod tests {
283    use super::*;
284
285    fn entry(pid: &str, role: &str, parent: Option<&str>) -> RegistryEntry {
286        RegistryEntry {
287            pid: pid.to_string(),
288            role: Some(role.to_string()),
289            parent_pid: parent.map(str::to_string),
290            module_hash: None,
291            protocol_binding: None,
292        }
293    }
294
295    #[test]
296    fn descendant_observability_resolves_the_exact_owning_root() {
297        let coordinator = entry("coordinator", "fleet_coordinator", None);
298        let root = entry("root", "root", Some("coordinator"));
299        let parent = entry("parent", "hub", Some("root"));
300        let child = entry("child", "leaf", Some("parent"));
301        let registry = vec![coordinator, root.clone(), parent, child.clone()];
302
303        assert_eq!(
304            fleet_subnet_root_entry(&registry, &child)
305                .expect("owning Root")
306                .pid,
307            root.pid
308        );
309    }
310}
canic_host/observability/mod.rs

canic_host/observability/
mod.rs
