canic_host/observability/
mod.rs

1//! Module: observability
2//!
3//! Responsibility: route exact Fleet telemetry through controller-authenticated Canic surfaces.
4//! Does not own: Fleet discovery, metric interpretation, or report rendering.
5//! Boundary: Root observations query the Root directly; descendants use its protected relay.
6
7use crate::{
8    CanisterProtocolError, call_canister_with_arg,
9    fleet_ensure::CurrentFleetResolution,
10    icp::IcpCli,
11    protocol_binding::{ProtocolBindingError, resolve_registry_protocol_binding},
12    query_canister_with_arg,
13    registry::RegistryEntry,
14};
15use candid::{Principal, types::principal::PrincipalError};
16use canic_contracts::dto::canister::CanisterInspectionRequest;
17use canic_contracts::dto::observability::CanisterObservabilityRequest;
18use canic_contracts::dto::observability::CanisterObservabilityResponse;
19use canic_contracts::dto::observability::FleetCanisterObservabilityRequest;
20use canic_contracts::dto::wire::projection::funding_observation::Command as RootCommandFragment;
21use canic_contracts::dto::wire::projection::funding_observation::Response as RootCommandResponseFragment;
22use canic_contracts::dto::wire::projection::protected_observability::RootStatusRequestFragment;
23use canic_contracts::dto::wire::projection::protected_observability::RootStatusResponseFragment;
24use canic_contracts::dto::wire::projection::protected_observability::StoreStatusRequestFragment;
25use canic_contracts::dto::wire::projection::protected_observability::StoreStatusResponseFragment;
26use canic_contracts::ids::CanisterRole;
27use canic_contracts::protocol;
28use std::{collections::BTreeSet, path::Path};
29use thiserror::Error as ThisError;
30
31/// Failure to route one protected observation through exact current Fleet authority.
32
33#[derive(Debug, ThisError)]
34pub enum FleetObservabilityError {
35    #[error("Canister {canister} has no owning Fleet Subnet Root in current topology")]
36    MissingRoot { canister: String },
37
38    #[error("current Fleet observability topology contains a parent cycle at {canister}")]
39    ParentCycle { canister: String },
40
41    #[error("invalid {field} Principal {value}: {source}")]
42    Principal {
43        field: &'static str,
44        value: String,
45        #[source]
46        source: PrincipalError,
47    },
48
49    #[error(transparent)]
50    Protocol(#[from] CanisterProtocolError),
51
52    #[error(transparent)]
53    ProtocolBinding(#[from] ProtocolBindingError),
54
55    #[error("Fleet Subnet Root does not expose cycle top-up history")]
56    RootCycleTopupsUnsupported,
57
58    #[error("Wasm Store does not maintain child funding grants")]
59    StoreChildFundingUnsupported,
60
61    #[error("Wasm Store does not expose cycle top-up history")]
62    StoreCycleTopupsUnsupported,
63
64    #[error("Wasm Store does not expose current memory allocations")]
65    StoreMemoryAllocationsUnsupported,
66
67    #[error("Wasm Store does not expose runtime metrics")]
68    StoreMetricsUnsupported,
69
70    #[error("Fleet Subnet Root returned a different observability response variant")]
71    UnexpectedRootResponse,
72
73    #[error("Canister {canister} cycle balance exceeds u128")]
74    CycleBalanceOverflow { canister: String },
75}
76
77/// Observe one current Fleet canister without granting its human operator lifecycle control.
78pub fn observe_fleet_canister(
79    icp: &IcpCli,
80    icp_root: &Path,
81    environment: &str,
82    fleet: &CurrentFleetResolution,
83    entry: &RegistryEntry,
84    request: CanisterObservabilityRequest,
85) -> Result<CanisterObservabilityResponse, FleetObservabilityError> {
86    if entry.role.as_deref() == Some(CanisterRole::ROOT.as_str()) {
87        return observe_root(icp, icp_root, environment, entry, request);
88    }
89
90    let root = fleet_subnet_root_entry(&fleet.registry.entries, entry)?;
91    let binding = resolve_registry_protocol_binding(icp_root, environment, root)?;
92    let root_canister = parse_principal("Fleet Subnet Root", &root.pid)?;
93    let target = parse_principal("observability target", &entry.pid)?;
94    if matches!(&request, CanisterObservabilityRequest::CycleBalance) {
95        return icp.measure_canister_inspection(root_canister, target, || {
96            crate::canister_protocol::inspection::preflight_inspection(
97                icp,
98                &binding.candid_path,
99                root_canister,
100                target,
101            )?;
102            let response: RootCommandResponseFragment = call_canister_with_arg(
103                icp,
104                &binding,
105                root_canister,
106                protocol::CANIC_ROOT_COMMAND,
107                &RootCommandFragment::InspectCanister(CanisterInspectionRequest {
108                    canister_id: target,
109                }),
110            )?;
111            let response = match response {
112                RootCommandResponseFragment::InspectCanister(response) => response,
113                RootCommandResponseFragment::InspectionReserveRequired(evidence) => {
114                    return Err(CanisterProtocolError::inspection_reserve(
115                        root_canister,
116                        target,
117                        evidence,
118                    )
119                    .into());
120                }
121                RootCommandResponseFragment::ObserveCanister(_) => {
122                    return Err(FleetObservabilityError::UnexpectedRootResponse);
123                }
124            };
125            let cycles = u128::try_from(response.cycles.0).map_err(|_| {
126                FleetObservabilityError::CycleBalanceOverflow {
127                    canister: entry.pid.clone(),
128                }
129            })?;
130            Ok(CanisterObservabilityResponse::CycleBalance(
131                canic_contracts::dto::role::CycleBalanceStatusResponse { cycles },
132            ))
133        });
134    }
135    if entry.role.as_deref() == Some(CanisterRole::WASM_STORE.as_str()) {
136        return observe_store(icp, icp_root, environment, entry, request);
137    }
138    let expected_child = child_funding_target(&request);
139    let response: RootCommandResponseFragment = call_canister_with_arg(
140        icp,
141        &binding,
142        root_canister,
143        protocol::CANIC_ROOT_COMMAND,
144        &RootCommandFragment::ObserveCanister(FleetCanisterObservabilityRequest {
145            canister_id: target,
146            request,
147        }),
148    )?;
149    let RootCommandResponseFragment::ObserveCanister(response) = response else {
150        return Err(FleetObservabilityError::UnexpectedRootResponse);
151    };
152    verify_child_funding_response(response, target, expected_child)
153}
154
155fn observe_store(
156    icp: &IcpCli,
157    icp_root: &Path,
158    environment: &str,
159    store: &RegistryEntry,
160    request: CanisterObservabilityRequest,
161) -> Result<CanisterObservabilityResponse, FleetObservabilityError> {
162    let request = match request {
163        CanisterObservabilityRequest::ChildFunding(_) => {
164            return Err(FleetObservabilityError::StoreChildFundingUnsupported);
165        }
166        CanisterObservabilityRequest::CycleHistory(page) => {
167            StoreStatusRequestFragment::CycleHistory(page)
168        }
169        CanisterObservabilityRequest::CycleTopups(_) => {
170            return Err(FleetObservabilityError::StoreCycleTopupsUnsupported);
171        }
172        CanisterObservabilityRequest::Metrics(_) => {
173            return Err(FleetObservabilityError::StoreMetricsUnsupported);
174        }
175        CanisterObservabilityRequest::MemoryAllocations => {
176            return Err(FleetObservabilityError::StoreMemoryAllocationsUnsupported);
177        }
178        CanisterObservabilityRequest::CycleBalance => {
179            unreachable!("Store CycleBalance uses Root management inspection");
180        }
181    };
182    let binding = resolve_registry_protocol_binding(icp_root, environment, store)?;
183    let store_canister = parse_principal("Wasm Store", &store.pid)?;
184    let response: StoreStatusResponseFragment = query_canister_with_arg(
185        icp,
186        &binding,
187        store_canister,
188        protocol::CANIC_OBSERVABILITY,
189        &request,
190    )?;
191    let StoreStatusResponseFragment::CycleHistory(page) = response;
192    Ok(CanisterObservabilityResponse::CycleHistory(page))
193}
194
195fn observe_root(
196    icp: &IcpCli,
197    icp_root: &Path,
198    environment: &str,
199    root: &RegistryEntry,
200    request: CanisterObservabilityRequest,
201) -> Result<CanisterObservabilityResponse, FleetObservabilityError> {
202    let expected_child = child_funding_target(&request);
203    let request = match request {
204        CanisterObservabilityRequest::ChildFunding(child) => {
205            RootStatusRequestFragment::ChildFunding(child)
206        }
207        CanisterObservabilityRequest::CycleBalance => RootStatusRequestFragment::CycleBalance,
208        CanisterObservabilityRequest::CycleHistory(page) => {
209            RootStatusRequestFragment::CycleHistory(page)
210        }
211        CanisterObservabilityRequest::CycleTopups(_) => {
212            return Err(FleetObservabilityError::RootCycleTopupsUnsupported);
213        }
214        CanisterObservabilityRequest::Metrics(request) => {
215            RootStatusRequestFragment::Metrics(request)
216        }
217        CanisterObservabilityRequest::MemoryAllocations => {
218            RootStatusRequestFragment::MemoryAllocations
219        }
220    };
221    let binding = resolve_registry_protocol_binding(icp_root, environment, root)?;
222    let root_canister = parse_principal("Fleet Subnet Root", &root.pid)?;
223    let response: RootStatusResponseFragment = query_canister_with_arg(
224        icp,
225        &binding,
226        root_canister,
227        protocol::CANIC_OBSERVABILITY,
228        &request,
229    )?;
230    let response = match response {
231        RootStatusResponseFragment::ChildFunding(response) => {
232            CanisterObservabilityResponse::ChildFunding(response)
233        }
234        RootStatusResponseFragment::CycleBalance(response) => {
235            CanisterObservabilityResponse::CycleBalance(response)
236        }
237        RootStatusResponseFragment::CycleHistory(response) => {
238            CanisterObservabilityResponse::CycleHistory(response)
239        }
240        RootStatusResponseFragment::Metrics(response) => {
241            CanisterObservabilityResponse::Metrics(response)
242        }
243        RootStatusResponseFragment::MemoryAllocations(response) => {
244            CanisterObservabilityResponse::MemoryAllocations(response)
245        }
246    };
247    verify_child_funding_response(response, root_canister, expected_child)
248}
249
250const fn child_funding_target(request: &CanisterObservabilityRequest) -> Option<Principal> {
251    match request {
252        CanisterObservabilityRequest::ChildFunding(child) => Some(*child),
253        _ => None,
254    }
255}
256
257fn verify_child_funding_response(
258    response: CanisterObservabilityResponse,
259    parent: Principal,
260    child: Option<Principal>,
261) -> Result<CanisterObservabilityResponse, FleetObservabilityError> {
262    if let Some(child) = child {
263        let CanisterObservabilityResponse::ChildFunding(value) = &response else {
264            return Err(FleetObservabilityError::UnexpectedRootResponse);
265        };
266        if (value.parent, value.child) != (parent, child) {
267            return Err(FleetObservabilityError::UnexpectedRootResponse);
268        }
269    }
270    Ok(response)
271}
272
273fn fleet_subnet_root_entry<'a>(
274    registry: &'a [RegistryEntry],
275    entry: &RegistryEntry,
276) -> Result<&'a RegistryEntry, FleetObservabilityError> {
277    let mut current = entry;
278    let mut visited = BTreeSet::new();
279    loop {
280        if !visited.insert(current.pid.as_str()) {
281            return Err(FleetObservabilityError::ParentCycle {
282                canister: current.pid.clone(),
283            });
284        }
285        if current.role.as_deref() == Some(CanisterRole::ROOT.as_str()) {
286            return registry
287                .iter()
288                .find(|candidate| candidate.pid == current.pid)
289                .ok_or_else(|| FleetObservabilityError::MissingRoot {
290                    canister: entry.pid.clone(),
291                });
292        }
293        let Some(parent) = current.parent_pid.as_deref() else {
294            return Err(FleetObservabilityError::MissingRoot {
295                canister: entry.pid.clone(),
296            });
297        };
298        current = registry
299            .iter()
300            .find(|candidate| candidate.pid == parent)
301            .ok_or_else(|| FleetObservabilityError::MissingRoot {
302                canister: entry.pid.clone(),
303            })?;
304    }
305}
306
307fn parse_principal(field: &'static str, value: &str) -> Result<Principal, FleetObservabilityError> {
308    Principal::from_text(value).map_err(|source| FleetObservabilityError::Principal {
309        field,
310        value: value.to_string(),
311        source,
312    })
313}
314
315#[cfg(test)]
316mod tests {
317    use super::*;
318
319    #[test]
320    fn child_funding_response_binds_parent_and_child_for_direct_and_relayed_reads() {
321        let parent = Principal::from_slice(&[1]);
322        let child = Principal::from_slice(&[2]);
323        let usage = canic_contracts::dto::observability::ChildFundingUsage {
324            parent,
325            child,
326            observed_at_ns: 10,
327            accounted_cycles: 0.into(),
328            last_accounted_at_secs: 0,
329            pending_operations: 0,
330            reserved_cycles: Some(0.into()),
331        };
332        assert!(
333            verify_child_funding_response(
334                CanisterObservabilityResponse::ChildFunding(usage.clone()),
335                parent,
336                Some(child)
337            )
338            .is_ok()
339        );
340        for (wrong_parent, wrong_child) in [(child, child), (parent, parent)] {
341            assert!(matches!(
342                verify_child_funding_response(
343                    CanisterObservabilityResponse::ChildFunding(usage.clone()),
344                    wrong_parent,
345                    Some(wrong_child)
346                ),
347                Err(FleetObservabilityError::UnexpectedRootResponse)
348            ));
349        }
350    }
351
352    fn entry(pid: &str, role: &str, parent: Option<&str>) -> RegistryEntry {
353        RegistryEntry {
354            pid: pid.to_string(),
355            role: Some(role.to_string()),
356            parent_pid: parent.map(str::to_string),
357            module_hash: None,
358            protocol_binding: None,
359        }
360    }
361
362    #[test]
363    fn descendant_observability_resolves_the_exact_owning_root() {
364        let coordinator = entry("coordinator", "fleet_coordinator", None);
365        let root = entry("root", "root", Some("coordinator"));
366        let parent = entry("parent", "hub", Some("root"));
367        let child = entry("child", "leaf", Some("parent"));
368        let registry = vec![coordinator, root.clone(), parent, child.clone()];
369
370        assert_eq!(
371            fleet_subnet_root_entry(&registry, &child)
372                .expect("owning Root")
373                .pid,
374            root.pid
375        );
376    }
377}
canic_host/observability/mod.rs

canic_host/observability/
mod.rs
