canic_host/deployment_truth/
report.rs

1use super::*;
2use std::collections::{BTreeMap, BTreeSet};
3
4///
5/// LocalDeploymentCheckRequest
6///
7#[derive(Clone, Debug, Eq, PartialEq)]
8pub struct LocalDeploymentCheckRequest {
9    pub deployment_name: String,
10    pub network: String,
11    pub workspace_root: std::path::PathBuf,
12    pub icp_root: std::path::PathBuf,
13    pub config_path: Option<std::path::PathBuf>,
14    pub observed_at: String,
15    pub runtime_variant: String,
16    pub build_profile: String,
17}
18
19/// Build local plan and inventory, then return the passive safety check bundle.
20pub fn check_local_deployment(
21    request: &LocalDeploymentCheckRequest,
22) -> Result<DeploymentCheckV1, DeploymentTruthError> {
23    let plan = build_local_deployment_plan(&LocalDeploymentPlanRequest {
24        deployment_name: request.deployment_name.clone(),
25        network: request.network.clone(),
26        workspace_root: request.workspace_root.clone(),
27        icp_root: request.icp_root.clone(),
28        config_path: request.config_path.clone(),
29        runtime_variant: request.runtime_variant.clone(),
30        build_profile: request.build_profile.clone(),
31    });
32    let inventory = collect_local_deployment_inventory(&LocalInventoryRequest {
33        deployment_name: request.deployment_name.clone(),
34        network: request.network.clone(),
35        workspace_root: request.workspace_root.clone(),
36        icp_root: request.icp_root.clone(),
37        config_path: request.config_path.clone(),
38        observed_at: request.observed_at.clone(),
39    })?;
40    let diff = compare_plan_to_inventory(&plan, &inventory);
41    let report = safety_report_from_diff(
42        format!(
43            "local:{}:{}:report",
44            request.network, request.deployment_name
45        ),
46        Some(format!(
47            "local:{}:{}:diff",
48            request.network, request.deployment_name
49        )),
50        &diff,
51    );
52
53    Ok(DeploymentCheckV1 {
54        schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
55        check_id: format!(
56            "local:{}:{}:check",
57            request.network, request.deployment_name
58        ),
59        plan,
60        inventory,
61        diff,
62        report,
63    })
64}
65
66/// Compare intended deployment state with observed inventory into a machine diff.
67#[must_use]
68pub fn compare_plan_to_inventory(
69    plan: &DeploymentPlanV1,
70    inventory: &DeploymentInventoryV1,
71) -> DeploymentDiffV1 {
72    let mut artifact_diff = Vec::new();
73    let mut controller_diff = Vec::new();
74    let mut pool_diff = Vec::new();
75    let mut embedded_config_diff = Vec::new();
76    let mut module_hash_diff = Vec::new();
77    let mut verifier_readiness_diff = Vec::new();
78    let mut hard_failures = Vec::new();
79    let mut warnings = Vec::new();
80
81    compare_identity(plan, inventory, &mut hard_failures);
82    compare_authority_profile(plan, &mut controller_diff, &mut hard_failures);
83    compare_artifacts(
84        plan,
85        inventory,
86        &mut artifact_diff,
87        &mut hard_failures,
88        &mut warnings,
89    );
90    compare_canisters(
91        plan,
92        inventory,
93        &mut controller_diff,
94        &mut hard_failures,
95        &mut warnings,
96    );
97    compare_pools(
98        plan,
99        inventory,
100        &mut pool_diff,
101        &mut hard_failures,
102        &mut warnings,
103    );
104    compare_module_hashes(
105        plan,
106        inventory,
107        &mut module_hash_diff,
108        &mut hard_failures,
109        &mut warnings,
110    );
111    compare_raw_config(
112        plan,
113        inventory,
114        &mut embedded_config_diff,
115        &mut hard_failures,
116    );
117    compare_embedded_config(
118        plan,
119        inventory,
120        &mut embedded_config_diff,
121        &mut hard_failures,
122        &mut warnings,
123    );
124    compare_verifier_readiness(
125        plan,
126        inventory,
127        &mut verifier_readiness_diff,
128        &mut hard_failures,
129        &mut warnings,
130    );
131    for assumption in &plan.unresolved_assumptions {
132        warnings.push(SafetyFindingV1 {
133            code: "plan_assumption".to_string(),
134            message: assumption.description.clone(),
135            severity: SafetySeverityV1::Warning,
136            subject: Some(assumption.key.clone()),
137        });
138    }
139    for gap in &inventory.unresolved_observations {
140        warnings.push(SafetyFindingV1 {
141            code: "observation_gap".to_string(),
142            message: gap.description.clone(),
143            severity: SafetySeverityV1::Warning,
144            subject: Some(gap.key.clone()),
145        });
146    }
147
148    let status = safety_status(&hard_failures, &warnings);
149    DeploymentDiffV1 {
150        schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
151        plan_identity: plan.deployment_identity.clone(),
152        observed_identity: inventory.observed_identity.clone(),
153        artifact_diff,
154        controller_diff,
155        pool_diff,
156        embedded_config_diff,
157        module_hash_diff,
158        verifier_readiness_diff,
159        resume_safety: ResumeSafetyV1 {
160            status,
161            reasons: resume_safety_reasons(&hard_failures, &warnings),
162        },
163        hard_failures,
164        warnings,
165        resumable_phases: Vec::new(),
166    }
167}
168
169/// Compare intended state, observed inventory, and a prior receipt into a
170/// resume-aware deployment diff.
171#[must_use]
172pub fn compare_plan_inventory_and_receipt(
173    plan: &DeploymentPlanV1,
174    inventory: &DeploymentInventoryV1,
175    receipt: &DeploymentReceiptV1,
176) -> DeploymentDiffV1 {
177    let mut diff = compare_plan_to_inventory(plan, inventory);
178    apply_receipt_resume_safety(plan, receipt, &mut diff);
179    diff
180}
181
182/// Render an operator-facing safety report from a machine deployment diff.
183#[must_use]
184pub fn safety_report_from_diff(
185    report_id: impl Into<String>,
186    diff_id: Option<String>,
187    diff: &DeploymentDiffV1,
188) -> SafetyReportV1 {
189    let status = safety_status(&diff.hard_failures, &diff.warnings);
190    SafetyReportV1 {
191        schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
192        report_id: report_id.into(),
193        diff_id,
194        status,
195        summary: safety_summary(status, diff.hard_failures.len(), diff.warnings.len()),
196        hard_failures: diff.hard_failures.clone(),
197        warnings: diff.warnings.clone(),
198        next_actions: safety_next_actions(status),
199    }
200}
201
202fn apply_receipt_resume_safety(
203    plan: &DeploymentPlanV1,
204    receipt: &DeploymentReceiptV1,
205    diff: &mut DeploymentDiffV1,
206) {
207    validate_receipt_identity(plan, receipt, &mut diff.hard_failures);
208    validate_receipt_command_result(receipt, &mut diff.hard_failures);
209    if !diff.hard_failures.is_empty() {
210        diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
211        diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
212        return;
213    }
214    let phase_failures = receipt_phase_failures(receipt);
215    for receipt in &receipt.phase_receipts {
216        if receipt.verified_postcondition.status != ObservationStatusV1::Observed {
217            diff.hard_failures.push(finding(
218                "receipt_postcondition_unverified",
219                format!(
220                    "receipt phase {} has no observed postcondition",
221                    receipt.phase
222                ),
223                SafetySeverityV1::HardFailure,
224                Some(receipt.phase.clone()),
225            ));
226            continue;
227        }
228        if phase_failures.contains(receipt.phase.as_str()) {
229            continue;
230        }
231        diff.resumable_phases.push(receipt.phase.clone());
232    }
233    diff.resumable_phases.sort();
234    diff.resumable_phases.dedup();
235    diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
236    diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
237}
238
239fn validate_receipt_identity(
240    plan: &DeploymentPlanV1,
241    receipt: &DeploymentReceiptV1,
242    hard_failures: &mut Vec<SafetyFindingV1>,
243) {
244    if receipt.plan_id != plan.plan_id {
245        hard_failures.push(finding(
246            "receipt_plan_mismatch",
247            format!(
248                "receipt plan {} does not match current plan {}",
249                receipt.plan_id, plan.plan_id
250            ),
251            SafetySeverityV1::HardFailure,
252            Some("receipt.plan_id".to_string()),
253        ));
254    }
255    if let (Some(expected), Some(observed)) = (
256        plan.deployment_identity.root_principal.as_ref(),
257        receipt.root_principal.as_ref(),
258    ) && expected != observed
259    {
260        hard_failures.push(finding(
261            "receipt_root_mismatch",
262            format!("receipt root {observed} does not match current plan root {expected}"),
263            SafetySeverityV1::HardFailure,
264            Some("receipt.root_principal".to_string()),
265        ));
266    }
267}
268
269fn validate_receipt_command_result(
270    receipt: &DeploymentReceiptV1,
271    hard_failures: &mut Vec<SafetyFindingV1>,
272) {
273    if let DeploymentCommandResultV1::Failed { code, message } = &receipt.command_result {
274        hard_failures.push(finding(
275            "receipt_failed_command",
276            format!("receipt command failed with {code}: {message}"),
277            SafetySeverityV1::HardFailure,
278            Some("receipt.command_result".to_string()),
279        ));
280    }
281}
282
283fn receipt_phase_failures(receipt: &DeploymentReceiptV1) -> BTreeSet<&str> {
284    let mut failures = BTreeSet::new();
285    for role_receipt in &receipt.role_phase_receipts {
286        if matches!(role_receipt.result, RolePhaseResultV1::Failed) {
287            failures.insert(role_receipt.phase.as_str());
288        }
289    }
290    failures
291}
292
293fn compare_identity(
294    plan: &DeploymentPlanV1,
295    inventory: &DeploymentInventoryV1,
296    hard_failures: &mut Vec<SafetyFindingV1>,
297) {
298    let Some(observed) = &inventory.observed_identity else {
299        hard_failures.push(finding(
300            "identity_unobserved",
301            "deployment identity was not observed",
302            SafetySeverityV1::HardFailure,
303            None,
304        ));
305        return;
306    };
307
308    if observed.network != plan.deployment_identity.network {
309        hard_failures.push(finding(
310            "network_mismatch",
311            format!(
312                "plan network {} differs from observed network {}",
313                plan.deployment_identity.network, observed.network
314            ),
315            SafetySeverityV1::HardFailure,
316            Some("deployment_identity.network".to_string()),
317        ));
318    }
319    if let (Some(expected), Some(actual)) = (
320        plan.deployment_identity.root_principal.as_ref(),
321        observed.root_principal.as_ref(),
322    ) && expected != actual
323    {
324        hard_failures.push(finding(
325            "root_trust_anchor_mismatch",
326            format!("plan root {expected} differs from observed root {actual}"),
327            SafetySeverityV1::HardFailure,
328            Some("deployment_identity.root_principal".to_string()),
329        ));
330    }
331    match (
332        plan.deployment_identity.deployment_manifest_digest.as_ref(),
333        observed.deployment_manifest_digest.as_ref(),
334    ) {
335        (Some(expected), Some(actual)) if expected != actual => {
336            hard_failures.push(finding(
337                "deployment_manifest_mismatch",
338                "deployment manifest digest differs from the observed local config",
339                SafetySeverityV1::HardFailure,
340                Some("deployment_identity.deployment_manifest_digest".to_string()),
341            ));
342        }
343        (Some(_), None) => {
344            hard_failures.push(finding(
345                "deployment_manifest_unobserved",
346                "deployment manifest digest was not observed",
347                SafetySeverityV1::HardFailure,
348                Some("deployment_identity.deployment_manifest_digest".to_string()),
349            ));
350        }
351        _ => {}
352    }
353}
354
355fn compare_authority_profile(
356    plan: &DeploymentPlanV1,
357    controller_diff: &mut Vec<DiffItemV1>,
358    hard_failures: &mut Vec<SafetyFindingV1>,
359) {
360    let mut reported = BTreeSet::new();
361    for controller in &plan.authority_profile.expected_controllers {
362        if !is_staging_or_emergency_controller(plan, controller) {
363            continue;
364        }
365        if !reported.insert(controller.as_str()) {
366            continue;
367        }
368        controller_diff.push(diff_item(
369            "controller_authority_overlap",
370            "authority_profile",
371            Some("expected-only".to_string()),
372            Some(controller.clone()),
373            SafetySeverityV1::HardFailure,
374        ));
375        hard_failures.push(finding(
376            "controller_authority_overlap",
377            format!(
378                "controller {controller} appears in both expected and staging/emergency authority"
379            ),
380            SafetySeverityV1::HardFailure,
381            Some("authority_profile".to_string()),
382        ));
383    }
384}
385
386fn compare_artifacts(
387    plan: &DeploymentPlanV1,
388    inventory: &DeploymentInventoryV1,
389    artifact_diff: &mut Vec<DiffItemV1>,
390    hard_failures: &mut Vec<SafetyFindingV1>,
391    warnings: &mut Vec<SafetyFindingV1>,
392) {
393    let observed_by_role = inventory
394        .observed_artifacts
395        .iter()
396        .map(|artifact| (artifact.role.as_str(), artifact))
397        .collect::<BTreeMap<_, _>>();
398
399    for expected in &plan.role_artifacts {
400        let Some(observed) = observed_by_role.get(expected.role.as_str()) else {
401            artifact_diff.push(diff_item(
402                "artifact",
403                &expected.role,
404                expected.wasm_gz_path.clone(),
405                None,
406                SafetySeverityV1::HardFailure,
407            ));
408            hard_failures.push(finding(
409                "artifact_missing",
410                format!("missing observed artifact for role {}", expected.role),
411                SafetySeverityV1::HardFailure,
412                Some(expected.role.clone()),
413            ));
414            continue;
415        };
416
417        compare_artifact_file_sha256(expected, observed, artifact_diff, hard_failures);
418
419        match (
420            expected.wasm_gz_sha256.as_ref(),
421            observed.payload_sha256.as_ref(),
422        ) {
423            (Some(want), Some(got)) if want != got => {
424                artifact_diff.push(diff_item(
425                    "artifact_sha256",
426                    &expected.role,
427                    Some(want.clone()),
428                    Some(got.clone()),
429                    SafetySeverityV1::HardFailure,
430                ));
431                hard_failures.push(finding(
432                    "artifact_digest_mismatch",
433                    format!("artifact digest mismatch for role {}", expected.role),
434                    SafetySeverityV1::HardFailure,
435                    Some(expected.role.clone()),
436                ));
437            }
438            (Some(want), None) => warnings.push(finding(
439                "artifact_digest_unobserved",
440                format!(
441                    "expected artifact digest {want} for role {} was not observed",
442                    expected.role
443                ),
444                SafetySeverityV1::Warning,
445                Some(expected.role.clone()),
446            )),
447            _ => {}
448        }
449    }
450}
451
452fn compare_artifact_file_sha256(
453    expected: &RoleArtifactV1,
454    observed: &ObservedArtifactV1,
455    artifact_diff: &mut Vec<DiffItemV1>,
456    hard_failures: &mut Vec<SafetyFindingV1>,
457) {
458    match (
459        expected.observed_wasm_gz_file_sha256.as_ref(),
460        observed.file_sha256.as_ref(),
461    ) {
462        (Some(want), Some(got)) if want != got => {
463            artifact_diff.push(diff_item(
464                "artifact_file_sha256",
465                &expected.role,
466                Some(want.clone()),
467                Some(got.clone()),
468                SafetySeverityV1::HardFailure,
469            ));
470            hard_failures.push(finding(
471                "artifact_file_digest_mismatch",
472                format!(
473                    "observed artifact file digest changed during deployment truth check for role {}",
474                    expected.role
475                ),
476                SafetySeverityV1::HardFailure,
477                Some(expected.role.clone()),
478            ));
479        }
480        (_, Some(got)) => {
481            artifact_diff.push(diff_item(
482                "artifact_file_sha256",
483                &expected.role,
484                expected.observed_wasm_gz_file_sha256.clone(),
485                Some(got.clone()),
486                SafetySeverityV1::Info,
487            ));
488        }
489        _ => {}
490    }
491}
492
493fn compare_canisters(
494    plan: &DeploymentPlanV1,
495    inventory: &DeploymentInventoryV1,
496    controller_diff: &mut Vec<DiffItemV1>,
497    hard_failures: &mut Vec<SafetyFindingV1>,
498    warnings: &mut Vec<SafetyFindingV1>,
499) {
500    for expected in &plan.expected_canisters {
501        let observed = expected.canister_id.as_ref().map_or_else(
502            || {
503                inventory
504                    .observed_canisters
505                    .iter()
506                    .find(|canister| canister.role.as_deref() == Some(expected.role.as_str()))
507            },
508            |id| {
509                inventory
510                    .observed_canisters
511                    .iter()
512                    .find(|canister| &canister.canister_id == id)
513            },
514        );
515        let Some(observed) = observed else {
516            let severity = if expected.canister_id.is_some() {
517                SafetySeverityV1::HardFailure
518            } else {
519                SafetySeverityV1::Warning
520            };
521            controller_diff.push(diff_item(
522                "canister",
523                &expected.role,
524                expected.canister_id.clone(),
525                None,
526                severity,
527            ));
528            let finding = finding(
529                if expected.canister_id.is_some() {
530                    "canister_missing"
531                } else {
532                    "canister_unobserved"
533                },
534                format!("missing observed canister for role {}", expected.role),
535                severity,
536                Some(expected.role.clone()),
537            );
538            if expected.canister_id.is_some() {
539                hard_failures.push(finding);
540            } else {
541                warnings.push(finding);
542            }
543            continue;
544        };
545        if matches!(
546            observed.control_class,
547            CanisterControlClassV1::UnknownUnsafe | CanisterControlClassV1::UserControlled
548        ) && expected.control_class == CanisterControlClassV1::DeploymentControlled
549        {
550            controller_diff.push(diff_item(
551                "control_class",
552                &expected.role,
553                Some("DeploymentControlled".to_string()),
554                Some(format!("{:?}", observed.control_class)),
555                SafetySeverityV1::HardFailure,
556            ));
557            hard_failures.push(finding(
558                "unsafe_control_class",
559                format!("role {} has unsafe observed control class", expected.role),
560                SafetySeverityV1::HardFailure,
561                Some(expected.role.clone()),
562            ));
563        }
564        compare_role_controllers(plan, observed, controller_diff, hard_failures, warnings);
565    }
566}
567
568fn compare_pools(
569    plan: &DeploymentPlanV1,
570    inventory: &DeploymentInventoryV1,
571    pool_diff: &mut Vec<DiffItemV1>,
572    hard_failures: &mut Vec<SafetyFindingV1>,
573    warnings: &mut Vec<SafetyFindingV1>,
574) {
575    let mut matched_observed = BTreeSet::new();
576    for expected in &plan.expected_pool {
577        compare_expected_pool(
578            expected,
579            inventory,
580            pool_diff,
581            hard_failures,
582            warnings,
583            &mut matched_observed,
584        );
585    }
586
587    for observed in &inventory.observed_pool {
588        warn_extra_observed_pool(plan, observed, pool_diff, warnings, &matched_observed);
589    }
590}
591
592fn compare_expected_pool<'a>(
593    expected: &ExpectedPoolCanisterV1,
594    inventory: &'a DeploymentInventoryV1,
595    pool_diff: &mut Vec<DiffItemV1>,
596    hard_failures: &mut Vec<SafetyFindingV1>,
597    warnings: &mut Vec<SafetyFindingV1>,
598    matched_observed: &mut BTreeSet<&'a str>,
599) {
600    let observed = expected
601        .canister_id
602        .as_ref()
603        .and_then(|id| {
604            inventory
605                .observed_pool
606                .iter()
607                .find(|pool| &pool.canister_id == id)
608        })
609        .or_else(|| {
610            inventory
611                .observed_pool
612                .iter()
613                .find(|pool| pool_matches_expected_pool(pool, expected))
614        });
615    let Some(observed) = observed else {
616        record_missing_pool(expected, pool_diff, hard_failures, warnings);
617        return;
618    };
619
620    matched_observed.insert(observed.canister_id.as_str());
621    record_pool_id_mismatch(expected, observed, pool_diff, hard_failures);
622    record_unsafe_pool_control_class(observed, pool_diff, hard_failures);
623}
624
625fn record_missing_pool(
626    expected: &ExpectedPoolCanisterV1,
627    pool_diff: &mut Vec<DiffItemV1>,
628    hard_failures: &mut Vec<SafetyFindingV1>,
629    warnings: &mut Vec<SafetyFindingV1>,
630) {
631    let severity = if expected.canister_id.is_some() {
632        SafetySeverityV1::HardFailure
633    } else {
634        SafetySeverityV1::Warning
635    };
636    let subject = expected_pool_subject(expected);
637    pool_diff.push(diff_item(
638        "pool_canister",
639        &subject,
640        expected.canister_id.clone(),
641        None,
642        severity,
643    ));
644    let finding = finding(
645        if expected.canister_id.is_some() {
646            "pool_canister_missing"
647        } else {
648            "pool_canister_unobserved"
649        },
650        format!("missing observed pool canister for {subject}"),
651        severity,
652        Some(subject),
653    );
654    if expected.canister_id.is_some() {
655        hard_failures.push(finding);
656    } else {
657        warnings.push(finding);
658    }
659}
660
661fn record_pool_id_mismatch(
662    expected: &ExpectedPoolCanisterV1,
663    observed: &ObservedPoolCanisterV1,
664    pool_diff: &mut Vec<DiffItemV1>,
665    hard_failures: &mut Vec<SafetyFindingV1>,
666) {
667    if let Some(expected_id) = expected.canister_id.as_ref()
668        && observed.canister_id != *expected_id
669    {
670        let subject = observed_pool_subject(observed);
671        pool_diff.push(diff_item(
672            "pool_canister_id",
673            &subject,
674            Some(expected_id.clone()),
675            Some(observed.canister_id.clone()),
676            SafetySeverityV1::HardFailure,
677        ));
678        hard_failures.push(finding(
679            "pool_canister_id_mismatch",
680            format!(
681                "pool canister {subject} has observed id {}, expected {expected_id}",
682                observed.canister_id
683            ),
684            SafetySeverityV1::HardFailure,
685            Some(subject),
686        ));
687    }
688}
689
690fn record_unsafe_pool_control_class(
691    observed: &ObservedPoolCanisterV1,
692    pool_diff: &mut Vec<DiffItemV1>,
693    hard_failures: &mut Vec<SafetyFindingV1>,
694) {
695    if !matches!(
696        observed.control_class,
697        CanisterControlClassV1::UnknownUnsafe | CanisterControlClassV1::UserControlled
698    ) {
699        return;
700    }
701    let subject = observed_pool_subject(observed);
702    pool_diff.push(diff_item(
703        "pool_control_class",
704        &subject,
705        Some("CanicManagedPool".to_string()),
706        Some(format!("{:?}", observed.control_class)),
707        SafetySeverityV1::HardFailure,
708    ));
709    hard_failures.push(finding(
710        "unsafe_pool_control_class",
711        format!("pool canister {subject} has unsafe observed control class"),
712        SafetySeverityV1::HardFailure,
713        Some(subject),
714    ));
715}
716
717fn warn_extra_observed_pool(
718    plan: &DeploymentPlanV1,
719    observed: &ObservedPoolCanisterV1,
720    pool_diff: &mut Vec<DiffItemV1>,
721    warnings: &mut Vec<SafetyFindingV1>,
722    matched_observed: &BTreeSet<&str>,
723) {
724    if matched_observed.contains(observed.canister_id.as_str())
725        || plan.expected_pool.iter().any(|expected| {
726            expected.canister_id.as_ref() == Some(&observed.canister_id)
727                || pool_matches_expected_pool(observed, expected)
728        })
729    {
730        return;
731    }
732    let subject = observed_pool_subject(observed);
733    pool_diff.push(diff_item(
734        "pool_extra",
735        &subject,
736        None,
737        Some(observed.canister_id.clone()),
738        SafetySeverityV1::Warning,
739    ));
740    warnings.push(finding(
741        "extra_pool_canister_observed",
742        format!("observed undeclared pool canister {subject}"),
743        SafetySeverityV1::Warning,
744        Some(subject),
745    ));
746}
747
748fn pool_matches_expected_pool(
749    observed: &ObservedPoolCanisterV1,
750    expected: &ExpectedPoolCanisterV1,
751) -> bool {
752    observed.pool == expected.pool
753        && expected
754            .role
755            .as_ref()
756            .is_none_or(|role| observed.role.as_ref() == Some(role))
757}
758
759fn expected_pool_subject(expected: &ExpectedPoolCanisterV1) -> String {
760    expected.role.as_ref().map_or_else(
761        || expected.pool.clone(),
762        |role| format!("{}:{role}", expected.pool),
763    )
764}
765
766fn observed_pool_subject(observed: &ObservedPoolCanisterV1) -> String {
767    observed.role.as_ref().map_or_else(
768        || observed.pool.clone(),
769        |role| format!("{}:{role}", observed.pool),
770    )
771}
772
773fn compare_role_controllers(
774    plan: &DeploymentPlanV1,
775    observed: &ObservedCanisterV1,
776    controller_diff: &mut Vec<DiffItemV1>,
777    hard_failures: &mut Vec<SafetyFindingV1>,
778    warnings: &mut Vec<SafetyFindingV1>,
779) {
780    let role = observed.role.as_deref().unwrap_or("unknown");
781    if observed.controllers.is_empty()
782        && observed.role_assignment_source.as_deref() != Some("icp_canister_status")
783    {
784        warnings.push(finding(
785            "controllers_unobserved",
786            format!("controllers were not observed for role {role}"),
787            SafetySeverityV1::Warning,
788            Some(role.to_string()),
789        ));
790        return;
791    }
792    for expected in &plan.authority_profile.expected_controllers {
793        if observed
794            .controllers
795            .iter()
796            .any(|controller| controller == expected)
797        {
798            continue;
799        }
800        controller_diff.push(diff_item(
801            "controller_missing",
802            role,
803            Some(expected.clone()),
804            Some(controller_set_label(&observed.controllers)),
805            SafetySeverityV1::HardFailure,
806        ));
807        hard_failures.push(finding(
808            "expected_controller_missing",
809            format!("role {role} is missing expected controller {expected}"),
810            SafetySeverityV1::HardFailure,
811            Some(role.to_string()),
812        ));
813    }
814
815    for observed_controller in &observed.controllers {
816        if is_declared_controller(plan, observed_controller) {
817            continue;
818        }
819        controller_diff.push(diff_item(
820            "controller_extra",
821            role,
822            Some(controller_set_label(
823                &plan.authority_profile.expected_controllers,
824            )),
825            Some(observed_controller.clone()),
826            SafetySeverityV1::Warning,
827        ));
828        warnings.push(finding(
829            "extra_controller_observed",
830            format!("role {role} has controller outside the expected authority profile"),
831            SafetySeverityV1::Warning,
832            Some(role.to_string()),
833        ));
834    }
835}
836
837fn is_declared_controller(plan: &DeploymentPlanV1, controller: &str) -> bool {
838    plan.authority_profile
839        .expected_controllers
840        .iter()
841        .chain(plan.authority_profile.staging_controllers.iter())
842        .chain(plan.authority_profile.emergency_controllers.iter())
843        .any(|expected| expected == controller)
844}
845
846fn is_staging_or_emergency_controller(plan: &DeploymentPlanV1, controller: &str) -> bool {
847    plan.authority_profile
848        .staging_controllers
849        .iter()
850        .chain(plan.authority_profile.emergency_controllers.iter())
851        .any(|declared| declared == controller)
852}
853
854fn controller_set_label(controllers: &[String]) -> String {
855    if controllers.is_empty() {
856        return "<none>".to_string();
857    }
858    controllers.join(",")
859}
860
861fn compare_module_hashes(
862    plan: &DeploymentPlanV1,
863    inventory: &DeploymentInventoryV1,
864    module_hash_diff: &mut Vec<DiffItemV1>,
865    hard_failures: &mut Vec<SafetyFindingV1>,
866    warnings: &mut Vec<SafetyFindingV1>,
867) {
868    let observed_by_role = inventory
869        .observed_canisters
870        .iter()
871        .filter_map(|canister| canister.role.as_deref().map(|role| (role, canister)))
872        .collect::<BTreeMap<_, _>>();
873
874    for artifact in &plan.role_artifacts {
875        let Some(expected) = artifact.installed_module_hash.as_ref() else {
876            continue;
877        };
878        let Some(observed_canister) = observed_by_role.get(artifact.role.as_str()) else {
879            continue;
880        };
881        match observed_canister.module_hash.as_ref() {
882            Some(observed) if observed != expected => {
883                module_hash_diff.push(diff_item(
884                    "installed_module_hash",
885                    &artifact.role,
886                    Some(expected.clone()),
887                    Some(observed.clone()),
888                    SafetySeverityV1::HardFailure,
889                ));
890                hard_failures.push(finding(
891                    "installed_module_hash_mismatch",
892                    format!("installed module hash differs for role {}", artifact.role),
893                    SafetySeverityV1::HardFailure,
894                    Some(artifact.role.clone()),
895                ));
896            }
897            None => warnings.push(finding(
898                "installed_module_hash_unobserved",
899                format!(
900                    "installed module hash was not observed for role {}",
901                    artifact.role
902                ),
903                SafetySeverityV1::Warning,
904                Some(artifact.role.clone()),
905            )),
906            _ => {}
907        }
908    }
909}
910
911fn compare_raw_config(
912    plan: &DeploymentPlanV1,
913    inventory: &DeploymentInventoryV1,
914    embedded_config_diff: &mut Vec<DiffItemV1>,
915    hard_failures: &mut Vec<SafetyFindingV1>,
916) {
917    let mut expected = plan
918        .role_artifacts
919        .iter()
920        .filter_map(|artifact| artifact.raw_config_sha256.as_ref())
921        .collect::<Vec<_>>();
922    expected.sort_unstable();
923    expected.dedup();
924    let [expected] = expected.as_slice() else {
925        if expected.len() > 1 {
926            hard_failures.push(finding(
927                "raw_config_plan_inconsistent",
928                "planned role artifacts disagree on raw config digest",
929                SafetySeverityV1::HardFailure,
930                Some("role_artifacts.raw_config_sha256".to_string()),
931            ));
932        }
933        return;
934    };
935
936    if let Some(observed) = &inventory.local_config.raw_config_sha256
937        && observed != *expected
938    {
939        embedded_config_diff.push(diff_item(
940            "raw_config_sha256",
941            "deployment",
942            Some((*expected).clone()),
943            Some(observed.clone()),
944            SafetySeverityV1::HardFailure,
945        ));
946        hard_failures.push(finding(
947            "raw_config_digest_mismatch",
948            "raw local config digest changed during deployment truth check",
949            SafetySeverityV1::HardFailure,
950            Some("local_config.raw_sha256".to_string()),
951        ));
952    }
953}
954
955fn compare_embedded_config(
956    plan: &DeploymentPlanV1,
957    inventory: &DeploymentInventoryV1,
958    embedded_config_diff: &mut Vec<DiffItemV1>,
959    hard_failures: &mut Vec<SafetyFindingV1>,
960    warnings: &mut Vec<SafetyFindingV1>,
961) {
962    let Some(expected) = &plan.deployment_identity.canonical_runtime_config_digest else {
963        return;
964    };
965    match &inventory.local_config.canonical_embedded_config_sha256 {
966        Some(observed) if observed != expected => {
967            embedded_config_diff.push(diff_item(
968                "canonical_config",
969                "deployment",
970                Some(expected.clone()),
971                Some(observed.clone()),
972                SafetySeverityV1::HardFailure,
973            ));
974            hard_failures.push(finding(
975                "canonical_config_mismatch",
976                "canonical runtime config digest differs from the plan",
977                SafetySeverityV1::HardFailure,
978                Some("local_config".to_string()),
979            ));
980        }
981        None => warnings.push(finding(
982            "canonical_config_unobserved",
983            "canonical runtime config digest was not observed",
984            SafetySeverityV1::Warning,
985            Some("local_config".to_string()),
986        )),
987        _ => {}
988    }
989}
990
991fn compare_verifier_readiness(
992    plan: &DeploymentPlanV1,
993    inventory: &DeploymentInventoryV1,
994    verifier_readiness_diff: &mut Vec<DiffItemV1>,
995    hard_failures: &mut Vec<SafetyFindingV1>,
996    warnings: &mut Vec<SafetyFindingV1>,
997) {
998    if !plan.expected_verifier_readiness.required {
999        return;
1000    }
1001    if inventory.observed_verifier_readiness.status == ObservationStatusV1::NotObserved {
1002        verifier_readiness_diff.push(diff_item(
1003            "verifier_readiness",
1004            "deployment",
1005            Some("required".to_string()),
1006            Some("not_observed".to_string()),
1007            SafetySeverityV1::Warning,
1008        ));
1009        warnings.push(finding(
1010            "verifier_readiness_unobserved",
1011            "verifier readiness was required but not observed",
1012            SafetySeverityV1::Warning,
1013            Some("verifier_readiness".to_string()),
1014        ));
1015    }
1016
1017    let observed_by_role = inventory
1018        .observed_verifier_readiness
1019        .role_epochs
1020        .iter()
1021        .map(|epoch| (epoch.role.as_str(), epoch))
1022        .collect::<BTreeMap<_, _>>();
1023    for expected in &plan.expected_verifier_readiness.expected_role_epochs {
1024        match observed_by_role.get(expected.role.as_str()) {
1025            Some(observed)
1026                if observed.status == ObservationStatusV1::Observed
1027                    && observed.observed_epoch >= Some(expected.minimum_epoch) => {}
1028            Some(observed)
1029                if observed.status == ObservationStatusV1::Observed
1030                    && observed.observed_epoch.is_some() =>
1031            {
1032                let observed_epoch = observed.observed_epoch.expect("checked above");
1033                verifier_readiness_diff.push(diff_item(
1034                    "verifier_role_epoch",
1035                    &expected.role,
1036                    Some(expected.minimum_epoch.to_string()),
1037                    Some(observed_epoch.to_string()),
1038                    SafetySeverityV1::HardFailure,
1039                ));
1040                hard_failures.push(finding(
1041                    "verifier_role_epoch_stale",
1042                    format!(
1043                        "verifier role {} has epoch {observed_epoch}, expected at least {}",
1044                        expected.role, expected.minimum_epoch
1045                    ),
1046                    SafetySeverityV1::HardFailure,
1047                    Some(expected.role.clone()),
1048                ));
1049            }
1050            _ => {
1051                verifier_readiness_diff.push(diff_item(
1052                    "verifier_role_epoch",
1053                    &expected.role,
1054                    Some(expected.minimum_epoch.to_string()),
1055                    Some("not_observed".to_string()),
1056                    SafetySeverityV1::Warning,
1057                ));
1058                warnings.push(finding(
1059                    "verifier_role_epoch_unobserved",
1060                    format!("verifier role {} epoch was not observed", expected.role),
1061                    SafetySeverityV1::Warning,
1062                    Some(expected.role.clone()),
1063                ));
1064            }
1065        }
1066    }
1067}
1068
1069fn finding(
1070    code: impl Into<String>,
1071    message: impl Into<String>,
1072    severity: SafetySeverityV1,
1073    subject: Option<String>,
1074) -> SafetyFindingV1 {
1075    SafetyFindingV1 {
1076        code: code.into(),
1077        message: message.into(),
1078        severity,
1079        subject,
1080    }
1081}
1082
1083fn diff_item(
1084    category: impl Into<String>,
1085    subject: impl Into<String>,
1086    expected: Option<String>,
1087    observed: Option<String>,
1088    severity: SafetySeverityV1,
1089) -> DiffItemV1 {
1090    DiffItemV1 {
1091        category: category.into(),
1092        subject: subject.into(),
1093        expected,
1094        observed,
1095        severity,
1096    }
1097}
1098
1099const fn safety_status(
1100    hard_failures: &[SafetyFindingV1],
1101    warnings: &[SafetyFindingV1],
1102) -> SafetyStatusV1 {
1103    if !hard_failures.is_empty() {
1104        SafetyStatusV1::Blocked
1105    } else if !warnings.is_empty() {
1106        SafetyStatusV1::Warning
1107    } else {
1108        SafetyStatusV1::Safe
1109    }
1110}
1111
1112fn resume_safety_reasons(
1113    hard_failures: &[SafetyFindingV1],
1114    warnings: &[SafetyFindingV1],
1115) -> Vec<String> {
1116    if !hard_failures.is_empty() {
1117        return hard_failures
1118            .iter()
1119            .map(|finding| finding.message.clone())
1120            .collect();
1121    }
1122    if !warnings.is_empty() {
1123        return warnings
1124            .iter()
1125            .map(|finding| finding.message.clone())
1126            .collect();
1127    }
1128    vec!["no blocking deployment truth differences were found".to_string()]
1129}
1130
1131fn safety_summary(
1132    status: SafetyStatusV1,
1133    hard_failure_count: usize,
1134    warning_count: usize,
1135) -> String {
1136    match status {
1137        SafetyStatusV1::Safe => "deployment inventory matches the checked plan".to_string(),
1138        SafetyStatusV1::Warning => {
1139            format!("deployment inventory has {warning_count} warning(s)")
1140        }
1141        SafetyStatusV1::Blocked => {
1142            format!(
1143                "deployment inventory has {hard_failure_count} blocking issue(s) and {warning_count} warning(s)"
1144            )
1145        }
1146        SafetyStatusV1::NotEvaluated => "deployment safety has not been evaluated".to_string(),
1147    }
1148}
1149
1150fn safety_next_actions(status: SafetyStatusV1) -> Vec<String> {
1151    match status {
1152        SafetyStatusV1::Safe => Vec::new(),
1153        SafetyStatusV1::Warning => {
1154            vec!["review deployment warnings before continuing".to_string()]
1155        }
1156        SafetyStatusV1::Blocked => {
1157            vec!["resolve blocking deployment truth differences before mutation".to_string()]
1158        }
1159        SafetyStatusV1::NotEvaluated => vec!["collect deployment inventory".to_string()],
1160    }
1161}
canic_host/deployment_truth/report.rs

canic_host/deployment_truth/
report.rs
