Skip to main content

canic_host/deployment_truth/
report.rs

1use super::*;
2use crate::subnet_catalog::{SubnetCatalogCacheRequest, load_cached_subnet_catalog};
3use canic_subnet_catalog::{MAINNET_NETWORK, SubnetKind};
4use std::collections::{BTreeMap, BTreeSet};
5
6///
7/// DuplicateEvidenceGroup
8///
9struct DuplicateEvidenceGroup {
10    subject: String,
11    count: usize,
12    evidence_label: String,
13    is_conflict: bool,
14}
15
16///
17/// LocalDeploymentCheckRequest
18///
19#[derive(Clone, Debug, Eq, PartialEq)]
20pub struct LocalDeploymentCheckRequest {
21    pub deployment_name: String,
22    pub network: String,
23    pub workspace_root: std::path::PathBuf,
24    pub icp_root: std::path::PathBuf,
25    pub config_path: Option<std::path::PathBuf>,
26    pub observed_at: String,
27    pub runtime_variant: String,
28    pub build_profile: String,
29}
30
31/// Build local plan and inventory, then return the passive safety check bundle.
32pub fn check_local_deployment(
33    request: &LocalDeploymentCheckRequest,
34) -> Result<DeploymentCheckV1, DeploymentTruthError> {
35    let plan = build_local_deployment_plan(&LocalDeploymentPlanRequest {
36        deployment_name: request.deployment_name.clone(),
37        network: request.network.clone(),
38        workspace_root: request.workspace_root.clone(),
39        icp_root: request.icp_root.clone(),
40        config_path: request.config_path.clone(),
41        runtime_variant: request.runtime_variant.clone(),
42        build_profile: request.build_profile.clone(),
43    });
44    let inventory = collect_local_deployment_inventory(&LocalInventoryRequest {
45        deployment_name: request.deployment_name.clone(),
46        network: request.network.clone(),
47        workspace_root: request.workspace_root.clone(),
48        icp_root: request.icp_root.clone(),
49        config_path: request.config_path.clone(),
50        observed_at: request.observed_at.clone(),
51    })?;
52    let mut diff = compare_plan_to_inventory(&plan, &inventory);
53    apply_root_canister_signature_subnet_check(
54        &mut diff,
55        &inventory,
56        &request.network,
57        &request.icp_root,
58    );
59    let report = safety_report_from_diff(
60        format!(
61            "local:{}:{}:report",
62            request.network, request.deployment_name
63        ),
64        Some(format!(
65            "local:{}:{}:diff",
66            request.network, request.deployment_name
67        )),
68        &diff,
69    );
70
71    Ok(DeploymentCheckV1 {
72        schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
73        check_id: format!(
74            "local:{}:{}:check",
75            request.network, request.deployment_name
76        ),
77        plan,
78        inventory,
79        diff,
80        report,
81    })
82}
83
84pub(super) fn apply_root_canister_signature_subnet_check(
85    diff: &mut DeploymentDiffV1,
86    inventory: &DeploymentInventoryV1,
87    network: &str,
88    icp_root: &std::path::Path,
89) {
90    if network != MAINNET_NETWORK {
91        return;
92    }
93    let Some(root) = &inventory.observed_root else {
94        return;
95    };
96    let request = SubnetCatalogCacheRequest {
97        icp_root: icp_root.to_path_buf(),
98        network: network.to_string(),
99    };
100    let catalog = match load_cached_subnet_catalog(&request) {
101        Ok(cached) => cached.catalog,
102        Err(err) => {
103            diff.hard_failures.push(finding(
104                "root_auth_subnet_evidence_missing",
105                format!(
106                    "cannot verify root canister-signature subnet kind for {}: {err}",
107                    root.observed_canister_id
108                ),
109                SafetySeverityV1::HardFailure,
110                Some(root.observed_canister_id.clone()),
111            ));
112            refresh_resume_safety(diff);
113            return;
114        }
115    };
116    let resolved = match catalog.resolve_canister(&root.observed_canister_id) {
117        Ok(resolved) => resolved,
118        Err(err) => {
119            diff.hard_failures.push(finding(
120                "root_auth_subnet_resolution_failed",
121                format!(
122                    "cannot resolve root canister-signature subnet kind for {}: {err}",
123                    root.observed_canister_id
124                ),
125                SafetySeverityV1::HardFailure,
126                Some(root.observed_canister_id.clone()),
127            ));
128            refresh_resume_safety(diff);
129            return;
130        }
131    };
132    if resolved.subnet.subnet_kind == SubnetKind::CloudEngine {
133        diff.hard_failures.push(finding(
134            "root_auth_cloud_engine_subnet",
135            format!(
136                "root canister {} resolves to cloud_engine subnet {}; IC canister signatures from cloud_engine subnets are invalid",
137                root.observed_canister_id, resolved.subnet.subnet_principal
138            ),
139            SafetySeverityV1::HardFailure,
140            Some(root.observed_canister_id.clone()),
141        ));
142        refresh_resume_safety(diff);
143    }
144}
145
146fn refresh_resume_safety(diff: &mut DeploymentDiffV1) {
147    diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
148    diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
149}
150
151/// Compare intended deployment state with observed inventory into a machine diff.
152#[must_use]
153pub fn compare_plan_to_inventory(
154    plan: &DeploymentPlanV1,
155    inventory: &DeploymentInventoryV1,
156) -> DeploymentDiffV1 {
157    let mut artifact_diff = Vec::new();
158    let mut controller_diff = Vec::new();
159    let mut pool_diff = Vec::new();
160    let mut embedded_config_diff = Vec::new();
161    let mut module_hash_diff = Vec::new();
162    let mut verifier_readiness_diff = Vec::new();
163    let mut hard_failures = Vec::new();
164    let mut warnings = Vec::new();
165
166    compare_identity(plan, inventory, &mut hard_failures);
167    compare_authority_profile(plan, &mut controller_diff, &mut hard_failures);
168    compare_artifacts(
169        plan,
170        inventory,
171        &mut artifact_diff,
172        &mut hard_failures,
173        &mut warnings,
174    );
175    compare_observed_canister_id_conflicts(
176        inventory,
177        &mut controller_diff,
178        &mut hard_failures,
179        &mut warnings,
180    );
181    compare_observed_canister_pool_role_conflicts(inventory, &mut pool_diff, &mut hard_failures);
182    compare_canisters(
183        plan,
184        inventory,
185        &mut controller_diff,
186        &mut hard_failures,
187        &mut warnings,
188    );
189    compare_pools(
190        plan,
191        inventory,
192        &mut pool_diff,
193        &mut hard_failures,
194        &mut warnings,
195    );
196    compare_module_hashes(
197        plan,
198        inventory,
199        &mut module_hash_diff,
200        &mut hard_failures,
201        &mut warnings,
202    );
203    compare_raw_config(
204        plan,
205        inventory,
206        &mut embedded_config_diff,
207        &mut hard_failures,
208    );
209    compare_embedded_config(
210        plan,
211        inventory,
212        &mut embedded_config_diff,
213        &mut hard_failures,
214        &mut warnings,
215    );
216    compare_verifier_readiness(
217        plan,
218        inventory,
219        &mut verifier_readiness_diff,
220        &mut hard_failures,
221        &mut warnings,
222    );
223    record_plan_assumptions(plan, &mut hard_failures, &mut warnings);
224    for gap in &inventory.unresolved_observations {
225        warnings.push(SafetyFindingV1 {
226            code: "observation_gap".to_string(),
227            message: gap.description.clone(),
228            severity: SafetySeverityV1::Warning,
229            subject: Some(gap.key.clone()),
230        });
231    }
232
233    let status = safety_status(&hard_failures, &warnings);
234    DeploymentDiffV1 {
235        schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
236        plan_identity: plan.deployment_identity.clone(),
237        observed_identity: inventory.observed_identity.clone(),
238        artifact_diff,
239        controller_diff,
240        pool_diff,
241        embedded_config_diff,
242        module_hash_diff,
243        verifier_readiness_diff,
244        resume_safety: ResumeSafetyV1 {
245            status,
246            reasons: resume_safety_reasons(&hard_failures, &warnings),
247        },
248        hard_failures,
249        warnings,
250        resumable_phases: Vec::new(),
251    }
252}
253
254fn record_plan_assumptions(
255    plan: &DeploymentPlanV1,
256    hard_failures: &mut Vec<SafetyFindingV1>,
257    warnings: &mut Vec<SafetyFindingV1>,
258) {
259    for assumption in &plan.unresolved_assumptions {
260        if assumption.key == "local_state.unverified_root_canister_id" {
261            hard_failures.push(SafetyFindingV1 {
262                code: "unverified_deployment_root".to_string(),
263                message: assumption.description.clone(),
264                severity: SafetySeverityV1::HardFailure,
265                subject: Some(assumption.key.clone()),
266            });
267        } else {
268            warnings.push(SafetyFindingV1 {
269                code: "plan_assumption".to_string(),
270                message: assumption.description.clone(),
271                severity: SafetySeverityV1::Warning,
272                subject: Some(assumption.key.clone()),
273            });
274        }
275    }
276}
277
278/// Compare intended state, observed inventory, and a prior receipt into a
279/// resume-aware deployment diff.
280#[must_use]
281pub fn compare_plan_inventory_and_receipt(
282    plan: &DeploymentPlanV1,
283    inventory: &DeploymentInventoryV1,
284    receipt: &DeploymentReceiptV1,
285) -> DeploymentDiffV1 {
286    let mut diff = compare_plan_to_inventory(plan, inventory);
287    apply_receipt_resume_safety(plan, receipt, &mut diff);
288    diff
289}
290
291/// Render an operator-facing safety report from a machine deployment diff.
292#[must_use]
293pub fn safety_report_from_diff(
294    report_id: impl Into<String>,
295    diff_id: Option<String>,
296    diff: &DeploymentDiffV1,
297) -> SafetyReportV1 {
298    let status = safety_status(&diff.hard_failures, &diff.warnings);
299    SafetyReportV1 {
300        schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
301        report_id: report_id.into(),
302        diff_id,
303        status,
304        summary: safety_summary(status, diff.hard_failures.len(), diff.warnings.len()),
305        hard_failures: diff.hard_failures.clone(),
306        warnings: diff.warnings.clone(),
307        next_actions: safety_next_actions(status),
308    }
309}
310
311fn apply_receipt_resume_safety(
312    plan: &DeploymentPlanV1,
313    receipt: &DeploymentReceiptV1,
314    diff: &mut DeploymentDiffV1,
315) {
316    validate_receipt_identity(plan, receipt, &mut diff.hard_failures);
317    validate_receipt_command_result(receipt, &mut diff.hard_failures);
318    validate_receipt_execution_status(receipt, &mut diff.hard_failures);
319    let phase_conflicts =
320        validate_receipt_phase_duplicates(receipt, &mut diff.hard_failures, &mut diff.warnings);
321    let role_phase_conflicts = validate_receipt_role_phase_duplicates(
322        receipt,
323        &mut diff.hard_failures,
324        &mut diff.warnings,
325    );
326    if !diff.hard_failures.is_empty() {
327        diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
328        diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
329        return;
330    }
331    let phase_failures = receipt_phase_failures(receipt);
332    for receipt in &receipt.phase_receipts {
333        if phase_conflicts.contains(&receipt.phase) {
334            continue;
335        }
336        if receipt.verified_postcondition.status != ObservationStatusV1::Observed {
337            diff.hard_failures.push(finding(
338                "receipt_postcondition_unverified",
339                format!(
340                    "receipt phase {} has no observed postcondition",
341                    receipt.phase
342                ),
343                SafetySeverityV1::HardFailure,
344                Some(receipt.phase.clone()),
345            ));
346            continue;
347        }
348        if phase_failures.contains(receipt.phase.as_str()) {
349            continue;
350        }
351        if role_phase_conflicts.contains(receipt.phase.as_str()) {
352            continue;
353        }
354        diff.resumable_phases.push(receipt.phase.clone());
355    }
356    diff.resumable_phases.sort();
357    diff.resumable_phases.dedup();
358    diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
359    diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
360}
361
362fn validate_receipt_phase_duplicates(
363    receipt: &DeploymentReceiptV1,
364    hard_failures: &mut Vec<SafetyFindingV1>,
365    warnings: &mut Vec<SafetyFindingV1>,
366) -> BTreeSet<String> {
367    let mut conflicting_phases = BTreeSet::new();
368    for group in duplicate_evidence_groups(
369        &receipt.phase_receipts,
370        |phase_receipt| phase_receipt.phase.as_str().to_string(),
371        receipt_phase_evidence_label,
372        " | ",
373    ) {
374        if group.is_conflict {
375            conflicting_phases.insert(group.subject.clone());
376            hard_failures.push(finding(
377                "receipt_phase_conflict",
378                format!(
379                    "receipt phase {} has conflicting evidence: {}",
380                    group.subject, group.evidence_label
381                ),
382                SafetySeverityV1::HardFailure,
383                Some(group.subject),
384            ));
385        } else {
386            warnings.push(finding(
387                "duplicate_receipt_phase",
388                format!(
389                    "receipt phase {} was reported {} times with identical evidence",
390                    group.subject, group.count
391                ),
392                SafetySeverityV1::Warning,
393                Some(group.subject),
394            ));
395        }
396    }
397    conflicting_phases
398}
399
400fn receipt_phase_evidence_label(receipt: &PhaseReceiptV1) -> String {
401    format!(
402        "status={:?};evidence={}",
403        receipt.verified_postcondition.status,
404        receipt.verified_postcondition.evidence.join(",")
405    )
406}
407
408fn validate_receipt_role_phase_duplicates(
409    receipt: &DeploymentReceiptV1,
410    hard_failures: &mut Vec<SafetyFindingV1>,
411    warnings: &mut Vec<SafetyFindingV1>,
412) -> BTreeSet<String> {
413    let mut conflicting_phases = BTreeSet::new();
414    for group in duplicate_evidence_groups(
415        &receipt.role_phase_receipts,
416        role_phase_subject,
417        role_phase_evidence_label,
418        " | ",
419    ) {
420        if group.is_conflict {
421            if let Some(phase) = group
422                .subject
423                .rsplit_once(':')
424                .map(|(_, phase)| phase.to_string())
425            {
426                conflicting_phases.insert(phase);
427            }
428            hard_failures.push(finding(
429                "receipt_role_phase_conflict",
430                format!(
431                    "receipt role phase {} has conflicting evidence: {}",
432                    group.subject, group.evidence_label
433                ),
434                SafetySeverityV1::HardFailure,
435                Some(group.subject),
436            ));
437        } else {
438            warnings.push(finding(
439                "duplicate_receipt_role_phase",
440                format!(
441                    "receipt role phase {} was reported {} times with identical evidence",
442                    group.subject, group.count
443                ),
444                SafetySeverityV1::Warning,
445                Some(group.subject),
446            ));
447        }
448    }
449    conflicting_phases
450}
451
452fn role_phase_subject(receipt: &RolePhaseReceiptV1) -> String {
453    format!("{}:{}", receipt.role, receipt.phase)
454}
455
456fn role_phase_evidence_label(receipt: &RolePhaseReceiptV1) -> String {
457    format!(
458        "result={:?};previous={};target={};observed={};artifact={};config={};error={}",
459        receipt.result,
460        receipt.previous_module_hash.as_deref().unwrap_or("<none>"),
461        receipt.target_module_hash.as_deref().unwrap_or("<none>"),
462        receipt
463            .observed_module_hash_after
464            .as_deref()
465            .unwrap_or("<none>"),
466        receipt.artifact_digest.as_deref().unwrap_or("<none>"),
467        receipt
468            .canonical_embedded_config_sha256
469            .as_deref()
470            .unwrap_or("<none>"),
471        receipt.error.as_deref().unwrap_or("<none>")
472    )
473}
474
475fn validate_receipt_identity(
476    plan: &DeploymentPlanV1,
477    receipt: &DeploymentReceiptV1,
478    hard_failures: &mut Vec<SafetyFindingV1>,
479) {
480    if receipt.plan_id != plan.plan_id {
481        hard_failures.push(finding(
482            "receipt_plan_mismatch",
483            format!(
484                "receipt plan {} does not match current plan {}",
485                receipt.plan_id, plan.plan_id
486            ),
487            SafetySeverityV1::HardFailure,
488            Some("receipt.plan_id".to_string()),
489        ));
490    }
491    if let (Some(expected), Some(observed)) = (
492        plan.deployment_identity.root_principal.as_ref(),
493        receipt.root_principal.as_ref(),
494    ) && expected != observed
495    {
496        hard_failures.push(finding(
497            "receipt_root_mismatch",
498            format!("receipt root {observed} does not match current plan root {expected}"),
499            SafetySeverityV1::HardFailure,
500            Some("receipt.root_principal".to_string()),
501        ));
502    }
503}
504
505fn validate_receipt_command_result(
506    receipt: &DeploymentReceiptV1,
507    hard_failures: &mut Vec<SafetyFindingV1>,
508) {
509    if let DeploymentCommandResultV1::Failed { code, message } = &receipt.command_result {
510        hard_failures.push(finding(
511            "receipt_failed_command",
512            format!("receipt command failed with {code}: {message}"),
513            SafetySeverityV1::HardFailure,
514            Some("receipt.command_result".to_string()),
515        ));
516    }
517}
518
519fn validate_receipt_execution_status(
520    receipt: &DeploymentReceiptV1,
521    hard_failures: &mut Vec<SafetyFindingV1>,
522) {
523    let derived_status = deployment_execution_status_for_receipt_parts(
524        &receipt.command_result,
525        &receipt.role_phase_receipts,
526    );
527    let status_is_consistent = match receipt.operation_status {
528        DeploymentExecutionStatusV1::FailedAfterMutation
529            if matches!(
530                derived_status,
531                DeploymentExecutionStatusV1::FailedBeforeMutation
532            ) =>
533        {
534            receipt.role_phase_receipts.is_empty()
535        }
536        _ => receipt.operation_status == derived_status,
537    };
538
539    if !status_is_consistent {
540        hard_failures.push(finding(
541            "receipt_execution_status_mismatch",
542            format!(
543                "receipt operation status {:?} does not match command result and role-phase evidence {:?}",
544                receipt.operation_status, derived_status
545            ),
546            SafetySeverityV1::HardFailure,
547            Some("receipt.operation_status".to_string()),
548        ));
549    }
550}
551
552fn receipt_phase_failures(receipt: &DeploymentReceiptV1) -> BTreeSet<&str> {
553    let mut failures = BTreeSet::new();
554    for role_receipt in &receipt.role_phase_receipts {
555        if matches!(role_receipt.result, RolePhaseResultV1::Failed) {
556            failures.insert(role_receipt.phase.as_str());
557        }
558    }
559    failures
560}
561
562fn compare_identity(
563    plan: &DeploymentPlanV1,
564    inventory: &DeploymentInventoryV1,
565    hard_failures: &mut Vec<SafetyFindingV1>,
566) {
567    let Some(observed) = &inventory.observed_identity else {
568        hard_failures.push(finding(
569            "identity_unobserved",
570            "deployment identity was not observed",
571            SafetySeverityV1::HardFailure,
572            None,
573        ));
574        return;
575    };
576
577    if observed.network != plan.deployment_identity.network {
578        hard_failures.push(finding(
579            "network_mismatch",
580            format!(
581                "plan network {} differs from observed network {}",
582                plan.deployment_identity.network, observed.network
583            ),
584            SafetySeverityV1::HardFailure,
585            Some("deployment_identity.network".to_string()),
586        ));
587    }
588    if let (Some(expected), Some(actual)) = (
589        plan.deployment_identity.root_principal.as_ref(),
590        observed.root_principal.as_ref(),
591    ) && expected != actual
592    {
593        hard_failures.push(finding(
594            "root_trust_anchor_mismatch",
595            format!("plan root {expected} differs from observed root {actual}"),
596            SafetySeverityV1::HardFailure,
597            Some("deployment_identity.root_principal".to_string()),
598        ));
599    }
600    match (
601        plan.deployment_identity.deployment_manifest_digest.as_ref(),
602        observed.deployment_manifest_digest.as_ref(),
603    ) {
604        (Some(expected), Some(actual)) if expected != actual => {
605            hard_failures.push(finding(
606                "deployment_manifest_mismatch",
607                "deployment manifest digest differs from the observed local config",
608                SafetySeverityV1::HardFailure,
609                Some("deployment_identity.deployment_manifest_digest".to_string()),
610            ));
611        }
612        (Some(_), None) => {
613            hard_failures.push(finding(
614                "deployment_manifest_unobserved",
615                "deployment manifest digest was not observed",
616                SafetySeverityV1::HardFailure,
617                Some("deployment_identity.deployment_manifest_digest".to_string()),
618            ));
619        }
620        _ => {}
621    }
622}
623
624fn compare_authority_profile(
625    plan: &DeploymentPlanV1,
626    controller_diff: &mut Vec<DiffItemV1>,
627    hard_failures: &mut Vec<SafetyFindingV1>,
628) {
629    let mut reported = BTreeSet::new();
630    for controller in &plan.authority_profile.expected_controllers {
631        if !is_staging_or_emergency_controller(plan, controller) {
632            continue;
633        }
634        if !reported.insert(controller.as_str()) {
635            continue;
636        }
637        controller_diff.push(diff_item(
638            "controller_authority_overlap",
639            "authority_profile",
640            Some("expected-only".to_string()),
641            Some(controller.clone()),
642            SafetySeverityV1::HardFailure,
643        ));
644        hard_failures.push(finding(
645            "controller_authority_overlap",
646            format!(
647                "controller {controller} appears in both expected and staging/emergency authority"
648            ),
649            SafetySeverityV1::HardFailure,
650            Some("authority_profile".to_string()),
651        ));
652    }
653}
654
655fn compare_artifacts(
656    plan: &DeploymentPlanV1,
657    inventory: &DeploymentInventoryV1,
658    artifact_diff: &mut Vec<DiffItemV1>,
659    hard_failures: &mut Vec<SafetyFindingV1>,
660    warnings: &mut Vec<SafetyFindingV1>,
661) {
662    let planned_conflicting_roles =
663        compare_planned_artifact_role_conflicts(plan, artifact_diff, hard_failures, warnings);
664    let conflicting_roles =
665        compare_observed_artifact_role_conflicts(inventory, artifact_diff, hard_failures, warnings);
666    let mut observed_by_role = BTreeMap::new();
667    for artifact in &inventory.observed_artifacts {
668        if conflicting_roles.contains(&artifact.role) {
669            continue;
670        }
671        observed_by_role
672            .entry(artifact.role.as_str())
673            .or_insert(artifact);
674    }
675
676    let mut compared_roles = BTreeSet::new();
677    for expected in &plan.role_artifacts {
678        if planned_conflicting_roles.contains(&expected.role)
679            || conflicting_roles.contains(&expected.role)
680            || !compared_roles.insert(expected.role.as_str())
681        {
682            continue;
683        }
684        let Some(observed) = observed_by_role.get(expected.role.as_str()) else {
685            record_missing_artifact(expected, artifact_diff, hard_failures);
686            continue;
687        };
688
689        compare_artifact_file_sha256(expected, observed, artifact_diff, hard_failures);
690        compare_artifact_payload_sha256(expected, observed, artifact_diff, hard_failures, warnings);
691    }
692}
693
694fn compare_planned_artifact_role_conflicts(
695    plan: &DeploymentPlanV1,
696    artifact_diff: &mut Vec<DiffItemV1>,
697    hard_failures: &mut Vec<SafetyFindingV1>,
698    warnings: &mut Vec<SafetyFindingV1>,
699) -> BTreeSet<String> {
700    let mut conflicting_roles = BTreeSet::new();
701    for group in duplicate_evidence_groups(
702        &plan.role_artifacts,
703        |planned| planned.role.as_str().to_string(),
704        planned_artifact_evidence_label,
705        " | ",
706    ) {
707        if group.is_conflict {
708            conflicting_roles.insert(group.subject.clone());
709            artifact_diff.push(diff_item(
710                "planned_artifact_role_conflict",
711                &group.subject,
712                Some("one planned artifact".to_string()),
713                Some(group.evidence_label.clone()),
714                SafetySeverityV1::HardFailure,
715            ));
716            hard_failures.push(finding(
717                "planned_artifact_role_conflict",
718                format!(
719                    "planned artifact role {} has conflicting evidence: {}",
720                    group.subject, group.evidence_label
721                ),
722                SafetySeverityV1::HardFailure,
723                Some(group.subject),
724            ));
725        } else {
726            artifact_diff.push(diff_item(
727                "planned_artifact_duplicate",
728                &group.subject,
729                Some(group.evidence_label.clone()),
730                Some(group.count.to_string()),
731                SafetySeverityV1::Warning,
732            ));
733            warnings.push(finding(
734                "duplicate_planned_artifact_role",
735                format!(
736                    "planned artifact role {} was declared {} times with identical evidence",
737                    group.subject, group.count
738                ),
739                SafetySeverityV1::Warning,
740                Some(group.subject),
741            ));
742        }
743    }
744    conflicting_roles
745}
746
747fn planned_artifact_evidence_label(planned: &RoleArtifactV1) -> String {
748    format!(
749        "wasm_gz_path={};wasm_gz={};file={};module={};raw_config={};canonical={}",
750        planned.wasm_gz_path.as_deref().unwrap_or("<none>"),
751        planned.wasm_gz_sha256.as_deref().unwrap_or("<none>"),
752        planned
753            .observed_wasm_gz_file_sha256
754            .as_deref()
755            .unwrap_or("<none>"),
756        planned.installed_module_hash.as_deref().unwrap_or("<none>"),
757        planned.raw_config_sha256.as_deref().unwrap_or("<none>"),
758        planned
759            .canonical_embedded_config_sha256
760            .as_deref()
761            .unwrap_or("<none>")
762    )
763}
764
765fn compare_observed_artifact_role_conflicts(
766    inventory: &DeploymentInventoryV1,
767    artifact_diff: &mut Vec<DiffItemV1>,
768    hard_failures: &mut Vec<SafetyFindingV1>,
769    warnings: &mut Vec<SafetyFindingV1>,
770) -> BTreeSet<String> {
771    let mut conflicting_roles = BTreeSet::new();
772    for group in duplicate_evidence_groups(
773        &inventory.observed_artifacts,
774        |observed| observed.role.as_str().to_string(),
775        observed_artifact_evidence_label,
776        " | ",
777    ) {
778        if group.is_conflict {
779            conflicting_roles.insert(group.subject.clone());
780            artifact_diff.push(diff_item(
781                "artifact_role_conflict",
782                &group.subject,
783                Some("one artifact observation".to_string()),
784                Some(group.evidence_label.clone()),
785                SafetySeverityV1::HardFailure,
786            ));
787            hard_failures.push(finding(
788                "artifact_role_conflict",
789                format!(
790                    "observed artifact role {} has conflicting evidence: {}",
791                    group.subject, group.evidence_label
792                ),
793                SafetySeverityV1::HardFailure,
794                Some(group.subject),
795            ));
796        } else {
797            artifact_diff.push(diff_item(
798                "artifact_duplicate",
799                &group.subject,
800                Some(group.evidence_label.clone()),
801                Some(group.count.to_string()),
802                SafetySeverityV1::Warning,
803            ));
804            warnings.push(finding(
805                "duplicate_artifact_observed",
806                format!(
807                    "observed artifact role {} was reported {} times with identical evidence",
808                    group.subject, group.count
809                ),
810                SafetySeverityV1::Warning,
811                Some(group.subject),
812            ));
813        }
814    }
815    conflicting_roles
816}
817
818fn observed_artifact_evidence_label(observed: &ObservedArtifactV1) -> String {
819    format!(
820        "path={};file={};payload={};size={};source={:?}",
821        observed.artifact_path,
822        observed.file_sha256.as_deref().unwrap_or("<none>"),
823        observed.payload_sha256.as_deref().unwrap_or("<none>"),
824        observed
825            .payload_size_bytes
826            .map_or_else(|| "<none>".to_string(), |size| size.to_string()),
827        observed.source
828    )
829}
830
831fn record_missing_artifact(
832    expected: &RoleArtifactV1,
833    artifact_diff: &mut Vec<DiffItemV1>,
834    hard_failures: &mut Vec<SafetyFindingV1>,
835) {
836    artifact_diff.push(diff_item(
837        "artifact",
838        &expected.role,
839        expected.wasm_gz_path.clone(),
840        None,
841        SafetySeverityV1::HardFailure,
842    ));
843    hard_failures.push(finding(
844        "artifact_missing",
845        format!("missing observed artifact for role {}", expected.role),
846        SafetySeverityV1::HardFailure,
847        Some(expected.role.clone()),
848    ));
849}
850
851fn compare_artifact_file_sha256(
852    expected: &RoleArtifactV1,
853    observed: &ObservedArtifactV1,
854    artifact_diff: &mut Vec<DiffItemV1>,
855    hard_failures: &mut Vec<SafetyFindingV1>,
856) {
857    match (
858        expected.observed_wasm_gz_file_sha256.as_ref(),
859        observed.file_sha256.as_ref(),
860    ) {
861        (Some(want), Some(got)) if want != got => {
862            artifact_diff.push(diff_item(
863                "artifact_file_sha256",
864                &expected.role,
865                Some(want.clone()),
866                Some(got.clone()),
867                SafetySeverityV1::HardFailure,
868            ));
869            hard_failures.push(finding(
870                "artifact_file_digest_mismatch",
871                format!(
872                    "observed artifact file digest changed during deployment truth check for role {}",
873                    expected.role
874                ),
875                SafetySeverityV1::HardFailure,
876                Some(expected.role.clone()),
877            ));
878        }
879        (_, Some(got)) => {
880            artifact_diff.push(diff_item(
881                "artifact_file_sha256",
882                &expected.role,
883                expected.observed_wasm_gz_file_sha256.clone(),
884                Some(got.clone()),
885                SafetySeverityV1::Info,
886            ));
887        }
888        _ => {}
889    }
890}
891
892fn compare_artifact_payload_sha256(
893    expected: &RoleArtifactV1,
894    observed: &ObservedArtifactV1,
895    artifact_diff: &mut Vec<DiffItemV1>,
896    hard_failures: &mut Vec<SafetyFindingV1>,
897    warnings: &mut Vec<SafetyFindingV1>,
898) {
899    match (
900        expected.wasm_gz_sha256.as_ref(),
901        observed.payload_sha256.as_ref(),
902    ) {
903        (Some(want), Some(got)) if want != got => {
904            artifact_diff.push(diff_item(
905                "artifact_sha256",
906                &expected.role,
907                Some(want.clone()),
908                Some(got.clone()),
909                SafetySeverityV1::HardFailure,
910            ));
911            hard_failures.push(finding(
912                "artifact_digest_mismatch",
913                format!("artifact digest mismatch for role {}", expected.role),
914                SafetySeverityV1::HardFailure,
915                Some(expected.role.clone()),
916            ));
917        }
918        (Some(want), None) => warnings.push(finding(
919            "artifact_digest_unobserved",
920            format!(
921                "expected artifact digest {want} for role {} was not observed",
922                expected.role
923            ),
924            SafetySeverityV1::Warning,
925            Some(expected.role.clone()),
926        )),
927        _ => {}
928    }
929}
930
931fn compare_observed_canister_id_conflicts(
932    inventory: &DeploymentInventoryV1,
933    controller_diff: &mut Vec<DiffItemV1>,
934    hard_failures: &mut Vec<SafetyFindingV1>,
935    warnings: &mut Vec<SafetyFindingV1>,
936) {
937    for group in duplicate_evidence_groups(
938        &inventory.observed_canisters,
939        |observed| observed.canister_id.as_str().to_string(),
940        observed_role_label,
941        ",",
942    ) {
943        if group.is_conflict {
944            controller_diff.push(diff_item(
945                "canister_id_role_conflict",
946                &group.subject,
947                None,
948                Some(group.evidence_label.clone()),
949                SafetySeverityV1::HardFailure,
950            ));
951            hard_failures.push(finding(
952                "canister_id_role_conflict",
953                format!(
954                    "observed canister {} has conflicting roles {}",
955                    group.subject, group.evidence_label
956                ),
957                SafetySeverityV1::HardFailure,
958                Some(group.subject),
959            ));
960        } else {
961            controller_diff.push(diff_item(
962                "canister_duplicate",
963                &group.subject,
964                Some(group.evidence_label.clone()),
965                Some(group.count.to_string()),
966                SafetySeverityV1::Warning,
967            ));
968            warnings.push(finding(
969                "duplicate_canister_observed",
970                format!(
971                    "observed canister {} was reported {} times for role {}",
972                    group.subject, group.count, group.evidence_label
973                ),
974                SafetySeverityV1::Warning,
975                Some(group.subject),
976            ));
977        }
978    }
979}
980
981fn observed_role_label(observed: &ObservedCanisterV1) -> String {
982    observed
983        .role
984        .clone()
985        .unwrap_or_else(|| "<unknown>".to_string())
986}
987
988fn compare_canisters(
989    plan: &DeploymentPlanV1,
990    inventory: &DeploymentInventoryV1,
991    controller_diff: &mut Vec<DiffItemV1>,
992    hard_failures: &mut Vec<SafetyFindingV1>,
993    warnings: &mut Vec<SafetyFindingV1>,
994) {
995    let planned_conflicts =
996        compare_planned_canister_conflicts(plan, controller_diff, hard_failures, warnings);
997    let mut matched_observed = BTreeSet::new();
998    let mut compared_planned = BTreeSet::new();
999    for expected in &plan.expected_canisters {
1000        if planned_conflicts.role_conflicts.contains(&expected.role)
1001            || expected
1002                .canister_id
1003                .as_ref()
1004                .is_some_and(|id| planned_conflicts.id_conflicts.contains(id))
1005            || !compared_planned.insert(planned_canister_evidence_label(expected))
1006        {
1007            continue;
1008        }
1009        let observed = expected.canister_id.as_ref().map_or_else(
1010            || {
1011                let role_matches = inventory
1012                    .observed_canisters
1013                    .iter()
1014                    .filter(|canister| canister.role.as_deref() == Some(expected.role.as_str()))
1015                    .collect::<Vec<_>>();
1016                if role_matches.len() > 1 {
1017                    record_ambiguous_canister_role(
1018                        expected,
1019                        &role_matches,
1020                        controller_diff,
1021                        hard_failures,
1022                    );
1023                    None
1024                } else {
1025                    role_matches.into_iter().next()
1026                }
1027            },
1028            |id| {
1029                inventory
1030                    .observed_canisters
1031                    .iter()
1032                    .find(|canister| &canister.canister_id == id)
1033            },
1034        );
1035        let Some(observed) = observed else {
1036            record_missing_canister(expected, controller_diff, hard_failures, warnings);
1037            continue;
1038        };
1039        matched_observed.insert(observed.canister_id.as_str());
1040        compare_observed_role(expected, observed, controller_diff, hard_failures);
1041        record_unsafe_canister_control_class(expected, observed, controller_diff, hard_failures);
1042        compare_role_controllers(plan, observed, controller_diff, hard_failures, warnings);
1043    }
1044    warn_extra_observed_canisters(
1045        plan,
1046        inventory,
1047        controller_diff,
1048        warnings,
1049        &matched_observed,
1050    );
1051}
1052
1053struct PlannedCanisterConflicts {
1054    role_conflicts: BTreeSet<String>,
1055    id_conflicts: BTreeSet<String>,
1056}
1057
1058fn compare_planned_canister_conflicts(
1059    plan: &DeploymentPlanV1,
1060    controller_diff: &mut Vec<DiffItemV1>,
1061    hard_failures: &mut Vec<SafetyFindingV1>,
1062    warnings: &mut Vec<SafetyFindingV1>,
1063) -> PlannedCanisterConflicts {
1064    let mut role_conflicts = BTreeSet::new();
1065    let mut id_conflicts = BTreeSet::new();
1066
1067    for group in duplicate_evidence_groups(
1068        &plan.expected_canisters,
1069        |planned| planned.role.as_str().to_string(),
1070        planned_canister_evidence_label,
1071        " | ",
1072    ) {
1073        if group.is_conflict {
1074            role_conflicts.insert(group.subject.clone());
1075            controller_diff.push(diff_item(
1076                "planned_canister_role_conflict",
1077                &group.subject,
1078                Some("one planned canister".to_string()),
1079                Some(group.evidence_label.clone()),
1080                SafetySeverityV1::HardFailure,
1081            ));
1082            hard_failures.push(finding(
1083                "planned_canister_role_conflict",
1084                format!(
1085                    "planned canister role {} has conflicting evidence: {}",
1086                    group.subject, group.evidence_label
1087                ),
1088                SafetySeverityV1::HardFailure,
1089                Some(group.subject),
1090            ));
1091        } else {
1092            controller_diff.push(diff_item(
1093                "planned_canister_duplicate",
1094                &group.subject,
1095                Some(group.evidence_label.clone()),
1096                Some(group.count.to_string()),
1097                SafetySeverityV1::Warning,
1098            ));
1099            warnings.push(finding(
1100                "duplicate_planned_canister_role",
1101                format!(
1102                    "planned canister role {} was declared {} times with identical evidence",
1103                    group.subject, group.count
1104                ),
1105                SafetySeverityV1::Warning,
1106                Some(group.subject),
1107            ));
1108        }
1109    }
1110
1111    for group in conflicting_assignment_groups(
1112        &plan.expected_canisters,
1113        |planned| planned.canister_id.clone(),
1114        |planned| planned.role.clone(),
1115        ",",
1116    ) {
1117        id_conflicts.insert(group.subject.clone());
1118        controller_diff.push(diff_item(
1119            "planned_canister_id_conflict",
1120            &group.subject,
1121            Some("one planned role".to_string()),
1122            Some(group.evidence_label.clone()),
1123            SafetySeverityV1::HardFailure,
1124        ));
1125        hard_failures.push(finding(
1126            "planned_canister_id_conflict",
1127            format!(
1128                "planned canister id {} is assigned to conflicting roles {}",
1129                group.subject, group.evidence_label
1130            ),
1131            SafetySeverityV1::HardFailure,
1132            Some(group.subject),
1133        ));
1134    }
1135
1136    PlannedCanisterConflicts {
1137        role_conflicts,
1138        id_conflicts,
1139    }
1140}
1141
1142fn planned_canister_evidence_label(planned: &ExpectedCanisterV1) -> String {
1143    format!(
1144        "role={};id={};control={:?}",
1145        planned.role,
1146        planned.canister_id.as_deref().unwrap_or("<none>"),
1147        planned.control_class
1148    )
1149}
1150
1151fn record_missing_canister(
1152    expected: &ExpectedCanisterV1,
1153    controller_diff: &mut Vec<DiffItemV1>,
1154    hard_failures: &mut Vec<SafetyFindingV1>,
1155    warnings: &mut Vec<SafetyFindingV1>,
1156) {
1157    let severity = if expected.canister_id.is_some() {
1158        SafetySeverityV1::HardFailure
1159    } else {
1160        SafetySeverityV1::Warning
1161    };
1162    controller_diff.push(diff_item(
1163        "canister",
1164        &expected.role,
1165        expected.canister_id.clone(),
1166        None,
1167        severity,
1168    ));
1169    let finding = finding(
1170        if expected.canister_id.is_some() {
1171            "canister_missing"
1172        } else {
1173            "canister_unobserved"
1174        },
1175        format!("missing observed canister for role {}", expected.role),
1176        severity,
1177        Some(expected.role.clone()),
1178    );
1179    if expected.canister_id.is_some() {
1180        hard_failures.push(finding);
1181    } else {
1182        warnings.push(finding);
1183    }
1184}
1185
1186fn record_unsafe_canister_control_class(
1187    expected: &ExpectedCanisterV1,
1188    observed: &ObservedCanisterV1,
1189    controller_diff: &mut Vec<DiffItemV1>,
1190    hard_failures: &mut Vec<SafetyFindingV1>,
1191) {
1192    if !matches!(
1193        observed.control_class,
1194        CanisterControlClassV1::UnknownUnsafe | CanisterControlClassV1::UserControlled
1195    ) || expected.control_class != CanisterControlClassV1::DeploymentControlled
1196    {
1197        return;
1198    }
1199    controller_diff.push(diff_item(
1200        "control_class",
1201        &expected.role,
1202        Some("DeploymentControlled".to_string()),
1203        Some(format!("{:?}", observed.control_class)),
1204        SafetySeverityV1::HardFailure,
1205    ));
1206    hard_failures.push(finding(
1207        "unsafe_control_class",
1208        format!("role {} has unsafe observed control class", expected.role),
1209        SafetySeverityV1::HardFailure,
1210        Some(expected.role.clone()),
1211    ));
1212}
1213
1214fn record_ambiguous_canister_role(
1215    expected: &ExpectedCanisterV1,
1216    observed_matches: &[&ObservedCanisterV1],
1217    controller_diff: &mut Vec<DiffItemV1>,
1218    hard_failures: &mut Vec<SafetyFindingV1>,
1219) {
1220    let observed_ids = observed_matches
1221        .iter()
1222        .map(|canister| canister.canister_id.as_str())
1223        .collect::<Vec<_>>()
1224        .join(",");
1225    controller_diff.push(diff_item(
1226        "canister_role_ambiguous",
1227        &expected.role,
1228        Some("one observed canister".to_string()),
1229        Some(observed_ids.clone()),
1230        SafetySeverityV1::HardFailure,
1231    ));
1232    hard_failures.push(finding(
1233        "canister_role_ambiguous",
1234        format!(
1235            "expected role {} has multiple observed canisters: {observed_ids}",
1236            expected.role
1237        ),
1238        SafetySeverityV1::HardFailure,
1239        Some(expected.role.clone()),
1240    ));
1241}
1242
1243fn compare_observed_role(
1244    expected: &ExpectedCanisterV1,
1245    observed: &ObservedCanisterV1,
1246    controller_diff: &mut Vec<DiffItemV1>,
1247    hard_failures: &mut Vec<SafetyFindingV1>,
1248) {
1249    let Some(observed_role) = observed.role.as_deref() else {
1250        return;
1251    };
1252    if observed_role == expected.role {
1253        return;
1254    }
1255    controller_diff.push(diff_item(
1256        "role_mismatch",
1257        &expected.role,
1258        Some(expected.role.clone()),
1259        Some(observed_role.to_string()),
1260        SafetySeverityV1::HardFailure,
1261    ));
1262    hard_failures.push(finding(
1263        "canister_role_mismatch",
1264        format!(
1265            "expected canister {} to have role {}, observed role {observed_role}",
1266            observed.canister_id, expected.role
1267        ),
1268        SafetySeverityV1::HardFailure,
1269        Some(expected.role.clone()),
1270    ));
1271}
1272
1273fn warn_extra_observed_canisters(
1274    plan: &DeploymentPlanV1,
1275    inventory: &DeploymentInventoryV1,
1276    controller_diff: &mut Vec<DiffItemV1>,
1277    warnings: &mut Vec<SafetyFindingV1>,
1278    matched_observed: &BTreeSet<&str>,
1279) {
1280    let expected_pool_roles = plan
1281        .expected_pool
1282        .iter()
1283        .filter_map(|pool| pool.role.as_deref())
1284        .collect::<BTreeSet<_>>();
1285
1286    for observed in &inventory.observed_canisters {
1287        if matched_observed.contains(observed.canister_id.as_str()) {
1288            continue;
1289        }
1290        if let Some(role) = observed.role.as_deref()
1291            && expected_pool_roles.contains(role)
1292        {
1293            continue;
1294        }
1295        let subject = observed_canister_subject(observed);
1296        controller_diff.push(diff_item(
1297            "canister_extra",
1298            &subject,
1299            None,
1300            Some(observed.canister_id.clone()),
1301            SafetySeverityV1::Warning,
1302        ));
1303        warnings.push(finding(
1304            "extra_canister_observed",
1305            format!("observed undeclared canister {subject}"),
1306            SafetySeverityV1::Warning,
1307            Some(subject),
1308        ));
1309    }
1310}
1311
1312fn observed_canister_subject(observed: &ObservedCanisterV1) -> String {
1313    observed
1314        .role
1315        .clone()
1316        .unwrap_or_else(|| observed.canister_id.clone())
1317}
1318
1319fn compare_observed_canister_pool_role_conflicts(
1320    inventory: &DeploymentInventoryV1,
1321    pool_diff: &mut Vec<DiffItemV1>,
1322    hard_failures: &mut Vec<SafetyFindingV1>,
1323) {
1324    let mut pools_by_id = BTreeMap::<&str, Vec<&ObservedPoolCanisterV1>>::new();
1325    for observed_pool in &inventory.observed_pool {
1326        pools_by_id
1327            .entry(observed_pool.canister_id.as_str())
1328            .or_default()
1329            .push(observed_pool);
1330    }
1331
1332    for observed_canister in &inventory.observed_canisters {
1333        let Some(canister_role) = observed_canister.role.as_deref() else {
1334            continue;
1335        };
1336        let Some(observed_pools) = pools_by_id.get(observed_canister.canister_id.as_str()) else {
1337            continue;
1338        };
1339        for observed_pool in observed_pools {
1340            let Some(pool_role) = observed_pool.role.as_deref() else {
1341                continue;
1342            };
1343            if pool_role == canister_role {
1344                continue;
1345            }
1346            let observed_label = format!(
1347                "canister={};pool={}",
1348                observed_canister_subject(observed_canister),
1349                observed_pool_subject(observed_pool)
1350            );
1351            pool_diff.push(diff_item(
1352                "canister_pool_role_conflict",
1353                &observed_canister.canister_id,
1354                None,
1355                Some(observed_label.clone()),
1356                SafetySeverityV1::HardFailure,
1357            ));
1358            hard_failures.push(finding(
1359                "canister_pool_role_conflict",
1360                format!(
1361                    "observed canister {} has conflicting canister/pool roles {observed_label}",
1362                    observed_canister.canister_id
1363                ),
1364                SafetySeverityV1::HardFailure,
1365                Some(observed_canister.canister_id.clone()),
1366            ));
1367        }
1368    }
1369}
1370
1371fn compare_pools(
1372    plan: &DeploymentPlanV1,
1373    inventory: &DeploymentInventoryV1,
1374    pool_diff: &mut Vec<DiffItemV1>,
1375    hard_failures: &mut Vec<SafetyFindingV1>,
1376    warnings: &mut Vec<SafetyFindingV1>,
1377) {
1378    let planned_conflicts =
1379        compare_planned_pool_conflicts(plan, pool_diff, hard_failures, warnings);
1380    compare_observed_pool_id_conflicts(inventory, pool_diff, hard_failures, warnings);
1381    let mut matched_observed = BTreeSet::new();
1382    let mut compared_planned = BTreeSet::new();
1383    for expected in &plan.expected_pool {
1384        if planned_conflicts
1385            .subject_conflicts
1386            .contains(&expected_pool_subject(expected))
1387            || expected
1388                .canister_id
1389                .as_ref()
1390                .is_some_and(|id| planned_conflicts.id_conflicts.contains(id))
1391            || !compared_planned.insert(planned_pool_evidence_label(expected))
1392        {
1393            continue;
1394        }
1395        compare_expected_pool(
1396            expected,
1397            inventory,
1398            pool_diff,
1399            hard_failures,
1400            warnings,
1401            &mut matched_observed,
1402        );
1403    }
1404
1405    for observed in &inventory.observed_pool {
1406        warn_extra_observed_pool(plan, observed, pool_diff, warnings, &matched_observed);
1407    }
1408}
1409
1410struct PlannedPoolConflicts {
1411    subject_conflicts: BTreeSet<String>,
1412    id_conflicts: BTreeSet<String>,
1413}
1414
1415fn compare_planned_pool_conflicts(
1416    plan: &DeploymentPlanV1,
1417    pool_diff: &mut Vec<DiffItemV1>,
1418    hard_failures: &mut Vec<SafetyFindingV1>,
1419    warnings: &mut Vec<SafetyFindingV1>,
1420) -> PlannedPoolConflicts {
1421    let mut subject_conflicts = BTreeSet::new();
1422    let mut id_conflicts = BTreeSet::new();
1423
1424    for group in duplicate_evidence_groups(
1425        &plan.expected_pool,
1426        expected_pool_subject,
1427        planned_pool_evidence_label,
1428        " | ",
1429    ) {
1430        if group.is_conflict {
1431            subject_conflicts.insert(group.subject.clone());
1432            pool_diff.push(diff_item(
1433                "planned_pool_conflict",
1434                &group.subject,
1435                Some("one planned pool canister".to_string()),
1436                Some(group.evidence_label.clone()),
1437                SafetySeverityV1::HardFailure,
1438            ));
1439            hard_failures.push(finding(
1440                "planned_pool_conflict",
1441                format!(
1442                    "planned pool {} has conflicting evidence: {}",
1443                    group.subject, group.evidence_label
1444                ),
1445                SafetySeverityV1::HardFailure,
1446                Some(group.subject),
1447            ));
1448        } else {
1449            pool_diff.push(diff_item(
1450                "planned_pool_duplicate",
1451                &group.subject,
1452                Some(group.evidence_label.clone()),
1453                Some(group.count.to_string()),
1454                SafetySeverityV1::Warning,
1455            ));
1456            warnings.push(finding(
1457                "duplicate_planned_pool",
1458                format!(
1459                    "planned pool {} was declared {} times with identical evidence",
1460                    group.subject, group.count
1461                ),
1462                SafetySeverityV1::Warning,
1463                Some(group.subject),
1464            ));
1465        }
1466    }
1467
1468    for group in conflicting_assignment_groups(
1469        &plan.expected_pool,
1470        |planned| planned.canister_id.clone(),
1471        expected_pool_subject,
1472        ",",
1473    ) {
1474        id_conflicts.insert(group.subject.clone());
1475        pool_diff.push(diff_item(
1476            "planned_pool_id_conflict",
1477            &group.subject,
1478            Some("one planned pool identity".to_string()),
1479            Some(group.evidence_label.clone()),
1480            SafetySeverityV1::HardFailure,
1481        ));
1482        hard_failures.push(finding(
1483            "planned_pool_id_conflict",
1484            format!(
1485                "planned pool id {} is assigned to conflicting identities {}",
1486                group.subject, group.evidence_label
1487            ),
1488            SafetySeverityV1::HardFailure,
1489            Some(group.subject),
1490        ));
1491    }
1492
1493    PlannedPoolConflicts {
1494        subject_conflicts,
1495        id_conflicts,
1496    }
1497}
1498
1499fn planned_pool_evidence_label(planned: &ExpectedPoolCanisterV1) -> String {
1500    format!(
1501        "pool={};role={};id={}",
1502        planned.pool,
1503        planned.role.as_deref().unwrap_or("<none>"),
1504        planned.canister_id.as_deref().unwrap_or("<none>")
1505    )
1506}
1507
1508fn compare_observed_pool_id_conflicts(
1509    inventory: &DeploymentInventoryV1,
1510    pool_diff: &mut Vec<DiffItemV1>,
1511    hard_failures: &mut Vec<SafetyFindingV1>,
1512    warnings: &mut Vec<SafetyFindingV1>,
1513) {
1514    for group in duplicate_evidence_groups(
1515        &inventory.observed_pool,
1516        |observed| observed.canister_id.as_str().to_string(),
1517        observed_pool_subject,
1518        ",",
1519    ) {
1520        if group.is_conflict {
1521            pool_diff.push(diff_item(
1522                "pool_canister_id_conflict",
1523                &group.subject,
1524                None,
1525                Some(group.evidence_label.clone()),
1526                SafetySeverityV1::HardFailure,
1527            ));
1528            hard_failures.push(finding(
1529                "pool_canister_id_conflict",
1530                format!(
1531                    "observed pool canister {} has conflicting pool identities {}",
1532                    group.subject, group.evidence_label
1533                ),
1534                SafetySeverityV1::HardFailure,
1535                Some(group.subject),
1536            ));
1537        } else {
1538            pool_diff.push(diff_item(
1539                "pool_canister_duplicate",
1540                &group.subject,
1541                Some(group.evidence_label.clone()),
1542                Some(group.count.to_string()),
1543                SafetySeverityV1::Warning,
1544            ));
1545            warnings.push(finding(
1546                "duplicate_pool_canister_observed",
1547                format!(
1548                    "observed pool canister {} was reported {} times for {}",
1549                    group.subject, group.count, group.evidence_label
1550                ),
1551                SafetySeverityV1::Warning,
1552                Some(group.subject),
1553            ));
1554        }
1555    }
1556}
1557
1558fn compare_expected_pool<'a>(
1559    expected: &ExpectedPoolCanisterV1,
1560    inventory: &'a DeploymentInventoryV1,
1561    pool_diff: &mut Vec<DiffItemV1>,
1562    hard_failures: &mut Vec<SafetyFindingV1>,
1563    warnings: &mut Vec<SafetyFindingV1>,
1564    matched_observed: &mut BTreeSet<&'a str>,
1565) {
1566    let observed = expected
1567        .canister_id
1568        .as_ref()
1569        .and_then(|id| {
1570            inventory
1571                .observed_pool
1572                .iter()
1573                .find(|pool| &pool.canister_id == id)
1574        })
1575        .or_else(|| {
1576            inventory
1577                .observed_pool
1578                .iter()
1579                .find(|pool| pool_matches_expected_pool(pool, expected))
1580        });
1581    let Some(observed) = observed else {
1582        record_missing_pool(expected, pool_diff, hard_failures, warnings);
1583        return;
1584    };
1585
1586    matched_observed.insert(observed.canister_id.as_str());
1587    record_pool_id_mismatch(expected, observed, pool_diff, hard_failures);
1588    record_unsafe_pool_control_class(observed, pool_diff, hard_failures);
1589}
1590
1591fn record_missing_pool(
1592    expected: &ExpectedPoolCanisterV1,
1593    pool_diff: &mut Vec<DiffItemV1>,
1594    hard_failures: &mut Vec<SafetyFindingV1>,
1595    warnings: &mut Vec<SafetyFindingV1>,
1596) {
1597    let severity = if expected.canister_id.is_some() {
1598        SafetySeverityV1::HardFailure
1599    } else {
1600        SafetySeverityV1::Warning
1601    };
1602    let subject = expected_pool_subject(expected);
1603    pool_diff.push(diff_item(
1604        "pool_canister",
1605        &subject,
1606        expected.canister_id.clone(),
1607        None,
1608        severity,
1609    ));
1610    let finding = finding(
1611        if expected.canister_id.is_some() {
1612            "pool_canister_missing"
1613        } else {
1614            "pool_canister_unobserved"
1615        },
1616        format!("missing observed pool canister for {subject}"),
1617        severity,
1618        Some(subject),
1619    );
1620    if expected.canister_id.is_some() {
1621        hard_failures.push(finding);
1622    } else {
1623        warnings.push(finding);
1624    }
1625}
1626
1627fn record_pool_id_mismatch(
1628    expected: &ExpectedPoolCanisterV1,
1629    observed: &ObservedPoolCanisterV1,
1630    pool_diff: &mut Vec<DiffItemV1>,
1631    hard_failures: &mut Vec<SafetyFindingV1>,
1632) {
1633    if let Some(expected_id) = expected.canister_id.as_ref()
1634        && observed.canister_id != *expected_id
1635    {
1636        let subject = observed_pool_subject(observed);
1637        pool_diff.push(diff_item(
1638            "pool_canister_id",
1639            &subject,
1640            Some(expected_id.clone()),
1641            Some(observed.canister_id.clone()),
1642            SafetySeverityV1::HardFailure,
1643        ));
1644        hard_failures.push(finding(
1645            "pool_canister_id_mismatch",
1646            format!(
1647                "pool canister {subject} has observed id {}, expected {expected_id}",
1648                observed.canister_id
1649            ),
1650            SafetySeverityV1::HardFailure,
1651            Some(subject),
1652        ));
1653    }
1654}
1655
1656fn record_unsafe_pool_control_class(
1657    observed: &ObservedPoolCanisterV1,
1658    pool_diff: &mut Vec<DiffItemV1>,
1659    hard_failures: &mut Vec<SafetyFindingV1>,
1660) {
1661    if !matches!(
1662        observed.control_class,
1663        CanisterControlClassV1::UnknownUnsafe | CanisterControlClassV1::UserControlled
1664    ) {
1665        return;
1666    }
1667    let subject = observed_pool_subject(observed);
1668    pool_diff.push(diff_item(
1669        "pool_control_class",
1670        &subject,
1671        Some("CanicManagedPool".to_string()),
1672        Some(format!("{:?}", observed.control_class)),
1673        SafetySeverityV1::HardFailure,
1674    ));
1675    hard_failures.push(finding(
1676        "unsafe_pool_control_class",
1677        format!("pool canister {subject} has unsafe observed control class"),
1678        SafetySeverityV1::HardFailure,
1679        Some(subject),
1680    ));
1681}
1682
1683fn warn_extra_observed_pool(
1684    plan: &DeploymentPlanV1,
1685    observed: &ObservedPoolCanisterV1,
1686    pool_diff: &mut Vec<DiffItemV1>,
1687    warnings: &mut Vec<SafetyFindingV1>,
1688    matched_observed: &BTreeSet<&str>,
1689) {
1690    if matched_observed.contains(observed.canister_id.as_str())
1691        || plan.expected_pool.iter().any(|expected| {
1692            expected.canister_id.as_ref() == Some(&observed.canister_id)
1693                || pool_matches_expected_pool(observed, expected)
1694        })
1695    {
1696        return;
1697    }
1698    let subject = observed_pool_subject(observed);
1699    pool_diff.push(diff_item(
1700        "pool_extra",
1701        &subject,
1702        None,
1703        Some(observed.canister_id.clone()),
1704        SafetySeverityV1::Warning,
1705    ));
1706    warnings.push(finding(
1707        "extra_pool_canister_observed",
1708        format!("observed undeclared pool canister {subject}"),
1709        SafetySeverityV1::Warning,
1710        Some(subject),
1711    ));
1712}
1713
1714fn pool_matches_expected_pool(
1715    observed: &ObservedPoolCanisterV1,
1716    expected: &ExpectedPoolCanisterV1,
1717) -> bool {
1718    observed.pool == expected.pool
1719        && expected
1720            .role
1721            .as_ref()
1722            .is_none_or(|role| observed.role.as_ref() == Some(role))
1723}
1724
1725fn expected_pool_subject(expected: &ExpectedPoolCanisterV1) -> String {
1726    expected.role.as_ref().map_or_else(
1727        || expected.pool.clone(),
1728        |role| format!("{}:{role}", expected.pool),
1729    )
1730}
1731
1732fn observed_pool_subject(observed: &ObservedPoolCanisterV1) -> String {
1733    observed.role.as_ref().map_or_else(
1734        || observed.pool.clone(),
1735        |role| format!("{}:{role}", observed.pool),
1736    )
1737}
1738
1739fn compare_role_controllers(
1740    plan: &DeploymentPlanV1,
1741    observed: &ObservedCanisterV1,
1742    controller_diff: &mut Vec<DiffItemV1>,
1743    hard_failures: &mut Vec<SafetyFindingV1>,
1744    warnings: &mut Vec<SafetyFindingV1>,
1745) {
1746    let role = observed.role.as_deref().unwrap_or("unknown");
1747    if observed.controllers.is_empty() && !observed_source_includes_live_status(observed) {
1748        warnings.push(finding(
1749            "controllers_unobserved",
1750            format!("controllers were not observed for role {role}"),
1751            SafetySeverityV1::Warning,
1752            Some(role.to_string()),
1753        ));
1754        return;
1755    }
1756    for expected in &plan.authority_profile.expected_controllers {
1757        if observed
1758            .controllers
1759            .iter()
1760            .any(|controller| controller == expected)
1761        {
1762            continue;
1763        }
1764        record_missing_expected_controller(
1765            role,
1766            expected,
1767            &observed.controllers,
1768            controller_diff,
1769            hard_failures,
1770        );
1771    }
1772
1773    for observed_controller in &observed.controllers {
1774        if is_declared_controller(plan, observed_controller) {
1775            continue;
1776        }
1777        record_extra_controller(role, observed_controller, plan, controller_diff, warnings);
1778    }
1779}
1780
1781fn record_missing_expected_controller(
1782    role: &str,
1783    expected: &str,
1784    observed_controllers: &[String],
1785    controller_diff: &mut Vec<DiffItemV1>,
1786    hard_failures: &mut Vec<SafetyFindingV1>,
1787) {
1788    controller_diff.push(diff_item(
1789        "controller_missing",
1790        role,
1791        Some(expected.to_string()),
1792        Some(controller_set_label(observed_controllers)),
1793        SafetySeverityV1::HardFailure,
1794    ));
1795    hard_failures.push(finding(
1796        "expected_controller_missing",
1797        format!("role {role} is missing expected controller {expected}"),
1798        SafetySeverityV1::HardFailure,
1799        Some(role.to_string()),
1800    ));
1801}
1802
1803fn record_extra_controller(
1804    role: &str,
1805    observed_controller: &str,
1806    plan: &DeploymentPlanV1,
1807    controller_diff: &mut Vec<DiffItemV1>,
1808    warnings: &mut Vec<SafetyFindingV1>,
1809) {
1810    controller_diff.push(diff_item(
1811        "controller_extra",
1812        role,
1813        Some(controller_set_label(
1814            &plan.authority_profile.expected_controllers,
1815        )),
1816        Some(observed_controller.to_string()),
1817        SafetySeverityV1::Warning,
1818    ));
1819    warnings.push(finding(
1820        "extra_controller_observed",
1821        format!("role {role} has controller outside the expected authority profile"),
1822        SafetySeverityV1::Warning,
1823        Some(role.to_string()),
1824    ));
1825}
1826
1827fn observed_source_includes_live_status(observed: &ObservedCanisterV1) -> bool {
1828    observed
1829        .role_assignment_source
1830        .as_deref()
1831        .is_some_and(|source| source.contains("icp_canister_status"))
1832}
1833
1834fn is_declared_controller(plan: &DeploymentPlanV1, controller: &str) -> bool {
1835    plan.authority_profile
1836        .expected_controllers
1837        .iter()
1838        .chain(plan.authority_profile.staging_controllers.iter())
1839        .chain(plan.authority_profile.emergency_controllers.iter())
1840        .any(|expected| expected == controller)
1841}
1842
1843fn is_staging_or_emergency_controller(plan: &DeploymentPlanV1, controller: &str) -> bool {
1844    plan.authority_profile
1845        .staging_controllers
1846        .iter()
1847        .chain(plan.authority_profile.emergency_controllers.iter())
1848        .any(|declared| declared == controller)
1849}
1850
1851fn controller_set_label(controllers: &[String]) -> String {
1852    if controllers.is_empty() {
1853        return "<none>".to_string();
1854    }
1855    controllers.join(",")
1856}
1857
1858fn compare_module_hashes(
1859    plan: &DeploymentPlanV1,
1860    inventory: &DeploymentInventoryV1,
1861    module_hash_diff: &mut Vec<DiffItemV1>,
1862    hard_failures: &mut Vec<SafetyFindingV1>,
1863    warnings: &mut Vec<SafetyFindingV1>,
1864) {
1865    for artifact in &plan.role_artifacts {
1866        let Some(expected) = artifact.installed_module_hash.as_ref() else {
1867            continue;
1868        };
1869        let Some(observed_canister) = observed_canister_for_module_hash(
1870            plan,
1871            inventory,
1872            &artifact.role,
1873            module_hash_diff,
1874            hard_failures,
1875        ) else {
1876            continue;
1877        };
1878        match observed_canister.module_hash.as_ref() {
1879            Some(observed) if observed != expected => record_module_hash_mismatch(
1880                &artifact.role,
1881                expected,
1882                observed,
1883                module_hash_diff,
1884                hard_failures,
1885            ),
1886            None => record_module_hash_unobserved(&artifact.role, warnings),
1887            _ => {}
1888        }
1889    }
1890}
1891
1892fn observed_canister_for_module_hash<'a>(
1893    plan: &DeploymentPlanV1,
1894    inventory: &'a DeploymentInventoryV1,
1895    role: &str,
1896    module_hash_diff: &mut Vec<DiffItemV1>,
1897    hard_failures: &mut Vec<SafetyFindingV1>,
1898) -> Option<&'a ObservedCanisterV1> {
1899    if let Some(expected_id) = expected_canister_id_for_role(plan, role) {
1900        return inventory
1901            .observed_canisters
1902            .iter()
1903            .find(|canister| canister.canister_id == expected_id);
1904    }
1905
1906    let role_matches = inventory
1907        .observed_canisters
1908        .iter()
1909        .filter(|canister| canister.role.as_deref() == Some(role))
1910        .collect::<Vec<_>>();
1911    if role_matches.len() > 1 {
1912        record_ambiguous_module_hash_role(role, &role_matches, module_hash_diff, hard_failures);
1913        return None;
1914    }
1915
1916    role_matches.into_iter().next()
1917}
1918
1919fn record_module_hash_mismatch(
1920    role: &str,
1921    expected: &str,
1922    observed: &str,
1923    module_hash_diff: &mut Vec<DiffItemV1>,
1924    hard_failures: &mut Vec<SafetyFindingV1>,
1925) {
1926    module_hash_diff.push(diff_item(
1927        "installed_module_hash",
1928        role,
1929        Some(expected.to_string()),
1930        Some(observed.to_string()),
1931        SafetySeverityV1::HardFailure,
1932    ));
1933    hard_failures.push(finding(
1934        "installed_module_hash_mismatch",
1935        format!("installed module hash differs for role {role}"),
1936        SafetySeverityV1::HardFailure,
1937        Some(role.to_string()),
1938    ));
1939}
1940
1941fn record_module_hash_unobserved(role: &str, warnings: &mut Vec<SafetyFindingV1>) {
1942    warnings.push(finding(
1943        "installed_module_hash_unobserved",
1944        format!("installed module hash was not observed for role {role}"),
1945        SafetySeverityV1::Warning,
1946        Some(role.to_string()),
1947    ));
1948}
1949
1950fn record_ambiguous_module_hash_role(
1951    role: &str,
1952    role_matches: &[&ObservedCanisterV1],
1953    module_hash_diff: &mut Vec<DiffItemV1>,
1954    hard_failures: &mut Vec<SafetyFindingV1>,
1955) {
1956    let observed_ids = role_matches
1957        .iter()
1958        .map(|canister| canister.canister_id.as_str())
1959        .collect::<Vec<_>>()
1960        .join(",");
1961    module_hash_diff.push(diff_item(
1962        "installed_module_hash_ambiguous",
1963        role,
1964        Some("one observed canister".to_string()),
1965        Some(observed_ids.clone()),
1966        SafetySeverityV1::HardFailure,
1967    ));
1968    hard_failures.push(finding(
1969        "installed_module_hash_ambiguous",
1970        format!(
1971            "installed module hash for role {role} has multiple observed canisters: {observed_ids}"
1972        ),
1973        SafetySeverityV1::HardFailure,
1974        Some(role.to_string()),
1975    ));
1976}
1977
1978fn expected_canister_id_for_role<'a>(plan: &'a DeploymentPlanV1, role: &str) -> Option<&'a str> {
1979    plan.expected_canisters
1980        .iter()
1981        .find(|canister| canister.role == role)
1982        .and_then(|canister| canister.canister_id.as_deref())
1983}
1984
1985fn compare_raw_config(
1986    plan: &DeploymentPlanV1,
1987    inventory: &DeploymentInventoryV1,
1988    embedded_config_diff: &mut Vec<DiffItemV1>,
1989    hard_failures: &mut Vec<SafetyFindingV1>,
1990) {
1991    let mut expected = plan
1992        .role_artifacts
1993        .iter()
1994        .filter_map(|artifact| artifact.raw_config_sha256.as_ref())
1995        .collect::<Vec<_>>();
1996    expected.sort_unstable();
1997    expected.dedup();
1998    let [expected] = expected.as_slice() else {
1999        if expected.len() > 1 {
2000            hard_failures.push(finding(
2001                "raw_config_plan_inconsistent",
2002                "planned role artifacts disagree on raw config digest",
2003                SafetySeverityV1::HardFailure,
2004                Some("role_artifacts.raw_config_sha256".to_string()),
2005            ));
2006        }
2007        return;
2008    };
2009
2010    if let Some(observed) = &inventory.local_config.raw_config_sha256
2011        && observed != *expected
2012    {
2013        record_raw_config_mismatch(expected, observed, embedded_config_diff, hard_failures);
2014    }
2015}
2016
2017fn record_raw_config_mismatch(
2018    expected: &str,
2019    observed: &str,
2020    embedded_config_diff: &mut Vec<DiffItemV1>,
2021    hard_failures: &mut Vec<SafetyFindingV1>,
2022) {
2023    embedded_config_diff.push(diff_item(
2024        "raw_config_sha256",
2025        "deployment",
2026        Some(expected.to_string()),
2027        Some(observed.to_string()),
2028        SafetySeverityV1::HardFailure,
2029    ));
2030    hard_failures.push(finding(
2031        "raw_config_digest_mismatch",
2032        "raw local config digest changed during deployment truth check",
2033        SafetySeverityV1::HardFailure,
2034        Some("local_config.raw_sha256".to_string()),
2035    ));
2036}
2037
2038fn compare_embedded_config(
2039    plan: &DeploymentPlanV1,
2040    inventory: &DeploymentInventoryV1,
2041    embedded_config_diff: &mut Vec<DiffItemV1>,
2042    hard_failures: &mut Vec<SafetyFindingV1>,
2043    warnings: &mut Vec<SafetyFindingV1>,
2044) {
2045    let Some(expected) = &plan.deployment_identity.canonical_runtime_config_digest else {
2046        return;
2047    };
2048    match &inventory.local_config.canonical_embedded_config_sha256 {
2049        Some(observed) if observed != expected => {
2050            record_canonical_config_mismatch(
2051                expected,
2052                observed,
2053                embedded_config_diff,
2054                hard_failures,
2055            );
2056        }
2057        None => record_canonical_config_unobserved(warnings),
2058        _ => {}
2059    }
2060}
2061
2062fn record_canonical_config_mismatch(
2063    expected: &str,
2064    observed: &str,
2065    embedded_config_diff: &mut Vec<DiffItemV1>,
2066    hard_failures: &mut Vec<SafetyFindingV1>,
2067) {
2068    embedded_config_diff.push(diff_item(
2069        "canonical_config",
2070        "deployment",
2071        Some(expected.to_string()),
2072        Some(observed.to_string()),
2073        SafetySeverityV1::HardFailure,
2074    ));
2075    hard_failures.push(finding(
2076        "canonical_config_mismatch",
2077        "canonical runtime config digest differs from the plan",
2078        SafetySeverityV1::HardFailure,
2079        Some("local_config".to_string()),
2080    ));
2081}
2082
2083fn record_canonical_config_unobserved(warnings: &mut Vec<SafetyFindingV1>) {
2084    warnings.push(finding(
2085        "canonical_config_unobserved",
2086        "canonical runtime config digest was not observed",
2087        SafetySeverityV1::Warning,
2088        Some("local_config".to_string()),
2089    ));
2090}
2091
2092fn compare_verifier_readiness(
2093    plan: &DeploymentPlanV1,
2094    inventory: &DeploymentInventoryV1,
2095    verifier_readiness_diff: &mut Vec<DiffItemV1>,
2096    hard_failures: &mut Vec<SafetyFindingV1>,
2097    warnings: &mut Vec<SafetyFindingV1>,
2098) {
2099    if !plan.expected_verifier_readiness.required {
2100        return;
2101    }
2102    if inventory.observed_verifier_readiness.status == ObservationStatusV1::NotObserved {
2103        verifier_readiness_diff.push(diff_item(
2104            "verifier_readiness",
2105            "deployment",
2106            Some("required".to_string()),
2107            Some("not_observed".to_string()),
2108            SafetySeverityV1::Warning,
2109        ));
2110        warnings.push(finding(
2111            "verifier_readiness_unobserved",
2112            "verifier readiness was required but not observed",
2113            SafetySeverityV1::Warning,
2114            Some("verifier_readiness".to_string()),
2115        ));
2116    }
2117
2118    let planned_conflicting_roles = compare_planned_verifier_epoch_conflicts(
2119        plan,
2120        verifier_readiness_diff,
2121        hard_failures,
2122        warnings,
2123    );
2124    let conflicting_roles = compare_observed_verifier_epoch_conflicts(
2125        inventory,
2126        verifier_readiness_diff,
2127        hard_failures,
2128        warnings,
2129    );
2130    let mut observed_by_role = BTreeMap::new();
2131    for epoch in &inventory.observed_verifier_readiness.role_epochs {
2132        if conflicting_roles.contains(&epoch.role) {
2133            continue;
2134        }
2135        observed_by_role.entry(epoch.role.as_str()).or_insert(epoch);
2136    }
2137    let mut compared_roles = BTreeSet::new();
2138    for expected in &plan.expected_verifier_readiness.expected_role_epochs {
2139        if planned_conflicting_roles.contains(&expected.role)
2140            || conflicting_roles.contains(&expected.role)
2141            || !compared_roles.insert(expected.role.as_str())
2142        {
2143            continue;
2144        }
2145        let observed = observed_by_role.get(expected.role.as_str());
2146        if let Some(observed_epoch) = observed.and_then(|observed| {
2147            (observed.status == ObservationStatusV1::Observed)
2148                .then_some(observed.observed_epoch)
2149                .flatten()
2150        }) {
2151            if observed_epoch < expected.minimum_epoch {
2152                record_stale_verifier_role_epoch(
2153                    expected,
2154                    observed_epoch,
2155                    verifier_readiness_diff,
2156                    hard_failures,
2157                );
2158            }
2159        } else {
2160            record_unobserved_verifier_role_epoch(expected, verifier_readiness_diff, warnings);
2161        }
2162    }
2163}
2164
2165fn record_stale_verifier_role_epoch(
2166    expected: &RoleEpochExpectationV1,
2167    observed_epoch: u64,
2168    verifier_readiness_diff: &mut Vec<DiffItemV1>,
2169    hard_failures: &mut Vec<SafetyFindingV1>,
2170) {
2171    verifier_readiness_diff.push(diff_item(
2172        "verifier_role_epoch",
2173        &expected.role,
2174        Some(expected.minimum_epoch.to_string()),
2175        Some(observed_epoch.to_string()),
2176        SafetySeverityV1::HardFailure,
2177    ));
2178    hard_failures.push(finding(
2179        "verifier_role_epoch_stale",
2180        format!(
2181            "verifier role {} has epoch {observed_epoch}, expected at least {}",
2182            expected.role, expected.minimum_epoch
2183        ),
2184        SafetySeverityV1::HardFailure,
2185        Some(expected.role.clone()),
2186    ));
2187}
2188
2189fn record_unobserved_verifier_role_epoch(
2190    expected: &RoleEpochExpectationV1,
2191    verifier_readiness_diff: &mut Vec<DiffItemV1>,
2192    warnings: &mut Vec<SafetyFindingV1>,
2193) {
2194    verifier_readiness_diff.push(diff_item(
2195        "verifier_role_epoch",
2196        &expected.role,
2197        Some(expected.minimum_epoch.to_string()),
2198        Some("not_observed".to_string()),
2199        SafetySeverityV1::Warning,
2200    ));
2201    warnings.push(finding(
2202        "verifier_role_epoch_unobserved",
2203        format!("verifier role {} epoch was not observed", expected.role),
2204        SafetySeverityV1::Warning,
2205        Some(expected.role.clone()),
2206    ));
2207}
2208
2209fn compare_planned_verifier_epoch_conflicts(
2210    plan: &DeploymentPlanV1,
2211    verifier_readiness_diff: &mut Vec<DiffItemV1>,
2212    hard_failures: &mut Vec<SafetyFindingV1>,
2213    warnings: &mut Vec<SafetyFindingV1>,
2214) -> BTreeSet<String> {
2215    let mut conflicting_roles = BTreeSet::new();
2216    for group in duplicate_evidence_groups(
2217        &plan.expected_verifier_readiness.expected_role_epochs,
2218        |expected| expected.role.as_str().to_string(),
2219        |expected| expected.minimum_epoch.to_string(),
2220        ",",
2221    ) {
2222        if group.is_conflict {
2223            conflicting_roles.insert(group.subject.clone());
2224            verifier_readiness_diff.push(diff_item(
2225                "planned_verifier_role_epoch_conflict",
2226                &group.subject,
2227                Some("one minimum epoch".to_string()),
2228                Some(group.evidence_label.clone()),
2229                SafetySeverityV1::HardFailure,
2230            ));
2231            hard_failures.push(finding(
2232                "planned_verifier_role_epoch_conflict",
2233                format!(
2234                    "planned verifier role {} has conflicting minimum epochs: {}",
2235                    group.subject, group.evidence_label
2236                ),
2237                SafetySeverityV1::HardFailure,
2238                Some(group.subject),
2239            ));
2240        } else {
2241            verifier_readiness_diff.push(diff_item(
2242                "planned_verifier_role_epoch_duplicate",
2243                &group.subject,
2244                Some(group.evidence_label.clone()),
2245                Some(group.count.to_string()),
2246                SafetySeverityV1::Warning,
2247            ));
2248            warnings.push(finding(
2249                "duplicate_planned_verifier_role_epoch",
2250                format!(
2251                    "planned verifier role {} epoch was declared {} times with identical evidence",
2252                    group.subject, group.count
2253                ),
2254                SafetySeverityV1::Warning,
2255                Some(group.subject),
2256            ));
2257        }
2258    }
2259    conflicting_roles
2260}
2261
2262fn compare_observed_verifier_epoch_conflicts(
2263    inventory: &DeploymentInventoryV1,
2264    verifier_readiness_diff: &mut Vec<DiffItemV1>,
2265    hard_failures: &mut Vec<SafetyFindingV1>,
2266    warnings: &mut Vec<SafetyFindingV1>,
2267) -> BTreeSet<String> {
2268    let mut conflicting_roles = BTreeSet::new();
2269    for group in duplicate_evidence_groups(
2270        &inventory.observed_verifier_readiness.role_epochs,
2271        |observed| observed.role.as_str().to_string(),
2272        verifier_epoch_evidence_label,
2273        ",",
2274    ) {
2275        if group.is_conflict {
2276            conflicting_roles.insert(group.subject.clone());
2277            verifier_readiness_diff.push(diff_item(
2278                "verifier_role_epoch_conflict",
2279                &group.subject,
2280                Some("one epoch observation".to_string()),
2281                Some(group.evidence_label.clone()),
2282                SafetySeverityV1::HardFailure,
2283            ));
2284            hard_failures.push(finding(
2285                "verifier_role_epoch_conflict",
2286                format!(
2287                    "verifier role {} has conflicting epoch observations: {}",
2288                    group.subject, group.evidence_label
2289                ),
2290                SafetySeverityV1::HardFailure,
2291                Some(group.subject),
2292            ));
2293        } else {
2294            verifier_readiness_diff.push(diff_item(
2295                "verifier_role_epoch_duplicate",
2296                &group.subject,
2297                Some(group.evidence_label.clone()),
2298                Some(group.count.to_string()),
2299                SafetySeverityV1::Warning,
2300            ));
2301            warnings.push(finding(
2302                "duplicate_verifier_role_epoch_observed",
2303                format!(
2304                    "verifier role {} epoch was reported {} times with identical evidence",
2305                    group.subject, group.count
2306                ),
2307                SafetySeverityV1::Warning,
2308                Some(group.subject),
2309            ));
2310        }
2311    }
2312    conflicting_roles
2313}
2314
2315fn verifier_epoch_evidence_label(observed: &RoleEpochObservationV1) -> String {
2316    format!(
2317        "epoch={};status={:?}",
2318        observed
2319            .observed_epoch
2320            .map_or_else(|| "<none>".to_string(), |epoch| epoch.to_string()),
2321        observed.status
2322    )
2323}
2324
2325fn finding(
2326    code: impl Into<String>,
2327    message: impl Into<String>,
2328    severity: SafetySeverityV1,
2329    subject: Option<String>,
2330) -> SafetyFindingV1 {
2331    SafetyFindingV1 {
2332        code: code.into(),
2333        message: message.into(),
2334        severity,
2335        subject,
2336    }
2337}
2338
2339fn diff_item(
2340    category: impl Into<String>,
2341    subject: impl Into<String>,
2342    expected: Option<String>,
2343    observed: Option<String>,
2344    severity: SafetySeverityV1,
2345) -> DiffItemV1 {
2346    DiffItemV1 {
2347        category: category.into(),
2348        subject: subject.into(),
2349        expected,
2350        observed,
2351        severity,
2352    }
2353}
2354
2355fn duplicate_evidence_groups<T>(
2356    items: &[T],
2357    subject: impl Fn(&T) -> String,
2358    evidence: impl Fn(&T) -> String,
2359    evidence_separator: &str,
2360) -> Vec<DuplicateEvidenceGroup> {
2361    let mut groups = Vec::new();
2362    for (subject, entries) in group_by_subject(items, |item| Some(subject(item))) {
2363        if entries.len() <= 1 {
2364            continue;
2365        }
2366        let evidence_values = entries
2367            .iter()
2368            .map(|entry| evidence(entry))
2369            .collect::<BTreeSet<_>>();
2370        groups.push(DuplicateEvidenceGroup {
2371            subject,
2372            count: entries.len(),
2373            evidence_label: evidence_values
2374                .iter()
2375                .cloned()
2376                .collect::<Vec<_>>()
2377                .join(evidence_separator),
2378            is_conflict: evidence_values.len() > 1,
2379        });
2380    }
2381    groups
2382}
2383
2384fn conflicting_assignment_groups<T>(
2385    items: &[T],
2386    subject: impl Fn(&T) -> Option<String>,
2387    value: impl Fn(&T) -> String,
2388    value_separator: &str,
2389) -> Vec<DuplicateEvidenceGroup> {
2390    let mut groups = Vec::new();
2391    for (subject, entries) in group_by_subject(items, subject) {
2392        if entries.len() <= 1 {
2393            continue;
2394        }
2395        let values = entries
2396            .iter()
2397            .map(|entry| value(entry))
2398            .collect::<BTreeSet<_>>();
2399        if values.len() <= 1 {
2400            continue;
2401        }
2402        groups.push(DuplicateEvidenceGroup {
2403            subject,
2404            count: entries.len(),
2405            evidence_label: values
2406                .iter()
2407                .cloned()
2408                .collect::<Vec<_>>()
2409                .join(value_separator),
2410            is_conflict: true,
2411        });
2412    }
2413    groups
2414}
2415
2416fn group_by_subject<T>(
2417    items: &[T],
2418    subject: impl Fn(&T) -> Option<String>,
2419) -> BTreeMap<String, Vec<&T>> {
2420    let mut by_subject = BTreeMap::<String, Vec<&T>>::new();
2421    for item in items {
2422        if let Some(subject) = subject(item) {
2423            by_subject.entry(subject).or_default().push(item);
2424        }
2425    }
2426    by_subject
2427}
2428
2429const fn safety_status(
2430    hard_failures: &[SafetyFindingV1],
2431    warnings: &[SafetyFindingV1],
2432) -> SafetyStatusV1 {
2433    if !hard_failures.is_empty() {
2434        SafetyStatusV1::Blocked
2435    } else if !warnings.is_empty() {
2436        SafetyStatusV1::Warning
2437    } else {
2438        SafetyStatusV1::Safe
2439    }
2440}
2441
2442fn resume_safety_reasons(
2443    hard_failures: &[SafetyFindingV1],
2444    warnings: &[SafetyFindingV1],
2445) -> Vec<String> {
2446    if !hard_failures.is_empty() {
2447        return hard_failures
2448            .iter()
2449            .map(|finding| finding.message.clone())
2450            .collect();
2451    }
2452    if !warnings.is_empty() {
2453        return warnings
2454            .iter()
2455            .map(|finding| finding.message.clone())
2456            .collect();
2457    }
2458    vec!["no blocking deployment truth differences were found".to_string()]
2459}
2460
2461fn safety_summary(
2462    status: SafetyStatusV1,
2463    hard_failure_count: usize,
2464    warning_count: usize,
2465) -> String {
2466    match status {
2467        SafetyStatusV1::Safe => "deployment inventory matches the checked plan".to_string(),
2468        SafetyStatusV1::Warning => {
2469            format!("deployment inventory has {warning_count} warning(s)")
2470        }
2471        SafetyStatusV1::Blocked => {
2472            format!(
2473                "deployment inventory has {hard_failure_count} blocking issue(s) and {warning_count} warning(s)"
2474            )
2475        }
2476        SafetyStatusV1::NotEvaluated => "deployment safety has not been evaluated".to_string(),
2477    }
2478}
2479
2480fn safety_next_actions(status: SafetyStatusV1) -> Vec<String> {
2481    match status {
2482        SafetyStatusV1::Safe => Vec::new(),
2483        SafetyStatusV1::Warning => {
2484            vec!["review deployment warnings before continuing".to_string()]
2485        }
2486        SafetyStatusV1::Blocked => {
2487            vec!["resolve blocking deployment truth differences before mutation".to_string()]
2488        }
2489        SafetyStatusV1::NotEvaluated => vec!["collect deployment inventory".to_string()],
2490    }
2491}