1use super::*;
2use crate::subnet_catalog::{SubnetCatalogCacheRequest, load_cached_subnet_catalog};
3use canic_subnet_catalog::{MAINNET_NETWORK, SubnetKind};
4use std::collections::{BTreeMap, BTreeSet};
5
6struct DuplicateEvidenceGroup {
10 subject: String,
11 count: usize,
12 evidence_label: String,
13 is_conflict: bool,
14}
15
16#[derive(Clone, Debug, Eq, PartialEq)]
20pub struct LocalDeploymentCheckRequest {
21 pub deployment_name: String,
22 pub network: String,
23 pub workspace_root: std::path::PathBuf,
24 pub icp_root: std::path::PathBuf,
25 pub config_path: Option<std::path::PathBuf>,
26 pub observed_at: String,
27 pub runtime_variant: String,
28 pub build_profile: String,
29}
30
31pub fn check_local_deployment(
33 request: &LocalDeploymentCheckRequest,
34) -> Result<DeploymentCheckV1, DeploymentTruthError> {
35 let plan = build_local_deployment_plan(&LocalDeploymentPlanRequest {
36 deployment_name: request.deployment_name.clone(),
37 network: request.network.clone(),
38 workspace_root: request.workspace_root.clone(),
39 icp_root: request.icp_root.clone(),
40 config_path: request.config_path.clone(),
41 runtime_variant: request.runtime_variant.clone(),
42 build_profile: request.build_profile.clone(),
43 });
44 let inventory = collect_local_deployment_inventory(&LocalInventoryRequest {
45 deployment_name: request.deployment_name.clone(),
46 network: request.network.clone(),
47 workspace_root: request.workspace_root.clone(),
48 icp_root: request.icp_root.clone(),
49 config_path: request.config_path.clone(),
50 observed_at: request.observed_at.clone(),
51 })?;
52 let mut diff = compare_plan_to_inventory(&plan, &inventory);
53 apply_root_canister_signature_subnet_check(
54 &mut diff,
55 &inventory,
56 &request.network,
57 &request.icp_root,
58 );
59 let report = safety_report_from_diff(
60 format!(
61 "local:{}:{}:report",
62 request.network, request.deployment_name
63 ),
64 Some(format!(
65 "local:{}:{}:diff",
66 request.network, request.deployment_name
67 )),
68 &diff,
69 );
70
71 Ok(DeploymentCheckV1 {
72 schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
73 check_id: format!(
74 "local:{}:{}:check",
75 request.network, request.deployment_name
76 ),
77 plan,
78 inventory,
79 diff,
80 report,
81 })
82}
83
84pub(super) fn apply_root_canister_signature_subnet_check(
85 diff: &mut DeploymentDiffV1,
86 inventory: &DeploymentInventoryV1,
87 network: &str,
88 icp_root: &std::path::Path,
89) {
90 if network != MAINNET_NETWORK {
91 return;
92 }
93 let Some(root) = &inventory.observed_root else {
94 return;
95 };
96 let request = SubnetCatalogCacheRequest {
97 icp_root: icp_root.to_path_buf(),
98 network: network.to_string(),
99 };
100 let catalog = match load_cached_subnet_catalog(&request) {
101 Ok(cached) => cached.catalog,
102 Err(err) => {
103 diff.hard_failures.push(finding(
104 "root_auth_subnet_evidence_missing",
105 format!(
106 "cannot verify root canister-signature subnet kind for {}: {err}",
107 root.observed_canister_id
108 ),
109 SafetySeverityV1::HardFailure,
110 Some(root.observed_canister_id.clone()),
111 ));
112 refresh_resume_safety(diff);
113 return;
114 }
115 };
116 let resolved = match catalog.resolve_canister(&root.observed_canister_id) {
117 Ok(resolved) => resolved,
118 Err(err) => {
119 diff.hard_failures.push(finding(
120 "root_auth_subnet_resolution_failed",
121 format!(
122 "cannot resolve root canister-signature subnet kind for {}: {err}",
123 root.observed_canister_id
124 ),
125 SafetySeverityV1::HardFailure,
126 Some(root.observed_canister_id.clone()),
127 ));
128 refresh_resume_safety(diff);
129 return;
130 }
131 };
132 if resolved.subnet.subnet_kind == SubnetKind::CloudEngine {
133 diff.hard_failures.push(finding(
134 "root_auth_cloud_engine_subnet",
135 format!(
136 "root canister {} resolves to cloud_engine subnet {}; IC canister signatures from cloud_engine subnets are invalid",
137 root.observed_canister_id, resolved.subnet.subnet_principal
138 ),
139 SafetySeverityV1::HardFailure,
140 Some(root.observed_canister_id.clone()),
141 ));
142 refresh_resume_safety(diff);
143 }
144}
145
146fn refresh_resume_safety(diff: &mut DeploymentDiffV1) {
147 diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
148 diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
149}
150
151#[must_use]
153pub fn compare_plan_to_inventory(
154 plan: &DeploymentPlanV1,
155 inventory: &DeploymentInventoryV1,
156) -> DeploymentDiffV1 {
157 let mut artifact_diff = Vec::new();
158 let mut controller_diff = Vec::new();
159 let mut pool_diff = Vec::new();
160 let mut embedded_config_diff = Vec::new();
161 let mut module_hash_diff = Vec::new();
162 let mut verifier_readiness_diff = Vec::new();
163 let mut hard_failures = Vec::new();
164 let mut warnings = Vec::new();
165
166 compare_identity(plan, inventory, &mut hard_failures);
167 compare_authority_profile(plan, &mut controller_diff, &mut hard_failures);
168 compare_artifacts(
169 plan,
170 inventory,
171 &mut artifact_diff,
172 &mut hard_failures,
173 &mut warnings,
174 );
175 compare_observed_canister_id_conflicts(
176 inventory,
177 &mut controller_diff,
178 &mut hard_failures,
179 &mut warnings,
180 );
181 compare_observed_canister_pool_role_conflicts(inventory, &mut pool_diff, &mut hard_failures);
182 compare_canisters(
183 plan,
184 inventory,
185 &mut controller_diff,
186 &mut hard_failures,
187 &mut warnings,
188 );
189 compare_pools(
190 plan,
191 inventory,
192 &mut pool_diff,
193 &mut hard_failures,
194 &mut warnings,
195 );
196 compare_module_hashes(
197 plan,
198 inventory,
199 &mut module_hash_diff,
200 &mut hard_failures,
201 &mut warnings,
202 );
203 compare_raw_config(
204 plan,
205 inventory,
206 &mut embedded_config_diff,
207 &mut hard_failures,
208 );
209 compare_embedded_config(
210 plan,
211 inventory,
212 &mut embedded_config_diff,
213 &mut hard_failures,
214 &mut warnings,
215 );
216 compare_verifier_readiness(
217 plan,
218 inventory,
219 &mut verifier_readiness_diff,
220 &mut hard_failures,
221 &mut warnings,
222 );
223 record_plan_assumptions(plan, &mut hard_failures, &mut warnings);
224 for gap in &inventory.unresolved_observations {
225 warnings.push(SafetyFindingV1 {
226 code: "observation_gap".to_string(),
227 message: gap.description.clone(),
228 severity: SafetySeverityV1::Warning,
229 subject: Some(gap.key.clone()),
230 });
231 }
232
233 let status = safety_status(&hard_failures, &warnings);
234 DeploymentDiffV1 {
235 schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
236 plan_identity: plan.deployment_identity.clone(),
237 observed_identity: inventory.observed_identity.clone(),
238 artifact_diff,
239 controller_diff,
240 pool_diff,
241 embedded_config_diff,
242 module_hash_diff,
243 verifier_readiness_diff,
244 resume_safety: ResumeSafetyV1 {
245 status,
246 reasons: resume_safety_reasons(&hard_failures, &warnings),
247 },
248 hard_failures,
249 warnings,
250 resumable_phases: Vec::new(),
251 }
252}
253
254fn record_plan_assumptions(
255 plan: &DeploymentPlanV1,
256 hard_failures: &mut Vec<SafetyFindingV1>,
257 warnings: &mut Vec<SafetyFindingV1>,
258) {
259 for assumption in &plan.unresolved_assumptions {
260 if assumption.key == "local_state.unverified_root_canister_id" {
261 hard_failures.push(SafetyFindingV1 {
262 code: "unverified_deployment_root".to_string(),
263 message: assumption.description.clone(),
264 severity: SafetySeverityV1::HardFailure,
265 subject: Some(assumption.key.clone()),
266 });
267 } else {
268 warnings.push(SafetyFindingV1 {
269 code: "plan_assumption".to_string(),
270 message: assumption.description.clone(),
271 severity: SafetySeverityV1::Warning,
272 subject: Some(assumption.key.clone()),
273 });
274 }
275 }
276}
277
278#[must_use]
281pub fn compare_plan_inventory_and_receipt(
282 plan: &DeploymentPlanV1,
283 inventory: &DeploymentInventoryV1,
284 receipt: &DeploymentReceiptV1,
285) -> DeploymentDiffV1 {
286 let mut diff = compare_plan_to_inventory(plan, inventory);
287 apply_receipt_resume_safety(plan, receipt, &mut diff);
288 diff
289}
290
291#[must_use]
293pub fn safety_report_from_diff(
294 report_id: impl Into<String>,
295 diff_id: Option<String>,
296 diff: &DeploymentDiffV1,
297) -> SafetyReportV1 {
298 let status = safety_status(&diff.hard_failures, &diff.warnings);
299 SafetyReportV1 {
300 schema_version: DEPLOYMENT_TRUTH_SCHEMA_VERSION,
301 report_id: report_id.into(),
302 diff_id,
303 status,
304 summary: safety_summary(status, diff.hard_failures.len(), diff.warnings.len()),
305 hard_failures: diff.hard_failures.clone(),
306 warnings: diff.warnings.clone(),
307 next_actions: safety_next_actions(status),
308 }
309}
310
311fn apply_receipt_resume_safety(
312 plan: &DeploymentPlanV1,
313 receipt: &DeploymentReceiptV1,
314 diff: &mut DeploymentDiffV1,
315) {
316 validate_receipt_identity(plan, receipt, &mut diff.hard_failures);
317 validate_receipt_command_result(receipt, &mut diff.hard_failures);
318 validate_receipt_execution_status(receipt, &mut diff.hard_failures);
319 let phase_conflicts =
320 validate_receipt_phase_duplicates(receipt, &mut diff.hard_failures, &mut diff.warnings);
321 let role_phase_conflicts = validate_receipt_role_phase_duplicates(
322 receipt,
323 &mut diff.hard_failures,
324 &mut diff.warnings,
325 );
326 if !diff.hard_failures.is_empty() {
327 diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
328 diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
329 return;
330 }
331 let phase_failures = receipt_phase_failures(receipt);
332 for receipt in &receipt.phase_receipts {
333 if phase_conflicts.contains(&receipt.phase) {
334 continue;
335 }
336 if receipt.verified_postcondition.status != ObservationStatusV1::Observed {
337 diff.hard_failures.push(finding(
338 "receipt_postcondition_unverified",
339 format!(
340 "receipt phase {} has no observed postcondition",
341 receipt.phase
342 ),
343 SafetySeverityV1::HardFailure,
344 Some(receipt.phase.clone()),
345 ));
346 continue;
347 }
348 if phase_failures.contains(receipt.phase.as_str()) {
349 continue;
350 }
351 if role_phase_conflicts.contains(receipt.phase.as_str()) {
352 continue;
353 }
354 diff.resumable_phases.push(receipt.phase.clone());
355 }
356 diff.resumable_phases.sort();
357 diff.resumable_phases.dedup();
358 diff.resume_safety.status = safety_status(&diff.hard_failures, &diff.warnings);
359 diff.resume_safety.reasons = resume_safety_reasons(&diff.hard_failures, &diff.warnings);
360}
361
362fn validate_receipt_phase_duplicates(
363 receipt: &DeploymentReceiptV1,
364 hard_failures: &mut Vec<SafetyFindingV1>,
365 warnings: &mut Vec<SafetyFindingV1>,
366) -> BTreeSet<String> {
367 let mut conflicting_phases = BTreeSet::new();
368 for group in duplicate_evidence_groups(
369 &receipt.phase_receipts,
370 |phase_receipt| phase_receipt.phase.as_str().to_string(),
371 receipt_phase_evidence_label,
372 " | ",
373 ) {
374 if group.is_conflict {
375 conflicting_phases.insert(group.subject.clone());
376 hard_failures.push(finding(
377 "receipt_phase_conflict",
378 format!(
379 "receipt phase {} has conflicting evidence: {}",
380 group.subject, group.evidence_label
381 ),
382 SafetySeverityV1::HardFailure,
383 Some(group.subject),
384 ));
385 } else {
386 warnings.push(finding(
387 "duplicate_receipt_phase",
388 format!(
389 "receipt phase {} was reported {} times with identical evidence",
390 group.subject, group.count
391 ),
392 SafetySeverityV1::Warning,
393 Some(group.subject),
394 ));
395 }
396 }
397 conflicting_phases
398}
399
400fn receipt_phase_evidence_label(receipt: &PhaseReceiptV1) -> String {
401 format!(
402 "status={:?};evidence={}",
403 receipt.verified_postcondition.status,
404 receipt.verified_postcondition.evidence.join(",")
405 )
406}
407
408fn validate_receipt_role_phase_duplicates(
409 receipt: &DeploymentReceiptV1,
410 hard_failures: &mut Vec<SafetyFindingV1>,
411 warnings: &mut Vec<SafetyFindingV1>,
412) -> BTreeSet<String> {
413 let mut conflicting_phases = BTreeSet::new();
414 for group in duplicate_evidence_groups(
415 &receipt.role_phase_receipts,
416 role_phase_subject,
417 role_phase_evidence_label,
418 " | ",
419 ) {
420 if group.is_conflict {
421 if let Some(phase) = group
422 .subject
423 .rsplit_once(':')
424 .map(|(_, phase)| phase.to_string())
425 {
426 conflicting_phases.insert(phase);
427 }
428 hard_failures.push(finding(
429 "receipt_role_phase_conflict",
430 format!(
431 "receipt role phase {} has conflicting evidence: {}",
432 group.subject, group.evidence_label
433 ),
434 SafetySeverityV1::HardFailure,
435 Some(group.subject),
436 ));
437 } else {
438 warnings.push(finding(
439 "duplicate_receipt_role_phase",
440 format!(
441 "receipt role phase {} was reported {} times with identical evidence",
442 group.subject, group.count
443 ),
444 SafetySeverityV1::Warning,
445 Some(group.subject),
446 ));
447 }
448 }
449 conflicting_phases
450}
451
452fn role_phase_subject(receipt: &RolePhaseReceiptV1) -> String {
453 format!("{}:{}", receipt.role, receipt.phase)
454}
455
456fn role_phase_evidence_label(receipt: &RolePhaseReceiptV1) -> String {
457 format!(
458 "result={:?};previous={};target={};observed={};artifact={};config={};error={}",
459 receipt.result,
460 receipt.previous_module_hash.as_deref().unwrap_or("<none>"),
461 receipt.target_module_hash.as_deref().unwrap_or("<none>"),
462 receipt
463 .observed_module_hash_after
464 .as_deref()
465 .unwrap_or("<none>"),
466 receipt.artifact_digest.as_deref().unwrap_or("<none>"),
467 receipt
468 .canonical_embedded_config_sha256
469 .as_deref()
470 .unwrap_or("<none>"),
471 receipt.error.as_deref().unwrap_or("<none>")
472 )
473}
474
475fn validate_receipt_identity(
476 plan: &DeploymentPlanV1,
477 receipt: &DeploymentReceiptV1,
478 hard_failures: &mut Vec<SafetyFindingV1>,
479) {
480 if receipt.plan_id != plan.plan_id {
481 hard_failures.push(finding(
482 "receipt_plan_mismatch",
483 format!(
484 "receipt plan {} does not match current plan {}",
485 receipt.plan_id, plan.plan_id
486 ),
487 SafetySeverityV1::HardFailure,
488 Some("receipt.plan_id".to_string()),
489 ));
490 }
491 if let (Some(expected), Some(observed)) = (
492 plan.deployment_identity.root_principal.as_ref(),
493 receipt.root_principal.as_ref(),
494 ) && expected != observed
495 {
496 hard_failures.push(finding(
497 "receipt_root_mismatch",
498 format!("receipt root {observed} does not match current plan root {expected}"),
499 SafetySeverityV1::HardFailure,
500 Some("receipt.root_principal".to_string()),
501 ));
502 }
503}
504
505fn validate_receipt_command_result(
506 receipt: &DeploymentReceiptV1,
507 hard_failures: &mut Vec<SafetyFindingV1>,
508) {
509 if let DeploymentCommandResultV1::Failed { code, message } = &receipt.command_result {
510 hard_failures.push(finding(
511 "receipt_failed_command",
512 format!("receipt command failed with {code}: {message}"),
513 SafetySeverityV1::HardFailure,
514 Some("receipt.command_result".to_string()),
515 ));
516 }
517}
518
519fn validate_receipt_execution_status(
520 receipt: &DeploymentReceiptV1,
521 hard_failures: &mut Vec<SafetyFindingV1>,
522) {
523 let derived_status = deployment_execution_status_for_receipt_parts(
524 &receipt.command_result,
525 &receipt.role_phase_receipts,
526 );
527 let status_is_consistent = match receipt.operation_status {
528 DeploymentExecutionStatusV1::FailedAfterMutation
529 if matches!(
530 derived_status,
531 DeploymentExecutionStatusV1::FailedBeforeMutation
532 ) =>
533 {
534 receipt.role_phase_receipts.is_empty()
535 }
536 _ => receipt.operation_status == derived_status,
537 };
538
539 if !status_is_consistent {
540 hard_failures.push(finding(
541 "receipt_execution_status_mismatch",
542 format!(
543 "receipt operation status {:?} does not match command result and role-phase evidence {:?}",
544 receipt.operation_status, derived_status
545 ),
546 SafetySeverityV1::HardFailure,
547 Some("receipt.operation_status".to_string()),
548 ));
549 }
550}
551
552fn receipt_phase_failures(receipt: &DeploymentReceiptV1) -> BTreeSet<&str> {
553 let mut failures = BTreeSet::new();
554 for role_receipt in &receipt.role_phase_receipts {
555 if matches!(role_receipt.result, RolePhaseResultV1::Failed) {
556 failures.insert(role_receipt.phase.as_str());
557 }
558 }
559 failures
560}
561
562fn compare_identity(
563 plan: &DeploymentPlanV1,
564 inventory: &DeploymentInventoryV1,
565 hard_failures: &mut Vec<SafetyFindingV1>,
566) {
567 let Some(observed) = &inventory.observed_identity else {
568 hard_failures.push(finding(
569 "identity_unobserved",
570 "deployment identity was not observed",
571 SafetySeverityV1::HardFailure,
572 None,
573 ));
574 return;
575 };
576
577 if observed.network != plan.deployment_identity.network {
578 hard_failures.push(finding(
579 "network_mismatch",
580 format!(
581 "plan network {} differs from observed network {}",
582 plan.deployment_identity.network, observed.network
583 ),
584 SafetySeverityV1::HardFailure,
585 Some("deployment_identity.network".to_string()),
586 ));
587 }
588 if let (Some(expected), Some(actual)) = (
589 plan.deployment_identity.root_principal.as_ref(),
590 observed.root_principal.as_ref(),
591 ) && expected != actual
592 {
593 hard_failures.push(finding(
594 "root_trust_anchor_mismatch",
595 format!("plan root {expected} differs from observed root {actual}"),
596 SafetySeverityV1::HardFailure,
597 Some("deployment_identity.root_principal".to_string()),
598 ));
599 }
600 match (
601 plan.deployment_identity.deployment_manifest_digest.as_ref(),
602 observed.deployment_manifest_digest.as_ref(),
603 ) {
604 (Some(expected), Some(actual)) if expected != actual => {
605 hard_failures.push(finding(
606 "deployment_manifest_mismatch",
607 "deployment manifest digest differs from the observed local config",
608 SafetySeverityV1::HardFailure,
609 Some("deployment_identity.deployment_manifest_digest".to_string()),
610 ));
611 }
612 (Some(_), None) => {
613 hard_failures.push(finding(
614 "deployment_manifest_unobserved",
615 "deployment manifest digest was not observed",
616 SafetySeverityV1::HardFailure,
617 Some("deployment_identity.deployment_manifest_digest".to_string()),
618 ));
619 }
620 _ => {}
621 }
622}
623
624fn compare_authority_profile(
625 plan: &DeploymentPlanV1,
626 controller_diff: &mut Vec<DiffItemV1>,
627 hard_failures: &mut Vec<SafetyFindingV1>,
628) {
629 let mut reported = BTreeSet::new();
630 for controller in &plan.authority_profile.expected_controllers {
631 if !is_staging_or_emergency_controller(plan, controller) {
632 continue;
633 }
634 if !reported.insert(controller.as_str()) {
635 continue;
636 }
637 controller_diff.push(diff_item(
638 "controller_authority_overlap",
639 "authority_profile",
640 Some("expected-only".to_string()),
641 Some(controller.clone()),
642 SafetySeverityV1::HardFailure,
643 ));
644 hard_failures.push(finding(
645 "controller_authority_overlap",
646 format!(
647 "controller {controller} appears in both expected and staging/emergency authority"
648 ),
649 SafetySeverityV1::HardFailure,
650 Some("authority_profile".to_string()),
651 ));
652 }
653}
654
655fn compare_artifacts(
656 plan: &DeploymentPlanV1,
657 inventory: &DeploymentInventoryV1,
658 artifact_diff: &mut Vec<DiffItemV1>,
659 hard_failures: &mut Vec<SafetyFindingV1>,
660 warnings: &mut Vec<SafetyFindingV1>,
661) {
662 let planned_conflicting_roles =
663 compare_planned_artifact_role_conflicts(plan, artifact_diff, hard_failures, warnings);
664 let conflicting_roles =
665 compare_observed_artifact_role_conflicts(inventory, artifact_diff, hard_failures, warnings);
666 let mut observed_by_role = BTreeMap::new();
667 for artifact in &inventory.observed_artifacts {
668 if conflicting_roles.contains(&artifact.role) {
669 continue;
670 }
671 observed_by_role
672 .entry(artifact.role.as_str())
673 .or_insert(artifact);
674 }
675
676 let mut compared_roles = BTreeSet::new();
677 for expected in &plan.role_artifacts {
678 if planned_conflicting_roles.contains(&expected.role)
679 || conflicting_roles.contains(&expected.role)
680 || !compared_roles.insert(expected.role.as_str())
681 {
682 continue;
683 }
684 let Some(observed) = observed_by_role.get(expected.role.as_str()) else {
685 record_missing_artifact(expected, artifact_diff, hard_failures);
686 continue;
687 };
688
689 compare_artifact_file_sha256(expected, observed, artifact_diff, hard_failures);
690 compare_artifact_payload_sha256(expected, observed, artifact_diff, hard_failures, warnings);
691 }
692}
693
694fn compare_planned_artifact_role_conflicts(
695 plan: &DeploymentPlanV1,
696 artifact_diff: &mut Vec<DiffItemV1>,
697 hard_failures: &mut Vec<SafetyFindingV1>,
698 warnings: &mut Vec<SafetyFindingV1>,
699) -> BTreeSet<String> {
700 let mut conflicting_roles = BTreeSet::new();
701 for group in duplicate_evidence_groups(
702 &plan.role_artifacts,
703 |planned| planned.role.as_str().to_string(),
704 planned_artifact_evidence_label,
705 " | ",
706 ) {
707 if group.is_conflict {
708 conflicting_roles.insert(group.subject.clone());
709 artifact_diff.push(diff_item(
710 "planned_artifact_role_conflict",
711 &group.subject,
712 Some("one planned artifact".to_string()),
713 Some(group.evidence_label.clone()),
714 SafetySeverityV1::HardFailure,
715 ));
716 hard_failures.push(finding(
717 "planned_artifact_role_conflict",
718 format!(
719 "planned artifact role {} has conflicting evidence: {}",
720 group.subject, group.evidence_label
721 ),
722 SafetySeverityV1::HardFailure,
723 Some(group.subject),
724 ));
725 } else {
726 artifact_diff.push(diff_item(
727 "planned_artifact_duplicate",
728 &group.subject,
729 Some(group.evidence_label.clone()),
730 Some(group.count.to_string()),
731 SafetySeverityV1::Warning,
732 ));
733 warnings.push(finding(
734 "duplicate_planned_artifact_role",
735 format!(
736 "planned artifact role {} was declared {} times with identical evidence",
737 group.subject, group.count
738 ),
739 SafetySeverityV1::Warning,
740 Some(group.subject),
741 ));
742 }
743 }
744 conflicting_roles
745}
746
747fn planned_artifact_evidence_label(planned: &RoleArtifactV1) -> String {
748 format!(
749 "wasm_gz_path={};wasm_gz={};file={};module={};raw_config={};canonical={}",
750 planned.wasm_gz_path.as_deref().unwrap_or("<none>"),
751 planned.wasm_gz_sha256.as_deref().unwrap_or("<none>"),
752 planned
753 .observed_wasm_gz_file_sha256
754 .as_deref()
755 .unwrap_or("<none>"),
756 planned.installed_module_hash.as_deref().unwrap_or("<none>"),
757 planned.raw_config_sha256.as_deref().unwrap_or("<none>"),
758 planned
759 .canonical_embedded_config_sha256
760 .as_deref()
761 .unwrap_or("<none>")
762 )
763}
764
765fn compare_observed_artifact_role_conflicts(
766 inventory: &DeploymentInventoryV1,
767 artifact_diff: &mut Vec<DiffItemV1>,
768 hard_failures: &mut Vec<SafetyFindingV1>,
769 warnings: &mut Vec<SafetyFindingV1>,
770) -> BTreeSet<String> {
771 let mut conflicting_roles = BTreeSet::new();
772 for group in duplicate_evidence_groups(
773 &inventory.observed_artifacts,
774 |observed| observed.role.as_str().to_string(),
775 observed_artifact_evidence_label,
776 " | ",
777 ) {
778 if group.is_conflict {
779 conflicting_roles.insert(group.subject.clone());
780 artifact_diff.push(diff_item(
781 "artifact_role_conflict",
782 &group.subject,
783 Some("one artifact observation".to_string()),
784 Some(group.evidence_label.clone()),
785 SafetySeverityV1::HardFailure,
786 ));
787 hard_failures.push(finding(
788 "artifact_role_conflict",
789 format!(
790 "observed artifact role {} has conflicting evidence: {}",
791 group.subject, group.evidence_label
792 ),
793 SafetySeverityV1::HardFailure,
794 Some(group.subject),
795 ));
796 } else {
797 artifact_diff.push(diff_item(
798 "artifact_duplicate",
799 &group.subject,
800 Some(group.evidence_label.clone()),
801 Some(group.count.to_string()),
802 SafetySeverityV1::Warning,
803 ));
804 warnings.push(finding(
805 "duplicate_artifact_observed",
806 format!(
807 "observed artifact role {} was reported {} times with identical evidence",
808 group.subject, group.count
809 ),
810 SafetySeverityV1::Warning,
811 Some(group.subject),
812 ));
813 }
814 }
815 conflicting_roles
816}
817
818fn observed_artifact_evidence_label(observed: &ObservedArtifactV1) -> String {
819 format!(
820 "path={};file={};payload={};size={};source={:?}",
821 observed.artifact_path,
822 observed.file_sha256.as_deref().unwrap_or("<none>"),
823 observed.payload_sha256.as_deref().unwrap_or("<none>"),
824 observed
825 .payload_size_bytes
826 .map_or_else(|| "<none>".to_string(), |size| size.to_string()),
827 observed.source
828 )
829}
830
831fn record_missing_artifact(
832 expected: &RoleArtifactV1,
833 artifact_diff: &mut Vec<DiffItemV1>,
834 hard_failures: &mut Vec<SafetyFindingV1>,
835) {
836 artifact_diff.push(diff_item(
837 "artifact",
838 &expected.role,
839 expected.wasm_gz_path.clone(),
840 None,
841 SafetySeverityV1::HardFailure,
842 ));
843 hard_failures.push(finding(
844 "artifact_missing",
845 format!("missing observed artifact for role {}", expected.role),
846 SafetySeverityV1::HardFailure,
847 Some(expected.role.clone()),
848 ));
849}
850
851fn compare_artifact_file_sha256(
852 expected: &RoleArtifactV1,
853 observed: &ObservedArtifactV1,
854 artifact_diff: &mut Vec<DiffItemV1>,
855 hard_failures: &mut Vec<SafetyFindingV1>,
856) {
857 match (
858 expected.observed_wasm_gz_file_sha256.as_ref(),
859 observed.file_sha256.as_ref(),
860 ) {
861 (Some(want), Some(got)) if want != got => {
862 artifact_diff.push(diff_item(
863 "artifact_file_sha256",
864 &expected.role,
865 Some(want.clone()),
866 Some(got.clone()),
867 SafetySeverityV1::HardFailure,
868 ));
869 hard_failures.push(finding(
870 "artifact_file_digest_mismatch",
871 format!(
872 "observed artifact file digest changed during deployment truth check for role {}",
873 expected.role
874 ),
875 SafetySeverityV1::HardFailure,
876 Some(expected.role.clone()),
877 ));
878 }
879 (_, Some(got)) => {
880 artifact_diff.push(diff_item(
881 "artifact_file_sha256",
882 &expected.role,
883 expected.observed_wasm_gz_file_sha256.clone(),
884 Some(got.clone()),
885 SafetySeverityV1::Info,
886 ));
887 }
888 _ => {}
889 }
890}
891
892fn compare_artifact_payload_sha256(
893 expected: &RoleArtifactV1,
894 observed: &ObservedArtifactV1,
895 artifact_diff: &mut Vec<DiffItemV1>,
896 hard_failures: &mut Vec<SafetyFindingV1>,
897 warnings: &mut Vec<SafetyFindingV1>,
898) {
899 match (
900 expected.wasm_gz_sha256.as_ref(),
901 observed.payload_sha256.as_ref(),
902 ) {
903 (Some(want), Some(got)) if want != got => {
904 artifact_diff.push(diff_item(
905 "artifact_sha256",
906 &expected.role,
907 Some(want.clone()),
908 Some(got.clone()),
909 SafetySeverityV1::HardFailure,
910 ));
911 hard_failures.push(finding(
912 "artifact_digest_mismatch",
913 format!("artifact digest mismatch for role {}", expected.role),
914 SafetySeverityV1::HardFailure,
915 Some(expected.role.clone()),
916 ));
917 }
918 (Some(want), None) => warnings.push(finding(
919 "artifact_digest_unobserved",
920 format!(
921 "expected artifact digest {want} for role {} was not observed",
922 expected.role
923 ),
924 SafetySeverityV1::Warning,
925 Some(expected.role.clone()),
926 )),
927 _ => {}
928 }
929}
930
931fn compare_observed_canister_id_conflicts(
932 inventory: &DeploymentInventoryV1,
933 controller_diff: &mut Vec<DiffItemV1>,
934 hard_failures: &mut Vec<SafetyFindingV1>,
935 warnings: &mut Vec<SafetyFindingV1>,
936) {
937 for group in duplicate_evidence_groups(
938 &inventory.observed_canisters,
939 |observed| observed.canister_id.as_str().to_string(),
940 observed_role_label,
941 ",",
942 ) {
943 if group.is_conflict {
944 controller_diff.push(diff_item(
945 "canister_id_role_conflict",
946 &group.subject,
947 None,
948 Some(group.evidence_label.clone()),
949 SafetySeverityV1::HardFailure,
950 ));
951 hard_failures.push(finding(
952 "canister_id_role_conflict",
953 format!(
954 "observed canister {} has conflicting roles {}",
955 group.subject, group.evidence_label
956 ),
957 SafetySeverityV1::HardFailure,
958 Some(group.subject),
959 ));
960 } else {
961 controller_diff.push(diff_item(
962 "canister_duplicate",
963 &group.subject,
964 Some(group.evidence_label.clone()),
965 Some(group.count.to_string()),
966 SafetySeverityV1::Warning,
967 ));
968 warnings.push(finding(
969 "duplicate_canister_observed",
970 format!(
971 "observed canister {} was reported {} times for role {}",
972 group.subject, group.count, group.evidence_label
973 ),
974 SafetySeverityV1::Warning,
975 Some(group.subject),
976 ));
977 }
978 }
979}
980
981fn observed_role_label(observed: &ObservedCanisterV1) -> String {
982 observed
983 .role
984 .clone()
985 .unwrap_or_else(|| "<unknown>".to_string())
986}
987
988fn compare_canisters(
989 plan: &DeploymentPlanV1,
990 inventory: &DeploymentInventoryV1,
991 controller_diff: &mut Vec<DiffItemV1>,
992 hard_failures: &mut Vec<SafetyFindingV1>,
993 warnings: &mut Vec<SafetyFindingV1>,
994) {
995 let planned_conflicts =
996 compare_planned_canister_conflicts(plan, controller_diff, hard_failures, warnings);
997 let mut matched_observed = BTreeSet::new();
998 let mut compared_planned = BTreeSet::new();
999 for expected in &plan.expected_canisters {
1000 if planned_conflicts.role_conflicts.contains(&expected.role)
1001 || expected
1002 .canister_id
1003 .as_ref()
1004 .is_some_and(|id| planned_conflicts.id_conflicts.contains(id))
1005 || !compared_planned.insert(planned_canister_evidence_label(expected))
1006 {
1007 continue;
1008 }
1009 let observed = expected.canister_id.as_ref().map_or_else(
1010 || {
1011 let role_matches = inventory
1012 .observed_canisters
1013 .iter()
1014 .filter(|canister| canister.role.as_deref() == Some(expected.role.as_str()))
1015 .collect::<Vec<_>>();
1016 if role_matches.len() > 1 {
1017 record_ambiguous_canister_role(
1018 expected,
1019 &role_matches,
1020 controller_diff,
1021 hard_failures,
1022 );
1023 None
1024 } else {
1025 role_matches.into_iter().next()
1026 }
1027 },
1028 |id| {
1029 inventory
1030 .observed_canisters
1031 .iter()
1032 .find(|canister| &canister.canister_id == id)
1033 },
1034 );
1035 let Some(observed) = observed else {
1036 record_missing_canister(expected, controller_diff, hard_failures, warnings);
1037 continue;
1038 };
1039 matched_observed.insert(observed.canister_id.as_str());
1040 compare_observed_role(expected, observed, controller_diff, hard_failures);
1041 record_unsafe_canister_control_class(expected, observed, controller_diff, hard_failures);
1042 compare_role_controllers(plan, observed, controller_diff, hard_failures, warnings);
1043 }
1044 warn_extra_observed_canisters(
1045 plan,
1046 inventory,
1047 controller_diff,
1048 warnings,
1049 &matched_observed,
1050 );
1051}
1052
1053struct PlannedCanisterConflicts {
1054 role_conflicts: BTreeSet<String>,
1055 id_conflicts: BTreeSet<String>,
1056}
1057
1058fn compare_planned_canister_conflicts(
1059 plan: &DeploymentPlanV1,
1060 controller_diff: &mut Vec<DiffItemV1>,
1061 hard_failures: &mut Vec<SafetyFindingV1>,
1062 warnings: &mut Vec<SafetyFindingV1>,
1063) -> PlannedCanisterConflicts {
1064 let mut role_conflicts = BTreeSet::new();
1065 let mut id_conflicts = BTreeSet::new();
1066
1067 for group in duplicate_evidence_groups(
1068 &plan.expected_canisters,
1069 |planned| planned.role.as_str().to_string(),
1070 planned_canister_evidence_label,
1071 " | ",
1072 ) {
1073 if group.is_conflict {
1074 role_conflicts.insert(group.subject.clone());
1075 controller_diff.push(diff_item(
1076 "planned_canister_role_conflict",
1077 &group.subject,
1078 Some("one planned canister".to_string()),
1079 Some(group.evidence_label.clone()),
1080 SafetySeverityV1::HardFailure,
1081 ));
1082 hard_failures.push(finding(
1083 "planned_canister_role_conflict",
1084 format!(
1085 "planned canister role {} has conflicting evidence: {}",
1086 group.subject, group.evidence_label
1087 ),
1088 SafetySeverityV1::HardFailure,
1089 Some(group.subject),
1090 ));
1091 } else {
1092 controller_diff.push(diff_item(
1093 "planned_canister_duplicate",
1094 &group.subject,
1095 Some(group.evidence_label.clone()),
1096 Some(group.count.to_string()),
1097 SafetySeverityV1::Warning,
1098 ));
1099 warnings.push(finding(
1100 "duplicate_planned_canister_role",
1101 format!(
1102 "planned canister role {} was declared {} times with identical evidence",
1103 group.subject, group.count
1104 ),
1105 SafetySeverityV1::Warning,
1106 Some(group.subject),
1107 ));
1108 }
1109 }
1110
1111 for group in conflicting_assignment_groups(
1112 &plan.expected_canisters,
1113 |planned| planned.canister_id.clone(),
1114 |planned| planned.role.clone(),
1115 ",",
1116 ) {
1117 id_conflicts.insert(group.subject.clone());
1118 controller_diff.push(diff_item(
1119 "planned_canister_id_conflict",
1120 &group.subject,
1121 Some("one planned role".to_string()),
1122 Some(group.evidence_label.clone()),
1123 SafetySeverityV1::HardFailure,
1124 ));
1125 hard_failures.push(finding(
1126 "planned_canister_id_conflict",
1127 format!(
1128 "planned canister id {} is assigned to conflicting roles {}",
1129 group.subject, group.evidence_label
1130 ),
1131 SafetySeverityV1::HardFailure,
1132 Some(group.subject),
1133 ));
1134 }
1135
1136 PlannedCanisterConflicts {
1137 role_conflicts,
1138 id_conflicts,
1139 }
1140}
1141
1142fn planned_canister_evidence_label(planned: &ExpectedCanisterV1) -> String {
1143 format!(
1144 "role={};id={};control={:?}",
1145 planned.role,
1146 planned.canister_id.as_deref().unwrap_or("<none>"),
1147 planned.control_class
1148 )
1149}
1150
1151fn record_missing_canister(
1152 expected: &ExpectedCanisterV1,
1153 controller_diff: &mut Vec<DiffItemV1>,
1154 hard_failures: &mut Vec<SafetyFindingV1>,
1155 warnings: &mut Vec<SafetyFindingV1>,
1156) {
1157 let severity = if expected.canister_id.is_some() {
1158 SafetySeverityV1::HardFailure
1159 } else {
1160 SafetySeverityV1::Warning
1161 };
1162 controller_diff.push(diff_item(
1163 "canister",
1164 &expected.role,
1165 expected.canister_id.clone(),
1166 None,
1167 severity,
1168 ));
1169 let finding = finding(
1170 if expected.canister_id.is_some() {
1171 "canister_missing"
1172 } else {
1173 "canister_unobserved"
1174 },
1175 format!("missing observed canister for role {}", expected.role),
1176 severity,
1177 Some(expected.role.clone()),
1178 );
1179 if expected.canister_id.is_some() {
1180 hard_failures.push(finding);
1181 } else {
1182 warnings.push(finding);
1183 }
1184}
1185
1186fn record_unsafe_canister_control_class(
1187 expected: &ExpectedCanisterV1,
1188 observed: &ObservedCanisterV1,
1189 controller_diff: &mut Vec<DiffItemV1>,
1190 hard_failures: &mut Vec<SafetyFindingV1>,
1191) {
1192 if !matches!(
1193 observed.control_class,
1194 CanisterControlClassV1::UnknownUnsafe | CanisterControlClassV1::UserControlled
1195 ) || expected.control_class != CanisterControlClassV1::DeploymentControlled
1196 {
1197 return;
1198 }
1199 controller_diff.push(diff_item(
1200 "control_class",
1201 &expected.role,
1202 Some("DeploymentControlled".to_string()),
1203 Some(format!("{:?}", observed.control_class)),
1204 SafetySeverityV1::HardFailure,
1205 ));
1206 hard_failures.push(finding(
1207 "unsafe_control_class",
1208 format!("role {} has unsafe observed control class", expected.role),
1209 SafetySeverityV1::HardFailure,
1210 Some(expected.role.clone()),
1211 ));
1212}
1213
1214fn record_ambiguous_canister_role(
1215 expected: &ExpectedCanisterV1,
1216 observed_matches: &[&ObservedCanisterV1],
1217 controller_diff: &mut Vec<DiffItemV1>,
1218 hard_failures: &mut Vec<SafetyFindingV1>,
1219) {
1220 let observed_ids = observed_matches
1221 .iter()
1222 .map(|canister| canister.canister_id.as_str())
1223 .collect::<Vec<_>>()
1224 .join(",");
1225 controller_diff.push(diff_item(
1226 "canister_role_ambiguous",
1227 &expected.role,
1228 Some("one observed canister".to_string()),
1229 Some(observed_ids.clone()),
1230 SafetySeverityV1::HardFailure,
1231 ));
1232 hard_failures.push(finding(
1233 "canister_role_ambiguous",
1234 format!(
1235 "expected role {} has multiple observed canisters: {observed_ids}",
1236 expected.role
1237 ),
1238 SafetySeverityV1::HardFailure,
1239 Some(expected.role.clone()),
1240 ));
1241}
1242
1243fn compare_observed_role(
1244 expected: &ExpectedCanisterV1,
1245 observed: &ObservedCanisterV1,
1246 controller_diff: &mut Vec<DiffItemV1>,
1247 hard_failures: &mut Vec<SafetyFindingV1>,
1248) {
1249 let Some(observed_role) = observed.role.as_deref() else {
1250 return;
1251 };
1252 if observed_role == expected.role {
1253 return;
1254 }
1255 controller_diff.push(diff_item(
1256 "role_mismatch",
1257 &expected.role,
1258 Some(expected.role.clone()),
1259 Some(observed_role.to_string()),
1260 SafetySeverityV1::HardFailure,
1261 ));
1262 hard_failures.push(finding(
1263 "canister_role_mismatch",
1264 format!(
1265 "expected canister {} to have role {}, observed role {observed_role}",
1266 observed.canister_id, expected.role
1267 ),
1268 SafetySeverityV1::HardFailure,
1269 Some(expected.role.clone()),
1270 ));
1271}
1272
1273fn warn_extra_observed_canisters(
1274 plan: &DeploymentPlanV1,
1275 inventory: &DeploymentInventoryV1,
1276 controller_diff: &mut Vec<DiffItemV1>,
1277 warnings: &mut Vec<SafetyFindingV1>,
1278 matched_observed: &BTreeSet<&str>,
1279) {
1280 let expected_pool_roles = plan
1281 .expected_pool
1282 .iter()
1283 .filter_map(|pool| pool.role.as_deref())
1284 .collect::<BTreeSet<_>>();
1285
1286 for observed in &inventory.observed_canisters {
1287 if matched_observed.contains(observed.canister_id.as_str()) {
1288 continue;
1289 }
1290 if let Some(role) = observed.role.as_deref()
1291 && expected_pool_roles.contains(role)
1292 {
1293 continue;
1294 }
1295 let subject = observed_canister_subject(observed);
1296 controller_diff.push(diff_item(
1297 "canister_extra",
1298 &subject,
1299 None,
1300 Some(observed.canister_id.clone()),
1301 SafetySeverityV1::Warning,
1302 ));
1303 warnings.push(finding(
1304 "extra_canister_observed",
1305 format!("observed undeclared canister {subject}"),
1306 SafetySeverityV1::Warning,
1307 Some(subject),
1308 ));
1309 }
1310}
1311
1312fn observed_canister_subject(observed: &ObservedCanisterV1) -> String {
1313 observed
1314 .role
1315 .clone()
1316 .unwrap_or_else(|| observed.canister_id.clone())
1317}
1318
1319fn compare_observed_canister_pool_role_conflicts(
1320 inventory: &DeploymentInventoryV1,
1321 pool_diff: &mut Vec<DiffItemV1>,
1322 hard_failures: &mut Vec<SafetyFindingV1>,
1323) {
1324 let mut pools_by_id = BTreeMap::<&str, Vec<&ObservedPoolCanisterV1>>::new();
1325 for observed_pool in &inventory.observed_pool {
1326 pools_by_id
1327 .entry(observed_pool.canister_id.as_str())
1328 .or_default()
1329 .push(observed_pool);
1330 }
1331
1332 for observed_canister in &inventory.observed_canisters {
1333 let Some(canister_role) = observed_canister.role.as_deref() else {
1334 continue;
1335 };
1336 let Some(observed_pools) = pools_by_id.get(observed_canister.canister_id.as_str()) else {
1337 continue;
1338 };
1339 for observed_pool in observed_pools {
1340 let Some(pool_role) = observed_pool.role.as_deref() else {
1341 continue;
1342 };
1343 if pool_role == canister_role {
1344 continue;
1345 }
1346 let observed_label = format!(
1347 "canister={};pool={}",
1348 observed_canister_subject(observed_canister),
1349 observed_pool_subject(observed_pool)
1350 );
1351 pool_diff.push(diff_item(
1352 "canister_pool_role_conflict",
1353 &observed_canister.canister_id,
1354 None,
1355 Some(observed_label.clone()),
1356 SafetySeverityV1::HardFailure,
1357 ));
1358 hard_failures.push(finding(
1359 "canister_pool_role_conflict",
1360 format!(
1361 "observed canister {} has conflicting canister/pool roles {observed_label}",
1362 observed_canister.canister_id
1363 ),
1364 SafetySeverityV1::HardFailure,
1365 Some(observed_canister.canister_id.clone()),
1366 ));
1367 }
1368 }
1369}
1370
1371fn compare_pools(
1372 plan: &DeploymentPlanV1,
1373 inventory: &DeploymentInventoryV1,
1374 pool_diff: &mut Vec<DiffItemV1>,
1375 hard_failures: &mut Vec<SafetyFindingV1>,
1376 warnings: &mut Vec<SafetyFindingV1>,
1377) {
1378 let planned_conflicts =
1379 compare_planned_pool_conflicts(plan, pool_diff, hard_failures, warnings);
1380 compare_observed_pool_id_conflicts(inventory, pool_diff, hard_failures, warnings);
1381 let mut matched_observed = BTreeSet::new();
1382 let mut compared_planned = BTreeSet::new();
1383 for expected in &plan.expected_pool {
1384 if planned_conflicts
1385 .subject_conflicts
1386 .contains(&expected_pool_subject(expected))
1387 || expected
1388 .canister_id
1389 .as_ref()
1390 .is_some_and(|id| planned_conflicts.id_conflicts.contains(id))
1391 || !compared_planned.insert(planned_pool_evidence_label(expected))
1392 {
1393 continue;
1394 }
1395 compare_expected_pool(
1396 expected,
1397 inventory,
1398 pool_diff,
1399 hard_failures,
1400 warnings,
1401 &mut matched_observed,
1402 );
1403 }
1404
1405 for observed in &inventory.observed_pool {
1406 warn_extra_observed_pool(plan, observed, pool_diff, warnings, &matched_observed);
1407 }
1408}
1409
1410struct PlannedPoolConflicts {
1411 subject_conflicts: BTreeSet<String>,
1412 id_conflicts: BTreeSet<String>,
1413}
1414
1415fn compare_planned_pool_conflicts(
1416 plan: &DeploymentPlanV1,
1417 pool_diff: &mut Vec<DiffItemV1>,
1418 hard_failures: &mut Vec<SafetyFindingV1>,
1419 warnings: &mut Vec<SafetyFindingV1>,
1420) -> PlannedPoolConflicts {
1421 let mut subject_conflicts = BTreeSet::new();
1422 let mut id_conflicts = BTreeSet::new();
1423
1424 for group in duplicate_evidence_groups(
1425 &plan.expected_pool,
1426 expected_pool_subject,
1427 planned_pool_evidence_label,
1428 " | ",
1429 ) {
1430 if group.is_conflict {
1431 subject_conflicts.insert(group.subject.clone());
1432 pool_diff.push(diff_item(
1433 "planned_pool_conflict",
1434 &group.subject,
1435 Some("one planned pool canister".to_string()),
1436 Some(group.evidence_label.clone()),
1437 SafetySeverityV1::HardFailure,
1438 ));
1439 hard_failures.push(finding(
1440 "planned_pool_conflict",
1441 format!(
1442 "planned pool {} has conflicting evidence: {}",
1443 group.subject, group.evidence_label
1444 ),
1445 SafetySeverityV1::HardFailure,
1446 Some(group.subject),
1447 ));
1448 } else {
1449 pool_diff.push(diff_item(
1450 "planned_pool_duplicate",
1451 &group.subject,
1452 Some(group.evidence_label.clone()),
1453 Some(group.count.to_string()),
1454 SafetySeverityV1::Warning,
1455 ));
1456 warnings.push(finding(
1457 "duplicate_planned_pool",
1458 format!(
1459 "planned pool {} was declared {} times with identical evidence",
1460 group.subject, group.count
1461 ),
1462 SafetySeverityV1::Warning,
1463 Some(group.subject),
1464 ));
1465 }
1466 }
1467
1468 for group in conflicting_assignment_groups(
1469 &plan.expected_pool,
1470 |planned| planned.canister_id.clone(),
1471 expected_pool_subject,
1472 ",",
1473 ) {
1474 id_conflicts.insert(group.subject.clone());
1475 pool_diff.push(diff_item(
1476 "planned_pool_id_conflict",
1477 &group.subject,
1478 Some("one planned pool identity".to_string()),
1479 Some(group.evidence_label.clone()),
1480 SafetySeverityV1::HardFailure,
1481 ));
1482 hard_failures.push(finding(
1483 "planned_pool_id_conflict",
1484 format!(
1485 "planned pool id {} is assigned to conflicting identities {}",
1486 group.subject, group.evidence_label
1487 ),
1488 SafetySeverityV1::HardFailure,
1489 Some(group.subject),
1490 ));
1491 }
1492
1493 PlannedPoolConflicts {
1494 subject_conflicts,
1495 id_conflicts,
1496 }
1497}
1498
1499fn planned_pool_evidence_label(planned: &ExpectedPoolCanisterV1) -> String {
1500 format!(
1501 "pool={};role={};id={}",
1502 planned.pool,
1503 planned.role.as_deref().unwrap_or("<none>"),
1504 planned.canister_id.as_deref().unwrap_or("<none>")
1505 )
1506}
1507
1508fn compare_observed_pool_id_conflicts(
1509 inventory: &DeploymentInventoryV1,
1510 pool_diff: &mut Vec<DiffItemV1>,
1511 hard_failures: &mut Vec<SafetyFindingV1>,
1512 warnings: &mut Vec<SafetyFindingV1>,
1513) {
1514 for group in duplicate_evidence_groups(
1515 &inventory.observed_pool,
1516 |observed| observed.canister_id.as_str().to_string(),
1517 observed_pool_subject,
1518 ",",
1519 ) {
1520 if group.is_conflict {
1521 pool_diff.push(diff_item(
1522 "pool_canister_id_conflict",
1523 &group.subject,
1524 None,
1525 Some(group.evidence_label.clone()),
1526 SafetySeverityV1::HardFailure,
1527 ));
1528 hard_failures.push(finding(
1529 "pool_canister_id_conflict",
1530 format!(
1531 "observed pool canister {} has conflicting pool identities {}",
1532 group.subject, group.evidence_label
1533 ),
1534 SafetySeverityV1::HardFailure,
1535 Some(group.subject),
1536 ));
1537 } else {
1538 pool_diff.push(diff_item(
1539 "pool_canister_duplicate",
1540 &group.subject,
1541 Some(group.evidence_label.clone()),
1542 Some(group.count.to_string()),
1543 SafetySeverityV1::Warning,
1544 ));
1545 warnings.push(finding(
1546 "duplicate_pool_canister_observed",
1547 format!(
1548 "observed pool canister {} was reported {} times for {}",
1549 group.subject, group.count, group.evidence_label
1550 ),
1551 SafetySeverityV1::Warning,
1552 Some(group.subject),
1553 ));
1554 }
1555 }
1556}
1557
1558fn compare_expected_pool<'a>(
1559 expected: &ExpectedPoolCanisterV1,
1560 inventory: &'a DeploymentInventoryV1,
1561 pool_diff: &mut Vec<DiffItemV1>,
1562 hard_failures: &mut Vec<SafetyFindingV1>,
1563 warnings: &mut Vec<SafetyFindingV1>,
1564 matched_observed: &mut BTreeSet<&'a str>,
1565) {
1566 let observed = expected
1567 .canister_id
1568 .as_ref()
1569 .and_then(|id| {
1570 inventory
1571 .observed_pool
1572 .iter()
1573 .find(|pool| &pool.canister_id == id)
1574 })
1575 .or_else(|| {
1576 inventory
1577 .observed_pool
1578 .iter()
1579 .find(|pool| pool_matches_expected_pool(pool, expected))
1580 });
1581 let Some(observed) = observed else {
1582 record_missing_pool(expected, pool_diff, hard_failures, warnings);
1583 return;
1584 };
1585
1586 matched_observed.insert(observed.canister_id.as_str());
1587 record_pool_id_mismatch(expected, observed, pool_diff, hard_failures);
1588 record_unsafe_pool_control_class(observed, pool_diff, hard_failures);
1589}
1590
1591fn record_missing_pool(
1592 expected: &ExpectedPoolCanisterV1,
1593 pool_diff: &mut Vec<DiffItemV1>,
1594 hard_failures: &mut Vec<SafetyFindingV1>,
1595 warnings: &mut Vec<SafetyFindingV1>,
1596) {
1597 let severity = if expected.canister_id.is_some() {
1598 SafetySeverityV1::HardFailure
1599 } else {
1600 SafetySeverityV1::Warning
1601 };
1602 let subject = expected_pool_subject(expected);
1603 pool_diff.push(diff_item(
1604 "pool_canister",
1605 &subject,
1606 expected.canister_id.clone(),
1607 None,
1608 severity,
1609 ));
1610 let finding = finding(
1611 if expected.canister_id.is_some() {
1612 "pool_canister_missing"
1613 } else {
1614 "pool_canister_unobserved"
1615 },
1616 format!("missing observed pool canister for {subject}"),
1617 severity,
1618 Some(subject),
1619 );
1620 if expected.canister_id.is_some() {
1621 hard_failures.push(finding);
1622 } else {
1623 warnings.push(finding);
1624 }
1625}
1626
1627fn record_pool_id_mismatch(
1628 expected: &ExpectedPoolCanisterV1,
1629 observed: &ObservedPoolCanisterV1,
1630 pool_diff: &mut Vec<DiffItemV1>,
1631 hard_failures: &mut Vec<SafetyFindingV1>,
1632) {
1633 if let Some(expected_id) = expected.canister_id.as_ref()
1634 && observed.canister_id != *expected_id
1635 {
1636 let subject = observed_pool_subject(observed);
1637 pool_diff.push(diff_item(
1638 "pool_canister_id",
1639 &subject,
1640 Some(expected_id.clone()),
1641 Some(observed.canister_id.clone()),
1642 SafetySeverityV1::HardFailure,
1643 ));
1644 hard_failures.push(finding(
1645 "pool_canister_id_mismatch",
1646 format!(
1647 "pool canister {subject} has observed id {}, expected {expected_id}",
1648 observed.canister_id
1649 ),
1650 SafetySeverityV1::HardFailure,
1651 Some(subject),
1652 ));
1653 }
1654}
1655
1656fn record_unsafe_pool_control_class(
1657 observed: &ObservedPoolCanisterV1,
1658 pool_diff: &mut Vec<DiffItemV1>,
1659 hard_failures: &mut Vec<SafetyFindingV1>,
1660) {
1661 if !matches!(
1662 observed.control_class,
1663 CanisterControlClassV1::UnknownUnsafe | CanisterControlClassV1::UserControlled
1664 ) {
1665 return;
1666 }
1667 let subject = observed_pool_subject(observed);
1668 pool_diff.push(diff_item(
1669 "pool_control_class",
1670 &subject,
1671 Some("CanicManagedPool".to_string()),
1672 Some(format!("{:?}", observed.control_class)),
1673 SafetySeverityV1::HardFailure,
1674 ));
1675 hard_failures.push(finding(
1676 "unsafe_pool_control_class",
1677 format!("pool canister {subject} has unsafe observed control class"),
1678 SafetySeverityV1::HardFailure,
1679 Some(subject),
1680 ));
1681}
1682
1683fn warn_extra_observed_pool(
1684 plan: &DeploymentPlanV1,
1685 observed: &ObservedPoolCanisterV1,
1686 pool_diff: &mut Vec<DiffItemV1>,
1687 warnings: &mut Vec<SafetyFindingV1>,
1688 matched_observed: &BTreeSet<&str>,
1689) {
1690 if matched_observed.contains(observed.canister_id.as_str())
1691 || plan.expected_pool.iter().any(|expected| {
1692 expected.canister_id.as_ref() == Some(&observed.canister_id)
1693 || pool_matches_expected_pool(observed, expected)
1694 })
1695 {
1696 return;
1697 }
1698 let subject = observed_pool_subject(observed);
1699 pool_diff.push(diff_item(
1700 "pool_extra",
1701 &subject,
1702 None,
1703 Some(observed.canister_id.clone()),
1704 SafetySeverityV1::Warning,
1705 ));
1706 warnings.push(finding(
1707 "extra_pool_canister_observed",
1708 format!("observed undeclared pool canister {subject}"),
1709 SafetySeverityV1::Warning,
1710 Some(subject),
1711 ));
1712}
1713
1714fn pool_matches_expected_pool(
1715 observed: &ObservedPoolCanisterV1,
1716 expected: &ExpectedPoolCanisterV1,
1717) -> bool {
1718 observed.pool == expected.pool
1719 && expected
1720 .role
1721 .as_ref()
1722 .is_none_or(|role| observed.role.as_ref() == Some(role))
1723}
1724
1725fn expected_pool_subject(expected: &ExpectedPoolCanisterV1) -> String {
1726 expected.role.as_ref().map_or_else(
1727 || expected.pool.clone(),
1728 |role| format!("{}:{role}", expected.pool),
1729 )
1730}
1731
1732fn observed_pool_subject(observed: &ObservedPoolCanisterV1) -> String {
1733 observed.role.as_ref().map_or_else(
1734 || observed.pool.clone(),
1735 |role| format!("{}:{role}", observed.pool),
1736 )
1737}
1738
1739fn compare_role_controllers(
1740 plan: &DeploymentPlanV1,
1741 observed: &ObservedCanisterV1,
1742 controller_diff: &mut Vec<DiffItemV1>,
1743 hard_failures: &mut Vec<SafetyFindingV1>,
1744 warnings: &mut Vec<SafetyFindingV1>,
1745) {
1746 let role = observed.role.as_deref().unwrap_or("unknown");
1747 if observed.controllers.is_empty() && !observed_source_includes_live_status(observed) {
1748 warnings.push(finding(
1749 "controllers_unobserved",
1750 format!("controllers were not observed for role {role}"),
1751 SafetySeverityV1::Warning,
1752 Some(role.to_string()),
1753 ));
1754 return;
1755 }
1756 for expected in &plan.authority_profile.expected_controllers {
1757 if observed
1758 .controllers
1759 .iter()
1760 .any(|controller| controller == expected)
1761 {
1762 continue;
1763 }
1764 record_missing_expected_controller(
1765 role,
1766 expected,
1767 &observed.controllers,
1768 controller_diff,
1769 hard_failures,
1770 );
1771 }
1772
1773 for observed_controller in &observed.controllers {
1774 if is_declared_controller(plan, observed_controller) {
1775 continue;
1776 }
1777 record_extra_controller(role, observed_controller, plan, controller_diff, warnings);
1778 }
1779}
1780
1781fn record_missing_expected_controller(
1782 role: &str,
1783 expected: &str,
1784 observed_controllers: &[String],
1785 controller_diff: &mut Vec<DiffItemV1>,
1786 hard_failures: &mut Vec<SafetyFindingV1>,
1787) {
1788 controller_diff.push(diff_item(
1789 "controller_missing",
1790 role,
1791 Some(expected.to_string()),
1792 Some(controller_set_label(observed_controllers)),
1793 SafetySeverityV1::HardFailure,
1794 ));
1795 hard_failures.push(finding(
1796 "expected_controller_missing",
1797 format!("role {role} is missing expected controller {expected}"),
1798 SafetySeverityV1::HardFailure,
1799 Some(role.to_string()),
1800 ));
1801}
1802
1803fn record_extra_controller(
1804 role: &str,
1805 observed_controller: &str,
1806 plan: &DeploymentPlanV1,
1807 controller_diff: &mut Vec<DiffItemV1>,
1808 warnings: &mut Vec<SafetyFindingV1>,
1809) {
1810 controller_diff.push(diff_item(
1811 "controller_extra",
1812 role,
1813 Some(controller_set_label(
1814 &plan.authority_profile.expected_controllers,
1815 )),
1816 Some(observed_controller.to_string()),
1817 SafetySeverityV1::Warning,
1818 ));
1819 warnings.push(finding(
1820 "extra_controller_observed",
1821 format!("role {role} has controller outside the expected authority profile"),
1822 SafetySeverityV1::Warning,
1823 Some(role.to_string()),
1824 ));
1825}
1826
1827fn observed_source_includes_live_status(observed: &ObservedCanisterV1) -> bool {
1828 observed
1829 .role_assignment_source
1830 .as_deref()
1831 .is_some_and(|source| source.contains("icp_canister_status"))
1832}
1833
1834fn is_declared_controller(plan: &DeploymentPlanV1, controller: &str) -> bool {
1835 plan.authority_profile
1836 .expected_controllers
1837 .iter()
1838 .chain(plan.authority_profile.staging_controllers.iter())
1839 .chain(plan.authority_profile.emergency_controllers.iter())
1840 .any(|expected| expected == controller)
1841}
1842
1843fn is_staging_or_emergency_controller(plan: &DeploymentPlanV1, controller: &str) -> bool {
1844 plan.authority_profile
1845 .staging_controllers
1846 .iter()
1847 .chain(plan.authority_profile.emergency_controllers.iter())
1848 .any(|declared| declared == controller)
1849}
1850
1851fn controller_set_label(controllers: &[String]) -> String {
1852 if controllers.is_empty() {
1853 return "<none>".to_string();
1854 }
1855 controllers.join(",")
1856}
1857
1858fn compare_module_hashes(
1859 plan: &DeploymentPlanV1,
1860 inventory: &DeploymentInventoryV1,
1861 module_hash_diff: &mut Vec<DiffItemV1>,
1862 hard_failures: &mut Vec<SafetyFindingV1>,
1863 warnings: &mut Vec<SafetyFindingV1>,
1864) {
1865 for artifact in &plan.role_artifacts {
1866 let Some(expected) = artifact.installed_module_hash.as_ref() else {
1867 continue;
1868 };
1869 let Some(observed_canister) = observed_canister_for_module_hash(
1870 plan,
1871 inventory,
1872 &artifact.role,
1873 module_hash_diff,
1874 hard_failures,
1875 ) else {
1876 continue;
1877 };
1878 match observed_canister.module_hash.as_ref() {
1879 Some(observed) if observed != expected => record_module_hash_mismatch(
1880 &artifact.role,
1881 expected,
1882 observed,
1883 module_hash_diff,
1884 hard_failures,
1885 ),
1886 None => record_module_hash_unobserved(&artifact.role, warnings),
1887 _ => {}
1888 }
1889 }
1890}
1891
1892fn observed_canister_for_module_hash<'a>(
1893 plan: &DeploymentPlanV1,
1894 inventory: &'a DeploymentInventoryV1,
1895 role: &str,
1896 module_hash_diff: &mut Vec<DiffItemV1>,
1897 hard_failures: &mut Vec<SafetyFindingV1>,
1898) -> Option<&'a ObservedCanisterV1> {
1899 if let Some(expected_id) = expected_canister_id_for_role(plan, role) {
1900 return inventory
1901 .observed_canisters
1902 .iter()
1903 .find(|canister| canister.canister_id == expected_id);
1904 }
1905
1906 let role_matches = inventory
1907 .observed_canisters
1908 .iter()
1909 .filter(|canister| canister.role.as_deref() == Some(role))
1910 .collect::<Vec<_>>();
1911 if role_matches.len() > 1 {
1912 record_ambiguous_module_hash_role(role, &role_matches, module_hash_diff, hard_failures);
1913 return None;
1914 }
1915
1916 role_matches.into_iter().next()
1917}
1918
1919fn record_module_hash_mismatch(
1920 role: &str,
1921 expected: &str,
1922 observed: &str,
1923 module_hash_diff: &mut Vec<DiffItemV1>,
1924 hard_failures: &mut Vec<SafetyFindingV1>,
1925) {
1926 module_hash_diff.push(diff_item(
1927 "installed_module_hash",
1928 role,
1929 Some(expected.to_string()),
1930 Some(observed.to_string()),
1931 SafetySeverityV1::HardFailure,
1932 ));
1933 hard_failures.push(finding(
1934 "installed_module_hash_mismatch",
1935 format!("installed module hash differs for role {role}"),
1936 SafetySeverityV1::HardFailure,
1937 Some(role.to_string()),
1938 ));
1939}
1940
1941fn record_module_hash_unobserved(role: &str, warnings: &mut Vec<SafetyFindingV1>) {
1942 warnings.push(finding(
1943 "installed_module_hash_unobserved",
1944 format!("installed module hash was not observed for role {role}"),
1945 SafetySeverityV1::Warning,
1946 Some(role.to_string()),
1947 ));
1948}
1949
1950fn record_ambiguous_module_hash_role(
1951 role: &str,
1952 role_matches: &[&ObservedCanisterV1],
1953 module_hash_diff: &mut Vec<DiffItemV1>,
1954 hard_failures: &mut Vec<SafetyFindingV1>,
1955) {
1956 let observed_ids = role_matches
1957 .iter()
1958 .map(|canister| canister.canister_id.as_str())
1959 .collect::<Vec<_>>()
1960 .join(",");
1961 module_hash_diff.push(diff_item(
1962 "installed_module_hash_ambiguous",
1963 role,
1964 Some("one observed canister".to_string()),
1965 Some(observed_ids.clone()),
1966 SafetySeverityV1::HardFailure,
1967 ));
1968 hard_failures.push(finding(
1969 "installed_module_hash_ambiguous",
1970 format!(
1971 "installed module hash for role {role} has multiple observed canisters: {observed_ids}"
1972 ),
1973 SafetySeverityV1::HardFailure,
1974 Some(role.to_string()),
1975 ));
1976}
1977
1978fn expected_canister_id_for_role<'a>(plan: &'a DeploymentPlanV1, role: &str) -> Option<&'a str> {
1979 plan.expected_canisters
1980 .iter()
1981 .find(|canister| canister.role == role)
1982 .and_then(|canister| canister.canister_id.as_deref())
1983}
1984
1985fn compare_raw_config(
1986 plan: &DeploymentPlanV1,
1987 inventory: &DeploymentInventoryV1,
1988 embedded_config_diff: &mut Vec<DiffItemV1>,
1989 hard_failures: &mut Vec<SafetyFindingV1>,
1990) {
1991 let mut expected = plan
1992 .role_artifacts
1993 .iter()
1994 .filter_map(|artifact| artifact.raw_config_sha256.as_ref())
1995 .collect::<Vec<_>>();
1996 expected.sort_unstable();
1997 expected.dedup();
1998 let [expected] = expected.as_slice() else {
1999 if expected.len() > 1 {
2000 hard_failures.push(finding(
2001 "raw_config_plan_inconsistent",
2002 "planned role artifacts disagree on raw config digest",
2003 SafetySeverityV1::HardFailure,
2004 Some("role_artifacts.raw_config_sha256".to_string()),
2005 ));
2006 }
2007 return;
2008 };
2009
2010 if let Some(observed) = &inventory.local_config.raw_config_sha256
2011 && observed != *expected
2012 {
2013 record_raw_config_mismatch(expected, observed, embedded_config_diff, hard_failures);
2014 }
2015}
2016
2017fn record_raw_config_mismatch(
2018 expected: &str,
2019 observed: &str,
2020 embedded_config_diff: &mut Vec<DiffItemV1>,
2021 hard_failures: &mut Vec<SafetyFindingV1>,
2022) {
2023 embedded_config_diff.push(diff_item(
2024 "raw_config_sha256",
2025 "deployment",
2026 Some(expected.to_string()),
2027 Some(observed.to_string()),
2028 SafetySeverityV1::HardFailure,
2029 ));
2030 hard_failures.push(finding(
2031 "raw_config_digest_mismatch",
2032 "raw local config digest changed during deployment truth check",
2033 SafetySeverityV1::HardFailure,
2034 Some("local_config.raw_sha256".to_string()),
2035 ));
2036}
2037
2038fn compare_embedded_config(
2039 plan: &DeploymentPlanV1,
2040 inventory: &DeploymentInventoryV1,
2041 embedded_config_diff: &mut Vec<DiffItemV1>,
2042 hard_failures: &mut Vec<SafetyFindingV1>,
2043 warnings: &mut Vec<SafetyFindingV1>,
2044) {
2045 let Some(expected) = &plan.deployment_identity.canonical_runtime_config_digest else {
2046 return;
2047 };
2048 match &inventory.local_config.canonical_embedded_config_sha256 {
2049 Some(observed) if observed != expected => {
2050 record_canonical_config_mismatch(
2051 expected,
2052 observed,
2053 embedded_config_diff,
2054 hard_failures,
2055 );
2056 }
2057 None => record_canonical_config_unobserved(warnings),
2058 _ => {}
2059 }
2060}
2061
2062fn record_canonical_config_mismatch(
2063 expected: &str,
2064 observed: &str,
2065 embedded_config_diff: &mut Vec<DiffItemV1>,
2066 hard_failures: &mut Vec<SafetyFindingV1>,
2067) {
2068 embedded_config_diff.push(diff_item(
2069 "canonical_config",
2070 "deployment",
2071 Some(expected.to_string()),
2072 Some(observed.to_string()),
2073 SafetySeverityV1::HardFailure,
2074 ));
2075 hard_failures.push(finding(
2076 "canonical_config_mismatch",
2077 "canonical runtime config digest differs from the plan",
2078 SafetySeverityV1::HardFailure,
2079 Some("local_config".to_string()),
2080 ));
2081}
2082
2083fn record_canonical_config_unobserved(warnings: &mut Vec<SafetyFindingV1>) {
2084 warnings.push(finding(
2085 "canonical_config_unobserved",
2086 "canonical runtime config digest was not observed",
2087 SafetySeverityV1::Warning,
2088 Some("local_config".to_string()),
2089 ));
2090}
2091
2092fn compare_verifier_readiness(
2093 plan: &DeploymentPlanV1,
2094 inventory: &DeploymentInventoryV1,
2095 verifier_readiness_diff: &mut Vec<DiffItemV1>,
2096 hard_failures: &mut Vec<SafetyFindingV1>,
2097 warnings: &mut Vec<SafetyFindingV1>,
2098) {
2099 if !plan.expected_verifier_readiness.required {
2100 return;
2101 }
2102 if inventory.observed_verifier_readiness.status == ObservationStatusV1::NotObserved {
2103 verifier_readiness_diff.push(diff_item(
2104 "verifier_readiness",
2105 "deployment",
2106 Some("required".to_string()),
2107 Some("not_observed".to_string()),
2108 SafetySeverityV1::Warning,
2109 ));
2110 warnings.push(finding(
2111 "verifier_readiness_unobserved",
2112 "verifier readiness was required but not observed",
2113 SafetySeverityV1::Warning,
2114 Some("verifier_readiness".to_string()),
2115 ));
2116 }
2117
2118 let planned_conflicting_roles = compare_planned_verifier_epoch_conflicts(
2119 plan,
2120 verifier_readiness_diff,
2121 hard_failures,
2122 warnings,
2123 );
2124 let conflicting_roles = compare_observed_verifier_epoch_conflicts(
2125 inventory,
2126 verifier_readiness_diff,
2127 hard_failures,
2128 warnings,
2129 );
2130 let mut observed_by_role = BTreeMap::new();
2131 for epoch in &inventory.observed_verifier_readiness.role_epochs {
2132 if conflicting_roles.contains(&epoch.role) {
2133 continue;
2134 }
2135 observed_by_role.entry(epoch.role.as_str()).or_insert(epoch);
2136 }
2137 let mut compared_roles = BTreeSet::new();
2138 for expected in &plan.expected_verifier_readiness.expected_role_epochs {
2139 if planned_conflicting_roles.contains(&expected.role)
2140 || conflicting_roles.contains(&expected.role)
2141 || !compared_roles.insert(expected.role.as_str())
2142 {
2143 continue;
2144 }
2145 let observed = observed_by_role.get(expected.role.as_str());
2146 if let Some(observed_epoch) = observed.and_then(|observed| {
2147 (observed.status == ObservationStatusV1::Observed)
2148 .then_some(observed.observed_epoch)
2149 .flatten()
2150 }) {
2151 if observed_epoch < expected.minimum_epoch {
2152 record_stale_verifier_role_epoch(
2153 expected,
2154 observed_epoch,
2155 verifier_readiness_diff,
2156 hard_failures,
2157 );
2158 }
2159 } else {
2160 record_unobserved_verifier_role_epoch(expected, verifier_readiness_diff, warnings);
2161 }
2162 }
2163}
2164
2165fn record_stale_verifier_role_epoch(
2166 expected: &RoleEpochExpectationV1,
2167 observed_epoch: u64,
2168 verifier_readiness_diff: &mut Vec<DiffItemV1>,
2169 hard_failures: &mut Vec<SafetyFindingV1>,
2170) {
2171 verifier_readiness_diff.push(diff_item(
2172 "verifier_role_epoch",
2173 &expected.role,
2174 Some(expected.minimum_epoch.to_string()),
2175 Some(observed_epoch.to_string()),
2176 SafetySeverityV1::HardFailure,
2177 ));
2178 hard_failures.push(finding(
2179 "verifier_role_epoch_stale",
2180 format!(
2181 "verifier role {} has epoch {observed_epoch}, expected at least {}",
2182 expected.role, expected.minimum_epoch
2183 ),
2184 SafetySeverityV1::HardFailure,
2185 Some(expected.role.clone()),
2186 ));
2187}
2188
2189fn record_unobserved_verifier_role_epoch(
2190 expected: &RoleEpochExpectationV1,
2191 verifier_readiness_diff: &mut Vec<DiffItemV1>,
2192 warnings: &mut Vec<SafetyFindingV1>,
2193) {
2194 verifier_readiness_diff.push(diff_item(
2195 "verifier_role_epoch",
2196 &expected.role,
2197 Some(expected.minimum_epoch.to_string()),
2198 Some("not_observed".to_string()),
2199 SafetySeverityV1::Warning,
2200 ));
2201 warnings.push(finding(
2202 "verifier_role_epoch_unobserved",
2203 format!("verifier role {} epoch was not observed", expected.role),
2204 SafetySeverityV1::Warning,
2205 Some(expected.role.clone()),
2206 ));
2207}
2208
2209fn compare_planned_verifier_epoch_conflicts(
2210 plan: &DeploymentPlanV1,
2211 verifier_readiness_diff: &mut Vec<DiffItemV1>,
2212 hard_failures: &mut Vec<SafetyFindingV1>,
2213 warnings: &mut Vec<SafetyFindingV1>,
2214) -> BTreeSet<String> {
2215 let mut conflicting_roles = BTreeSet::new();
2216 for group in duplicate_evidence_groups(
2217 &plan.expected_verifier_readiness.expected_role_epochs,
2218 |expected| expected.role.as_str().to_string(),
2219 |expected| expected.minimum_epoch.to_string(),
2220 ",",
2221 ) {
2222 if group.is_conflict {
2223 conflicting_roles.insert(group.subject.clone());
2224 verifier_readiness_diff.push(diff_item(
2225 "planned_verifier_role_epoch_conflict",
2226 &group.subject,
2227 Some("one minimum epoch".to_string()),
2228 Some(group.evidence_label.clone()),
2229 SafetySeverityV1::HardFailure,
2230 ));
2231 hard_failures.push(finding(
2232 "planned_verifier_role_epoch_conflict",
2233 format!(
2234 "planned verifier role {} has conflicting minimum epochs: {}",
2235 group.subject, group.evidence_label
2236 ),
2237 SafetySeverityV1::HardFailure,
2238 Some(group.subject),
2239 ));
2240 } else {
2241 verifier_readiness_diff.push(diff_item(
2242 "planned_verifier_role_epoch_duplicate",
2243 &group.subject,
2244 Some(group.evidence_label.clone()),
2245 Some(group.count.to_string()),
2246 SafetySeverityV1::Warning,
2247 ));
2248 warnings.push(finding(
2249 "duplicate_planned_verifier_role_epoch",
2250 format!(
2251 "planned verifier role {} epoch was declared {} times with identical evidence",
2252 group.subject, group.count
2253 ),
2254 SafetySeverityV1::Warning,
2255 Some(group.subject),
2256 ));
2257 }
2258 }
2259 conflicting_roles
2260}
2261
2262fn compare_observed_verifier_epoch_conflicts(
2263 inventory: &DeploymentInventoryV1,
2264 verifier_readiness_diff: &mut Vec<DiffItemV1>,
2265 hard_failures: &mut Vec<SafetyFindingV1>,
2266 warnings: &mut Vec<SafetyFindingV1>,
2267) -> BTreeSet<String> {
2268 let mut conflicting_roles = BTreeSet::new();
2269 for group in duplicate_evidence_groups(
2270 &inventory.observed_verifier_readiness.role_epochs,
2271 |observed| observed.role.as_str().to_string(),
2272 verifier_epoch_evidence_label,
2273 ",",
2274 ) {
2275 if group.is_conflict {
2276 conflicting_roles.insert(group.subject.clone());
2277 verifier_readiness_diff.push(diff_item(
2278 "verifier_role_epoch_conflict",
2279 &group.subject,
2280 Some("one epoch observation".to_string()),
2281 Some(group.evidence_label.clone()),
2282 SafetySeverityV1::HardFailure,
2283 ));
2284 hard_failures.push(finding(
2285 "verifier_role_epoch_conflict",
2286 format!(
2287 "verifier role {} has conflicting epoch observations: {}",
2288 group.subject, group.evidence_label
2289 ),
2290 SafetySeverityV1::HardFailure,
2291 Some(group.subject),
2292 ));
2293 } else {
2294 verifier_readiness_diff.push(diff_item(
2295 "verifier_role_epoch_duplicate",
2296 &group.subject,
2297 Some(group.evidence_label.clone()),
2298 Some(group.count.to_string()),
2299 SafetySeverityV1::Warning,
2300 ));
2301 warnings.push(finding(
2302 "duplicate_verifier_role_epoch_observed",
2303 format!(
2304 "verifier role {} epoch was reported {} times with identical evidence",
2305 group.subject, group.count
2306 ),
2307 SafetySeverityV1::Warning,
2308 Some(group.subject),
2309 ));
2310 }
2311 }
2312 conflicting_roles
2313}
2314
2315fn verifier_epoch_evidence_label(observed: &RoleEpochObservationV1) -> String {
2316 format!(
2317 "epoch={};status={:?}",
2318 observed
2319 .observed_epoch
2320 .map_or_else(|| "<none>".to_string(), |epoch| epoch.to_string()),
2321 observed.status
2322 )
2323}
2324
2325fn finding(
2326 code: impl Into<String>,
2327 message: impl Into<String>,
2328 severity: SafetySeverityV1,
2329 subject: Option<String>,
2330) -> SafetyFindingV1 {
2331 SafetyFindingV1 {
2332 code: code.into(),
2333 message: message.into(),
2334 severity,
2335 subject,
2336 }
2337}
2338
2339fn diff_item(
2340 category: impl Into<String>,
2341 subject: impl Into<String>,
2342 expected: Option<String>,
2343 observed: Option<String>,
2344 severity: SafetySeverityV1,
2345) -> DiffItemV1 {
2346 DiffItemV1 {
2347 category: category.into(),
2348 subject: subject.into(),
2349 expected,
2350 observed,
2351 severity,
2352 }
2353}
2354
2355fn duplicate_evidence_groups<T>(
2356 items: &[T],
2357 subject: impl Fn(&T) -> String,
2358 evidence: impl Fn(&T) -> String,
2359 evidence_separator: &str,
2360) -> Vec<DuplicateEvidenceGroup> {
2361 let mut groups = Vec::new();
2362 for (subject, entries) in group_by_subject(items, |item| Some(subject(item))) {
2363 if entries.len() <= 1 {
2364 continue;
2365 }
2366 let evidence_values = entries
2367 .iter()
2368 .map(|entry| evidence(entry))
2369 .collect::<BTreeSet<_>>();
2370 groups.push(DuplicateEvidenceGroup {
2371 subject,
2372 count: entries.len(),
2373 evidence_label: evidence_values
2374 .iter()
2375 .cloned()
2376 .collect::<Vec<_>>()
2377 .join(evidence_separator),
2378 is_conflict: evidence_values.len() > 1,
2379 });
2380 }
2381 groups
2382}
2383
2384fn conflicting_assignment_groups<T>(
2385 items: &[T],
2386 subject: impl Fn(&T) -> Option<String>,
2387 value: impl Fn(&T) -> String,
2388 value_separator: &str,
2389) -> Vec<DuplicateEvidenceGroup> {
2390 let mut groups = Vec::new();
2391 for (subject, entries) in group_by_subject(items, subject) {
2392 if entries.len() <= 1 {
2393 continue;
2394 }
2395 let values = entries
2396 .iter()
2397 .map(|entry| value(entry))
2398 .collect::<BTreeSet<_>>();
2399 if values.len() <= 1 {
2400 continue;
2401 }
2402 groups.push(DuplicateEvidenceGroup {
2403 subject,
2404 count: entries.len(),
2405 evidence_label: values
2406 .iter()
2407 .cloned()
2408 .collect::<Vec<_>>()
2409 .join(value_separator),
2410 is_conflict: true,
2411 });
2412 }
2413 groups
2414}
2415
2416fn group_by_subject<T>(
2417 items: &[T],
2418 subject: impl Fn(&T) -> Option<String>,
2419) -> BTreeMap<String, Vec<&T>> {
2420 let mut by_subject = BTreeMap::<String, Vec<&T>>::new();
2421 for item in items {
2422 if let Some(subject) = subject(item) {
2423 by_subject.entry(subject).or_default().push(item);
2424 }
2425 }
2426 by_subject
2427}
2428
2429const fn safety_status(
2430 hard_failures: &[SafetyFindingV1],
2431 warnings: &[SafetyFindingV1],
2432) -> SafetyStatusV1 {
2433 if !hard_failures.is_empty() {
2434 SafetyStatusV1::Blocked
2435 } else if !warnings.is_empty() {
2436 SafetyStatusV1::Warning
2437 } else {
2438 SafetyStatusV1::Safe
2439 }
2440}
2441
2442fn resume_safety_reasons(
2443 hard_failures: &[SafetyFindingV1],
2444 warnings: &[SafetyFindingV1],
2445) -> Vec<String> {
2446 if !hard_failures.is_empty() {
2447 return hard_failures
2448 .iter()
2449 .map(|finding| finding.message.clone())
2450 .collect();
2451 }
2452 if !warnings.is_empty() {
2453 return warnings
2454 .iter()
2455 .map(|finding| finding.message.clone())
2456 .collect();
2457 }
2458 vec!["no blocking deployment truth differences were found".to_string()]
2459}
2460
2461fn safety_summary(
2462 status: SafetyStatusV1,
2463 hard_failure_count: usize,
2464 warning_count: usize,
2465) -> String {
2466 match status {
2467 SafetyStatusV1::Safe => "deployment inventory matches the checked plan".to_string(),
2468 SafetyStatusV1::Warning => {
2469 format!("deployment inventory has {warning_count} warning(s)")
2470 }
2471 SafetyStatusV1::Blocked => {
2472 format!(
2473 "deployment inventory has {hard_failure_count} blocking issue(s) and {warning_count} warning(s)"
2474 )
2475 }
2476 SafetyStatusV1::NotEvaluated => "deployment safety has not been evaluated".to_string(),
2477 }
2478}
2479
2480fn safety_next_actions(status: SafetyStatusV1) -> Vec<String> {
2481 match status {
2482 SafetyStatusV1::Safe => Vec::new(),
2483 SafetyStatusV1::Warning => {
2484 vec!["review deployment warnings before continuing".to_string()]
2485 }
2486 SafetyStatusV1::Blocked => {
2487 vec!["resolve blocking deployment truth differences before mutation".to_string()]
2488 }
2489 SafetyStatusV1::NotEvaluated => vec!["collect deployment inventory".to_string()],
2490 }
2491}