1use crate::{
2 artifacts::{ArtifactChecksum, ArtifactChecksumError},
3 manifest::{
4 FleetBackupManifest, FleetMember, IdentityMode, ManifestDesignConformanceReport,
5 ManifestValidationError, SourceSnapshot, VerificationCheck, VerificationPlan,
6 },
7};
8use candid::Principal;
9use serde::{Deserialize, Serialize};
10use std::{
11 collections::{BTreeMap, BTreeSet},
12 path::{Component, Path, PathBuf},
13 str::FromStr,
14};
15use thiserror::Error as ThisError;
16
17#[derive(Clone, Debug, Default, Deserialize, Serialize)]
22pub struct RestoreMapping {
23 pub members: Vec<RestoreMappingEntry>,
24}
25
26impl RestoreMapping {
27 fn target_for(&self, source_canister: &str) -> Option<&str> {
29 self.members
30 .iter()
31 .find(|entry| entry.source_canister == source_canister)
32 .map(|entry| entry.target_canister.as_str())
33 }
34}
35
36#[derive(Clone, Debug, Deserialize, Serialize)]
41pub struct RestoreMappingEntry {
42 pub source_canister: String,
43 pub target_canister: String,
44}
45
46#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
51pub struct RestorePlan {
52 pub backup_id: String,
53 pub source_environment: String,
54 pub source_root_canister: String,
55 pub topology_hash: String,
56 pub member_count: usize,
57 pub identity_summary: RestoreIdentitySummary,
58 pub snapshot_summary: RestoreSnapshotSummary,
59 pub verification_summary: RestoreVerificationSummary,
60 pub readiness_summary: RestoreReadinessSummary,
61 pub operation_summary: RestoreOperationSummary,
62 pub ordering_summary: RestoreOrderingSummary,
63 #[serde(default)]
64 pub design_conformance: Option<ManifestDesignConformanceReport>,
65 #[serde(default)]
66 pub fleet_verification_checks: Vec<VerificationCheck>,
67 pub phases: Vec<RestorePhase>,
68}
69
70impl RestorePlan {
71 #[must_use]
73 pub fn ordered_members(&self) -> Vec<&RestorePlanMember> {
74 self.phases
75 .iter()
76 .flat_map(|phase| phase.members.iter())
77 .collect()
78 }
79}
80
81#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
86pub struct RestoreStatus {
87 pub status_version: u16,
88 pub backup_id: String,
89 pub source_environment: String,
90 pub source_root_canister: String,
91 pub topology_hash: String,
92 pub ready: bool,
93 pub readiness_reasons: Vec<String>,
94 pub verification_required: bool,
95 pub member_count: usize,
96 pub phase_count: usize,
97 #[serde(default)]
98 pub planned_snapshot_uploads: usize,
99 pub planned_snapshot_loads: usize,
100 pub planned_code_reinstalls: usize,
101 pub planned_verification_checks: usize,
102 #[serde(default)]
103 pub planned_operations: usize,
104 pub phases: Vec<RestoreStatusPhase>,
105}
106
107impl RestoreStatus {
108 #[must_use]
110 pub fn from_plan(plan: &RestorePlan) -> Self {
111 Self {
112 status_version: 1,
113 backup_id: plan.backup_id.clone(),
114 source_environment: plan.source_environment.clone(),
115 source_root_canister: plan.source_root_canister.clone(),
116 topology_hash: plan.topology_hash.clone(),
117 ready: plan.readiness_summary.ready,
118 readiness_reasons: plan.readiness_summary.reasons.clone(),
119 verification_required: plan.verification_summary.verification_required,
120 member_count: plan.member_count,
121 phase_count: plan.ordering_summary.phase_count,
122 planned_snapshot_uploads: plan
123 .operation_summary
124 .effective_planned_snapshot_uploads(plan.member_count),
125 planned_snapshot_loads: plan.operation_summary.planned_snapshot_loads,
126 planned_code_reinstalls: plan.operation_summary.planned_code_reinstalls,
127 planned_verification_checks: plan.operation_summary.planned_verification_checks,
128 planned_operations: plan
129 .operation_summary
130 .effective_planned_operations(plan.member_count),
131 phases: plan
132 .phases
133 .iter()
134 .map(RestoreStatusPhase::from_plan_phase)
135 .collect(),
136 }
137 }
138}
139
140#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
145pub struct RestoreStatusPhase {
146 pub restore_group: u16,
147 pub members: Vec<RestoreStatusMember>,
148}
149
150impl RestoreStatusPhase {
151 fn from_plan_phase(phase: &RestorePhase) -> Self {
153 Self {
154 restore_group: phase.restore_group,
155 members: phase
156 .members
157 .iter()
158 .map(RestoreStatusMember::from_plan_member)
159 .collect(),
160 }
161 }
162}
163
164#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
169pub struct RestoreStatusMember {
170 pub source_canister: String,
171 pub target_canister: String,
172 pub role: String,
173 pub restore_group: u16,
174 pub phase_order: usize,
175 pub snapshot_id: String,
176 pub artifact_path: String,
177 pub state: RestoreMemberState,
178}
179
180impl RestoreStatusMember {
181 fn from_plan_member(member: &RestorePlanMember) -> Self {
183 Self {
184 source_canister: member.source_canister.clone(),
185 target_canister: member.target_canister.clone(),
186 role: member.role.clone(),
187 restore_group: member.restore_group,
188 phase_order: member.phase_order,
189 snapshot_id: member.source_snapshot.snapshot_id.clone(),
190 artifact_path: member.source_snapshot.artifact_path.clone(),
191 state: RestoreMemberState::Planned,
192 }
193 }
194}
195
196#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
201#[serde(rename_all = "kebab-case")]
202pub enum RestoreMemberState {
203 Planned,
204}
205
206#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
211pub struct RestoreApplyDryRun {
212 pub dry_run_version: u16,
213 pub backup_id: String,
214 pub ready: bool,
215 pub readiness_reasons: Vec<String>,
216 pub member_count: usize,
217 pub phase_count: usize,
218 pub status_supplied: bool,
219 #[serde(default)]
220 pub planned_snapshot_uploads: usize,
221 pub planned_snapshot_loads: usize,
222 pub planned_code_reinstalls: usize,
223 pub planned_verification_checks: usize,
224 #[serde(default)]
225 pub planned_operations: usize,
226 pub rendered_operations: usize,
227 #[serde(default)]
228 pub operation_counts: RestoreApplyOperationKindCounts,
229 pub artifact_validation: Option<RestoreApplyArtifactValidation>,
230 pub phases: Vec<RestoreApplyDryRunPhase>,
231}
232
233impl RestoreApplyDryRun {
234 pub fn try_from_plan(
236 plan: &RestorePlan,
237 status: Option<&RestoreStatus>,
238 ) -> Result<Self, RestoreApplyDryRunError> {
239 if let Some(status) = status {
240 validate_restore_status_matches_plan(plan, status)?;
241 }
242
243 Ok(Self::from_validated_plan(plan, status))
244 }
245
246 pub fn try_from_plan_with_artifacts(
248 plan: &RestorePlan,
249 status: Option<&RestoreStatus>,
250 backup_root: &Path,
251 ) -> Result<Self, RestoreApplyDryRunError> {
252 let mut dry_run = Self::try_from_plan(plan, status)?;
253 dry_run.artifact_validation = Some(validate_restore_apply_artifacts(plan, backup_root)?);
254 Ok(dry_run)
255 }
256
257 fn from_validated_plan(plan: &RestorePlan, status: Option<&RestoreStatus>) -> Self {
259 let mut next_sequence = 0;
260 let phases = plan
261 .phases
262 .iter()
263 .map(|phase| RestoreApplyDryRunPhase::from_plan_phase(phase, &mut next_sequence))
264 .collect::<Vec<_>>();
265 let mut phases = phases;
266 append_fleet_verification_operations(plan, &mut phases, &mut next_sequence);
267 let rendered_operations = phases
268 .iter()
269 .map(|phase| phase.operations.len())
270 .sum::<usize>();
271 let operation_counts = RestoreApplyOperationKindCounts::from_dry_run_phases(&phases);
272
273 Self {
274 dry_run_version: 1,
275 backup_id: plan.backup_id.clone(),
276 ready: status.map_or(plan.readiness_summary.ready, |status| status.ready),
277 readiness_reasons: status.map_or_else(
278 || plan.readiness_summary.reasons.clone(),
279 |status| status.readiness_reasons.clone(),
280 ),
281 member_count: plan.member_count,
282 phase_count: plan.ordering_summary.phase_count,
283 status_supplied: status.is_some(),
284 planned_snapshot_uploads: plan
285 .operation_summary
286 .effective_planned_snapshot_uploads(plan.member_count),
287 planned_snapshot_loads: plan.operation_summary.planned_snapshot_loads,
288 planned_code_reinstalls: plan.operation_summary.planned_code_reinstalls,
289 planned_verification_checks: plan.operation_summary.planned_verification_checks,
290 planned_operations: plan
291 .operation_summary
292 .effective_planned_operations(plan.member_count),
293 rendered_operations,
294 operation_counts,
295 artifact_validation: None,
296 phases,
297 }
298 }
299}
300
301#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
306pub struct RestoreApplyJournal {
307 pub journal_version: u16,
308 pub backup_id: String,
309 pub ready: bool,
310 pub blocked_reasons: Vec<String>,
311 pub operation_count: usize,
312 #[serde(default)]
313 pub operation_counts: RestoreApplyOperationKindCounts,
314 pub pending_operations: usize,
315 pub ready_operations: usize,
316 pub blocked_operations: usize,
317 pub completed_operations: usize,
318 pub failed_operations: usize,
319 pub operations: Vec<RestoreApplyJournalOperation>,
320}
321
322impl RestoreApplyJournal {
323 #[must_use]
325 pub fn from_dry_run(dry_run: &RestoreApplyDryRun) -> Self {
326 let blocked_reasons = restore_apply_blocked_reasons(dry_run);
327 let initial_state = if blocked_reasons.is_empty() {
328 RestoreApplyOperationState::Ready
329 } else {
330 RestoreApplyOperationState::Blocked
331 };
332 let operations = dry_run
333 .phases
334 .iter()
335 .flat_map(|phase| phase.operations.iter())
336 .map(|operation| {
337 RestoreApplyJournalOperation::from_dry_run_operation(
338 operation,
339 initial_state.clone(),
340 &blocked_reasons,
341 )
342 })
343 .collect::<Vec<_>>();
344 let ready_operations = operations
345 .iter()
346 .filter(|operation| operation.state == RestoreApplyOperationState::Ready)
347 .count();
348 let blocked_operations = operations
349 .iter()
350 .filter(|operation| operation.state == RestoreApplyOperationState::Blocked)
351 .count();
352 let operation_counts = RestoreApplyOperationKindCounts::from_operations(&operations);
353
354 Self {
355 journal_version: 1,
356 backup_id: dry_run.backup_id.clone(),
357 ready: blocked_reasons.is_empty(),
358 blocked_reasons,
359 operation_count: operations.len(),
360 operation_counts,
361 pending_operations: 0,
362 ready_operations,
363 blocked_operations,
364 completed_operations: 0,
365 failed_operations: 0,
366 operations,
367 }
368 }
369
370 pub fn validate(&self) -> Result<(), RestoreApplyJournalError> {
372 validate_apply_journal_version(self.journal_version)?;
373 validate_apply_journal_nonempty("backup_id", &self.backup_id)?;
374 validate_apply_journal_count(
375 "operation_count",
376 self.operation_count,
377 self.operations.len(),
378 )?;
379
380 let state_counts = RestoreApplyJournalStateCounts::from_operations(&self.operations);
381 let operation_counts = RestoreApplyOperationKindCounts::from_operations(&self.operations);
382 self.operation_counts
383 .validate_matches_if_supplied(&operation_counts)?;
384 validate_apply_journal_count(
385 "pending_operations",
386 self.pending_operations,
387 state_counts.pending,
388 )?;
389 validate_apply_journal_count(
390 "ready_operations",
391 self.ready_operations,
392 state_counts.ready,
393 )?;
394 validate_apply_journal_count(
395 "blocked_operations",
396 self.blocked_operations,
397 state_counts.blocked,
398 )?;
399 validate_apply_journal_count(
400 "completed_operations",
401 self.completed_operations,
402 state_counts.completed,
403 )?;
404 validate_apply_journal_count(
405 "failed_operations",
406 self.failed_operations,
407 state_counts.failed,
408 )?;
409
410 if self.ready && (!self.blocked_reasons.is_empty() || self.blocked_operations > 0) {
411 return Err(RestoreApplyJournalError::ReadyJournalHasBlockingState);
412 }
413
414 validate_apply_journal_sequences(&self.operations)?;
415 for operation in &self.operations {
416 operation.validate()?;
417 }
418
419 Ok(())
420 }
421
422 #[must_use]
424 pub fn status(&self) -> RestoreApplyJournalStatus {
425 RestoreApplyJournalStatus::from_journal(self)
426 }
427
428 #[must_use]
430 pub fn report(&self) -> RestoreApplyJournalReport {
431 RestoreApplyJournalReport::from_journal(self)
432 }
433
434 #[must_use]
436 pub fn next_ready_operation(&self) -> Option<&RestoreApplyJournalOperation> {
437 self.operations
438 .iter()
439 .filter(|operation| operation.state == RestoreApplyOperationState::Ready)
440 .min_by_key(|operation| operation.sequence)
441 }
442
443 #[must_use]
445 pub fn next_transition_operation(&self) -> Option<&RestoreApplyJournalOperation> {
446 self.operations
447 .iter()
448 .filter(|operation| {
449 matches!(
450 operation.state,
451 RestoreApplyOperationState::Ready | RestoreApplyOperationState::Pending
452 )
453 })
454 .min_by_key(|operation| operation.sequence)
455 }
456
457 #[must_use]
459 pub fn next_operation(&self) -> RestoreApplyNextOperation {
460 RestoreApplyNextOperation::from_journal(self)
461 }
462
463 #[must_use]
465 pub fn next_command_preview(&self) -> RestoreApplyCommandPreview {
466 RestoreApplyCommandPreview::from_journal(self)
467 }
468
469 #[must_use]
471 pub fn next_command_preview_with_config(
472 &self,
473 config: &RestoreApplyCommandConfig,
474 ) -> RestoreApplyCommandPreview {
475 RestoreApplyCommandPreview::from_journal_with_config(self, config)
476 }
477
478 pub fn mark_next_operation_pending(&mut self) -> Result<(), RestoreApplyJournalError> {
480 self.mark_next_operation_pending_at(None)
481 }
482
483 pub fn mark_next_operation_pending_at(
485 &mut self,
486 updated_at: Option<String>,
487 ) -> Result<(), RestoreApplyJournalError> {
488 let sequence = self
489 .next_transition_sequence()
490 .ok_or(RestoreApplyJournalError::NoTransitionableOperation)?;
491 self.mark_operation_pending_at(sequence, updated_at)
492 }
493
494 pub fn mark_operation_pending(
496 &mut self,
497 sequence: usize,
498 ) -> Result<(), RestoreApplyJournalError> {
499 self.mark_operation_pending_at(sequence, None)
500 }
501
502 pub fn mark_operation_pending_at(
504 &mut self,
505 sequence: usize,
506 updated_at: Option<String>,
507 ) -> Result<(), RestoreApplyJournalError> {
508 self.transition_operation(
509 sequence,
510 RestoreApplyOperationState::Pending,
511 Vec::new(),
512 updated_at,
513 )
514 }
515
516 pub fn mark_next_operation_ready(&mut self) -> Result<(), RestoreApplyJournalError> {
518 self.mark_next_operation_ready_at(None)
519 }
520
521 pub fn mark_next_operation_ready_at(
523 &mut self,
524 updated_at: Option<String>,
525 ) -> Result<(), RestoreApplyJournalError> {
526 let operation = self
527 .next_transition_operation()
528 .ok_or(RestoreApplyJournalError::NoTransitionableOperation)?;
529 if operation.state != RestoreApplyOperationState::Pending {
530 return Err(RestoreApplyJournalError::NoPendingOperation);
531 }
532
533 self.mark_operation_ready_at(operation.sequence, updated_at)
534 }
535
536 pub fn mark_operation_ready(
538 &mut self,
539 sequence: usize,
540 ) -> Result<(), RestoreApplyJournalError> {
541 self.mark_operation_ready_at(sequence, None)
542 }
543
544 pub fn mark_operation_ready_at(
546 &mut self,
547 sequence: usize,
548 updated_at: Option<String>,
549 ) -> Result<(), RestoreApplyJournalError> {
550 self.transition_operation(
551 sequence,
552 RestoreApplyOperationState::Ready,
553 Vec::new(),
554 updated_at,
555 )
556 }
557
558 pub fn mark_operation_completed(
560 &mut self,
561 sequence: usize,
562 ) -> Result<(), RestoreApplyJournalError> {
563 self.mark_operation_completed_at(sequence, None)
564 }
565
566 pub fn mark_operation_completed_at(
568 &mut self,
569 sequence: usize,
570 updated_at: Option<String>,
571 ) -> Result<(), RestoreApplyJournalError> {
572 self.transition_operation(
573 sequence,
574 RestoreApplyOperationState::Completed,
575 Vec::new(),
576 updated_at,
577 )
578 }
579
580 pub fn mark_operation_failed(
582 &mut self,
583 sequence: usize,
584 reason: String,
585 ) -> Result<(), RestoreApplyJournalError> {
586 self.mark_operation_failed_at(sequence, reason, None)
587 }
588
589 pub fn mark_operation_failed_at(
591 &mut self,
592 sequence: usize,
593 reason: String,
594 updated_at: Option<String>,
595 ) -> Result<(), RestoreApplyJournalError> {
596 if reason.trim().is_empty() {
597 return Err(RestoreApplyJournalError::FailureReasonRequired(sequence));
598 }
599
600 self.transition_operation(
601 sequence,
602 RestoreApplyOperationState::Failed,
603 vec![reason],
604 updated_at,
605 )
606 }
607
608 fn transition_operation(
610 &mut self,
611 sequence: usize,
612 next_state: RestoreApplyOperationState,
613 blocking_reasons: Vec<String>,
614 updated_at: Option<String>,
615 ) -> Result<(), RestoreApplyJournalError> {
616 let index = self
617 .operations
618 .iter()
619 .position(|operation| operation.sequence == sequence)
620 .ok_or(RestoreApplyJournalError::OperationNotFound(sequence))?;
621 let operation = &self.operations[index];
622
623 if !operation.can_transition_to(&next_state) {
624 return Err(RestoreApplyJournalError::InvalidOperationTransition {
625 sequence,
626 from: operation.state.clone(),
627 to: next_state,
628 });
629 }
630
631 self.validate_operation_transition_order(operation, &next_state)?;
632
633 let operation = &mut self.operations[index];
634 operation.state = next_state;
635 operation.blocking_reasons = blocking_reasons;
636 operation.state_updated_at = updated_at;
637 self.refresh_operation_counts();
638 self.validate()
639 }
640
641 fn validate_operation_transition_order(
643 &self,
644 operation: &RestoreApplyJournalOperation,
645 next_state: &RestoreApplyOperationState,
646 ) -> Result<(), RestoreApplyJournalError> {
647 if operation.state == *next_state {
648 return Ok(());
649 }
650
651 let next_sequence = self
652 .next_transition_sequence()
653 .ok_or(RestoreApplyJournalError::NoTransitionableOperation)?;
654
655 if operation.sequence == next_sequence {
656 return Ok(());
657 }
658
659 Err(RestoreApplyJournalError::OutOfOrderOperationTransition {
660 requested: operation.sequence,
661 next: next_sequence,
662 })
663 }
664
665 fn next_transition_sequence(&self) -> Option<usize> {
667 self.next_transition_operation()
668 .map(|operation| operation.sequence)
669 }
670
671 fn refresh_operation_counts(&mut self) {
673 let state_counts = RestoreApplyJournalStateCounts::from_operations(&self.operations);
674 self.operation_count = self.operations.len();
675 self.operation_counts = RestoreApplyOperationKindCounts::from_operations(&self.operations);
676 self.pending_operations = state_counts.pending;
677 self.ready_operations = state_counts.ready;
678 self.blocked_operations = state_counts.blocked;
679 self.completed_operations = state_counts.completed;
680 self.failed_operations = state_counts.failed;
681 }
682
683 const fn operation_counts_supplied(&self) -> bool {
685 !self.operation_counts.is_empty() || self.operations.is_empty()
686 }
687}
688
689const fn validate_apply_journal_version(version: u16) -> Result<(), RestoreApplyJournalError> {
691 if version == 1 {
692 return Ok(());
693 }
694
695 Err(RestoreApplyJournalError::UnsupportedVersion(version))
696}
697
698fn validate_apply_journal_nonempty(
700 field: &'static str,
701 value: &str,
702) -> Result<(), RestoreApplyJournalError> {
703 if !value.trim().is_empty() {
704 return Ok(());
705 }
706
707 Err(RestoreApplyJournalError::MissingField(field))
708}
709
710const fn validate_apply_journal_count(
712 field: &'static str,
713 reported: usize,
714 actual: usize,
715) -> Result<(), RestoreApplyJournalError> {
716 if reported == actual {
717 return Ok(());
718 }
719
720 Err(RestoreApplyJournalError::CountMismatch {
721 field,
722 reported,
723 actual,
724 })
725}
726
727fn validate_apply_journal_sequences(
729 operations: &[RestoreApplyJournalOperation],
730) -> Result<(), RestoreApplyJournalError> {
731 let mut sequences = BTreeSet::new();
732 for operation in operations {
733 if !sequences.insert(operation.sequence) {
734 return Err(RestoreApplyJournalError::DuplicateSequence(
735 operation.sequence,
736 ));
737 }
738 }
739
740 for expected in 0..operations.len() {
741 if !sequences.contains(&expected) {
742 return Err(RestoreApplyJournalError::MissingSequence(expected));
743 }
744 }
745
746 Ok(())
747}
748
749#[derive(Clone, Debug, Default, Eq, PartialEq)]
754struct RestoreApplyJournalStateCounts {
755 pending: usize,
756 ready: usize,
757 blocked: usize,
758 completed: usize,
759 failed: usize,
760}
761
762impl RestoreApplyJournalStateCounts {
763 fn from_operations(operations: &[RestoreApplyJournalOperation]) -> Self {
765 let mut counts = Self::default();
766 for operation in operations {
767 match operation.state {
768 RestoreApplyOperationState::Pending => counts.pending += 1,
769 RestoreApplyOperationState::Ready => counts.ready += 1,
770 RestoreApplyOperationState::Blocked => counts.blocked += 1,
771 RestoreApplyOperationState::Completed => counts.completed += 1,
772 RestoreApplyOperationState::Failed => counts.failed += 1,
773 }
774 }
775 counts
776 }
777}
778
779#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
784pub struct RestoreApplyOperationKindCounts {
785 pub snapshot_uploads: usize,
786 pub snapshot_loads: usize,
787 pub code_reinstalls: usize,
788 pub member_verifications: usize,
789 pub fleet_verifications: usize,
790 pub verification_operations: usize,
791}
792
793impl RestoreApplyOperationKindCounts {
794 #[must_use]
796 pub fn from_operations(operations: &[RestoreApplyJournalOperation]) -> Self {
797 let mut counts = Self::default();
798 for operation in operations {
799 counts.record(&operation.operation);
800 }
801 counts
802 }
803
804 pub fn validate_matches_if_supplied(
806 &self,
807 expected: &Self,
808 ) -> Result<(), RestoreApplyJournalError> {
809 if self.is_empty() && !expected.is_empty() {
810 return Ok(());
811 }
812
813 validate_apply_journal_count(
814 "operation_counts.snapshot_uploads",
815 self.snapshot_uploads,
816 expected.snapshot_uploads,
817 )?;
818 validate_apply_journal_count(
819 "operation_counts.snapshot_loads",
820 self.snapshot_loads,
821 expected.snapshot_loads,
822 )?;
823 validate_apply_journal_count(
824 "operation_counts.code_reinstalls",
825 self.code_reinstalls,
826 expected.code_reinstalls,
827 )?;
828 validate_apply_journal_count(
829 "operation_counts.member_verifications",
830 self.member_verifications,
831 expected.member_verifications,
832 )?;
833 validate_apply_journal_count(
834 "operation_counts.fleet_verifications",
835 self.fleet_verifications,
836 expected.fleet_verifications,
837 )?;
838 validate_apply_journal_count(
839 "operation_counts.verification_operations",
840 self.verification_operations,
841 expected.verification_operations,
842 )
843 }
844
845 const fn is_empty(&self) -> bool {
847 self.snapshot_uploads == 0
848 && self.snapshot_loads == 0
849 && self.code_reinstalls == 0
850 && self.member_verifications == 0
851 && self.fleet_verifications == 0
852 && self.verification_operations == 0
853 }
854
855 #[must_use]
857 pub fn from_dry_run_phases(phases: &[RestoreApplyDryRunPhase]) -> Self {
858 let mut counts = Self::default();
859 for operation in phases.iter().flat_map(|phase| {
860 phase
861 .operations
862 .iter()
863 .map(|operation| &operation.operation)
864 }) {
865 counts.record(operation);
866 }
867 counts
868 }
869
870 const fn record(&mut self, operation: &RestoreApplyOperationKind) {
872 match operation {
873 RestoreApplyOperationKind::UploadSnapshot => self.snapshot_uploads += 1,
874 RestoreApplyOperationKind::LoadSnapshot => self.snapshot_loads += 1,
875 RestoreApplyOperationKind::ReinstallCode => self.code_reinstalls += 1,
876 RestoreApplyOperationKind::VerifyMember => {
877 self.member_verifications += 1;
878 self.verification_operations += 1;
879 }
880 RestoreApplyOperationKind::VerifyFleet => {
881 self.fleet_verifications += 1;
882 self.verification_operations += 1;
883 }
884 }
885 }
886}
887
888fn restore_apply_blocked_reasons(dry_run: &RestoreApplyDryRun) -> Vec<String> {
890 let mut reasons = dry_run.readiness_reasons.clone();
891
892 match &dry_run.artifact_validation {
893 Some(validation) => {
894 if !validation.artifacts_present {
895 reasons.push("missing-artifacts".to_string());
896 }
897 if !validation.checksums_verified {
898 reasons.push("artifact-checksum-validation-incomplete".to_string());
899 }
900 }
901 None => reasons.push("missing-artifact-validation".to_string()),
902 }
903
904 reasons.sort();
905 reasons.dedup();
906 reasons
907}
908
909#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
914pub struct RestoreApplyJournalStatus {
915 pub status_version: u16,
916 pub backup_id: String,
917 pub ready: bool,
918 pub complete: bool,
919 pub blocked_reasons: Vec<String>,
920 pub operation_count: usize,
921 #[serde(default)]
922 pub operation_counts: RestoreApplyOperationKindCounts,
923 pub operation_counts_supplied: bool,
924 pub progress: RestoreApplyProgressSummary,
925 pub pending_summary: RestoreApplyPendingSummary,
926 pub pending_operations: usize,
927 pub ready_operations: usize,
928 pub blocked_operations: usize,
929 pub completed_operations: usize,
930 pub failed_operations: usize,
931 pub next_ready_sequence: Option<usize>,
932 pub next_ready_operation: Option<RestoreApplyOperationKind>,
933 pub next_transition_sequence: Option<usize>,
934 pub next_transition_state: Option<RestoreApplyOperationState>,
935 pub next_transition_operation: Option<RestoreApplyOperationKind>,
936 pub next_transition_updated_at: Option<String>,
937}
938
939impl RestoreApplyJournalStatus {
940 #[must_use]
942 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
943 let next_ready = journal.next_ready_operation();
944 let next_transition = journal.next_transition_operation();
945
946 Self {
947 status_version: 1,
948 backup_id: journal.backup_id.clone(),
949 ready: journal.ready,
950 complete: journal.operation_count > 0
951 && journal.completed_operations == journal.operation_count,
952 blocked_reasons: journal.blocked_reasons.clone(),
953 operation_count: journal.operation_count,
954 operation_counts: RestoreApplyOperationKindCounts::from_operations(&journal.operations),
955 operation_counts_supplied: journal.operation_counts_supplied(),
956 progress: RestoreApplyProgressSummary::from_journal(journal),
957 pending_summary: RestoreApplyPendingSummary::from_journal(journal),
958 pending_operations: journal.pending_operations,
959 ready_operations: journal.ready_operations,
960 blocked_operations: journal.blocked_operations,
961 completed_operations: journal.completed_operations,
962 failed_operations: journal.failed_operations,
963 next_ready_sequence: next_ready.map(|operation| operation.sequence),
964 next_ready_operation: next_ready.map(|operation| operation.operation.clone()),
965 next_transition_sequence: next_transition.map(|operation| operation.sequence),
966 next_transition_state: next_transition.map(|operation| operation.state.clone()),
967 next_transition_operation: next_transition.map(|operation| operation.operation.clone()),
968 next_transition_updated_at: next_transition
969 .and_then(|operation| operation.state_updated_at.clone()),
970 }
971 }
972}
973
974#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
979#[expect(
980 clippy::struct_excessive_bools,
981 reason = "apply reports intentionally expose stable JSON flags for operators and CI"
982)]
983pub struct RestoreApplyJournalReport {
984 pub report_version: u16,
985 pub backup_id: String,
986 pub outcome: RestoreApplyReportOutcome,
987 pub attention_required: bool,
988 pub ready: bool,
989 pub complete: bool,
990 pub blocked_reasons: Vec<String>,
991 pub operation_count: usize,
992 #[serde(default)]
993 pub operation_counts: RestoreApplyOperationKindCounts,
994 pub operation_counts_supplied: bool,
995 pub progress: RestoreApplyProgressSummary,
996 pub pending_summary: RestoreApplyPendingSummary,
997 pub pending_operations: usize,
998 pub ready_operations: usize,
999 pub blocked_operations: usize,
1000 pub completed_operations: usize,
1001 pub failed_operations: usize,
1002 pub next_transition: Option<RestoreApplyReportOperation>,
1003 pub pending: Vec<RestoreApplyReportOperation>,
1004 pub failed: Vec<RestoreApplyReportOperation>,
1005 pub blocked: Vec<RestoreApplyReportOperation>,
1006}
1007
1008impl RestoreApplyJournalReport {
1009 #[must_use]
1011 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1012 let complete =
1013 journal.operation_count > 0 && journal.completed_operations == journal.operation_count;
1014 let outcome = RestoreApplyReportOutcome::from_journal(journal, complete);
1015 let pending = report_operations_with_state(journal, RestoreApplyOperationState::Pending);
1016 let failed = report_operations_with_state(journal, RestoreApplyOperationState::Failed);
1017 let blocked = report_operations_with_state(journal, RestoreApplyOperationState::Blocked);
1018
1019 Self {
1020 report_version: 1,
1021 backup_id: journal.backup_id.clone(),
1022 outcome: outcome.clone(),
1023 attention_required: outcome.attention_required(),
1024 ready: journal.ready,
1025 complete,
1026 blocked_reasons: journal.blocked_reasons.clone(),
1027 operation_count: journal.operation_count,
1028 operation_counts: RestoreApplyOperationKindCounts::from_operations(&journal.operations),
1029 operation_counts_supplied: journal.operation_counts_supplied(),
1030 progress: RestoreApplyProgressSummary::from_journal(journal),
1031 pending_summary: RestoreApplyPendingSummary::from_journal(journal),
1032 pending_operations: journal.pending_operations,
1033 ready_operations: journal.ready_operations,
1034 blocked_operations: journal.blocked_operations,
1035 completed_operations: journal.completed_operations,
1036 failed_operations: journal.failed_operations,
1037 next_transition: journal
1038 .next_transition_operation()
1039 .map(RestoreApplyReportOperation::from_journal_operation),
1040 pending,
1041 failed,
1042 blocked,
1043 }
1044 }
1045}
1046
1047#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1052pub struct RestoreApplyPendingSummary {
1053 pub pending_operations: usize,
1054 pub pending_operation_available: bool,
1055 pub pending_sequence: Option<usize>,
1056 pub pending_operation: Option<RestoreApplyOperationKind>,
1057 pub pending_updated_at: Option<String>,
1058 pub pending_updated_at_known: bool,
1059}
1060
1061impl RestoreApplyPendingSummary {
1062 #[must_use]
1064 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1065 let pending = journal
1066 .operations
1067 .iter()
1068 .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1069 .min_by_key(|operation| operation.sequence);
1070 let pending_updated_at = pending.and_then(|operation| operation.state_updated_at.clone());
1071 let pending_updated_at_known = pending_updated_at
1072 .as_deref()
1073 .is_some_and(known_state_update_marker);
1074
1075 Self {
1076 pending_operations: journal.pending_operations,
1077 pending_operation_available: pending.is_some(),
1078 pending_sequence: pending.map(|operation| operation.sequence),
1079 pending_operation: pending.map(|operation| operation.operation.clone()),
1080 pending_updated_at,
1081 pending_updated_at_known,
1082 }
1083 }
1084}
1085
1086fn known_state_update_marker(value: &str) -> bool {
1088 !value.trim().is_empty() && value != "unknown"
1089}
1090
1091#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1096pub struct RestoreApplyProgressSummary {
1097 pub operation_count: usize,
1098 pub completed_operations: usize,
1099 pub remaining_operations: usize,
1100 pub transitionable_operations: usize,
1101 pub attention_operations: usize,
1102 pub completion_basis_points: usize,
1103}
1104
1105impl RestoreApplyProgressSummary {
1106 #[must_use]
1108 pub const fn from_journal(journal: &RestoreApplyJournal) -> Self {
1109 let remaining_operations = journal
1110 .operation_count
1111 .saturating_sub(journal.completed_operations);
1112 let transitionable_operations = journal.ready_operations + journal.pending_operations;
1113 let attention_operations =
1114 journal.pending_operations + journal.blocked_operations + journal.failed_operations;
1115 let completion_basis_points =
1116 completion_basis_points(journal.completed_operations, journal.operation_count);
1117
1118 Self {
1119 operation_count: journal.operation_count,
1120 completed_operations: journal.completed_operations,
1121 remaining_operations,
1122 transitionable_operations,
1123 attention_operations,
1124 completion_basis_points,
1125 }
1126 }
1127}
1128
1129const fn completion_basis_points(completed_operations: usize, operation_count: usize) -> usize {
1131 if operation_count == 0 {
1132 return 0;
1133 }
1134
1135 completed_operations.saturating_mul(10_000) / operation_count
1136}
1137
1138#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1143#[serde(rename_all = "kebab-case")]
1144pub enum RestoreApplyReportOutcome {
1145 Empty,
1146 Complete,
1147 Failed,
1148 Blocked,
1149 Pending,
1150 InProgress,
1151}
1152
1153impl RestoreApplyReportOutcome {
1154 const fn from_journal(journal: &RestoreApplyJournal, complete: bool) -> Self {
1156 if journal.operation_count == 0 {
1157 return Self::Empty;
1158 }
1159 if complete {
1160 return Self::Complete;
1161 }
1162 if journal.failed_operations > 0 {
1163 return Self::Failed;
1164 }
1165 if !journal.ready || journal.blocked_operations > 0 {
1166 return Self::Blocked;
1167 }
1168 if journal.pending_operations > 0 {
1169 return Self::Pending;
1170 }
1171 Self::InProgress
1172 }
1173
1174 const fn attention_required(&self) -> bool {
1176 matches!(self, Self::Failed | Self::Blocked | Self::Pending)
1177 }
1178}
1179
1180#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1185pub struct RestoreApplyReportOperation {
1186 pub sequence: usize,
1187 pub operation: RestoreApplyOperationKind,
1188 pub state: RestoreApplyOperationState,
1189 pub restore_group: u16,
1190 pub phase_order: usize,
1191 pub role: String,
1192 pub source_canister: String,
1193 pub target_canister: String,
1194 pub state_updated_at: Option<String>,
1195 pub reasons: Vec<String>,
1196}
1197
1198impl RestoreApplyReportOperation {
1199 fn from_journal_operation(operation: &RestoreApplyJournalOperation) -> Self {
1201 Self {
1202 sequence: operation.sequence,
1203 operation: operation.operation.clone(),
1204 state: operation.state.clone(),
1205 restore_group: operation.restore_group,
1206 phase_order: operation.phase_order,
1207 role: operation.role.clone(),
1208 source_canister: operation.source_canister.clone(),
1209 target_canister: operation.target_canister.clone(),
1210 state_updated_at: operation.state_updated_at.clone(),
1211 reasons: operation.blocking_reasons.clone(),
1212 }
1213 }
1214}
1215
1216fn report_operations_with_state(
1218 journal: &RestoreApplyJournal,
1219 state: RestoreApplyOperationState,
1220) -> Vec<RestoreApplyReportOperation> {
1221 journal
1222 .operations
1223 .iter()
1224 .filter(|operation| operation.state == state)
1225 .map(RestoreApplyReportOperation::from_journal_operation)
1226 .collect()
1227}
1228
1229#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1234pub struct RestoreApplyNextOperation {
1235 pub response_version: u16,
1236 pub backup_id: String,
1237 pub ready: bool,
1238 pub complete: bool,
1239 pub operation_available: bool,
1240 pub blocked_reasons: Vec<String>,
1241 pub operation: Option<RestoreApplyJournalOperation>,
1242}
1243
1244impl RestoreApplyNextOperation {
1245 #[must_use]
1247 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1248 let complete =
1249 journal.operation_count > 0 && journal.completed_operations == journal.operation_count;
1250 let operation = journal.next_transition_operation().cloned();
1251
1252 Self {
1253 response_version: 1,
1254 backup_id: journal.backup_id.clone(),
1255 ready: journal.ready,
1256 complete,
1257 operation_available: operation.is_some(),
1258 blocked_reasons: journal.blocked_reasons.clone(),
1259 operation,
1260 }
1261 }
1262}
1263
1264#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1269#[expect(
1270 clippy::struct_excessive_bools,
1271 reason = "runner preview exposes machine-readable availability and safety flags"
1272)]
1273pub struct RestoreApplyCommandPreview {
1274 pub response_version: u16,
1275 pub backup_id: String,
1276 pub ready: bool,
1277 pub complete: bool,
1278 pub operation_available: bool,
1279 pub command_available: bool,
1280 pub blocked_reasons: Vec<String>,
1281 pub operation: Option<RestoreApplyJournalOperation>,
1282 pub command: Option<RestoreApplyRunnerCommand>,
1283}
1284
1285impl RestoreApplyCommandPreview {
1286 #[must_use]
1288 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1289 Self::from_journal_with_config(journal, &RestoreApplyCommandConfig::default())
1290 }
1291
1292 #[must_use]
1294 pub fn from_journal_with_config(
1295 journal: &RestoreApplyJournal,
1296 config: &RestoreApplyCommandConfig,
1297 ) -> Self {
1298 let complete =
1299 journal.operation_count > 0 && journal.completed_operations == journal.operation_count;
1300 let operation = journal.next_transition_operation().cloned();
1301 let command = operation
1302 .as_ref()
1303 .and_then(|operation| RestoreApplyRunnerCommand::from_operation(operation, config));
1304
1305 Self {
1306 response_version: 1,
1307 backup_id: journal.backup_id.clone(),
1308 ready: journal.ready,
1309 complete,
1310 operation_available: operation.is_some(),
1311 command_available: command.is_some(),
1312 blocked_reasons: journal.blocked_reasons.clone(),
1313 operation,
1314 command,
1315 }
1316 }
1317}
1318
1319#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1324pub struct RestoreApplyCommandConfig {
1325 pub program: String,
1326 pub network: Option<String>,
1327}
1328
1329impl Default for RestoreApplyCommandConfig {
1330 fn default() -> Self {
1332 Self {
1333 program: "dfx".to_string(),
1334 network: None,
1335 }
1336 }
1337}
1338
1339#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1344pub struct RestoreApplyRunnerCommand {
1345 pub program: String,
1346 pub args: Vec<String>,
1347 pub mutates: bool,
1348 pub requires_stopped_canister: bool,
1349 pub note: String,
1350}
1351
1352impl RestoreApplyRunnerCommand {
1353 fn from_operation(
1355 operation: &RestoreApplyJournalOperation,
1356 config: &RestoreApplyCommandConfig,
1357 ) -> Option<Self> {
1358 match operation.operation {
1359 RestoreApplyOperationKind::UploadSnapshot => {
1360 let artifact_path = operation.artifact_path.as_ref()?;
1361 Some(Self {
1362 program: config.program.clone(),
1363 args: dfx_canister_args(
1364 config,
1365 vec![
1366 "snapshot".to_string(),
1367 "upload".to_string(),
1368 "--dir".to_string(),
1369 artifact_path.clone(),
1370 operation.target_canister.clone(),
1371 ],
1372 ),
1373 mutates: true,
1374 requires_stopped_canister: false,
1375 note: "uploads the downloaded snapshot artifact to the target canister"
1376 .to_string(),
1377 })
1378 }
1379 RestoreApplyOperationKind::LoadSnapshot => {
1380 let snapshot_id = operation.snapshot_id.as_ref()?;
1381 Some(Self {
1382 program: config.program.clone(),
1383 args: dfx_canister_args(
1384 config,
1385 vec![
1386 "snapshot".to_string(),
1387 "load".to_string(),
1388 operation.target_canister.clone(),
1389 snapshot_id.clone(),
1390 ],
1391 ),
1392 mutates: true,
1393 requires_stopped_canister: true,
1394 note: "loads the uploaded snapshot into the target canister".to_string(),
1395 })
1396 }
1397 RestoreApplyOperationKind::ReinstallCode => Some(Self {
1398 program: config.program.clone(),
1399 args: dfx_canister_args(
1400 config,
1401 vec![
1402 "install".to_string(),
1403 "--mode".to_string(),
1404 "reinstall".to_string(),
1405 "--yes".to_string(),
1406 operation.target_canister.clone(),
1407 ],
1408 ),
1409 mutates: true,
1410 requires_stopped_canister: false,
1411 note: "reinstalls target canister code using the local dfx project configuration"
1412 .to_string(),
1413 }),
1414 RestoreApplyOperationKind::VerifyMember | RestoreApplyOperationKind::VerifyFleet => {
1415 match operation.verification_kind.as_deref() {
1416 Some("status") => Some(Self {
1417 program: config.program.clone(),
1418 args: dfx_canister_args(
1419 config,
1420 vec!["status".to_string(), operation.target_canister.clone()],
1421 ),
1422 mutates: false,
1423 requires_stopped_canister: false,
1424 note: verification_command_note(
1425 &operation.operation,
1426 "checks target canister status",
1427 "checks target fleet root canister status",
1428 )
1429 .to_string(),
1430 }),
1431 Some(_) => {
1432 let method = operation.verification_method.as_ref()?;
1433 Some(Self {
1434 program: config.program.clone(),
1435 args: dfx_canister_args(
1436 config,
1437 vec![
1438 "call".to_string(),
1439 operation.target_canister.clone(),
1440 method.clone(),
1441 ],
1442 ),
1443 mutates: false,
1444 requires_stopped_canister: false,
1445 note: verification_command_note(
1446 &operation.operation,
1447 "calls the declared verification method",
1448 "calls the declared fleet verification method",
1449 )
1450 .to_string(),
1451 })
1452 }
1453 None => None,
1454 }
1455 }
1456 }
1457 }
1458}
1459
1460const fn verification_command_note(
1462 operation: &RestoreApplyOperationKind,
1463 member_note: &'static str,
1464 fleet_note: &'static str,
1465) -> &'static str {
1466 match operation {
1467 RestoreApplyOperationKind::VerifyFleet => fleet_note,
1468 RestoreApplyOperationKind::UploadSnapshot
1469 | RestoreApplyOperationKind::LoadSnapshot
1470 | RestoreApplyOperationKind::ReinstallCode
1471 | RestoreApplyOperationKind::VerifyMember => member_note,
1472 }
1473}
1474
1475fn dfx_canister_args(config: &RestoreApplyCommandConfig, mut tail: Vec<String>) -> Vec<String> {
1477 let mut args = vec!["canister".to_string()];
1478 if let Some(network) = &config.network {
1479 args.push("--network".to_string());
1480 args.push(network.clone());
1481 }
1482 args.append(&mut tail);
1483 args
1484}
1485
1486#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1491pub struct RestoreApplyJournalOperation {
1492 pub sequence: usize,
1493 pub operation: RestoreApplyOperationKind,
1494 pub state: RestoreApplyOperationState,
1495 #[serde(default, skip_serializing_if = "Option::is_none")]
1496 pub state_updated_at: Option<String>,
1497 pub blocking_reasons: Vec<String>,
1498 pub restore_group: u16,
1499 pub phase_order: usize,
1500 pub source_canister: String,
1501 pub target_canister: String,
1502 pub role: String,
1503 pub snapshot_id: Option<String>,
1504 pub artifact_path: Option<String>,
1505 pub verification_kind: Option<String>,
1506 pub verification_method: Option<String>,
1507}
1508
1509impl RestoreApplyJournalOperation {
1510 fn from_dry_run_operation(
1512 operation: &RestoreApplyDryRunOperation,
1513 state: RestoreApplyOperationState,
1514 blocked_reasons: &[String],
1515 ) -> Self {
1516 Self {
1517 sequence: operation.sequence,
1518 operation: operation.operation.clone(),
1519 state: state.clone(),
1520 state_updated_at: None,
1521 blocking_reasons: if state == RestoreApplyOperationState::Blocked {
1522 blocked_reasons.to_vec()
1523 } else {
1524 Vec::new()
1525 },
1526 restore_group: operation.restore_group,
1527 phase_order: operation.phase_order,
1528 source_canister: operation.source_canister.clone(),
1529 target_canister: operation.target_canister.clone(),
1530 role: operation.role.clone(),
1531 snapshot_id: operation.snapshot_id.clone(),
1532 artifact_path: operation.artifact_path.clone(),
1533 verification_kind: operation.verification_kind.clone(),
1534 verification_method: operation.verification_method.clone(),
1535 }
1536 }
1537
1538 fn validate(&self) -> Result<(), RestoreApplyJournalError> {
1540 validate_apply_journal_nonempty("operations[].source_canister", &self.source_canister)?;
1541 validate_apply_journal_nonempty("operations[].target_canister", &self.target_canister)?;
1542 validate_apply_journal_nonempty("operations[].role", &self.role)?;
1543 if let Some(updated_at) = &self.state_updated_at {
1544 validate_apply_journal_nonempty("operations[].state_updated_at", updated_at)?;
1545 }
1546 self.validate_operation_fields()?;
1547
1548 match self.state {
1549 RestoreApplyOperationState::Blocked if self.blocking_reasons.is_empty() => Err(
1550 RestoreApplyJournalError::BlockedOperationMissingReason(self.sequence),
1551 ),
1552 RestoreApplyOperationState::Failed if self.blocking_reasons.is_empty() => Err(
1553 RestoreApplyJournalError::FailureReasonRequired(self.sequence),
1554 ),
1555 RestoreApplyOperationState::Pending
1556 | RestoreApplyOperationState::Ready
1557 | RestoreApplyOperationState::Completed
1558 if !self.blocking_reasons.is_empty() =>
1559 {
1560 Err(RestoreApplyJournalError::UnblockedOperationHasReasons(
1561 self.sequence,
1562 ))
1563 }
1564 RestoreApplyOperationState::Blocked
1565 | RestoreApplyOperationState::Failed
1566 | RestoreApplyOperationState::Pending
1567 | RestoreApplyOperationState::Ready
1568 | RestoreApplyOperationState::Completed => Ok(()),
1569 }
1570 }
1571
1572 fn validate_operation_fields(&self) -> Result<(), RestoreApplyJournalError> {
1574 match self.operation {
1575 RestoreApplyOperationKind::UploadSnapshot => self
1576 .validate_required_field("operations[].artifact_path", self.artifact_path.as_ref())
1577 .map(|_| ()),
1578 RestoreApplyOperationKind::LoadSnapshot => self
1579 .validate_required_field("operations[].snapshot_id", self.snapshot_id.as_ref())
1580 .map(|_| ()),
1581 RestoreApplyOperationKind::ReinstallCode => Ok(()),
1582 RestoreApplyOperationKind::VerifyMember | RestoreApplyOperationKind::VerifyFleet => {
1583 let kind = self.validate_required_field(
1584 "operations[].verification_kind",
1585 self.verification_kind.as_ref(),
1586 )?;
1587 if kind == "status" {
1588 return Ok(());
1589 }
1590 self.validate_required_field(
1591 "operations[].verification_method",
1592 self.verification_method.as_ref(),
1593 )
1594 .map(|_| ())
1595 }
1596 }
1597 }
1598
1599 fn validate_required_field<'a>(
1601 &self,
1602 field: &'static str,
1603 value: Option<&'a String>,
1604 ) -> Result<&'a str, RestoreApplyJournalError> {
1605 let value = value.map(String::as_str).ok_or_else(|| {
1606 RestoreApplyJournalError::OperationMissingField {
1607 sequence: self.sequence,
1608 operation: self.operation.clone(),
1609 field,
1610 }
1611 })?;
1612 if value.trim().is_empty() {
1613 return Err(RestoreApplyJournalError::OperationMissingField {
1614 sequence: self.sequence,
1615 operation: self.operation.clone(),
1616 field,
1617 });
1618 }
1619
1620 Ok(value)
1621 }
1622
1623 const fn can_transition_to(&self, next_state: &RestoreApplyOperationState) -> bool {
1625 match (&self.state, next_state) {
1626 (
1627 RestoreApplyOperationState::Ready | RestoreApplyOperationState::Pending,
1628 RestoreApplyOperationState::Pending,
1629 )
1630 | (RestoreApplyOperationState::Pending, RestoreApplyOperationState::Ready)
1631 | (
1632 RestoreApplyOperationState::Ready
1633 | RestoreApplyOperationState::Pending
1634 | RestoreApplyOperationState::Completed,
1635 RestoreApplyOperationState::Completed,
1636 )
1637 | (
1638 RestoreApplyOperationState::Ready
1639 | RestoreApplyOperationState::Pending
1640 | RestoreApplyOperationState::Failed,
1641 RestoreApplyOperationState::Failed,
1642 ) => true,
1643 (
1644 RestoreApplyOperationState::Blocked
1645 | RestoreApplyOperationState::Completed
1646 | RestoreApplyOperationState::Failed
1647 | RestoreApplyOperationState::Pending
1648 | RestoreApplyOperationState::Ready,
1649 _,
1650 ) => false,
1651 }
1652 }
1653}
1654
1655#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1660#[serde(rename_all = "kebab-case")]
1661pub enum RestoreApplyOperationState {
1662 Pending,
1663 Ready,
1664 Blocked,
1665 Completed,
1666 Failed,
1667}
1668
1669#[derive(Debug, ThisError)]
1674pub enum RestoreApplyJournalError {
1675 #[error("unsupported restore apply journal version {0}")]
1676 UnsupportedVersion(u16),
1677
1678 #[error("restore apply journal field {0} is required")]
1679 MissingField(&'static str),
1680
1681 #[error("restore apply journal count {field} mismatch: reported={reported}, actual={actual}")]
1682 CountMismatch {
1683 field: &'static str,
1684 reported: usize,
1685 actual: usize,
1686 },
1687
1688 #[error("restore apply journal has duplicate operation sequence {0}")]
1689 DuplicateSequence(usize),
1690
1691 #[error("restore apply journal is missing operation sequence {0}")]
1692 MissingSequence(usize),
1693
1694 #[error("ready restore apply journal cannot include blocked reasons or blocked operations")]
1695 ReadyJournalHasBlockingState,
1696
1697 #[error("blocked restore apply journal operation {0} is missing a blocking reason")]
1698 BlockedOperationMissingReason(usize),
1699
1700 #[error("unblocked restore apply journal operation {0} cannot have blocking reasons")]
1701 UnblockedOperationHasReasons(usize),
1702
1703 #[error("restore apply journal operation {sequence} {operation:?} is missing field {field}")]
1704 OperationMissingField {
1705 sequence: usize,
1706 operation: RestoreApplyOperationKind,
1707 field: &'static str,
1708 },
1709
1710 #[error("restore apply journal operation {0} was not found")]
1711 OperationNotFound(usize),
1712
1713 #[error("restore apply journal operation {sequence} cannot transition from {from:?} to {to:?}")]
1714 InvalidOperationTransition {
1715 sequence: usize,
1716 from: RestoreApplyOperationState,
1717 to: RestoreApplyOperationState,
1718 },
1719
1720 #[error("failed restore apply journal operation {0} requires a reason")]
1721 FailureReasonRequired(usize),
1722
1723 #[error("restore apply journal has no operation that can be advanced")]
1724 NoTransitionableOperation,
1725
1726 #[error("restore apply journal has no pending operation to release")]
1727 NoPendingOperation,
1728
1729 #[error("restore apply journal operation {requested} cannot advance before operation {next}")]
1730 OutOfOrderOperationTransition { requested: usize, next: usize },
1731}
1732
1733fn validate_restore_apply_artifacts(
1735 plan: &RestorePlan,
1736 backup_root: &Path,
1737) -> Result<RestoreApplyArtifactValidation, RestoreApplyDryRunError> {
1738 let mut checks = Vec::new();
1739
1740 for member in plan.ordered_members() {
1741 checks.push(validate_restore_apply_artifact(member, backup_root)?);
1742 }
1743
1744 let members_with_expected_checksums = checks
1745 .iter()
1746 .filter(|check| check.checksum_expected.is_some())
1747 .count();
1748 let artifacts_present = checks.iter().all(|check| check.exists);
1749 let checksums_verified = members_with_expected_checksums == plan.member_count
1750 && checks.iter().all(|check| check.checksum_verified);
1751
1752 Ok(RestoreApplyArtifactValidation {
1753 backup_root: backup_root.to_string_lossy().to_string(),
1754 checked_members: checks.len(),
1755 artifacts_present,
1756 checksums_verified,
1757 members_with_expected_checksums,
1758 checks,
1759 })
1760}
1761
1762fn validate_restore_apply_artifact(
1764 member: &RestorePlanMember,
1765 backup_root: &Path,
1766) -> Result<RestoreApplyArtifactCheck, RestoreApplyDryRunError> {
1767 let artifact_path = safe_restore_artifact_path(
1768 &member.source_canister,
1769 &member.source_snapshot.artifact_path,
1770 )?;
1771 let resolved_path = backup_root.join(&artifact_path);
1772
1773 if !resolved_path.exists() {
1774 return Err(RestoreApplyDryRunError::ArtifactMissing {
1775 source_canister: member.source_canister.clone(),
1776 artifact_path: member.source_snapshot.artifact_path.clone(),
1777 resolved_path: resolved_path.to_string_lossy().to_string(),
1778 });
1779 }
1780
1781 let (checksum_actual, checksum_verified) =
1782 if let Some(expected) = &member.source_snapshot.checksum {
1783 let checksum = ArtifactChecksum::from_path(&resolved_path).map_err(|source| {
1784 RestoreApplyDryRunError::ArtifactChecksum {
1785 source_canister: member.source_canister.clone(),
1786 artifact_path: member.source_snapshot.artifact_path.clone(),
1787 source,
1788 }
1789 })?;
1790 checksum.verify(expected).map_err(|source| {
1791 RestoreApplyDryRunError::ArtifactChecksum {
1792 source_canister: member.source_canister.clone(),
1793 artifact_path: member.source_snapshot.artifact_path.clone(),
1794 source,
1795 }
1796 })?;
1797 (Some(checksum.hash), true)
1798 } else {
1799 (None, false)
1800 };
1801
1802 Ok(RestoreApplyArtifactCheck {
1803 source_canister: member.source_canister.clone(),
1804 target_canister: member.target_canister.clone(),
1805 snapshot_id: member.source_snapshot.snapshot_id.clone(),
1806 artifact_path: member.source_snapshot.artifact_path.clone(),
1807 resolved_path: resolved_path.to_string_lossy().to_string(),
1808 exists: true,
1809 checksum_algorithm: member.source_snapshot.checksum_algorithm.clone(),
1810 checksum_expected: member.source_snapshot.checksum.clone(),
1811 checksum_actual,
1812 checksum_verified,
1813 })
1814}
1815
1816fn safe_restore_artifact_path(
1818 source_canister: &str,
1819 artifact_path: &str,
1820) -> Result<PathBuf, RestoreApplyDryRunError> {
1821 let path = Path::new(artifact_path);
1822 let is_safe = path
1823 .components()
1824 .all(|component| matches!(component, Component::Normal(_) | Component::CurDir));
1825
1826 if is_safe {
1827 return Ok(path.to_path_buf());
1828 }
1829
1830 Err(RestoreApplyDryRunError::ArtifactPathEscapesBackup {
1831 source_canister: source_canister.to_string(),
1832 artifact_path: artifact_path.to_string(),
1833 })
1834}
1835
1836fn validate_restore_status_matches_plan(
1838 plan: &RestorePlan,
1839 status: &RestoreStatus,
1840) -> Result<(), RestoreApplyDryRunError> {
1841 validate_status_string_field("backup_id", &plan.backup_id, &status.backup_id)?;
1842 validate_status_string_field(
1843 "source_environment",
1844 &plan.source_environment,
1845 &status.source_environment,
1846 )?;
1847 validate_status_string_field(
1848 "source_root_canister",
1849 &plan.source_root_canister,
1850 &status.source_root_canister,
1851 )?;
1852 validate_status_string_field("topology_hash", &plan.topology_hash, &status.topology_hash)?;
1853 validate_status_usize_field("member_count", plan.member_count, status.member_count)?;
1854 validate_status_usize_field(
1855 "phase_count",
1856 plan.ordering_summary.phase_count,
1857 status.phase_count,
1858 )?;
1859 Ok(())
1860}
1861
1862fn validate_status_string_field(
1864 field: &'static str,
1865 plan: &str,
1866 status: &str,
1867) -> Result<(), RestoreApplyDryRunError> {
1868 if plan == status {
1869 return Ok(());
1870 }
1871
1872 Err(RestoreApplyDryRunError::StatusPlanMismatch {
1873 field,
1874 plan: plan.to_string(),
1875 status: status.to_string(),
1876 })
1877}
1878
1879const fn validate_status_usize_field(
1881 field: &'static str,
1882 plan: usize,
1883 status: usize,
1884) -> Result<(), RestoreApplyDryRunError> {
1885 if plan == status {
1886 return Ok(());
1887 }
1888
1889 Err(RestoreApplyDryRunError::StatusPlanCountMismatch {
1890 field,
1891 plan,
1892 status,
1893 })
1894}
1895
1896#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1901pub struct RestoreApplyArtifactValidation {
1902 pub backup_root: String,
1903 pub checked_members: usize,
1904 pub artifacts_present: bool,
1905 pub checksums_verified: bool,
1906 pub members_with_expected_checksums: usize,
1907 pub checks: Vec<RestoreApplyArtifactCheck>,
1908}
1909
1910#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1915pub struct RestoreApplyArtifactCheck {
1916 pub source_canister: String,
1917 pub target_canister: String,
1918 pub snapshot_id: String,
1919 pub artifact_path: String,
1920 pub resolved_path: String,
1921 pub exists: bool,
1922 pub checksum_algorithm: String,
1923 pub checksum_expected: Option<String>,
1924 pub checksum_actual: Option<String>,
1925 pub checksum_verified: bool,
1926}
1927
1928#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1933pub struct RestoreApplyDryRunPhase {
1934 pub restore_group: u16,
1935 pub operations: Vec<RestoreApplyDryRunOperation>,
1936}
1937
1938impl RestoreApplyDryRunPhase {
1939 fn from_plan_phase(phase: &RestorePhase, next_sequence: &mut usize) -> Self {
1941 let mut operations = Vec::new();
1942
1943 for member in &phase.members {
1944 push_member_operation(
1945 &mut operations,
1946 next_sequence,
1947 RestoreApplyOperationKind::UploadSnapshot,
1948 member,
1949 None,
1950 );
1951 push_member_operation(
1952 &mut operations,
1953 next_sequence,
1954 RestoreApplyOperationKind::LoadSnapshot,
1955 member,
1956 None,
1957 );
1958 push_member_operation(
1959 &mut operations,
1960 next_sequence,
1961 RestoreApplyOperationKind::ReinstallCode,
1962 member,
1963 None,
1964 );
1965
1966 for check in &member.verification_checks {
1967 push_member_operation(
1968 &mut operations,
1969 next_sequence,
1970 RestoreApplyOperationKind::VerifyMember,
1971 member,
1972 Some(check),
1973 );
1974 }
1975 }
1976
1977 Self {
1978 restore_group: phase.restore_group,
1979 operations,
1980 }
1981 }
1982}
1983
1984fn push_member_operation(
1986 operations: &mut Vec<RestoreApplyDryRunOperation>,
1987 next_sequence: &mut usize,
1988 operation: RestoreApplyOperationKind,
1989 member: &RestorePlanMember,
1990 check: Option<&VerificationCheck>,
1991) {
1992 let sequence = *next_sequence;
1993 *next_sequence += 1;
1994
1995 operations.push(RestoreApplyDryRunOperation {
1996 sequence,
1997 operation,
1998 restore_group: member.restore_group,
1999 phase_order: member.phase_order,
2000 source_canister: member.source_canister.clone(),
2001 target_canister: member.target_canister.clone(),
2002 role: member.role.clone(),
2003 snapshot_id: Some(member.source_snapshot.snapshot_id.clone()),
2004 artifact_path: Some(member.source_snapshot.artifact_path.clone()),
2005 verification_kind: check.map(|check| check.kind.clone()),
2006 verification_method: check.and_then(|check| check.method.clone()),
2007 });
2008}
2009
2010fn append_fleet_verification_operations(
2012 plan: &RestorePlan,
2013 phases: &mut [RestoreApplyDryRunPhase],
2014 next_sequence: &mut usize,
2015) {
2016 if plan.fleet_verification_checks.is_empty() {
2017 return;
2018 }
2019
2020 let Some(phase) = phases.last_mut() else {
2021 return;
2022 };
2023 let root = plan
2024 .phases
2025 .iter()
2026 .flat_map(|phase| phase.members.iter())
2027 .find(|member| member.source_canister == plan.source_root_canister);
2028 let source_canister = root.map_or_else(
2029 || plan.source_root_canister.clone(),
2030 |member| member.source_canister.clone(),
2031 );
2032 let target_canister = root.map_or_else(
2033 || plan.source_root_canister.clone(),
2034 |member| member.target_canister.clone(),
2035 );
2036 let restore_group = phase.restore_group;
2037
2038 for check in &plan.fleet_verification_checks {
2039 push_fleet_operation(
2040 &mut phase.operations,
2041 next_sequence,
2042 restore_group,
2043 &source_canister,
2044 &target_canister,
2045 check,
2046 );
2047 }
2048}
2049
2050fn push_fleet_operation(
2052 operations: &mut Vec<RestoreApplyDryRunOperation>,
2053 next_sequence: &mut usize,
2054 restore_group: u16,
2055 source_canister: &str,
2056 target_canister: &str,
2057 check: &VerificationCheck,
2058) {
2059 let sequence = *next_sequence;
2060 *next_sequence += 1;
2061 let phase_order = operations.len();
2062
2063 operations.push(RestoreApplyDryRunOperation {
2064 sequence,
2065 operation: RestoreApplyOperationKind::VerifyFleet,
2066 restore_group,
2067 phase_order,
2068 source_canister: source_canister.to_string(),
2069 target_canister: target_canister.to_string(),
2070 role: "fleet".to_string(),
2071 snapshot_id: None,
2072 artifact_path: None,
2073 verification_kind: Some(check.kind.clone()),
2074 verification_method: check.method.clone(),
2075 });
2076}
2077
2078#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2083pub struct RestoreApplyDryRunOperation {
2084 pub sequence: usize,
2085 pub operation: RestoreApplyOperationKind,
2086 pub restore_group: u16,
2087 pub phase_order: usize,
2088 pub source_canister: String,
2089 pub target_canister: String,
2090 pub role: String,
2091 pub snapshot_id: Option<String>,
2092 pub artifact_path: Option<String>,
2093 pub verification_kind: Option<String>,
2094 pub verification_method: Option<String>,
2095}
2096
2097#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2102#[serde(rename_all = "kebab-case")]
2103pub enum RestoreApplyOperationKind {
2104 UploadSnapshot,
2105 LoadSnapshot,
2106 ReinstallCode,
2107 VerifyMember,
2108 VerifyFleet,
2109}
2110
2111#[derive(Debug, ThisError)]
2116pub enum RestoreApplyDryRunError {
2117 #[error("restore status field {field} does not match plan: plan={plan}, status={status}")]
2118 StatusPlanMismatch {
2119 field: &'static str,
2120 plan: String,
2121 status: String,
2122 },
2123
2124 #[error("restore status field {field} does not match plan: plan={plan}, status={status}")]
2125 StatusPlanCountMismatch {
2126 field: &'static str,
2127 plan: usize,
2128 status: usize,
2129 },
2130
2131 #[error("restore artifact path for {source_canister} escapes backup root: {artifact_path}")]
2132 ArtifactPathEscapesBackup {
2133 source_canister: String,
2134 artifact_path: String,
2135 },
2136
2137 #[error(
2138 "restore artifact for {source_canister} is missing: {artifact_path} at {resolved_path}"
2139 )]
2140 ArtifactMissing {
2141 source_canister: String,
2142 artifact_path: String,
2143 resolved_path: String,
2144 },
2145
2146 #[error("restore artifact checksum failed for {source_canister} at {artifact_path}: {source}")]
2147 ArtifactChecksum {
2148 source_canister: String,
2149 artifact_path: String,
2150 #[source]
2151 source: ArtifactChecksumError,
2152 },
2153}
2154
2155#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2160pub struct RestoreIdentitySummary {
2161 pub mapping_supplied: bool,
2162 pub all_sources_mapped: bool,
2163 pub fixed_members: usize,
2164 pub relocatable_members: usize,
2165 pub in_place_members: usize,
2166 pub mapped_members: usize,
2167 pub remapped_members: usize,
2168}
2169
2170#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2175#[expect(
2176 clippy::struct_excessive_bools,
2177 reason = "restore summaries intentionally expose machine-readable readiness flags"
2178)]
2179pub struct RestoreSnapshotSummary {
2180 pub all_members_have_module_hash: bool,
2181 pub all_members_have_wasm_hash: bool,
2182 pub all_members_have_code_version: bool,
2183 pub all_members_have_checksum: bool,
2184 pub members_with_module_hash: usize,
2185 pub members_with_wasm_hash: usize,
2186 pub members_with_code_version: usize,
2187 pub members_with_checksum: usize,
2188}
2189
2190#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2195pub struct RestoreVerificationSummary {
2196 pub verification_required: bool,
2197 pub all_members_have_checks: bool,
2198 pub fleet_checks: usize,
2199 pub member_check_groups: usize,
2200 pub member_checks: usize,
2201 pub members_with_checks: usize,
2202 pub total_checks: usize,
2203}
2204
2205#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2210pub struct RestoreReadinessSummary {
2211 pub ready: bool,
2212 pub reasons: Vec<String>,
2213}
2214
2215#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2220pub struct RestoreOperationSummary {
2221 #[serde(default)]
2222 pub planned_snapshot_uploads: usize,
2223 pub planned_snapshot_loads: usize,
2224 pub planned_code_reinstalls: usize,
2225 pub planned_verification_checks: usize,
2226 #[serde(default)]
2227 pub planned_operations: usize,
2228 pub planned_phases: usize,
2229}
2230
2231impl RestoreOperationSummary {
2232 #[must_use]
2234 pub const fn effective_planned_snapshot_uploads(&self, member_count: usize) -> usize {
2235 if self.planned_snapshot_uploads == 0 && member_count > 0 {
2236 return member_count;
2237 }
2238
2239 self.planned_snapshot_uploads
2240 }
2241
2242 #[must_use]
2244 pub const fn effective_planned_operations(&self, member_count: usize) -> usize {
2245 if self.planned_operations == 0 {
2246 return self.effective_planned_snapshot_uploads(member_count)
2247 + self.planned_snapshot_loads
2248 + self.planned_code_reinstalls
2249 + self.planned_verification_checks;
2250 }
2251
2252 self.planned_operations
2253 }
2254}
2255
2256#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2261pub struct RestoreOrderingSummary {
2262 pub phase_count: usize,
2263 pub dependency_free_members: usize,
2264 pub in_group_parent_edges: usize,
2265 pub cross_group_parent_edges: usize,
2266}
2267
2268#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2273pub struct RestorePhase {
2274 pub restore_group: u16,
2275 pub members: Vec<RestorePlanMember>,
2276}
2277
2278#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2283pub struct RestorePlanMember {
2284 pub source_canister: String,
2285 pub target_canister: String,
2286 pub role: String,
2287 pub parent_source_canister: Option<String>,
2288 pub parent_target_canister: Option<String>,
2289 pub ordering_dependency: Option<RestoreOrderingDependency>,
2290 pub phase_order: usize,
2291 pub restore_group: u16,
2292 pub identity_mode: IdentityMode,
2293 pub verification_class: String,
2294 pub verification_checks: Vec<VerificationCheck>,
2295 pub source_snapshot: SourceSnapshot,
2296}
2297
2298#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2303pub struct RestoreOrderingDependency {
2304 pub source_canister: String,
2305 pub target_canister: String,
2306 pub relationship: RestoreOrderingRelationship,
2307}
2308
2309#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2314#[serde(rename_all = "kebab-case")]
2315pub enum RestoreOrderingRelationship {
2316 ParentInSameGroup,
2317 ParentInEarlierGroup,
2318}
2319
2320pub struct RestorePlanner;
2325
2326impl RestorePlanner {
2327 pub fn plan(
2329 manifest: &FleetBackupManifest,
2330 mapping: Option<&RestoreMapping>,
2331 ) -> Result<RestorePlan, RestorePlanError> {
2332 manifest.validate()?;
2333 if let Some(mapping) = mapping {
2334 validate_mapping(mapping)?;
2335 validate_mapping_sources(manifest, mapping)?;
2336 }
2337
2338 let members = resolve_members(manifest, mapping)?;
2339 let identity_summary = restore_identity_summary(&members, mapping.is_some());
2340 let snapshot_summary = restore_snapshot_summary(&members);
2341 let verification_summary = restore_verification_summary(manifest, &members);
2342 let readiness_summary = restore_readiness_summary(&snapshot_summary, &verification_summary);
2343 validate_restore_group_dependencies(&members)?;
2344 let phases = group_and_order_members(members)?;
2345 let ordering_summary = restore_ordering_summary(&phases);
2346 let operation_summary =
2347 restore_operation_summary(manifest.fleet.members.len(), &verification_summary, &phases);
2348
2349 Ok(RestorePlan {
2350 backup_id: manifest.backup_id.clone(),
2351 source_environment: manifest.source.environment.clone(),
2352 source_root_canister: manifest.source.root_canister.clone(),
2353 topology_hash: manifest.fleet.topology_hash.clone(),
2354 member_count: manifest.fleet.members.len(),
2355 identity_summary,
2356 snapshot_summary,
2357 verification_summary,
2358 readiness_summary,
2359 operation_summary,
2360 ordering_summary,
2361 design_conformance: Some(manifest.design_conformance_report()),
2362 fleet_verification_checks: manifest.verification.fleet_checks.clone(),
2363 phases,
2364 })
2365 }
2366}
2367
2368#[derive(Debug, ThisError)]
2373pub enum RestorePlanError {
2374 #[error(transparent)]
2375 InvalidManifest(#[from] ManifestValidationError),
2376
2377 #[error("field {field} must be a valid principal: {value}")]
2378 InvalidPrincipal { field: &'static str, value: String },
2379
2380 #[error("mapping contains duplicate source canister {0}")]
2381 DuplicateMappingSource(String),
2382
2383 #[error("mapping contains duplicate target canister {0}")]
2384 DuplicateMappingTarget(String),
2385
2386 #[error("mapping references unknown source canister {0}")]
2387 UnknownMappingSource(String),
2388
2389 #[error("mapping is missing source canister {0}")]
2390 MissingMappingSource(String),
2391
2392 #[error("fixed-identity member {source_canister} cannot be mapped to {target_canister}")]
2393 FixedIdentityRemap {
2394 source_canister: String,
2395 target_canister: String,
2396 },
2397
2398 #[error("restore plan contains duplicate target canister {0}")]
2399 DuplicatePlanTarget(String),
2400
2401 #[error("restore group {0} contains a parent cycle or unresolved dependency")]
2402 RestoreOrderCycle(u16),
2403
2404 #[error(
2405 "restore plan places parent {parent_source_canister} in group {parent_restore_group} after child {child_source_canister} in group {child_restore_group}"
2406 )]
2407 ParentRestoreGroupAfterChild {
2408 child_source_canister: String,
2409 parent_source_canister: String,
2410 child_restore_group: u16,
2411 parent_restore_group: u16,
2412 },
2413}
2414
2415fn validate_mapping(mapping: &RestoreMapping) -> Result<(), RestorePlanError> {
2417 let mut sources = BTreeSet::new();
2418 let mut targets = BTreeSet::new();
2419
2420 for entry in &mapping.members {
2421 validate_principal("mapping.members[].source_canister", &entry.source_canister)?;
2422 validate_principal("mapping.members[].target_canister", &entry.target_canister)?;
2423
2424 if !sources.insert(entry.source_canister.clone()) {
2425 return Err(RestorePlanError::DuplicateMappingSource(
2426 entry.source_canister.clone(),
2427 ));
2428 }
2429
2430 if !targets.insert(entry.target_canister.clone()) {
2431 return Err(RestorePlanError::DuplicateMappingTarget(
2432 entry.target_canister.clone(),
2433 ));
2434 }
2435 }
2436
2437 Ok(())
2438}
2439
2440fn validate_mapping_sources(
2442 manifest: &FleetBackupManifest,
2443 mapping: &RestoreMapping,
2444) -> Result<(), RestorePlanError> {
2445 let sources = manifest
2446 .fleet
2447 .members
2448 .iter()
2449 .map(|member| member.canister_id.as_str())
2450 .collect::<BTreeSet<_>>();
2451
2452 for entry in &mapping.members {
2453 if !sources.contains(entry.source_canister.as_str()) {
2454 return Err(RestorePlanError::UnknownMappingSource(
2455 entry.source_canister.clone(),
2456 ));
2457 }
2458 }
2459
2460 Ok(())
2461}
2462
2463fn resolve_members(
2465 manifest: &FleetBackupManifest,
2466 mapping: Option<&RestoreMapping>,
2467) -> Result<Vec<RestorePlanMember>, RestorePlanError> {
2468 let mut plan_members = Vec::with_capacity(manifest.fleet.members.len());
2469 let mut targets = BTreeSet::new();
2470 let mut source_to_target = BTreeMap::new();
2471
2472 for member in &manifest.fleet.members {
2473 let target = resolve_target(member, mapping)?;
2474 if !targets.insert(target.clone()) {
2475 return Err(RestorePlanError::DuplicatePlanTarget(target));
2476 }
2477
2478 source_to_target.insert(member.canister_id.clone(), target.clone());
2479 plan_members.push(RestorePlanMember {
2480 source_canister: member.canister_id.clone(),
2481 target_canister: target,
2482 role: member.role.clone(),
2483 parent_source_canister: member.parent_canister_id.clone(),
2484 parent_target_canister: None,
2485 ordering_dependency: None,
2486 phase_order: 0,
2487 restore_group: member.restore_group,
2488 identity_mode: member.identity_mode.clone(),
2489 verification_class: member.verification_class.clone(),
2490 verification_checks: concrete_member_verification_checks(
2491 member,
2492 &manifest.verification,
2493 ),
2494 source_snapshot: member.source_snapshot.clone(),
2495 });
2496 }
2497
2498 for member in &mut plan_members {
2499 member.parent_target_canister = member
2500 .parent_source_canister
2501 .as_ref()
2502 .and_then(|parent| source_to_target.get(parent))
2503 .cloned();
2504 }
2505
2506 Ok(plan_members)
2507}
2508
2509fn concrete_member_verification_checks(
2511 member: &FleetMember,
2512 verification: &VerificationPlan,
2513) -> Vec<VerificationCheck> {
2514 let mut checks = member
2515 .verification_checks
2516 .iter()
2517 .filter(|check| verification_check_applies_to_role(check, &member.role))
2518 .cloned()
2519 .collect::<Vec<_>>();
2520
2521 for group in &verification.member_checks {
2522 if group.role != member.role {
2523 continue;
2524 }
2525
2526 checks.extend(
2527 group
2528 .checks
2529 .iter()
2530 .filter(|check| verification_check_applies_to_role(check, &member.role))
2531 .cloned(),
2532 );
2533 }
2534
2535 checks
2536}
2537
2538fn verification_check_applies_to_role(check: &VerificationCheck, role: &str) -> bool {
2540 check.roles.is_empty() || check.roles.iter().any(|check_role| check_role == role)
2541}
2542
2543fn resolve_target(
2545 member: &FleetMember,
2546 mapping: Option<&RestoreMapping>,
2547) -> Result<String, RestorePlanError> {
2548 let target = match mapping {
2549 Some(mapping) => mapping
2550 .target_for(&member.canister_id)
2551 .ok_or_else(|| RestorePlanError::MissingMappingSource(member.canister_id.clone()))?
2552 .to_string(),
2553 None => member.canister_id.clone(),
2554 };
2555
2556 if matches!(member.identity_mode, IdentityMode::Fixed) && target != member.canister_id {
2557 return Err(RestorePlanError::FixedIdentityRemap {
2558 source_canister: member.canister_id.clone(),
2559 target_canister: target,
2560 });
2561 }
2562
2563 Ok(target)
2564}
2565
2566fn restore_identity_summary(
2568 members: &[RestorePlanMember],
2569 mapping_supplied: bool,
2570) -> RestoreIdentitySummary {
2571 let mut summary = RestoreIdentitySummary {
2572 mapping_supplied,
2573 all_sources_mapped: false,
2574 fixed_members: 0,
2575 relocatable_members: 0,
2576 in_place_members: 0,
2577 mapped_members: 0,
2578 remapped_members: 0,
2579 };
2580
2581 for member in members {
2582 match member.identity_mode {
2583 IdentityMode::Fixed => summary.fixed_members += 1,
2584 IdentityMode::Relocatable => summary.relocatable_members += 1,
2585 }
2586
2587 if member.source_canister == member.target_canister {
2588 summary.in_place_members += 1;
2589 } else {
2590 summary.remapped_members += 1;
2591 }
2592 if mapping_supplied {
2593 summary.mapped_members += 1;
2594 }
2595 }
2596
2597 summary.all_sources_mapped = mapping_supplied && summary.mapped_members == members.len();
2598
2599 summary
2600}
2601
2602fn restore_snapshot_summary(members: &[RestorePlanMember]) -> RestoreSnapshotSummary {
2604 let members_with_module_hash = members
2605 .iter()
2606 .filter(|member| member.source_snapshot.module_hash.is_some())
2607 .count();
2608 let members_with_wasm_hash = members
2609 .iter()
2610 .filter(|member| member.source_snapshot.wasm_hash.is_some())
2611 .count();
2612 let members_with_code_version = members
2613 .iter()
2614 .filter(|member| member.source_snapshot.code_version.is_some())
2615 .count();
2616 let members_with_checksum = members
2617 .iter()
2618 .filter(|member| member.source_snapshot.checksum.is_some())
2619 .count();
2620
2621 RestoreSnapshotSummary {
2622 all_members_have_module_hash: members_with_module_hash == members.len(),
2623 all_members_have_wasm_hash: members_with_wasm_hash == members.len(),
2624 all_members_have_code_version: members_with_code_version == members.len(),
2625 all_members_have_checksum: members_with_checksum == members.len(),
2626 members_with_module_hash,
2627 members_with_wasm_hash,
2628 members_with_code_version,
2629 members_with_checksum,
2630 }
2631}
2632
2633fn restore_readiness_summary(
2635 snapshot: &RestoreSnapshotSummary,
2636 verification: &RestoreVerificationSummary,
2637) -> RestoreReadinessSummary {
2638 let mut reasons = Vec::new();
2639
2640 if !snapshot.all_members_have_module_hash {
2641 reasons.push("missing-module-hash".to_string());
2642 }
2643 if !snapshot.all_members_have_wasm_hash {
2644 reasons.push("missing-wasm-hash".to_string());
2645 }
2646 if !snapshot.all_members_have_code_version {
2647 reasons.push("missing-code-version".to_string());
2648 }
2649 if !snapshot.all_members_have_checksum {
2650 reasons.push("missing-snapshot-checksum".to_string());
2651 }
2652 if !verification.all_members_have_checks {
2653 reasons.push("missing-verification-checks".to_string());
2654 }
2655
2656 RestoreReadinessSummary {
2657 ready: reasons.is_empty(),
2658 reasons,
2659 }
2660}
2661
2662fn restore_verification_summary(
2664 manifest: &FleetBackupManifest,
2665 members: &[RestorePlanMember],
2666) -> RestoreVerificationSummary {
2667 let fleet_checks = manifest.verification.fleet_checks.len();
2668 let member_check_groups = manifest.verification.member_checks.len();
2669 let member_checks = members
2670 .iter()
2671 .map(|member| member.verification_checks.len())
2672 .sum::<usize>();
2673 let members_with_checks = members
2674 .iter()
2675 .filter(|member| !member.verification_checks.is_empty())
2676 .count();
2677
2678 RestoreVerificationSummary {
2679 verification_required: true,
2680 all_members_have_checks: members_with_checks == members.len(),
2681 fleet_checks,
2682 member_check_groups,
2683 member_checks,
2684 members_with_checks,
2685 total_checks: fleet_checks + member_checks,
2686 }
2687}
2688
2689const fn restore_operation_summary(
2691 member_count: usize,
2692 verification_summary: &RestoreVerificationSummary,
2693 phases: &[RestorePhase],
2694) -> RestoreOperationSummary {
2695 RestoreOperationSummary {
2696 planned_snapshot_uploads: member_count,
2697 planned_snapshot_loads: member_count,
2698 planned_code_reinstalls: member_count,
2699 planned_verification_checks: verification_summary.total_checks,
2700 planned_operations: member_count
2701 + member_count
2702 + member_count
2703 + verification_summary.total_checks,
2704 planned_phases: phases.len(),
2705 }
2706}
2707
2708fn validate_restore_group_dependencies(
2710 members: &[RestorePlanMember],
2711) -> Result<(), RestorePlanError> {
2712 let groups_by_source = members
2713 .iter()
2714 .map(|member| (member.source_canister.as_str(), member.restore_group))
2715 .collect::<BTreeMap<_, _>>();
2716
2717 for member in members {
2718 let Some(parent) = &member.parent_source_canister else {
2719 continue;
2720 };
2721 let Some(parent_group) = groups_by_source.get(parent.as_str()) else {
2722 continue;
2723 };
2724
2725 if *parent_group > member.restore_group {
2726 return Err(RestorePlanError::ParentRestoreGroupAfterChild {
2727 child_source_canister: member.source_canister.clone(),
2728 parent_source_canister: parent.clone(),
2729 child_restore_group: member.restore_group,
2730 parent_restore_group: *parent_group,
2731 });
2732 }
2733 }
2734
2735 Ok(())
2736}
2737
2738fn group_and_order_members(
2740 members: Vec<RestorePlanMember>,
2741) -> Result<Vec<RestorePhase>, RestorePlanError> {
2742 let mut groups = BTreeMap::<u16, Vec<RestorePlanMember>>::new();
2743 for member in members {
2744 groups.entry(member.restore_group).or_default().push(member);
2745 }
2746
2747 groups
2748 .into_iter()
2749 .map(|(restore_group, members)| {
2750 let members = order_group(restore_group, members)?;
2751 Ok(RestorePhase {
2752 restore_group,
2753 members,
2754 })
2755 })
2756 .collect()
2757}
2758
2759fn order_group(
2761 restore_group: u16,
2762 members: Vec<RestorePlanMember>,
2763) -> Result<Vec<RestorePlanMember>, RestorePlanError> {
2764 let mut remaining = members;
2765 let group_sources = remaining
2766 .iter()
2767 .map(|member| member.source_canister.clone())
2768 .collect::<BTreeSet<_>>();
2769 let mut emitted = BTreeSet::new();
2770 let mut ordered = Vec::with_capacity(remaining.len());
2771
2772 while !remaining.is_empty() {
2773 let Some(index) = remaining
2774 .iter()
2775 .position(|member| parent_satisfied(member, &group_sources, &emitted))
2776 else {
2777 return Err(RestorePlanError::RestoreOrderCycle(restore_group));
2778 };
2779
2780 let mut member = remaining.remove(index);
2781 member.phase_order = ordered.len();
2782 member.ordering_dependency = ordering_dependency(&member, &group_sources);
2783 emitted.insert(member.source_canister.clone());
2784 ordered.push(member);
2785 }
2786
2787 Ok(ordered)
2788}
2789
2790fn ordering_dependency(
2792 member: &RestorePlanMember,
2793 group_sources: &BTreeSet<String>,
2794) -> Option<RestoreOrderingDependency> {
2795 let parent_source = member.parent_source_canister.as_ref()?;
2796 let parent_target = member.parent_target_canister.as_ref()?;
2797 let relationship = if group_sources.contains(parent_source) {
2798 RestoreOrderingRelationship::ParentInSameGroup
2799 } else {
2800 RestoreOrderingRelationship::ParentInEarlierGroup
2801 };
2802
2803 Some(RestoreOrderingDependency {
2804 source_canister: parent_source.clone(),
2805 target_canister: parent_target.clone(),
2806 relationship,
2807 })
2808}
2809
2810fn restore_ordering_summary(phases: &[RestorePhase]) -> RestoreOrderingSummary {
2812 let mut summary = RestoreOrderingSummary {
2813 phase_count: phases.len(),
2814 dependency_free_members: 0,
2815 in_group_parent_edges: 0,
2816 cross_group_parent_edges: 0,
2817 };
2818
2819 for member in phases.iter().flat_map(|phase| phase.members.iter()) {
2820 match &member.ordering_dependency {
2821 Some(dependency)
2822 if dependency.relationship == RestoreOrderingRelationship::ParentInSameGroup =>
2823 {
2824 summary.in_group_parent_edges += 1;
2825 }
2826 Some(dependency)
2827 if dependency.relationship == RestoreOrderingRelationship::ParentInEarlierGroup =>
2828 {
2829 summary.cross_group_parent_edges += 1;
2830 }
2831 Some(_) => {}
2832 None => summary.dependency_free_members += 1,
2833 }
2834 }
2835
2836 summary
2837}
2838
2839fn parent_satisfied(
2841 member: &RestorePlanMember,
2842 group_sources: &BTreeSet<String>,
2843 emitted: &BTreeSet<String>,
2844) -> bool {
2845 match &member.parent_source_canister {
2846 Some(parent) if group_sources.contains(parent) => emitted.contains(parent),
2847 _ => true,
2848 }
2849}
2850
2851fn validate_principal(field: &'static str, value: &str) -> Result<(), RestorePlanError> {
2853 Principal::from_str(value)
2854 .map(|_| ())
2855 .map_err(|_| RestorePlanError::InvalidPrincipal {
2856 field,
2857 value: value.to_string(),
2858 })
2859}
2860
2861#[cfg(test)]
2862mod tests {
2863 use super::*;
2864 use crate::manifest::{
2865 BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetSection,
2866 MemberVerificationChecks, SourceMetadata, SourceSnapshot, ToolMetadata, VerificationCheck,
2867 VerificationPlan,
2868 };
2869 use std::{
2870 env, fs,
2871 path::{Path, PathBuf},
2872 time::{SystemTime, UNIX_EPOCH},
2873 };
2874
2875 const ROOT: &str = "aaaaa-aa";
2876 const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2877 const CHILD_TWO: &str = "r7inp-6aaaa-aaaaa-aaabq-cai";
2878 const TARGET: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2879 const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2880
2881 fn command_preview_journal(
2883 operation: RestoreApplyOperationKind,
2884 verification_kind: Option<&str>,
2885 verification_method: Option<&str>,
2886 ) -> RestoreApplyJournal {
2887 let journal = RestoreApplyJournal {
2888 journal_version: 1,
2889 backup_id: "fbk_test_001".to_string(),
2890 ready: true,
2891 blocked_reasons: Vec::new(),
2892 operation_count: 1,
2893 operation_counts: RestoreApplyOperationKindCounts::default(),
2894 pending_operations: 0,
2895 ready_operations: 1,
2896 blocked_operations: 0,
2897 completed_operations: 0,
2898 failed_operations: 0,
2899 operations: vec![RestoreApplyJournalOperation {
2900 sequence: 0,
2901 operation,
2902 state: RestoreApplyOperationState::Ready,
2903 state_updated_at: None,
2904 blocking_reasons: Vec::new(),
2905 restore_group: 1,
2906 phase_order: 0,
2907 source_canister: ROOT.to_string(),
2908 target_canister: ROOT.to_string(),
2909 role: "root".to_string(),
2910 snapshot_id: Some("snap-root".to_string()),
2911 artifact_path: Some("artifacts/root".to_string()),
2912 verification_kind: verification_kind.map(str::to_string),
2913 verification_method: verification_method.map(str::to_string),
2914 }],
2915 };
2916
2917 journal.validate().expect("journal should validate");
2918 journal
2919 }
2920
2921 fn valid_manifest(identity_mode: IdentityMode) -> FleetBackupManifest {
2923 FleetBackupManifest {
2924 manifest_version: 1,
2925 backup_id: "fbk_test_001".to_string(),
2926 created_at: "2026-04-10T12:00:00Z".to_string(),
2927 tool: ToolMetadata {
2928 name: "canic".to_string(),
2929 version: "v1".to_string(),
2930 },
2931 source: SourceMetadata {
2932 environment: "local".to_string(),
2933 root_canister: ROOT.to_string(),
2934 },
2935 consistency: ConsistencySection {
2936 mode: ConsistencyMode::CrashConsistent,
2937 backup_units: vec![BackupUnit {
2938 unit_id: "whole-fleet".to_string(),
2939 kind: BackupUnitKind::WholeFleet,
2940 roles: vec!["root".to_string(), "app".to_string()],
2941 consistency_reason: None,
2942 dependency_closure: Vec::new(),
2943 topology_validation: "subtree-closed".to_string(),
2944 quiescence_strategy: None,
2945 }],
2946 },
2947 fleet: FleetSection {
2948 topology_hash_algorithm: "sha256".to_string(),
2949 topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
2950 discovery_topology_hash: HASH.to_string(),
2951 pre_snapshot_topology_hash: HASH.to_string(),
2952 topology_hash: HASH.to_string(),
2953 members: vec![
2954 fleet_member("app", CHILD, Some(ROOT), identity_mode, 1),
2955 fleet_member("root", ROOT, None, IdentityMode::Fixed, 1),
2956 ],
2957 },
2958 verification: VerificationPlan {
2959 fleet_checks: Vec::new(),
2960 member_checks: Vec::new(),
2961 },
2962 }
2963 }
2964
2965 fn fleet_member(
2967 role: &str,
2968 canister_id: &str,
2969 parent_canister_id: Option<&str>,
2970 identity_mode: IdentityMode,
2971 restore_group: u16,
2972 ) -> FleetMember {
2973 FleetMember {
2974 role: role.to_string(),
2975 canister_id: canister_id.to_string(),
2976 parent_canister_id: parent_canister_id.map(str::to_string),
2977 subnet_canister_id: None,
2978 controller_hint: Some(ROOT.to_string()),
2979 identity_mode,
2980 restore_group,
2981 verification_class: "basic".to_string(),
2982 verification_checks: vec![VerificationCheck {
2983 kind: "call".to_string(),
2984 method: Some("canic_ready".to_string()),
2985 roles: Vec::new(),
2986 }],
2987 source_snapshot: SourceSnapshot {
2988 snapshot_id: format!("snap-{role}"),
2989 module_hash: Some(HASH.to_string()),
2990 wasm_hash: Some(HASH.to_string()),
2991 code_version: Some("v0.30.0".to_string()),
2992 artifact_path: format!("artifacts/{role}"),
2993 checksum_algorithm: "sha256".to_string(),
2994 checksum: Some(HASH.to_string()),
2995 },
2996 }
2997 }
2998
2999 #[test]
3001 fn in_place_plan_orders_parent_before_child() {
3002 let manifest = valid_manifest(IdentityMode::Relocatable);
3003
3004 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3005 let ordered = plan.ordered_members();
3006
3007 assert_eq!(plan.backup_id, "fbk_test_001");
3008 assert_eq!(plan.source_environment, "local");
3009 assert_eq!(plan.source_root_canister, ROOT);
3010 assert_eq!(plan.topology_hash, HASH);
3011 assert_eq!(plan.member_count, 2);
3012 assert_eq!(plan.identity_summary.fixed_members, 1);
3013 assert_eq!(plan.identity_summary.relocatable_members, 1);
3014 assert_eq!(plan.identity_summary.in_place_members, 2);
3015 assert_eq!(plan.identity_summary.mapped_members, 0);
3016 assert_eq!(plan.identity_summary.remapped_members, 0);
3017 assert!(plan.verification_summary.verification_required);
3018 assert!(plan.verification_summary.all_members_have_checks);
3019 assert!(plan.readiness_summary.ready);
3020 assert!(plan.readiness_summary.reasons.is_empty());
3021 assert_eq!(plan.verification_summary.fleet_checks, 0);
3022 assert_eq!(plan.verification_summary.member_check_groups, 0);
3023 assert_eq!(plan.verification_summary.member_checks, 2);
3024 assert_eq!(plan.verification_summary.members_with_checks, 2);
3025 assert_eq!(plan.verification_summary.total_checks, 2);
3026 assert_eq!(plan.ordering_summary.phase_count, 1);
3027 assert_eq!(plan.ordering_summary.dependency_free_members, 1);
3028 assert_eq!(plan.ordering_summary.in_group_parent_edges, 1);
3029 assert_eq!(plan.ordering_summary.cross_group_parent_edges, 0);
3030 assert_eq!(ordered[0].phase_order, 0);
3031 assert_eq!(ordered[1].phase_order, 1);
3032 assert_eq!(ordered[0].source_canister, ROOT);
3033 assert_eq!(ordered[1].source_canister, CHILD);
3034 assert_eq!(
3035 ordered[1].ordering_dependency,
3036 Some(RestoreOrderingDependency {
3037 source_canister: ROOT.to_string(),
3038 target_canister: ROOT.to_string(),
3039 relationship: RestoreOrderingRelationship::ParentInSameGroup,
3040 })
3041 );
3042 }
3043
3044 #[test]
3046 fn plan_reports_parent_dependency_from_earlier_group() {
3047 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3048 manifest.fleet.members[0].restore_group = 2;
3049 manifest.fleet.members[1].restore_group = 1;
3050
3051 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3052 let ordered = plan.ordered_members();
3053
3054 assert_eq!(plan.phases.len(), 2);
3055 assert_eq!(plan.ordering_summary.phase_count, 2);
3056 assert_eq!(plan.ordering_summary.dependency_free_members, 1);
3057 assert_eq!(plan.ordering_summary.in_group_parent_edges, 0);
3058 assert_eq!(plan.ordering_summary.cross_group_parent_edges, 1);
3059 assert_eq!(ordered[0].source_canister, ROOT);
3060 assert_eq!(ordered[1].source_canister, CHILD);
3061 assert_eq!(
3062 ordered[1].ordering_dependency,
3063 Some(RestoreOrderingDependency {
3064 source_canister: ROOT.to_string(),
3065 target_canister: ROOT.to_string(),
3066 relationship: RestoreOrderingRelationship::ParentInEarlierGroup,
3067 })
3068 );
3069 }
3070
3071 #[test]
3073 fn plan_rejects_parent_in_later_restore_group() {
3074 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3075 manifest.fleet.members[0].restore_group = 1;
3076 manifest.fleet.members[1].restore_group = 2;
3077
3078 let err = RestorePlanner::plan(&manifest, None)
3079 .expect_err("parent-after-child group ordering should fail");
3080
3081 assert!(matches!(
3082 err,
3083 RestorePlanError::ParentRestoreGroupAfterChild { .. }
3084 ));
3085 }
3086
3087 #[test]
3089 fn fixed_identity_member_cannot_be_remapped() {
3090 let manifest = valid_manifest(IdentityMode::Fixed);
3091 let mapping = RestoreMapping {
3092 members: vec![
3093 RestoreMappingEntry {
3094 source_canister: ROOT.to_string(),
3095 target_canister: ROOT.to_string(),
3096 },
3097 RestoreMappingEntry {
3098 source_canister: CHILD.to_string(),
3099 target_canister: TARGET.to_string(),
3100 },
3101 ],
3102 };
3103
3104 let err = RestorePlanner::plan(&manifest, Some(&mapping))
3105 .expect_err("fixed member remap should fail");
3106
3107 assert!(matches!(err, RestorePlanError::FixedIdentityRemap { .. }));
3108 }
3109
3110 #[test]
3112 fn relocatable_member_can_be_mapped() {
3113 let manifest = valid_manifest(IdentityMode::Relocatable);
3114 let mapping = RestoreMapping {
3115 members: vec![
3116 RestoreMappingEntry {
3117 source_canister: ROOT.to_string(),
3118 target_canister: ROOT.to_string(),
3119 },
3120 RestoreMappingEntry {
3121 source_canister: CHILD.to_string(),
3122 target_canister: TARGET.to_string(),
3123 },
3124 ],
3125 };
3126
3127 let plan = RestorePlanner::plan(&manifest, Some(&mapping)).expect("plan should build");
3128 let child = plan
3129 .ordered_members()
3130 .into_iter()
3131 .find(|member| member.source_canister == CHILD)
3132 .expect("child member should be planned");
3133
3134 assert_eq!(plan.identity_summary.fixed_members, 1);
3135 assert_eq!(plan.identity_summary.relocatable_members, 1);
3136 assert_eq!(plan.identity_summary.in_place_members, 1);
3137 assert_eq!(plan.identity_summary.mapped_members, 2);
3138 assert_eq!(plan.identity_summary.remapped_members, 1);
3139 assert_eq!(child.target_canister, TARGET);
3140 assert_eq!(child.parent_target_canister, Some(ROOT.to_string()));
3141 }
3142
3143 #[test]
3145 fn plan_members_include_snapshot_and_verification_metadata() {
3146 let manifest = valid_manifest(IdentityMode::Relocatable);
3147
3148 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3149 let root = plan
3150 .ordered_members()
3151 .into_iter()
3152 .find(|member| member.source_canister == ROOT)
3153 .expect("root member should be planned");
3154
3155 assert_eq!(root.identity_mode, IdentityMode::Fixed);
3156 assert_eq!(root.verification_class, "basic");
3157 assert_eq!(root.verification_checks[0].kind, "call");
3158 assert_eq!(root.source_snapshot.snapshot_id, "snap-root");
3159 assert_eq!(root.source_snapshot.artifact_path, "artifacts/root");
3160 }
3161
3162 #[test]
3164 fn plan_includes_mapping_summary() {
3165 let manifest = valid_manifest(IdentityMode::Relocatable);
3166 let in_place = RestorePlanner::plan(&manifest, None).expect("plan should build");
3167
3168 assert!(!in_place.identity_summary.mapping_supplied);
3169 assert!(!in_place.identity_summary.all_sources_mapped);
3170 assert_eq!(in_place.identity_summary.mapped_members, 0);
3171
3172 let mapping = RestoreMapping {
3173 members: vec![
3174 RestoreMappingEntry {
3175 source_canister: ROOT.to_string(),
3176 target_canister: ROOT.to_string(),
3177 },
3178 RestoreMappingEntry {
3179 source_canister: CHILD.to_string(),
3180 target_canister: TARGET.to_string(),
3181 },
3182 ],
3183 };
3184 let mapped = RestorePlanner::plan(&manifest, Some(&mapping)).expect("plan should build");
3185
3186 assert!(mapped.identity_summary.mapping_supplied);
3187 assert!(mapped.identity_summary.all_sources_mapped);
3188 assert_eq!(mapped.identity_summary.mapped_members, 2);
3189 assert_eq!(mapped.identity_summary.remapped_members, 1);
3190 }
3191
3192 #[test]
3194 fn plan_includes_snapshot_summary() {
3195 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3196 manifest.fleet.members[1].source_snapshot.module_hash = None;
3197 manifest.fleet.members[1].source_snapshot.wasm_hash = None;
3198 manifest.fleet.members[1].source_snapshot.checksum = None;
3199
3200 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3201
3202 assert!(!plan.snapshot_summary.all_members_have_module_hash);
3203 assert!(!plan.snapshot_summary.all_members_have_wasm_hash);
3204 assert!(plan.snapshot_summary.all_members_have_code_version);
3205 assert!(!plan.snapshot_summary.all_members_have_checksum);
3206 assert_eq!(plan.snapshot_summary.members_with_module_hash, 1);
3207 assert_eq!(plan.snapshot_summary.members_with_wasm_hash, 1);
3208 assert_eq!(plan.snapshot_summary.members_with_code_version, 2);
3209 assert_eq!(plan.snapshot_summary.members_with_checksum, 1);
3210 assert!(!plan.readiness_summary.ready);
3211 assert_eq!(
3212 plan.readiness_summary.reasons,
3213 [
3214 "missing-module-hash",
3215 "missing-wasm-hash",
3216 "missing-snapshot-checksum"
3217 ]
3218 );
3219 }
3220
3221 #[test]
3223 fn plan_includes_verification_summary() {
3224 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3225 manifest.verification.fleet_checks.push(VerificationCheck {
3226 kind: "fleet-ready".to_string(),
3227 method: None,
3228 roles: Vec::new(),
3229 });
3230 manifest
3231 .verification
3232 .member_checks
3233 .push(MemberVerificationChecks {
3234 role: "app".to_string(),
3235 checks: vec![VerificationCheck {
3236 kind: "app-ready".to_string(),
3237 method: Some("ready".to_string()),
3238 roles: Vec::new(),
3239 }],
3240 });
3241
3242 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3243
3244 assert!(plan.verification_summary.verification_required);
3245 assert!(plan.verification_summary.all_members_have_checks);
3246 let app = plan
3247 .ordered_members()
3248 .into_iter()
3249 .find(|member| member.role == "app")
3250 .expect("app member should be planned");
3251 assert_eq!(app.verification_checks.len(), 2);
3252 assert_eq!(plan.fleet_verification_checks.len(), 1);
3253 assert_eq!(plan.fleet_verification_checks[0].kind, "fleet-ready");
3254 assert_eq!(plan.verification_summary.fleet_checks, 1);
3255 assert_eq!(plan.verification_summary.member_check_groups, 1);
3256 assert_eq!(plan.verification_summary.member_checks, 3);
3257 assert_eq!(plan.verification_summary.members_with_checks, 2);
3258 assert_eq!(plan.verification_summary.total_checks, 4);
3259 }
3260
3261 #[test]
3263 fn plan_includes_operation_summary() {
3264 let manifest = valid_manifest(IdentityMode::Relocatable);
3265
3266 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3267
3268 assert_eq!(plan.operation_summary.planned_snapshot_uploads, 2);
3269 assert_eq!(plan.operation_summary.planned_snapshot_loads, 2);
3270 assert_eq!(plan.operation_summary.planned_code_reinstalls, 2);
3271 assert_eq!(plan.operation_summary.planned_verification_checks, 2);
3272 assert_eq!(plan.operation_summary.planned_operations, 8);
3273 assert_eq!(plan.operation_summary.planned_phases, 1);
3274 }
3275
3276 #[test]
3278 fn plan_includes_design_conformance_report() {
3279 let manifest = valid_manifest(IdentityMode::Relocatable);
3280
3281 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3282 let design = plan
3283 .design_conformance
3284 .as_ref()
3285 .expect("new plans should carry design conformance");
3286
3287 assert!(design.design_v1_ready);
3288 assert!(design.topology.design_v1_ready);
3289 assert!(design.backup_units.design_v1_ready);
3290 assert!(design.quiescence.design_v1_ready);
3291 assert!(design.verification.design_v1_ready);
3292 assert!(design.snapshot_provenance.design_v1_ready);
3293 assert!(design.restore_order.design_v1_ready);
3294 }
3295
3296 #[test]
3298 fn restore_plan_defaults_missing_newer_restore_fields() {
3299 let manifest = valid_manifest(IdentityMode::Relocatable);
3300 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3301 let mut value = serde_json::to_value(&plan).expect("serialize plan");
3302 value
3303 .as_object_mut()
3304 .expect("plan should serialize as an object")
3305 .remove("fleet_verification_checks");
3306 value
3307 .as_object_mut()
3308 .expect("plan should serialize as an object")
3309 .remove("design_conformance");
3310 let operation_summary = value
3311 .get_mut("operation_summary")
3312 .and_then(serde_json::Value::as_object_mut)
3313 .expect("operation summary should serialize as an object");
3314 operation_summary.remove("planned_snapshot_uploads");
3315 operation_summary.remove("planned_operations");
3316
3317 let decoded: RestorePlan = serde_json::from_value(value).expect("decode old plan shape");
3318 let status = RestoreStatus::from_plan(&decoded);
3319 let dry_run =
3320 RestoreApplyDryRun::try_from_plan(&decoded, None).expect("old plan should dry-run");
3321
3322 assert!(decoded.fleet_verification_checks.is_empty());
3323 assert_eq!(decoded.design_conformance, None);
3324 assert_eq!(decoded.operation_summary.planned_snapshot_uploads, 0);
3325 assert_eq!(decoded.operation_summary.planned_operations, 0);
3326 assert_eq!(status.planned_snapshot_uploads, 2);
3327 assert_eq!(status.planned_operations, 8);
3328 assert_eq!(dry_run.planned_snapshot_uploads, 2);
3329 assert_eq!(dry_run.planned_operations, 8);
3330 assert_eq!(decoded.backup_id, plan.backup_id);
3331 assert_eq!(decoded.member_count, plan.member_count);
3332 }
3333
3334 #[test]
3336 fn restore_status_starts_all_members_as_planned() {
3337 let manifest = valid_manifest(IdentityMode::Relocatable);
3338
3339 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3340 let status = RestoreStatus::from_plan(&plan);
3341
3342 assert_eq!(status.status_version, 1);
3343 assert_eq!(status.backup_id.as_str(), plan.backup_id.as_str());
3344 assert_eq!(
3345 status.source_environment.as_str(),
3346 plan.source_environment.as_str()
3347 );
3348 assert_eq!(
3349 status.source_root_canister.as_str(),
3350 plan.source_root_canister.as_str()
3351 );
3352 assert_eq!(status.topology_hash.as_str(), plan.topology_hash.as_str());
3353 assert!(status.ready);
3354 assert!(status.readiness_reasons.is_empty());
3355 assert!(status.verification_required);
3356 assert_eq!(status.member_count, 2);
3357 assert_eq!(status.phase_count, 1);
3358 assert_eq!(status.planned_snapshot_uploads, 2);
3359 assert_eq!(status.planned_snapshot_loads, 2);
3360 assert_eq!(status.planned_code_reinstalls, 2);
3361 assert_eq!(status.planned_verification_checks, 2);
3362 assert_eq!(status.planned_operations, 8);
3363 assert_eq!(status.phases.len(), 1);
3364 assert_eq!(status.phases[0].restore_group, 1);
3365 assert_eq!(status.phases[0].members.len(), 2);
3366 assert_eq!(
3367 status.phases[0].members[0].state,
3368 RestoreMemberState::Planned
3369 );
3370 assert_eq!(status.phases[0].members[0].source_canister, ROOT);
3371 assert_eq!(status.phases[0].members[0].target_canister, ROOT);
3372 assert_eq!(status.phases[0].members[0].snapshot_id, "snap-root");
3373 assert_eq!(status.phases[0].members[0].artifact_path, "artifacts/root");
3374 assert_eq!(
3375 status.phases[0].members[1].state,
3376 RestoreMemberState::Planned
3377 );
3378 assert_eq!(status.phases[0].members[1].source_canister, CHILD);
3379 }
3380
3381 #[test]
3383 fn apply_dry_run_renders_ordered_member_operations() {
3384 let manifest = valid_manifest(IdentityMode::Relocatable);
3385
3386 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3387 let status = RestoreStatus::from_plan(&plan);
3388 let dry_run =
3389 RestoreApplyDryRun::try_from_plan(&plan, Some(&status)).expect("dry-run should build");
3390
3391 assert_eq!(dry_run.dry_run_version, 1);
3392 assert_eq!(dry_run.backup_id.as_str(), "fbk_test_001");
3393 assert!(dry_run.ready);
3394 assert!(dry_run.status_supplied);
3395 assert_eq!(dry_run.member_count, 2);
3396 assert_eq!(dry_run.phase_count, 1);
3397 assert_eq!(dry_run.planned_snapshot_uploads, 2);
3398 assert_eq!(dry_run.planned_snapshot_loads, 2);
3399 assert_eq!(dry_run.planned_code_reinstalls, 2);
3400 assert_eq!(dry_run.planned_verification_checks, 2);
3401 assert_eq!(dry_run.planned_operations, 8);
3402 assert_eq!(dry_run.rendered_operations, 8);
3403 assert_eq!(dry_run.operation_counts.snapshot_uploads, 2);
3404 assert_eq!(dry_run.operation_counts.snapshot_loads, 2);
3405 assert_eq!(dry_run.operation_counts.code_reinstalls, 2);
3406 assert_eq!(dry_run.operation_counts.member_verifications, 2);
3407 assert_eq!(dry_run.operation_counts.fleet_verifications, 0);
3408 assert_eq!(dry_run.operation_counts.verification_operations, 2);
3409 assert_eq!(dry_run.phases.len(), 1);
3410
3411 let operations = &dry_run.phases[0].operations;
3412 assert_eq!(operations[0].sequence, 0);
3413 assert_eq!(
3414 operations[0].operation,
3415 RestoreApplyOperationKind::UploadSnapshot
3416 );
3417 assert_eq!(operations[0].source_canister, ROOT);
3418 assert_eq!(operations[0].target_canister, ROOT);
3419 assert_eq!(operations[0].snapshot_id, Some("snap-root".to_string()));
3420 assert_eq!(
3421 operations[0].artifact_path,
3422 Some("artifacts/root".to_string())
3423 );
3424 assert_eq!(
3425 operations[1].operation,
3426 RestoreApplyOperationKind::LoadSnapshot
3427 );
3428 assert_eq!(
3429 operations[2].operation,
3430 RestoreApplyOperationKind::ReinstallCode
3431 );
3432 assert_eq!(
3433 operations[3].operation,
3434 RestoreApplyOperationKind::VerifyMember
3435 );
3436 assert_eq!(operations[3].verification_kind, Some("call".to_string()));
3437 assert_eq!(
3438 operations[3].verification_method,
3439 Some("canic_ready".to_string())
3440 );
3441 assert_eq!(operations[4].source_canister, CHILD);
3442 assert_eq!(
3443 operations[7].operation,
3444 RestoreApplyOperationKind::VerifyMember
3445 );
3446 }
3447
3448 #[test]
3450 fn apply_dry_run_renders_fleet_verification_operations() {
3451 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3452 manifest.verification.fleet_checks.push(VerificationCheck {
3453 kind: "fleet-ready".to_string(),
3454 method: Some("canic_fleet_ready".to_string()),
3455 roles: Vec::new(),
3456 });
3457
3458 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3459 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3460
3461 assert_eq!(plan.operation_summary.planned_verification_checks, 3);
3462 assert_eq!(dry_run.rendered_operations, 9);
3463 let operation = dry_run.phases[0]
3464 .operations
3465 .last()
3466 .expect("fleet verification operation should be rendered");
3467 assert_eq!(operation.sequence, 8);
3468 assert_eq!(operation.operation, RestoreApplyOperationKind::VerifyFleet);
3469 assert_eq!(operation.source_canister, ROOT);
3470 assert_eq!(operation.target_canister, ROOT);
3471 assert_eq!(operation.role, "fleet");
3472 assert_eq!(operation.snapshot_id, None);
3473 assert_eq!(operation.artifact_path, None);
3474 assert_eq!(operation.verification_kind, Some("fleet-ready".to_string()));
3475 assert_eq!(
3476 operation.verification_method,
3477 Some("canic_fleet_ready".to_string())
3478 );
3479 }
3480
3481 #[test]
3483 fn apply_dry_run_sequences_operations_across_phases() {
3484 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3485 manifest.fleet.members[0].restore_group = 2;
3486
3487 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3488 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3489
3490 assert_eq!(dry_run.phases.len(), 2);
3491 assert_eq!(dry_run.rendered_operations, 8);
3492 assert_eq!(dry_run.phases[0].operations[0].sequence, 0);
3493 assert_eq!(dry_run.phases[0].operations[3].sequence, 3);
3494 assert_eq!(dry_run.phases[1].operations[0].sequence, 4);
3495 assert_eq!(dry_run.phases[1].operations[3].sequence, 7);
3496 }
3497
3498 #[test]
3500 fn apply_dry_run_validates_artifacts_under_backup_root() {
3501 let root = temp_dir("canic-restore-apply-artifacts-ok");
3502 fs::create_dir_all(&root).expect("create temp root");
3503 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3504 set_member_artifact(
3505 &mut manifest,
3506 CHILD,
3507 &root,
3508 "artifacts/child",
3509 b"child-snapshot",
3510 );
3511 set_member_artifact(
3512 &mut manifest,
3513 ROOT,
3514 &root,
3515 "artifacts/root",
3516 b"root-snapshot",
3517 );
3518
3519 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3520 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3521 .expect("dry-run should validate artifacts");
3522
3523 let validation = dry_run
3524 .artifact_validation
3525 .expect("artifact validation should be present");
3526 assert_eq!(validation.checked_members, 2);
3527 assert!(validation.artifacts_present);
3528 assert!(validation.checksums_verified);
3529 assert_eq!(validation.members_with_expected_checksums, 2);
3530 assert_eq!(validation.checks[0].source_canister, ROOT);
3531 assert!(validation.checks[0].checksum_verified);
3532
3533 fs::remove_dir_all(root).expect("remove temp root");
3534 }
3535
3536 #[test]
3538 fn apply_journal_marks_validated_operations_ready() {
3539 let root = temp_dir("canic-restore-apply-journal-ready");
3540 fs::create_dir_all(&root).expect("create temp root");
3541 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3542 set_member_artifact(
3543 &mut manifest,
3544 CHILD,
3545 &root,
3546 "artifacts/child",
3547 b"child-snapshot",
3548 );
3549 set_member_artifact(
3550 &mut manifest,
3551 ROOT,
3552 &root,
3553 "artifacts/root",
3554 b"root-snapshot",
3555 );
3556
3557 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3558 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3559 .expect("dry-run should validate artifacts");
3560 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3561
3562 fs::remove_dir_all(root).expect("remove temp root");
3563 assert_eq!(journal.journal_version, 1);
3564 assert_eq!(journal.backup_id.as_str(), "fbk_test_001");
3565 assert!(journal.ready);
3566 assert!(journal.blocked_reasons.is_empty());
3567 assert_eq!(journal.operation_count, 8);
3568 assert_eq!(journal.ready_operations, 8);
3569 assert_eq!(journal.blocked_operations, 0);
3570 assert_eq!(journal.operations[0].sequence, 0);
3571 assert_eq!(
3572 journal.operations[0].state,
3573 RestoreApplyOperationState::Ready
3574 );
3575 assert!(journal.operations[0].blocking_reasons.is_empty());
3576 }
3577
3578 #[test]
3580 fn apply_journal_blocks_without_artifact_validation() {
3581 let manifest = valid_manifest(IdentityMode::Relocatable);
3582
3583 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3584 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3585 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3586
3587 assert!(!journal.ready);
3588 assert_eq!(journal.ready_operations, 0);
3589 assert_eq!(journal.blocked_operations, 8);
3590 assert!(
3591 journal
3592 .blocked_reasons
3593 .contains(&"missing-artifact-validation".to_string())
3594 );
3595 assert!(
3596 journal.operations[0]
3597 .blocking_reasons
3598 .contains(&"missing-artifact-validation".to_string())
3599 );
3600 }
3601
3602 #[test]
3604 fn apply_journal_status_reports_next_ready_operation() {
3605 let root = temp_dir("canic-restore-apply-journal-status");
3606 fs::create_dir_all(&root).expect("create temp root");
3607 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3608 set_member_artifact(
3609 &mut manifest,
3610 CHILD,
3611 &root,
3612 "artifacts/child",
3613 b"child-snapshot",
3614 );
3615 set_member_artifact(
3616 &mut manifest,
3617 ROOT,
3618 &root,
3619 "artifacts/root",
3620 b"root-snapshot",
3621 );
3622
3623 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3624 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3625 .expect("dry-run should validate artifacts");
3626 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3627 let status = journal.status();
3628 let report = journal.report();
3629
3630 fs::remove_dir_all(root).expect("remove temp root");
3631 assert_eq!(status.status_version, 1);
3632 assert_eq!(status.backup_id.as_str(), "fbk_test_001");
3633 assert!(status.ready);
3634 assert!(!status.complete);
3635 assert_eq!(status.operation_count, 8);
3636 assert_eq!(status.operation_counts.snapshot_uploads, 2);
3637 assert_eq!(status.operation_counts.snapshot_loads, 2);
3638 assert_eq!(status.operation_counts.code_reinstalls, 2);
3639 assert_eq!(status.operation_counts.member_verifications, 2);
3640 assert_eq!(status.operation_counts.fleet_verifications, 0);
3641 assert_eq!(status.operation_counts.verification_operations, 2);
3642 assert!(status.operation_counts_supplied);
3643 assert_eq!(journal.operation_counts, status.operation_counts);
3644 assert_eq!(report.operation_counts, status.operation_counts);
3645 assert!(report.operation_counts_supplied);
3646 assert_eq!(status.progress.operation_count, 8);
3647 assert_eq!(status.progress.completed_operations, 0);
3648 assert_eq!(status.progress.remaining_operations, 8);
3649 assert_eq!(status.progress.transitionable_operations, 8);
3650 assert_eq!(status.progress.attention_operations, 0);
3651 assert_eq!(status.progress.completion_basis_points, 0);
3652 assert_eq!(report.progress, status.progress);
3653 assert_eq!(status.pending_summary.pending_operations, 0);
3654 assert!(!status.pending_summary.pending_operation_available);
3655 assert_eq!(status.pending_summary.pending_sequence, None);
3656 assert_eq!(status.pending_summary.pending_operation, None);
3657 assert_eq!(status.pending_summary.pending_updated_at, None);
3658 assert!(!status.pending_summary.pending_updated_at_known);
3659 assert_eq!(report.pending_summary, status.pending_summary);
3660 assert_eq!(status.ready_operations, 8);
3661 assert_eq!(status.next_ready_sequence, Some(0));
3662 assert_eq!(
3663 status.next_ready_operation,
3664 Some(RestoreApplyOperationKind::UploadSnapshot)
3665 );
3666 assert_eq!(status.next_transition_sequence, Some(0));
3667 assert_eq!(
3668 status.next_transition_state,
3669 Some(RestoreApplyOperationState::Ready)
3670 );
3671 assert_eq!(
3672 status.next_transition_operation,
3673 Some(RestoreApplyOperationKind::UploadSnapshot)
3674 );
3675 }
3676
3677 #[test]
3679 fn apply_journal_next_operation_reports_full_ready_row() {
3680 let root = temp_dir("canic-restore-apply-journal-next");
3681 fs::create_dir_all(&root).expect("create temp root");
3682 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3683 set_member_artifact(
3684 &mut manifest,
3685 CHILD,
3686 &root,
3687 "artifacts/child",
3688 b"child-snapshot",
3689 );
3690 set_member_artifact(
3691 &mut manifest,
3692 ROOT,
3693 &root,
3694 "artifacts/root",
3695 b"root-snapshot",
3696 );
3697
3698 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3699 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3700 .expect("dry-run should validate artifacts");
3701 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3702 journal
3703 .mark_operation_completed(0)
3704 .expect("mark operation completed");
3705 let next = journal.next_operation();
3706
3707 fs::remove_dir_all(root).expect("remove temp root");
3708 assert!(next.ready);
3709 assert!(!next.complete);
3710 assert!(next.operation_available);
3711 let operation = next.operation.expect("next operation");
3712 assert_eq!(operation.sequence, 1);
3713 assert_eq!(operation.state, RestoreApplyOperationState::Ready);
3714 assert_eq!(operation.operation, RestoreApplyOperationKind::LoadSnapshot);
3715 assert_eq!(operation.source_canister, ROOT);
3716 }
3717
3718 #[test]
3720 fn apply_journal_next_operation_reports_blocked_state() {
3721 let manifest = valid_manifest(IdentityMode::Relocatable);
3722
3723 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3724 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3725 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3726 let next = journal.next_operation();
3727
3728 assert!(!next.ready);
3729 assert!(!next.operation_available);
3730 assert!(next.operation.is_none());
3731 assert!(
3732 next.blocked_reasons
3733 .contains(&"missing-artifact-validation".to_string())
3734 );
3735 }
3736
3737 #[test]
3739 fn apply_journal_command_preview_reports_upload_command() {
3740 let root = temp_dir("canic-restore-apply-command-upload");
3741 fs::create_dir_all(&root).expect("create temp root");
3742 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3743 set_member_artifact(
3744 &mut manifest,
3745 CHILD,
3746 &root,
3747 "artifacts/child",
3748 b"child-snapshot",
3749 );
3750 set_member_artifact(
3751 &mut manifest,
3752 ROOT,
3753 &root,
3754 "artifacts/root",
3755 b"root-snapshot",
3756 );
3757
3758 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3759 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3760 .expect("dry-run should validate artifacts");
3761 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3762 let preview = journal.next_command_preview();
3763
3764 fs::remove_dir_all(root).expect("remove temp root");
3765 assert!(preview.ready);
3766 assert!(preview.operation_available);
3767 assert!(preview.command_available);
3768 let command = preview.command.expect("command preview");
3769 assert_eq!(command.program, "dfx");
3770 assert_eq!(
3771 command.args,
3772 vec![
3773 "canister".to_string(),
3774 "snapshot".to_string(),
3775 "upload".to_string(),
3776 "--dir".to_string(),
3777 "artifacts/root".to_string(),
3778 ROOT.to_string(),
3779 ]
3780 );
3781 assert!(command.mutates);
3782 assert!(!command.requires_stopped_canister);
3783 }
3784
3785 #[test]
3787 fn apply_journal_command_preview_honors_command_config() {
3788 let root = temp_dir("canic-restore-apply-command-config");
3789 fs::create_dir_all(&root).expect("create temp root");
3790 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3791 set_member_artifact(
3792 &mut manifest,
3793 CHILD,
3794 &root,
3795 "artifacts/child",
3796 b"child-snapshot",
3797 );
3798 set_member_artifact(
3799 &mut manifest,
3800 ROOT,
3801 &root,
3802 "artifacts/root",
3803 b"root-snapshot",
3804 );
3805
3806 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3807 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3808 .expect("dry-run should validate artifacts");
3809 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3810 let preview = journal.next_command_preview_with_config(&RestoreApplyCommandConfig {
3811 program: "/tmp/dfx".to_string(),
3812 network: Some("local".to_string()),
3813 });
3814
3815 fs::remove_dir_all(root).expect("remove temp root");
3816 let command = preview.command.expect("command preview");
3817 assert_eq!(command.program, "/tmp/dfx");
3818 assert_eq!(
3819 command.args,
3820 vec![
3821 "canister".to_string(),
3822 "--network".to_string(),
3823 "local".to_string(),
3824 "snapshot".to_string(),
3825 "upload".to_string(),
3826 "--dir".to_string(),
3827 "artifacts/root".to_string(),
3828 ROOT.to_string(),
3829 ]
3830 );
3831 }
3832
3833 #[test]
3835 fn apply_journal_command_preview_reports_load_command() {
3836 let root = temp_dir("canic-restore-apply-command-load");
3837 fs::create_dir_all(&root).expect("create temp root");
3838 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3839 set_member_artifact(
3840 &mut manifest,
3841 CHILD,
3842 &root,
3843 "artifacts/child",
3844 b"child-snapshot",
3845 );
3846 set_member_artifact(
3847 &mut manifest,
3848 ROOT,
3849 &root,
3850 "artifacts/root",
3851 b"root-snapshot",
3852 );
3853
3854 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3855 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3856 .expect("dry-run should validate artifacts");
3857 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3858 journal
3859 .mark_operation_completed(0)
3860 .expect("mark upload completed");
3861 let preview = journal.next_command_preview();
3862
3863 fs::remove_dir_all(root).expect("remove temp root");
3864 let command = preview.command.expect("command preview");
3865 assert_eq!(
3866 command.args,
3867 vec![
3868 "canister".to_string(),
3869 "snapshot".to_string(),
3870 "load".to_string(),
3871 ROOT.to_string(),
3872 "snap-root".to_string(),
3873 ]
3874 );
3875 assert!(command.mutates);
3876 assert!(command.requires_stopped_canister);
3877 }
3878
3879 #[test]
3881 fn apply_journal_command_preview_reports_reinstall_command() {
3882 let journal = command_preview_journal(RestoreApplyOperationKind::ReinstallCode, None, None);
3883 let preview = journal.next_command_preview_with_config(&RestoreApplyCommandConfig {
3884 program: "dfx".to_string(),
3885 network: Some("local".to_string()),
3886 });
3887
3888 assert!(preview.command_available);
3889 let command = preview.command.expect("command preview");
3890 assert_eq!(
3891 command.args,
3892 vec![
3893 "canister".to_string(),
3894 "--network".to_string(),
3895 "local".to_string(),
3896 "install".to_string(),
3897 "--mode".to_string(),
3898 "reinstall".to_string(),
3899 "--yes".to_string(),
3900 ROOT.to_string(),
3901 ]
3902 );
3903 assert!(command.mutates);
3904 assert!(!command.requires_stopped_canister);
3905 }
3906
3907 #[test]
3909 fn apply_journal_command_preview_reports_status_verification_command() {
3910 let journal = command_preview_journal(
3911 RestoreApplyOperationKind::VerifyMember,
3912 Some("status"),
3913 None,
3914 );
3915 let preview = journal.next_command_preview();
3916
3917 assert!(preview.command_available);
3918 let command = preview.command.expect("command preview");
3919 assert_eq!(
3920 command.args,
3921 vec![
3922 "canister".to_string(),
3923 "status".to_string(),
3924 ROOT.to_string()
3925 ]
3926 );
3927 assert!(!command.mutates);
3928 assert!(!command.requires_stopped_canister);
3929 }
3930
3931 #[test]
3933 fn apply_journal_command_preview_reports_method_verification_command() {
3934 let journal = command_preview_journal(
3935 RestoreApplyOperationKind::VerifyMember,
3936 Some("query"),
3937 Some("health"),
3938 );
3939 let preview = journal.next_command_preview();
3940
3941 assert!(preview.command_available);
3942 let command = preview.command.expect("command preview");
3943 assert_eq!(
3944 command.args,
3945 vec![
3946 "canister".to_string(),
3947 "call".to_string(),
3948 ROOT.to_string(),
3949 "health".to_string(),
3950 ]
3951 );
3952 assert!(!command.mutates);
3953 assert!(!command.requires_stopped_canister);
3954 }
3955
3956 #[test]
3958 fn apply_journal_command_preview_reports_fleet_verification_command() {
3959 let journal = command_preview_journal(
3960 RestoreApplyOperationKind::VerifyFleet,
3961 Some("fleet-ready"),
3962 Some("canic_fleet_ready"),
3963 );
3964 let preview = journal.next_command_preview();
3965
3966 assert!(preview.command_available);
3967 let command = preview.command.expect("command preview");
3968 assert_eq!(
3969 command.args,
3970 vec![
3971 "canister".to_string(),
3972 "call".to_string(),
3973 ROOT.to_string(),
3974 "canic_fleet_ready".to_string(),
3975 ]
3976 );
3977 assert!(!command.mutates);
3978 assert!(!command.requires_stopped_canister);
3979 assert_eq!(command.note, "calls the declared fleet verification method");
3980 }
3981
3982 #[test]
3984 fn apply_journal_validation_rejects_method_verification_without_method() {
3985 let journal = RestoreApplyJournal {
3986 journal_version: 1,
3987 backup_id: "fbk_test_001".to_string(),
3988 ready: true,
3989 blocked_reasons: Vec::new(),
3990 operation_count: 1,
3991 operation_counts: RestoreApplyOperationKindCounts::default(),
3992 pending_operations: 0,
3993 ready_operations: 1,
3994 blocked_operations: 0,
3995 completed_operations: 0,
3996 failed_operations: 0,
3997 operations: vec![RestoreApplyJournalOperation {
3998 sequence: 0,
3999 operation: RestoreApplyOperationKind::VerifyMember,
4000 state: RestoreApplyOperationState::Ready,
4001 state_updated_at: None,
4002 blocking_reasons: Vec::new(),
4003 restore_group: 1,
4004 phase_order: 0,
4005 source_canister: ROOT.to_string(),
4006 target_canister: ROOT.to_string(),
4007 role: "root".to_string(),
4008 snapshot_id: Some("snap-root".to_string()),
4009 artifact_path: Some("artifacts/root".to_string()),
4010 verification_kind: Some("query".to_string()),
4011 verification_method: None,
4012 }],
4013 };
4014
4015 let err = journal
4016 .validate()
4017 .expect_err("method verification without method should fail");
4018
4019 assert!(matches!(
4020 err,
4021 RestoreApplyJournalError::OperationMissingField {
4022 sequence: 0,
4023 operation: RestoreApplyOperationKind::VerifyMember,
4024 field: "operations[].verification_method",
4025 }
4026 ));
4027 }
4028
4029 #[test]
4031 fn apply_journal_validation_rejects_count_mismatch() {
4032 let manifest = valid_manifest(IdentityMode::Relocatable);
4033
4034 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4035 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4036 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4037 journal.blocked_operations = 0;
4038
4039 let err = journal.validate().expect_err("count mismatch should fail");
4040
4041 assert!(matches!(
4042 err,
4043 RestoreApplyJournalError::CountMismatch {
4044 field: "blocked_operations",
4045 ..
4046 }
4047 ));
4048 }
4049
4050 #[test]
4052 fn apply_journal_validation_rejects_operation_kind_count_mismatch() {
4053 let mut journal =
4054 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4055 journal.operation_counts = RestoreApplyOperationKindCounts {
4056 snapshot_uploads: 0,
4057 snapshot_loads: 1,
4058 code_reinstalls: 0,
4059 member_verifications: 0,
4060 fleet_verifications: 0,
4061 verification_operations: 0,
4062 };
4063
4064 let err = journal
4065 .validate()
4066 .expect_err("operation-kind count mismatch should fail");
4067
4068 assert!(matches!(
4069 err,
4070 RestoreApplyJournalError::CountMismatch {
4071 field: "operation_counts.snapshot_uploads",
4072 reported: 0,
4073 actual: 1,
4074 }
4075 ));
4076 }
4077
4078 #[test]
4080 fn apply_journal_defaults_missing_operation_kind_counts() {
4081 let mut journal =
4082 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4083 journal.operation_counts =
4084 RestoreApplyOperationKindCounts::from_operations(&journal.operations);
4085 let mut value = serde_json::to_value(&journal).expect("serialize journal");
4086 value
4087 .as_object_mut()
4088 .expect("journal should serialize as an object")
4089 .remove("operation_counts");
4090
4091 let decoded: RestoreApplyJournal =
4092 serde_json::from_value(value).expect("decode old journal shape");
4093 decoded.validate().expect("old journal should validate");
4094 let status = decoded.status();
4095
4096 assert_eq!(
4097 decoded.operation_counts,
4098 RestoreApplyOperationKindCounts::default()
4099 );
4100 assert_eq!(status.operation_counts.snapshot_uploads, 1);
4101 assert_eq!(status.operation_counts.snapshot_loads, 0);
4102 assert!(!status.operation_counts_supplied);
4103 }
4104
4105 #[test]
4107 fn apply_journal_validation_rejects_duplicate_sequences() {
4108 let manifest = valid_manifest(IdentityMode::Relocatable);
4109
4110 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4111 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4112 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4113 journal.operations[1].sequence = journal.operations[0].sequence;
4114
4115 let err = journal
4116 .validate()
4117 .expect_err("duplicate sequence should fail");
4118
4119 assert!(matches!(
4120 err,
4121 RestoreApplyJournalError::DuplicateSequence(0)
4122 ));
4123 }
4124
4125 #[test]
4127 fn apply_journal_validation_rejects_failed_without_reason() {
4128 let manifest = valid_manifest(IdentityMode::Relocatable);
4129
4130 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4131 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4132 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4133 journal.operations[0].state = RestoreApplyOperationState::Failed;
4134 journal.operations[0].blocking_reasons = Vec::new();
4135 journal.blocked_operations -= 1;
4136 journal.failed_operations = 1;
4137
4138 let err = journal
4139 .validate()
4140 .expect_err("failed operation without reason should fail");
4141
4142 assert!(matches!(
4143 err,
4144 RestoreApplyJournalError::FailureReasonRequired(0)
4145 ));
4146 }
4147
4148 #[test]
4150 fn apply_journal_mark_next_operation_pending_claims_first_operation() {
4151 let mut journal =
4152 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4153
4154 journal
4155 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4156 .expect("mark operation pending");
4157 let status = journal.status();
4158 let report = journal.report();
4159 let next = journal.next_operation();
4160 let preview = journal.next_command_preview();
4161
4162 assert_eq!(journal.pending_operations, 1);
4163 assert_eq!(journal.ready_operations, 0);
4164 assert_eq!(
4165 journal.operations[0].state,
4166 RestoreApplyOperationState::Pending
4167 );
4168 assert_eq!(
4169 journal.operations[0].state_updated_at.as_deref(),
4170 Some("2026-05-04T12:00:00Z")
4171 );
4172 assert_eq!(status.next_ready_sequence, None);
4173 assert_eq!(status.next_transition_sequence, Some(0));
4174 assert_eq!(
4175 status.next_transition_state,
4176 Some(RestoreApplyOperationState::Pending)
4177 );
4178 assert_eq!(
4179 status.next_transition_updated_at.as_deref(),
4180 Some("2026-05-04T12:00:00Z")
4181 );
4182 assert_eq!(status.pending_summary.pending_operations, 1);
4183 assert!(status.pending_summary.pending_operation_available);
4184 assert_eq!(status.pending_summary.pending_sequence, Some(0));
4185 assert_eq!(
4186 status.pending_summary.pending_operation,
4187 Some(RestoreApplyOperationKind::UploadSnapshot)
4188 );
4189 assert_eq!(
4190 status.pending_summary.pending_updated_at.as_deref(),
4191 Some("2026-05-04T12:00:00Z")
4192 );
4193 assert!(status.pending_summary.pending_updated_at_known);
4194 assert_eq!(report.pending_summary, status.pending_summary);
4195 assert!(next.operation_available);
4196 assert_eq!(
4197 next.operation.expect("next operation").state,
4198 RestoreApplyOperationState::Pending
4199 );
4200 assert!(preview.operation_available);
4201 assert!(preview.command_available);
4202 assert_eq!(
4203 preview.operation.expect("preview operation").state,
4204 RestoreApplyOperationState::Pending
4205 );
4206 }
4207
4208 #[test]
4210 fn apply_journal_mark_next_operation_ready_unclaims_pending_operation() {
4211 let mut journal =
4212 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4213
4214 journal
4215 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4216 .expect("mark operation pending");
4217 journal
4218 .mark_next_operation_ready_at(Some("2026-05-04T12:01:00Z".to_string()))
4219 .expect("mark operation ready");
4220 let status = journal.status();
4221 let next = journal.next_operation();
4222
4223 assert_eq!(journal.pending_operations, 0);
4224 assert_eq!(journal.ready_operations, 1);
4225 assert_eq!(
4226 journal.operations[0].state,
4227 RestoreApplyOperationState::Ready
4228 );
4229 assert_eq!(
4230 journal.operations[0].state_updated_at.as_deref(),
4231 Some("2026-05-04T12:01:00Z")
4232 );
4233 assert_eq!(status.next_ready_sequence, Some(0));
4234 assert_eq!(status.next_transition_sequence, Some(0));
4235 assert_eq!(
4236 status.next_transition_state,
4237 Some(RestoreApplyOperationState::Ready)
4238 );
4239 assert_eq!(
4240 status.next_transition_updated_at.as_deref(),
4241 Some("2026-05-04T12:01:00Z")
4242 );
4243 assert_eq!(
4244 next.operation.expect("next operation").state,
4245 RestoreApplyOperationState::Ready
4246 );
4247 }
4248
4249 #[test]
4251 fn apply_journal_validation_rejects_empty_state_updated_at() {
4252 let mut journal =
4253 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4254
4255 journal.operations[0].state_updated_at = Some(String::new());
4256 let err = journal
4257 .validate()
4258 .expect_err("empty state update marker should fail");
4259
4260 assert!(matches!(
4261 err,
4262 RestoreApplyJournalError::MissingField("operations[].state_updated_at")
4263 ));
4264 }
4265
4266 #[test]
4268 fn apply_journal_validation_rejects_missing_operation_fields() {
4269 let mut upload =
4270 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4271 upload.operations[0].artifact_path = None;
4272 let err = upload
4273 .validate()
4274 .expect_err("upload without artifact path should fail");
4275 assert!(matches!(
4276 err,
4277 RestoreApplyJournalError::OperationMissingField {
4278 sequence: 0,
4279 operation: RestoreApplyOperationKind::UploadSnapshot,
4280 field: "operations[].artifact_path",
4281 }
4282 ));
4283
4284 let mut load = command_preview_journal(RestoreApplyOperationKind::LoadSnapshot, None, None);
4285 load.operations[0].snapshot_id = None;
4286 let err = load
4287 .validate()
4288 .expect_err("load without snapshot id should fail");
4289 assert!(matches!(
4290 err,
4291 RestoreApplyJournalError::OperationMissingField {
4292 sequence: 0,
4293 operation: RestoreApplyOperationKind::LoadSnapshot,
4294 field: "operations[].snapshot_id",
4295 }
4296 ));
4297
4298 let mut verify = command_preview_journal(
4299 RestoreApplyOperationKind::VerifyMember,
4300 Some("query"),
4301 Some("health"),
4302 );
4303 verify.operations[0].verification_method = None;
4304 let err = verify
4305 .validate()
4306 .expect_err("method verification without method should fail");
4307 assert!(matches!(
4308 err,
4309 RestoreApplyJournalError::OperationMissingField {
4310 sequence: 0,
4311 operation: RestoreApplyOperationKind::VerifyMember,
4312 field: "operations[].verification_method",
4313 }
4314 ));
4315 }
4316
4317 #[test]
4319 fn apply_journal_mark_next_operation_ready_rejects_without_pending_operation() {
4320 let mut journal =
4321 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4322
4323 let err = journal
4324 .mark_next_operation_ready()
4325 .expect_err("ready operation should not unclaim");
4326
4327 assert!(matches!(err, RestoreApplyJournalError::NoPendingOperation));
4328 assert_eq!(journal.ready_operations, 1);
4329 assert_eq!(journal.pending_operations, 0);
4330 }
4331
4332 #[test]
4334 fn apply_journal_mark_pending_rejects_out_of_order_operation() {
4335 let root = temp_dir("canic-restore-apply-journal-pending-out-of-order");
4336 fs::create_dir_all(&root).expect("create temp root");
4337 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4338 set_member_artifact(
4339 &mut manifest,
4340 CHILD,
4341 &root,
4342 "artifacts/child",
4343 b"child-snapshot",
4344 );
4345 set_member_artifact(
4346 &mut manifest,
4347 ROOT,
4348 &root,
4349 "artifacts/root",
4350 b"root-snapshot",
4351 );
4352
4353 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4354 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4355 .expect("dry-run should validate artifacts");
4356 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4357
4358 let err = journal
4359 .mark_operation_pending(1)
4360 .expect_err("out-of-order pending claim should fail");
4361
4362 fs::remove_dir_all(root).expect("remove temp root");
4363 assert!(matches!(
4364 err,
4365 RestoreApplyJournalError::OutOfOrderOperationTransition {
4366 requested: 1,
4367 next: 0
4368 }
4369 ));
4370 assert_eq!(journal.pending_operations, 0);
4371 assert_eq!(journal.ready_operations, 8);
4372 }
4373
4374 #[test]
4376 fn apply_journal_mark_completed_advances_next_ready_operation() {
4377 let root = temp_dir("canic-restore-apply-journal-completed");
4378 fs::create_dir_all(&root).expect("create temp root");
4379 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4380 set_member_artifact(
4381 &mut manifest,
4382 CHILD,
4383 &root,
4384 "artifacts/child",
4385 b"child-snapshot",
4386 );
4387 set_member_artifact(
4388 &mut manifest,
4389 ROOT,
4390 &root,
4391 "artifacts/root",
4392 b"root-snapshot",
4393 );
4394
4395 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4396 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4397 .expect("dry-run should validate artifacts");
4398 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4399
4400 journal
4401 .mark_operation_completed(0)
4402 .expect("mark operation completed");
4403 let status = journal.status();
4404
4405 fs::remove_dir_all(root).expect("remove temp root");
4406 assert_eq!(
4407 journal.operations[0].state,
4408 RestoreApplyOperationState::Completed
4409 );
4410 assert_eq!(journal.completed_operations, 1);
4411 assert_eq!(journal.ready_operations, 7);
4412 assert_eq!(status.next_ready_sequence, Some(1));
4413 assert_eq!(status.progress.completed_operations, 1);
4414 assert_eq!(status.progress.remaining_operations, 7);
4415 assert_eq!(status.progress.transitionable_operations, 7);
4416 assert_eq!(status.progress.attention_operations, 0);
4417 assert_eq!(status.progress.completion_basis_points, 1250);
4418 }
4419
4420 #[test]
4422 fn apply_journal_mark_completed_rejects_out_of_order_operation() {
4423 let root = temp_dir("canic-restore-apply-journal-out-of-order");
4424 fs::create_dir_all(&root).expect("create temp root");
4425 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4426 set_member_artifact(
4427 &mut manifest,
4428 CHILD,
4429 &root,
4430 "artifacts/child",
4431 b"child-snapshot",
4432 );
4433 set_member_artifact(
4434 &mut manifest,
4435 ROOT,
4436 &root,
4437 "artifacts/root",
4438 b"root-snapshot",
4439 );
4440
4441 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4442 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4443 .expect("dry-run should validate artifacts");
4444 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4445
4446 let err = journal
4447 .mark_operation_completed(1)
4448 .expect_err("out-of-order operation should fail");
4449
4450 fs::remove_dir_all(root).expect("remove temp root");
4451 assert!(matches!(
4452 err,
4453 RestoreApplyJournalError::OutOfOrderOperationTransition {
4454 requested: 1,
4455 next: 0
4456 }
4457 ));
4458 assert_eq!(journal.completed_operations, 0);
4459 assert_eq!(journal.ready_operations, 8);
4460 }
4461
4462 #[test]
4464 fn apply_journal_mark_failed_records_reason() {
4465 let root = temp_dir("canic-restore-apply-journal-failed");
4466 fs::create_dir_all(&root).expect("create temp root");
4467 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4468 set_member_artifact(
4469 &mut manifest,
4470 CHILD,
4471 &root,
4472 "artifacts/child",
4473 b"child-snapshot",
4474 );
4475 set_member_artifact(
4476 &mut manifest,
4477 ROOT,
4478 &root,
4479 "artifacts/root",
4480 b"root-snapshot",
4481 );
4482
4483 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4484 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4485 .expect("dry-run should validate artifacts");
4486 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4487
4488 journal
4489 .mark_operation_failed(0, "dfx-load-failed".to_string())
4490 .expect("mark operation failed");
4491
4492 fs::remove_dir_all(root).expect("remove temp root");
4493 assert_eq!(
4494 journal.operations[0].state,
4495 RestoreApplyOperationState::Failed
4496 );
4497 assert_eq!(
4498 journal.operations[0].blocking_reasons,
4499 vec!["dfx-load-failed".to_string()]
4500 );
4501 assert_eq!(journal.failed_operations, 1);
4502 assert_eq!(journal.ready_operations, 7);
4503 }
4504
4505 #[test]
4507 fn apply_journal_rejects_blocked_operation_completion() {
4508 let manifest = valid_manifest(IdentityMode::Relocatable);
4509
4510 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4511 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4512 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4513
4514 let err = journal
4515 .mark_operation_completed(0)
4516 .expect_err("blocked operation should not complete");
4517
4518 assert!(matches!(
4519 err,
4520 RestoreApplyJournalError::InvalidOperationTransition { sequence: 0, .. }
4521 ));
4522 }
4523
4524 #[test]
4526 fn apply_dry_run_rejects_missing_artifacts() {
4527 let root = temp_dir("canic-restore-apply-artifacts-missing");
4528 fs::create_dir_all(&root).expect("create temp root");
4529 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4530 manifest.fleet.members[0].source_snapshot.artifact_path = "missing-child".to_string();
4531
4532 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4533 let err = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4534 .expect_err("missing artifact should fail");
4535
4536 fs::remove_dir_all(root).expect("remove temp root");
4537 assert!(matches!(
4538 err,
4539 RestoreApplyDryRunError::ArtifactMissing { .. }
4540 ));
4541 }
4542
4543 #[test]
4545 fn apply_dry_run_rejects_artifact_path_traversal() {
4546 let root = temp_dir("canic-restore-apply-artifacts-traversal");
4547 fs::create_dir_all(&root).expect("create temp root");
4548 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4549 manifest.fleet.members[1].source_snapshot.artifact_path = "../outside".to_string();
4550
4551 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4552 let err = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4553 .expect_err("path traversal should fail");
4554
4555 fs::remove_dir_all(root).expect("remove temp root");
4556 assert!(matches!(
4557 err,
4558 RestoreApplyDryRunError::ArtifactPathEscapesBackup { .. }
4559 ));
4560 }
4561
4562 #[test]
4564 fn apply_dry_run_rejects_mismatched_status() {
4565 let manifest = valid_manifest(IdentityMode::Relocatable);
4566
4567 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4568 let mut status = RestoreStatus::from_plan(&plan);
4569 status.backup_id = "other-backup".to_string();
4570
4571 let err = RestoreApplyDryRun::try_from_plan(&plan, Some(&status))
4572 .expect_err("mismatched status should fail");
4573
4574 assert!(matches!(
4575 err,
4576 RestoreApplyDryRunError::StatusPlanMismatch {
4577 field: "backup_id",
4578 ..
4579 }
4580 ));
4581 }
4582
4583 #[test]
4585 fn plan_expands_role_verification_checks_per_matching_member() {
4586 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4587 manifest.fleet.members.push(fleet_member(
4588 "app",
4589 CHILD_TWO,
4590 Some(ROOT),
4591 IdentityMode::Relocatable,
4592 1,
4593 ));
4594 manifest
4595 .verification
4596 .member_checks
4597 .push(MemberVerificationChecks {
4598 role: "app".to_string(),
4599 checks: vec![VerificationCheck {
4600 kind: "app-ready".to_string(),
4601 method: Some("ready".to_string()),
4602 roles: Vec::new(),
4603 }],
4604 });
4605
4606 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4607
4608 assert_eq!(plan.verification_summary.fleet_checks, 0);
4609 assert_eq!(plan.verification_summary.member_check_groups, 1);
4610 assert_eq!(plan.verification_summary.member_checks, 5);
4611 assert_eq!(plan.verification_summary.members_with_checks, 3);
4612 assert_eq!(plan.verification_summary.total_checks, 5);
4613 }
4614
4615 #[test]
4617 fn plan_applies_member_verification_role_filters() {
4618 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4619 manifest.fleet.members[0]
4620 .verification_checks
4621 .push(VerificationCheck {
4622 kind: "root-only-inline".to_string(),
4623 method: Some("wrong_member".to_string()),
4624 roles: vec!["root".to_string()],
4625 });
4626 manifest
4627 .verification
4628 .member_checks
4629 .push(MemberVerificationChecks {
4630 role: "app".to_string(),
4631 checks: vec![
4632 VerificationCheck {
4633 kind: "app-role-check".to_string(),
4634 method: Some("app_ready".to_string()),
4635 roles: vec!["app".to_string()],
4636 },
4637 VerificationCheck {
4638 kind: "root-filtered-check".to_string(),
4639 method: Some("wrong_role".to_string()),
4640 roles: vec!["root".to_string()],
4641 },
4642 ],
4643 });
4644
4645 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4646 let app = plan
4647 .ordered_members()
4648 .into_iter()
4649 .find(|member| member.role == "app")
4650 .expect("app member should be planned");
4651 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4652 let app_verification_methods = dry_run.phases[0]
4653 .operations
4654 .iter()
4655 .filter(|operation| {
4656 operation.source_canister == CHILD
4657 && operation.operation == RestoreApplyOperationKind::VerifyMember
4658 })
4659 .filter_map(|operation| operation.verification_method.as_deref())
4660 .collect::<Vec<_>>();
4661
4662 assert_eq!(app.verification_checks.len(), 2);
4663 assert_eq!(
4664 app.verification_checks
4665 .iter()
4666 .map(|check| check.kind.as_str())
4667 .collect::<Vec<_>>(),
4668 ["call", "app-role-check"]
4669 );
4670 assert_eq!(plan.verification_summary.member_checks, 3);
4671 assert_eq!(plan.verification_summary.total_checks, 3);
4672 assert_eq!(dry_run.rendered_operations, 9);
4673 assert_eq!(app_verification_methods, ["canic_ready", "app_ready"]);
4674 }
4675
4676 #[test]
4678 fn mapped_restore_requires_complete_mapping() {
4679 let manifest = valid_manifest(IdentityMode::Relocatable);
4680 let mapping = RestoreMapping {
4681 members: vec![RestoreMappingEntry {
4682 source_canister: ROOT.to_string(),
4683 target_canister: ROOT.to_string(),
4684 }],
4685 };
4686
4687 let err = RestorePlanner::plan(&manifest, Some(&mapping))
4688 .expect_err("incomplete mapping should fail");
4689
4690 assert!(matches!(err, RestorePlanError::MissingMappingSource(_)));
4691 }
4692
4693 #[test]
4695 fn mapped_restore_rejects_unknown_mapping_sources() {
4696 let manifest = valid_manifest(IdentityMode::Relocatable);
4697 let unknown = "rdmx6-jaaaa-aaaaa-aaadq-cai";
4698 let mapping = RestoreMapping {
4699 members: vec![
4700 RestoreMappingEntry {
4701 source_canister: ROOT.to_string(),
4702 target_canister: ROOT.to_string(),
4703 },
4704 RestoreMappingEntry {
4705 source_canister: CHILD.to_string(),
4706 target_canister: TARGET.to_string(),
4707 },
4708 RestoreMappingEntry {
4709 source_canister: unknown.to_string(),
4710 target_canister: unknown.to_string(),
4711 },
4712 ],
4713 };
4714
4715 let err = RestorePlanner::plan(&manifest, Some(&mapping))
4716 .expect_err("unknown mapping source should fail");
4717
4718 assert!(matches!(err, RestorePlanError::UnknownMappingSource(_)));
4719 }
4720
4721 #[test]
4723 fn duplicate_mapping_targets_fail_validation() {
4724 let manifest = valid_manifest(IdentityMode::Relocatable);
4725 let mapping = RestoreMapping {
4726 members: vec![
4727 RestoreMappingEntry {
4728 source_canister: ROOT.to_string(),
4729 target_canister: ROOT.to_string(),
4730 },
4731 RestoreMappingEntry {
4732 source_canister: CHILD.to_string(),
4733 target_canister: ROOT.to_string(),
4734 },
4735 ],
4736 };
4737
4738 let err = RestorePlanner::plan(&manifest, Some(&mapping))
4739 .expect_err("duplicate targets should fail");
4740
4741 assert!(matches!(err, RestorePlanError::DuplicateMappingTarget(_)));
4742 }
4743
4744 fn set_member_artifact(
4746 manifest: &mut FleetBackupManifest,
4747 canister_id: &str,
4748 root: &Path,
4749 artifact_path: &str,
4750 bytes: &[u8],
4751 ) {
4752 let full_path = root.join(artifact_path);
4753 fs::create_dir_all(full_path.parent().expect("artifact parent")).expect("create parent");
4754 fs::write(&full_path, bytes).expect("write artifact");
4755 let checksum = ArtifactChecksum::from_bytes(bytes);
4756 let member = manifest
4757 .fleet
4758 .members
4759 .iter_mut()
4760 .find(|member| member.canister_id == canister_id)
4761 .expect("member should exist");
4762 member.source_snapshot.artifact_path = artifact_path.to_string();
4763 member.source_snapshot.checksum = Some(checksum.hash);
4764 }
4765
4766 fn temp_dir(name: &str) -> PathBuf {
4768 let nanos = SystemTime::now()
4769 .duration_since(UNIX_EPOCH)
4770 .expect("system time should be after epoch")
4771 .as_nanos();
4772 env::temp_dir().join(format!("{name}-{nanos}"))
4773 }
4774}