1use crate::{
2 artifacts::{ArtifactChecksum, ArtifactChecksumError},
3 manifest::{
4 FleetBackupManifest, FleetMember, IdentityMode, ManifestValidationError, SourceSnapshot,
5 VerificationCheck, VerificationPlan,
6 },
7};
8use candid::Principal;
9use serde::{Deserialize, Serialize};
10use std::{
11 collections::{BTreeMap, BTreeSet},
12 path::{Component, Path, PathBuf},
13 str::FromStr,
14};
15use thiserror::Error as ThisError;
16
17#[derive(Clone, Debug, Default, Deserialize, Serialize)]
22pub struct RestoreMapping {
23 pub members: Vec<RestoreMappingEntry>,
24}
25
26impl RestoreMapping {
27 fn target_for(&self, source_canister: &str) -> Option<&str> {
29 self.members
30 .iter()
31 .find(|entry| entry.source_canister == source_canister)
32 .map(|entry| entry.target_canister.as_str())
33 }
34}
35
36#[derive(Clone, Debug, Deserialize, Serialize)]
41pub struct RestoreMappingEntry {
42 pub source_canister: String,
43 pub target_canister: String,
44}
45
46#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
51pub struct RestorePlan {
52 pub backup_id: String,
53 pub source_environment: String,
54 pub source_root_canister: String,
55 pub topology_hash: String,
56 pub member_count: usize,
57 pub identity_summary: RestoreIdentitySummary,
58 pub snapshot_summary: RestoreSnapshotSummary,
59 pub verification_summary: RestoreVerificationSummary,
60 pub readiness_summary: RestoreReadinessSummary,
61 pub operation_summary: RestoreOperationSummary,
62 pub ordering_summary: RestoreOrderingSummary,
63 #[serde(default)]
64 pub fleet_verification_checks: Vec<VerificationCheck>,
65 pub phases: Vec<RestorePhase>,
66}
67
68impl RestorePlan {
69 #[must_use]
71 pub fn ordered_members(&self) -> Vec<&RestorePlanMember> {
72 self.phases
73 .iter()
74 .flat_map(|phase| phase.members.iter())
75 .collect()
76 }
77}
78
79#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
84pub struct RestoreStatus {
85 pub status_version: u16,
86 pub backup_id: String,
87 pub source_environment: String,
88 pub source_root_canister: String,
89 pub topology_hash: String,
90 pub ready: bool,
91 pub readiness_reasons: Vec<String>,
92 pub verification_required: bool,
93 pub member_count: usize,
94 pub phase_count: usize,
95 #[serde(default)]
96 pub planned_snapshot_uploads: usize,
97 pub planned_snapshot_loads: usize,
98 pub planned_code_reinstalls: usize,
99 pub planned_verification_checks: usize,
100 #[serde(default)]
101 pub planned_operations: usize,
102 pub phases: Vec<RestoreStatusPhase>,
103}
104
105impl RestoreStatus {
106 #[must_use]
108 pub fn from_plan(plan: &RestorePlan) -> Self {
109 Self {
110 status_version: 1,
111 backup_id: plan.backup_id.clone(),
112 source_environment: plan.source_environment.clone(),
113 source_root_canister: plan.source_root_canister.clone(),
114 topology_hash: plan.topology_hash.clone(),
115 ready: plan.readiness_summary.ready,
116 readiness_reasons: plan.readiness_summary.reasons.clone(),
117 verification_required: plan.verification_summary.verification_required,
118 member_count: plan.member_count,
119 phase_count: plan.ordering_summary.phase_count,
120 planned_snapshot_uploads: plan
121 .operation_summary
122 .effective_planned_snapshot_uploads(plan.member_count),
123 planned_snapshot_loads: plan.operation_summary.planned_snapshot_loads,
124 planned_code_reinstalls: plan.operation_summary.planned_code_reinstalls,
125 planned_verification_checks: plan.operation_summary.planned_verification_checks,
126 planned_operations: plan
127 .operation_summary
128 .effective_planned_operations(plan.member_count),
129 phases: plan
130 .phases
131 .iter()
132 .map(RestoreStatusPhase::from_plan_phase)
133 .collect(),
134 }
135 }
136}
137
138#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
143pub struct RestoreStatusPhase {
144 pub restore_group: u16,
145 pub members: Vec<RestoreStatusMember>,
146}
147
148impl RestoreStatusPhase {
149 fn from_plan_phase(phase: &RestorePhase) -> Self {
151 Self {
152 restore_group: phase.restore_group,
153 members: phase
154 .members
155 .iter()
156 .map(RestoreStatusMember::from_plan_member)
157 .collect(),
158 }
159 }
160}
161
162#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
167pub struct RestoreStatusMember {
168 pub source_canister: String,
169 pub target_canister: String,
170 pub role: String,
171 pub restore_group: u16,
172 pub phase_order: usize,
173 pub snapshot_id: String,
174 pub artifact_path: String,
175 pub state: RestoreMemberState,
176}
177
178impl RestoreStatusMember {
179 fn from_plan_member(member: &RestorePlanMember) -> Self {
181 Self {
182 source_canister: member.source_canister.clone(),
183 target_canister: member.target_canister.clone(),
184 role: member.role.clone(),
185 restore_group: member.restore_group,
186 phase_order: member.phase_order,
187 snapshot_id: member.source_snapshot.snapshot_id.clone(),
188 artifact_path: member.source_snapshot.artifact_path.clone(),
189 state: RestoreMemberState::Planned,
190 }
191 }
192}
193
194#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
199#[serde(rename_all = "kebab-case")]
200pub enum RestoreMemberState {
201 Planned,
202}
203
204#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
209pub struct RestoreApplyDryRun {
210 pub dry_run_version: u16,
211 pub backup_id: String,
212 pub ready: bool,
213 pub readiness_reasons: Vec<String>,
214 pub member_count: usize,
215 pub phase_count: usize,
216 pub status_supplied: bool,
217 #[serde(default)]
218 pub planned_snapshot_uploads: usize,
219 pub planned_snapshot_loads: usize,
220 pub planned_code_reinstalls: usize,
221 pub planned_verification_checks: usize,
222 #[serde(default)]
223 pub planned_operations: usize,
224 pub rendered_operations: usize,
225 #[serde(default)]
226 pub operation_counts: RestoreApplyOperationKindCounts,
227 pub artifact_validation: Option<RestoreApplyArtifactValidation>,
228 pub phases: Vec<RestoreApplyDryRunPhase>,
229}
230
231impl RestoreApplyDryRun {
232 pub fn try_from_plan(
234 plan: &RestorePlan,
235 status: Option<&RestoreStatus>,
236 ) -> Result<Self, RestoreApplyDryRunError> {
237 if let Some(status) = status {
238 validate_restore_status_matches_plan(plan, status)?;
239 }
240
241 Ok(Self::from_validated_plan(plan, status))
242 }
243
244 pub fn try_from_plan_with_artifacts(
246 plan: &RestorePlan,
247 status: Option<&RestoreStatus>,
248 backup_root: &Path,
249 ) -> Result<Self, RestoreApplyDryRunError> {
250 let mut dry_run = Self::try_from_plan(plan, status)?;
251 dry_run.artifact_validation = Some(validate_restore_apply_artifacts(plan, backup_root)?);
252 Ok(dry_run)
253 }
254
255 fn from_validated_plan(plan: &RestorePlan, status: Option<&RestoreStatus>) -> Self {
257 let mut next_sequence = 0;
258 let phases = plan
259 .phases
260 .iter()
261 .map(|phase| RestoreApplyDryRunPhase::from_plan_phase(phase, &mut next_sequence))
262 .collect::<Vec<_>>();
263 let mut phases = phases;
264 append_fleet_verification_operations(plan, &mut phases, &mut next_sequence);
265 let rendered_operations = phases
266 .iter()
267 .map(|phase| phase.operations.len())
268 .sum::<usize>();
269 let operation_counts = RestoreApplyOperationKindCounts::from_dry_run_phases(&phases);
270
271 Self {
272 dry_run_version: 1,
273 backup_id: plan.backup_id.clone(),
274 ready: status.map_or(plan.readiness_summary.ready, |status| status.ready),
275 readiness_reasons: status.map_or_else(
276 || plan.readiness_summary.reasons.clone(),
277 |status| status.readiness_reasons.clone(),
278 ),
279 member_count: plan.member_count,
280 phase_count: plan.ordering_summary.phase_count,
281 status_supplied: status.is_some(),
282 planned_snapshot_uploads: plan
283 .operation_summary
284 .effective_planned_snapshot_uploads(plan.member_count),
285 planned_snapshot_loads: plan.operation_summary.planned_snapshot_loads,
286 planned_code_reinstalls: plan.operation_summary.planned_code_reinstalls,
287 planned_verification_checks: plan.operation_summary.planned_verification_checks,
288 planned_operations: plan
289 .operation_summary
290 .effective_planned_operations(plan.member_count),
291 rendered_operations,
292 operation_counts,
293 artifact_validation: None,
294 phases,
295 }
296 }
297}
298
299#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
304pub struct RestoreApplyJournal {
305 pub journal_version: u16,
306 pub backup_id: String,
307 pub ready: bool,
308 pub blocked_reasons: Vec<String>,
309 pub operation_count: usize,
310 #[serde(default)]
311 pub operation_counts: RestoreApplyOperationKindCounts,
312 pub pending_operations: usize,
313 pub ready_operations: usize,
314 pub blocked_operations: usize,
315 pub completed_operations: usize,
316 pub failed_operations: usize,
317 pub operations: Vec<RestoreApplyJournalOperation>,
318}
319
320impl RestoreApplyJournal {
321 #[must_use]
323 pub fn from_dry_run(dry_run: &RestoreApplyDryRun) -> Self {
324 let blocked_reasons = restore_apply_blocked_reasons(dry_run);
325 let initial_state = if blocked_reasons.is_empty() {
326 RestoreApplyOperationState::Ready
327 } else {
328 RestoreApplyOperationState::Blocked
329 };
330 let operations = dry_run
331 .phases
332 .iter()
333 .flat_map(|phase| phase.operations.iter())
334 .map(|operation| {
335 RestoreApplyJournalOperation::from_dry_run_operation(
336 operation,
337 initial_state.clone(),
338 &blocked_reasons,
339 )
340 })
341 .collect::<Vec<_>>();
342 let ready_operations = operations
343 .iter()
344 .filter(|operation| operation.state == RestoreApplyOperationState::Ready)
345 .count();
346 let blocked_operations = operations
347 .iter()
348 .filter(|operation| operation.state == RestoreApplyOperationState::Blocked)
349 .count();
350 let operation_counts = RestoreApplyOperationKindCounts::from_operations(&operations);
351
352 Self {
353 journal_version: 1,
354 backup_id: dry_run.backup_id.clone(),
355 ready: blocked_reasons.is_empty(),
356 blocked_reasons,
357 operation_count: operations.len(),
358 operation_counts,
359 pending_operations: 0,
360 ready_operations,
361 blocked_operations,
362 completed_operations: 0,
363 failed_operations: 0,
364 operations,
365 }
366 }
367
368 pub fn validate(&self) -> Result<(), RestoreApplyJournalError> {
370 validate_apply_journal_version(self.journal_version)?;
371 validate_apply_journal_nonempty("backup_id", &self.backup_id)?;
372 validate_apply_journal_count(
373 "operation_count",
374 self.operation_count,
375 self.operations.len(),
376 )?;
377
378 let state_counts = RestoreApplyJournalStateCounts::from_operations(&self.operations);
379 let operation_counts = RestoreApplyOperationKindCounts::from_operations(&self.operations);
380 self.operation_counts
381 .validate_matches_if_supplied(&operation_counts)?;
382 validate_apply_journal_count(
383 "pending_operations",
384 self.pending_operations,
385 state_counts.pending,
386 )?;
387 validate_apply_journal_count(
388 "ready_operations",
389 self.ready_operations,
390 state_counts.ready,
391 )?;
392 validate_apply_journal_count(
393 "blocked_operations",
394 self.blocked_operations,
395 state_counts.blocked,
396 )?;
397 validate_apply_journal_count(
398 "completed_operations",
399 self.completed_operations,
400 state_counts.completed,
401 )?;
402 validate_apply_journal_count(
403 "failed_operations",
404 self.failed_operations,
405 state_counts.failed,
406 )?;
407
408 if self.ready && (!self.blocked_reasons.is_empty() || self.blocked_operations > 0) {
409 return Err(RestoreApplyJournalError::ReadyJournalHasBlockingState);
410 }
411
412 validate_apply_journal_sequences(&self.operations)?;
413 for operation in &self.operations {
414 operation.validate()?;
415 }
416
417 Ok(())
418 }
419
420 #[must_use]
422 pub fn status(&self) -> RestoreApplyJournalStatus {
423 RestoreApplyJournalStatus::from_journal(self)
424 }
425
426 #[must_use]
428 pub fn report(&self) -> RestoreApplyJournalReport {
429 RestoreApplyJournalReport::from_journal(self)
430 }
431
432 #[must_use]
434 pub fn next_ready_operation(&self) -> Option<&RestoreApplyJournalOperation> {
435 self.operations
436 .iter()
437 .filter(|operation| operation.state == RestoreApplyOperationState::Ready)
438 .min_by_key(|operation| operation.sequence)
439 }
440
441 #[must_use]
443 pub fn next_transition_operation(&self) -> Option<&RestoreApplyJournalOperation> {
444 self.operations
445 .iter()
446 .filter(|operation| {
447 matches!(
448 operation.state,
449 RestoreApplyOperationState::Ready | RestoreApplyOperationState::Pending
450 )
451 })
452 .min_by_key(|operation| operation.sequence)
453 }
454
455 #[must_use]
457 pub fn next_operation(&self) -> RestoreApplyNextOperation {
458 RestoreApplyNextOperation::from_journal(self)
459 }
460
461 #[must_use]
463 pub fn next_command_preview(&self) -> RestoreApplyCommandPreview {
464 RestoreApplyCommandPreview::from_journal(self)
465 }
466
467 #[must_use]
469 pub fn next_command_preview_with_config(
470 &self,
471 config: &RestoreApplyCommandConfig,
472 ) -> RestoreApplyCommandPreview {
473 RestoreApplyCommandPreview::from_journal_with_config(self, config)
474 }
475
476 pub fn mark_next_operation_pending(&mut self) -> Result<(), RestoreApplyJournalError> {
478 self.mark_next_operation_pending_at(None)
479 }
480
481 pub fn mark_next_operation_pending_at(
483 &mut self,
484 updated_at: Option<String>,
485 ) -> Result<(), RestoreApplyJournalError> {
486 let sequence = self
487 .next_transition_sequence()
488 .ok_or(RestoreApplyJournalError::NoTransitionableOperation)?;
489 self.mark_operation_pending_at(sequence, updated_at)
490 }
491
492 pub fn mark_operation_pending(
494 &mut self,
495 sequence: usize,
496 ) -> Result<(), RestoreApplyJournalError> {
497 self.mark_operation_pending_at(sequence, None)
498 }
499
500 pub fn mark_operation_pending_at(
502 &mut self,
503 sequence: usize,
504 updated_at: Option<String>,
505 ) -> Result<(), RestoreApplyJournalError> {
506 self.transition_operation(
507 sequence,
508 RestoreApplyOperationState::Pending,
509 Vec::new(),
510 updated_at,
511 )
512 }
513
514 pub fn mark_next_operation_ready(&mut self) -> Result<(), RestoreApplyJournalError> {
516 self.mark_next_operation_ready_at(None)
517 }
518
519 pub fn mark_next_operation_ready_at(
521 &mut self,
522 updated_at: Option<String>,
523 ) -> Result<(), RestoreApplyJournalError> {
524 let operation = self
525 .next_transition_operation()
526 .ok_or(RestoreApplyJournalError::NoTransitionableOperation)?;
527 if operation.state != RestoreApplyOperationState::Pending {
528 return Err(RestoreApplyJournalError::NoPendingOperation);
529 }
530
531 self.mark_operation_ready_at(operation.sequence, updated_at)
532 }
533
534 pub fn mark_operation_ready(
536 &mut self,
537 sequence: usize,
538 ) -> Result<(), RestoreApplyJournalError> {
539 self.mark_operation_ready_at(sequence, None)
540 }
541
542 pub fn mark_operation_ready_at(
544 &mut self,
545 sequence: usize,
546 updated_at: Option<String>,
547 ) -> Result<(), RestoreApplyJournalError> {
548 self.transition_operation(
549 sequence,
550 RestoreApplyOperationState::Ready,
551 Vec::new(),
552 updated_at,
553 )
554 }
555
556 pub fn mark_operation_completed(
558 &mut self,
559 sequence: usize,
560 ) -> Result<(), RestoreApplyJournalError> {
561 self.mark_operation_completed_at(sequence, None)
562 }
563
564 pub fn mark_operation_completed_at(
566 &mut self,
567 sequence: usize,
568 updated_at: Option<String>,
569 ) -> Result<(), RestoreApplyJournalError> {
570 self.transition_operation(
571 sequence,
572 RestoreApplyOperationState::Completed,
573 Vec::new(),
574 updated_at,
575 )
576 }
577
578 pub fn mark_operation_failed(
580 &mut self,
581 sequence: usize,
582 reason: String,
583 ) -> Result<(), RestoreApplyJournalError> {
584 self.mark_operation_failed_at(sequence, reason, None)
585 }
586
587 pub fn mark_operation_failed_at(
589 &mut self,
590 sequence: usize,
591 reason: String,
592 updated_at: Option<String>,
593 ) -> Result<(), RestoreApplyJournalError> {
594 if reason.trim().is_empty() {
595 return Err(RestoreApplyJournalError::FailureReasonRequired(sequence));
596 }
597
598 self.transition_operation(
599 sequence,
600 RestoreApplyOperationState::Failed,
601 vec![reason],
602 updated_at,
603 )
604 }
605
606 fn transition_operation(
608 &mut self,
609 sequence: usize,
610 next_state: RestoreApplyOperationState,
611 blocking_reasons: Vec<String>,
612 updated_at: Option<String>,
613 ) -> Result<(), RestoreApplyJournalError> {
614 let index = self
615 .operations
616 .iter()
617 .position(|operation| operation.sequence == sequence)
618 .ok_or(RestoreApplyJournalError::OperationNotFound(sequence))?;
619 let operation = &self.operations[index];
620
621 if !operation.can_transition_to(&next_state) {
622 return Err(RestoreApplyJournalError::InvalidOperationTransition {
623 sequence,
624 from: operation.state.clone(),
625 to: next_state,
626 });
627 }
628
629 self.validate_operation_transition_order(operation, &next_state)?;
630
631 let operation = &mut self.operations[index];
632 operation.state = next_state;
633 operation.blocking_reasons = blocking_reasons;
634 operation.state_updated_at = updated_at;
635 self.refresh_operation_counts();
636 self.validate()
637 }
638
639 fn validate_operation_transition_order(
641 &self,
642 operation: &RestoreApplyJournalOperation,
643 next_state: &RestoreApplyOperationState,
644 ) -> Result<(), RestoreApplyJournalError> {
645 if operation.state == *next_state {
646 return Ok(());
647 }
648
649 let next_sequence = self
650 .next_transition_sequence()
651 .ok_or(RestoreApplyJournalError::NoTransitionableOperation)?;
652
653 if operation.sequence == next_sequence {
654 return Ok(());
655 }
656
657 Err(RestoreApplyJournalError::OutOfOrderOperationTransition {
658 requested: operation.sequence,
659 next: next_sequence,
660 })
661 }
662
663 fn next_transition_sequence(&self) -> Option<usize> {
665 self.next_transition_operation()
666 .map(|operation| operation.sequence)
667 }
668
669 fn refresh_operation_counts(&mut self) {
671 let state_counts = RestoreApplyJournalStateCounts::from_operations(&self.operations);
672 self.operation_count = self.operations.len();
673 self.operation_counts = RestoreApplyOperationKindCounts::from_operations(&self.operations);
674 self.pending_operations = state_counts.pending;
675 self.ready_operations = state_counts.ready;
676 self.blocked_operations = state_counts.blocked;
677 self.completed_operations = state_counts.completed;
678 self.failed_operations = state_counts.failed;
679 }
680
681 const fn operation_counts_supplied(&self) -> bool {
683 !self.operation_counts.is_empty() || self.operations.is_empty()
684 }
685}
686
687const fn validate_apply_journal_version(version: u16) -> Result<(), RestoreApplyJournalError> {
689 if version == 1 {
690 return Ok(());
691 }
692
693 Err(RestoreApplyJournalError::UnsupportedVersion(version))
694}
695
696fn validate_apply_journal_nonempty(
698 field: &'static str,
699 value: &str,
700) -> Result<(), RestoreApplyJournalError> {
701 if !value.trim().is_empty() {
702 return Ok(());
703 }
704
705 Err(RestoreApplyJournalError::MissingField(field))
706}
707
708const fn validate_apply_journal_count(
710 field: &'static str,
711 reported: usize,
712 actual: usize,
713) -> Result<(), RestoreApplyJournalError> {
714 if reported == actual {
715 return Ok(());
716 }
717
718 Err(RestoreApplyJournalError::CountMismatch {
719 field,
720 reported,
721 actual,
722 })
723}
724
725fn validate_apply_journal_sequences(
727 operations: &[RestoreApplyJournalOperation],
728) -> Result<(), RestoreApplyJournalError> {
729 let mut sequences = BTreeSet::new();
730 for operation in operations {
731 if !sequences.insert(operation.sequence) {
732 return Err(RestoreApplyJournalError::DuplicateSequence(
733 operation.sequence,
734 ));
735 }
736 }
737
738 for expected in 0..operations.len() {
739 if !sequences.contains(&expected) {
740 return Err(RestoreApplyJournalError::MissingSequence(expected));
741 }
742 }
743
744 Ok(())
745}
746
747#[derive(Clone, Debug, Default, Eq, PartialEq)]
752struct RestoreApplyJournalStateCounts {
753 pending: usize,
754 ready: usize,
755 blocked: usize,
756 completed: usize,
757 failed: usize,
758}
759
760impl RestoreApplyJournalStateCounts {
761 fn from_operations(operations: &[RestoreApplyJournalOperation]) -> Self {
763 let mut counts = Self::default();
764 for operation in operations {
765 match operation.state {
766 RestoreApplyOperationState::Pending => counts.pending += 1,
767 RestoreApplyOperationState::Ready => counts.ready += 1,
768 RestoreApplyOperationState::Blocked => counts.blocked += 1,
769 RestoreApplyOperationState::Completed => counts.completed += 1,
770 RestoreApplyOperationState::Failed => counts.failed += 1,
771 }
772 }
773 counts
774 }
775}
776
777#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
782pub struct RestoreApplyOperationKindCounts {
783 pub snapshot_uploads: usize,
784 pub snapshot_loads: usize,
785 pub code_reinstalls: usize,
786 pub member_verifications: usize,
787 pub fleet_verifications: usize,
788 pub verification_operations: usize,
789}
790
791impl RestoreApplyOperationKindCounts {
792 #[must_use]
794 pub fn from_operations(operations: &[RestoreApplyJournalOperation]) -> Self {
795 let mut counts = Self::default();
796 for operation in operations {
797 counts.record(&operation.operation);
798 }
799 counts
800 }
801
802 pub fn validate_matches_if_supplied(
804 &self,
805 expected: &Self,
806 ) -> Result<(), RestoreApplyJournalError> {
807 if self.is_empty() && !expected.is_empty() {
808 return Ok(());
809 }
810
811 validate_apply_journal_count(
812 "operation_counts.snapshot_uploads",
813 self.snapshot_uploads,
814 expected.snapshot_uploads,
815 )?;
816 validate_apply_journal_count(
817 "operation_counts.snapshot_loads",
818 self.snapshot_loads,
819 expected.snapshot_loads,
820 )?;
821 validate_apply_journal_count(
822 "operation_counts.code_reinstalls",
823 self.code_reinstalls,
824 expected.code_reinstalls,
825 )?;
826 validate_apply_journal_count(
827 "operation_counts.member_verifications",
828 self.member_verifications,
829 expected.member_verifications,
830 )?;
831 validate_apply_journal_count(
832 "operation_counts.fleet_verifications",
833 self.fleet_verifications,
834 expected.fleet_verifications,
835 )?;
836 validate_apply_journal_count(
837 "operation_counts.verification_operations",
838 self.verification_operations,
839 expected.verification_operations,
840 )
841 }
842
843 const fn is_empty(&self) -> bool {
845 self.snapshot_uploads == 0
846 && self.snapshot_loads == 0
847 && self.code_reinstalls == 0
848 && self.member_verifications == 0
849 && self.fleet_verifications == 0
850 && self.verification_operations == 0
851 }
852
853 #[must_use]
855 pub fn from_dry_run_phases(phases: &[RestoreApplyDryRunPhase]) -> Self {
856 let mut counts = Self::default();
857 for operation in phases.iter().flat_map(|phase| {
858 phase
859 .operations
860 .iter()
861 .map(|operation| &operation.operation)
862 }) {
863 counts.record(operation);
864 }
865 counts
866 }
867
868 const fn record(&mut self, operation: &RestoreApplyOperationKind) {
870 match operation {
871 RestoreApplyOperationKind::UploadSnapshot => self.snapshot_uploads += 1,
872 RestoreApplyOperationKind::LoadSnapshot => self.snapshot_loads += 1,
873 RestoreApplyOperationKind::ReinstallCode => self.code_reinstalls += 1,
874 RestoreApplyOperationKind::VerifyMember => {
875 self.member_verifications += 1;
876 self.verification_operations += 1;
877 }
878 RestoreApplyOperationKind::VerifyFleet => {
879 self.fleet_verifications += 1;
880 self.verification_operations += 1;
881 }
882 }
883 }
884}
885
886fn restore_apply_blocked_reasons(dry_run: &RestoreApplyDryRun) -> Vec<String> {
888 let mut reasons = dry_run.readiness_reasons.clone();
889
890 match &dry_run.artifact_validation {
891 Some(validation) => {
892 if !validation.artifacts_present {
893 reasons.push("missing-artifacts".to_string());
894 }
895 if !validation.checksums_verified {
896 reasons.push("artifact-checksum-validation-incomplete".to_string());
897 }
898 }
899 None => reasons.push("missing-artifact-validation".to_string()),
900 }
901
902 reasons.sort();
903 reasons.dedup();
904 reasons
905}
906
907#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
912pub struct RestoreApplyJournalStatus {
913 pub status_version: u16,
914 pub backup_id: String,
915 pub ready: bool,
916 pub complete: bool,
917 pub blocked_reasons: Vec<String>,
918 pub operation_count: usize,
919 #[serde(default)]
920 pub operation_counts: RestoreApplyOperationKindCounts,
921 pub operation_counts_supplied: bool,
922 pub progress: RestoreApplyProgressSummary,
923 pub pending_summary: RestoreApplyPendingSummary,
924 pub pending_operations: usize,
925 pub ready_operations: usize,
926 pub blocked_operations: usize,
927 pub completed_operations: usize,
928 pub failed_operations: usize,
929 pub next_ready_sequence: Option<usize>,
930 pub next_ready_operation: Option<RestoreApplyOperationKind>,
931 pub next_transition_sequence: Option<usize>,
932 pub next_transition_state: Option<RestoreApplyOperationState>,
933 pub next_transition_operation: Option<RestoreApplyOperationKind>,
934 pub next_transition_updated_at: Option<String>,
935}
936
937impl RestoreApplyJournalStatus {
938 #[must_use]
940 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
941 let next_ready = journal.next_ready_operation();
942 let next_transition = journal.next_transition_operation();
943
944 Self {
945 status_version: 1,
946 backup_id: journal.backup_id.clone(),
947 ready: journal.ready,
948 complete: journal.operation_count > 0
949 && journal.completed_operations == journal.operation_count,
950 blocked_reasons: journal.blocked_reasons.clone(),
951 operation_count: journal.operation_count,
952 operation_counts: RestoreApplyOperationKindCounts::from_operations(&journal.operations),
953 operation_counts_supplied: journal.operation_counts_supplied(),
954 progress: RestoreApplyProgressSummary::from_journal(journal),
955 pending_summary: RestoreApplyPendingSummary::from_journal(journal),
956 pending_operations: journal.pending_operations,
957 ready_operations: journal.ready_operations,
958 blocked_operations: journal.blocked_operations,
959 completed_operations: journal.completed_operations,
960 failed_operations: journal.failed_operations,
961 next_ready_sequence: next_ready.map(|operation| operation.sequence),
962 next_ready_operation: next_ready.map(|operation| operation.operation.clone()),
963 next_transition_sequence: next_transition.map(|operation| operation.sequence),
964 next_transition_state: next_transition.map(|operation| operation.state.clone()),
965 next_transition_operation: next_transition.map(|operation| operation.operation.clone()),
966 next_transition_updated_at: next_transition
967 .and_then(|operation| operation.state_updated_at.clone()),
968 }
969 }
970}
971
972#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
977#[expect(
978 clippy::struct_excessive_bools,
979 reason = "apply reports intentionally expose stable JSON flags for operators and CI"
980)]
981pub struct RestoreApplyJournalReport {
982 pub report_version: u16,
983 pub backup_id: String,
984 pub outcome: RestoreApplyReportOutcome,
985 pub attention_required: bool,
986 pub ready: bool,
987 pub complete: bool,
988 pub blocked_reasons: Vec<String>,
989 pub operation_count: usize,
990 #[serde(default)]
991 pub operation_counts: RestoreApplyOperationKindCounts,
992 pub operation_counts_supplied: bool,
993 pub progress: RestoreApplyProgressSummary,
994 pub pending_summary: RestoreApplyPendingSummary,
995 pub pending_operations: usize,
996 pub ready_operations: usize,
997 pub blocked_operations: usize,
998 pub completed_operations: usize,
999 pub failed_operations: usize,
1000 pub next_transition: Option<RestoreApplyReportOperation>,
1001 pub pending: Vec<RestoreApplyReportOperation>,
1002 pub failed: Vec<RestoreApplyReportOperation>,
1003 pub blocked: Vec<RestoreApplyReportOperation>,
1004}
1005
1006impl RestoreApplyJournalReport {
1007 #[must_use]
1009 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1010 let complete =
1011 journal.operation_count > 0 && journal.completed_operations == journal.operation_count;
1012 let outcome = RestoreApplyReportOutcome::from_journal(journal, complete);
1013 let pending = report_operations_with_state(journal, RestoreApplyOperationState::Pending);
1014 let failed = report_operations_with_state(journal, RestoreApplyOperationState::Failed);
1015 let blocked = report_operations_with_state(journal, RestoreApplyOperationState::Blocked);
1016
1017 Self {
1018 report_version: 1,
1019 backup_id: journal.backup_id.clone(),
1020 outcome: outcome.clone(),
1021 attention_required: outcome.attention_required(),
1022 ready: journal.ready,
1023 complete,
1024 blocked_reasons: journal.blocked_reasons.clone(),
1025 operation_count: journal.operation_count,
1026 operation_counts: RestoreApplyOperationKindCounts::from_operations(&journal.operations),
1027 operation_counts_supplied: journal.operation_counts_supplied(),
1028 progress: RestoreApplyProgressSummary::from_journal(journal),
1029 pending_summary: RestoreApplyPendingSummary::from_journal(journal),
1030 pending_operations: journal.pending_operations,
1031 ready_operations: journal.ready_operations,
1032 blocked_operations: journal.blocked_operations,
1033 completed_operations: journal.completed_operations,
1034 failed_operations: journal.failed_operations,
1035 next_transition: journal
1036 .next_transition_operation()
1037 .map(RestoreApplyReportOperation::from_journal_operation),
1038 pending,
1039 failed,
1040 blocked,
1041 }
1042 }
1043}
1044
1045#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1050pub struct RestoreApplyPendingSummary {
1051 pub pending_operations: usize,
1052 pub pending_operation_available: bool,
1053 pub pending_sequence: Option<usize>,
1054 pub pending_operation: Option<RestoreApplyOperationKind>,
1055 pub pending_updated_at: Option<String>,
1056 pub pending_updated_at_known: bool,
1057}
1058
1059impl RestoreApplyPendingSummary {
1060 #[must_use]
1062 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1063 let pending = journal
1064 .operations
1065 .iter()
1066 .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1067 .min_by_key(|operation| operation.sequence);
1068 let pending_updated_at = pending.and_then(|operation| operation.state_updated_at.clone());
1069 let pending_updated_at_known = pending_updated_at
1070 .as_deref()
1071 .is_some_and(known_state_update_marker);
1072
1073 Self {
1074 pending_operations: journal.pending_operations,
1075 pending_operation_available: pending.is_some(),
1076 pending_sequence: pending.map(|operation| operation.sequence),
1077 pending_operation: pending.map(|operation| operation.operation.clone()),
1078 pending_updated_at,
1079 pending_updated_at_known,
1080 }
1081 }
1082}
1083
1084fn known_state_update_marker(value: &str) -> bool {
1086 !value.trim().is_empty() && value != "unknown"
1087}
1088
1089#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1094pub struct RestoreApplyProgressSummary {
1095 pub operation_count: usize,
1096 pub completed_operations: usize,
1097 pub remaining_operations: usize,
1098 pub transitionable_operations: usize,
1099 pub attention_operations: usize,
1100 pub completion_basis_points: usize,
1101}
1102
1103impl RestoreApplyProgressSummary {
1104 #[must_use]
1106 pub const fn from_journal(journal: &RestoreApplyJournal) -> Self {
1107 let remaining_operations = journal
1108 .operation_count
1109 .saturating_sub(journal.completed_operations);
1110 let transitionable_operations = journal.ready_operations + journal.pending_operations;
1111 let attention_operations =
1112 journal.pending_operations + journal.blocked_operations + journal.failed_operations;
1113 let completion_basis_points =
1114 completion_basis_points(journal.completed_operations, journal.operation_count);
1115
1116 Self {
1117 operation_count: journal.operation_count,
1118 completed_operations: journal.completed_operations,
1119 remaining_operations,
1120 transitionable_operations,
1121 attention_operations,
1122 completion_basis_points,
1123 }
1124 }
1125}
1126
1127const fn completion_basis_points(completed_operations: usize, operation_count: usize) -> usize {
1129 if operation_count == 0 {
1130 return 0;
1131 }
1132
1133 completed_operations.saturating_mul(10_000) / operation_count
1134}
1135
1136#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1141#[serde(rename_all = "kebab-case")]
1142pub enum RestoreApplyReportOutcome {
1143 Empty,
1144 Complete,
1145 Failed,
1146 Blocked,
1147 Pending,
1148 InProgress,
1149}
1150
1151impl RestoreApplyReportOutcome {
1152 const fn from_journal(journal: &RestoreApplyJournal, complete: bool) -> Self {
1154 if journal.operation_count == 0 {
1155 return Self::Empty;
1156 }
1157 if complete {
1158 return Self::Complete;
1159 }
1160 if journal.failed_operations > 0 {
1161 return Self::Failed;
1162 }
1163 if !journal.ready || journal.blocked_operations > 0 {
1164 return Self::Blocked;
1165 }
1166 if journal.pending_operations > 0 {
1167 return Self::Pending;
1168 }
1169 Self::InProgress
1170 }
1171
1172 const fn attention_required(&self) -> bool {
1174 matches!(self, Self::Failed | Self::Blocked | Self::Pending)
1175 }
1176}
1177
1178#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1183pub struct RestoreApplyReportOperation {
1184 pub sequence: usize,
1185 pub operation: RestoreApplyOperationKind,
1186 pub state: RestoreApplyOperationState,
1187 pub restore_group: u16,
1188 pub phase_order: usize,
1189 pub role: String,
1190 pub source_canister: String,
1191 pub target_canister: String,
1192 pub state_updated_at: Option<String>,
1193 pub reasons: Vec<String>,
1194}
1195
1196impl RestoreApplyReportOperation {
1197 fn from_journal_operation(operation: &RestoreApplyJournalOperation) -> Self {
1199 Self {
1200 sequence: operation.sequence,
1201 operation: operation.operation.clone(),
1202 state: operation.state.clone(),
1203 restore_group: operation.restore_group,
1204 phase_order: operation.phase_order,
1205 role: operation.role.clone(),
1206 source_canister: operation.source_canister.clone(),
1207 target_canister: operation.target_canister.clone(),
1208 state_updated_at: operation.state_updated_at.clone(),
1209 reasons: operation.blocking_reasons.clone(),
1210 }
1211 }
1212}
1213
1214fn report_operations_with_state(
1216 journal: &RestoreApplyJournal,
1217 state: RestoreApplyOperationState,
1218) -> Vec<RestoreApplyReportOperation> {
1219 journal
1220 .operations
1221 .iter()
1222 .filter(|operation| operation.state == state)
1223 .map(RestoreApplyReportOperation::from_journal_operation)
1224 .collect()
1225}
1226
1227#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1232pub struct RestoreApplyNextOperation {
1233 pub response_version: u16,
1234 pub backup_id: String,
1235 pub ready: bool,
1236 pub complete: bool,
1237 pub operation_available: bool,
1238 pub blocked_reasons: Vec<String>,
1239 pub operation: Option<RestoreApplyJournalOperation>,
1240}
1241
1242impl RestoreApplyNextOperation {
1243 #[must_use]
1245 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1246 let complete =
1247 journal.operation_count > 0 && journal.completed_operations == journal.operation_count;
1248 let operation = journal.next_transition_operation().cloned();
1249
1250 Self {
1251 response_version: 1,
1252 backup_id: journal.backup_id.clone(),
1253 ready: journal.ready,
1254 complete,
1255 operation_available: operation.is_some(),
1256 blocked_reasons: journal.blocked_reasons.clone(),
1257 operation,
1258 }
1259 }
1260}
1261
1262#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1267#[expect(
1268 clippy::struct_excessive_bools,
1269 reason = "runner preview exposes machine-readable availability and safety flags"
1270)]
1271pub struct RestoreApplyCommandPreview {
1272 pub response_version: u16,
1273 pub backup_id: String,
1274 pub ready: bool,
1275 pub complete: bool,
1276 pub operation_available: bool,
1277 pub command_available: bool,
1278 pub blocked_reasons: Vec<String>,
1279 pub operation: Option<RestoreApplyJournalOperation>,
1280 pub command: Option<RestoreApplyRunnerCommand>,
1281}
1282
1283impl RestoreApplyCommandPreview {
1284 #[must_use]
1286 pub fn from_journal(journal: &RestoreApplyJournal) -> Self {
1287 Self::from_journal_with_config(journal, &RestoreApplyCommandConfig::default())
1288 }
1289
1290 #[must_use]
1292 pub fn from_journal_with_config(
1293 journal: &RestoreApplyJournal,
1294 config: &RestoreApplyCommandConfig,
1295 ) -> Self {
1296 let complete =
1297 journal.operation_count > 0 && journal.completed_operations == journal.operation_count;
1298 let operation = journal.next_transition_operation().cloned();
1299 let command = operation
1300 .as_ref()
1301 .and_then(|operation| RestoreApplyRunnerCommand::from_operation(operation, config));
1302
1303 Self {
1304 response_version: 1,
1305 backup_id: journal.backup_id.clone(),
1306 ready: journal.ready,
1307 complete,
1308 operation_available: operation.is_some(),
1309 command_available: command.is_some(),
1310 blocked_reasons: journal.blocked_reasons.clone(),
1311 operation,
1312 command,
1313 }
1314 }
1315}
1316
1317#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1322pub struct RestoreApplyCommandConfig {
1323 pub program: String,
1324 pub network: Option<String>,
1325}
1326
1327impl Default for RestoreApplyCommandConfig {
1328 fn default() -> Self {
1330 Self {
1331 program: "dfx".to_string(),
1332 network: None,
1333 }
1334 }
1335}
1336
1337#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1342pub struct RestoreApplyRunnerCommand {
1343 pub program: String,
1344 pub args: Vec<String>,
1345 pub mutates: bool,
1346 pub requires_stopped_canister: bool,
1347 pub note: String,
1348}
1349
1350impl RestoreApplyRunnerCommand {
1351 fn from_operation(
1353 operation: &RestoreApplyJournalOperation,
1354 config: &RestoreApplyCommandConfig,
1355 ) -> Option<Self> {
1356 match operation.operation {
1357 RestoreApplyOperationKind::UploadSnapshot => {
1358 let artifact_path = operation.artifact_path.as_ref()?;
1359 Some(Self {
1360 program: config.program.clone(),
1361 args: dfx_canister_args(
1362 config,
1363 vec![
1364 "snapshot".to_string(),
1365 "upload".to_string(),
1366 "--dir".to_string(),
1367 artifact_path.clone(),
1368 operation.target_canister.clone(),
1369 ],
1370 ),
1371 mutates: true,
1372 requires_stopped_canister: false,
1373 note: "uploads the downloaded snapshot artifact to the target canister"
1374 .to_string(),
1375 })
1376 }
1377 RestoreApplyOperationKind::LoadSnapshot => {
1378 let snapshot_id = operation.snapshot_id.as_ref()?;
1379 Some(Self {
1380 program: config.program.clone(),
1381 args: dfx_canister_args(
1382 config,
1383 vec![
1384 "snapshot".to_string(),
1385 "load".to_string(),
1386 operation.target_canister.clone(),
1387 snapshot_id.clone(),
1388 ],
1389 ),
1390 mutates: true,
1391 requires_stopped_canister: true,
1392 note: "loads the uploaded snapshot into the target canister".to_string(),
1393 })
1394 }
1395 RestoreApplyOperationKind::ReinstallCode => Some(Self {
1396 program: config.program.clone(),
1397 args: dfx_canister_args(
1398 config,
1399 vec![
1400 "install".to_string(),
1401 "--mode".to_string(),
1402 "reinstall".to_string(),
1403 "--yes".to_string(),
1404 operation.target_canister.clone(),
1405 ],
1406 ),
1407 mutates: true,
1408 requires_stopped_canister: false,
1409 note: "reinstalls target canister code using the local dfx project configuration"
1410 .to_string(),
1411 }),
1412 RestoreApplyOperationKind::VerifyMember | RestoreApplyOperationKind::VerifyFleet => {
1413 match operation.verification_kind.as_deref() {
1414 Some("status") => Some(Self {
1415 program: config.program.clone(),
1416 args: dfx_canister_args(
1417 config,
1418 vec!["status".to_string(), operation.target_canister.clone()],
1419 ),
1420 mutates: false,
1421 requires_stopped_canister: false,
1422 note: verification_command_note(
1423 &operation.operation,
1424 "checks target canister status",
1425 "checks target fleet root canister status",
1426 )
1427 .to_string(),
1428 }),
1429 Some(_) => {
1430 let method = operation.verification_method.as_ref()?;
1431 Some(Self {
1432 program: config.program.clone(),
1433 args: dfx_canister_args(
1434 config,
1435 vec![
1436 "call".to_string(),
1437 operation.target_canister.clone(),
1438 method.clone(),
1439 ],
1440 ),
1441 mutates: false,
1442 requires_stopped_canister: false,
1443 note: verification_command_note(
1444 &operation.operation,
1445 "calls the declared verification method",
1446 "calls the declared fleet verification method",
1447 )
1448 .to_string(),
1449 })
1450 }
1451 None => None,
1452 }
1453 }
1454 }
1455 }
1456}
1457
1458const fn verification_command_note(
1460 operation: &RestoreApplyOperationKind,
1461 member_note: &'static str,
1462 fleet_note: &'static str,
1463) -> &'static str {
1464 match operation {
1465 RestoreApplyOperationKind::VerifyFleet => fleet_note,
1466 RestoreApplyOperationKind::UploadSnapshot
1467 | RestoreApplyOperationKind::LoadSnapshot
1468 | RestoreApplyOperationKind::ReinstallCode
1469 | RestoreApplyOperationKind::VerifyMember => member_note,
1470 }
1471}
1472
1473fn dfx_canister_args(config: &RestoreApplyCommandConfig, mut tail: Vec<String>) -> Vec<String> {
1475 let mut args = vec!["canister".to_string()];
1476 if let Some(network) = &config.network {
1477 args.push("--network".to_string());
1478 args.push(network.clone());
1479 }
1480 args.append(&mut tail);
1481 args
1482}
1483
1484#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1489pub struct RestoreApplyJournalOperation {
1490 pub sequence: usize,
1491 pub operation: RestoreApplyOperationKind,
1492 pub state: RestoreApplyOperationState,
1493 #[serde(default, skip_serializing_if = "Option::is_none")]
1494 pub state_updated_at: Option<String>,
1495 pub blocking_reasons: Vec<String>,
1496 pub restore_group: u16,
1497 pub phase_order: usize,
1498 pub source_canister: String,
1499 pub target_canister: String,
1500 pub role: String,
1501 pub snapshot_id: Option<String>,
1502 pub artifact_path: Option<String>,
1503 pub verification_kind: Option<String>,
1504 pub verification_method: Option<String>,
1505}
1506
1507impl RestoreApplyJournalOperation {
1508 fn from_dry_run_operation(
1510 operation: &RestoreApplyDryRunOperation,
1511 state: RestoreApplyOperationState,
1512 blocked_reasons: &[String],
1513 ) -> Self {
1514 Self {
1515 sequence: operation.sequence,
1516 operation: operation.operation.clone(),
1517 state: state.clone(),
1518 state_updated_at: None,
1519 blocking_reasons: if state == RestoreApplyOperationState::Blocked {
1520 blocked_reasons.to_vec()
1521 } else {
1522 Vec::new()
1523 },
1524 restore_group: operation.restore_group,
1525 phase_order: operation.phase_order,
1526 source_canister: operation.source_canister.clone(),
1527 target_canister: operation.target_canister.clone(),
1528 role: operation.role.clone(),
1529 snapshot_id: operation.snapshot_id.clone(),
1530 artifact_path: operation.artifact_path.clone(),
1531 verification_kind: operation.verification_kind.clone(),
1532 verification_method: operation.verification_method.clone(),
1533 }
1534 }
1535
1536 fn validate(&self) -> Result<(), RestoreApplyJournalError> {
1538 validate_apply_journal_nonempty("operations[].source_canister", &self.source_canister)?;
1539 validate_apply_journal_nonempty("operations[].target_canister", &self.target_canister)?;
1540 validate_apply_journal_nonempty("operations[].role", &self.role)?;
1541 if let Some(updated_at) = &self.state_updated_at {
1542 validate_apply_journal_nonempty("operations[].state_updated_at", updated_at)?;
1543 }
1544 self.validate_operation_fields()?;
1545
1546 match self.state {
1547 RestoreApplyOperationState::Blocked if self.blocking_reasons.is_empty() => Err(
1548 RestoreApplyJournalError::BlockedOperationMissingReason(self.sequence),
1549 ),
1550 RestoreApplyOperationState::Failed if self.blocking_reasons.is_empty() => Err(
1551 RestoreApplyJournalError::FailureReasonRequired(self.sequence),
1552 ),
1553 RestoreApplyOperationState::Pending
1554 | RestoreApplyOperationState::Ready
1555 | RestoreApplyOperationState::Completed
1556 if !self.blocking_reasons.is_empty() =>
1557 {
1558 Err(RestoreApplyJournalError::UnblockedOperationHasReasons(
1559 self.sequence,
1560 ))
1561 }
1562 RestoreApplyOperationState::Blocked
1563 | RestoreApplyOperationState::Failed
1564 | RestoreApplyOperationState::Pending
1565 | RestoreApplyOperationState::Ready
1566 | RestoreApplyOperationState::Completed => Ok(()),
1567 }
1568 }
1569
1570 fn validate_operation_fields(&self) -> Result<(), RestoreApplyJournalError> {
1572 match self.operation {
1573 RestoreApplyOperationKind::UploadSnapshot => self
1574 .validate_required_field("operations[].artifact_path", self.artifact_path.as_ref())
1575 .map(|_| ()),
1576 RestoreApplyOperationKind::LoadSnapshot => self
1577 .validate_required_field("operations[].snapshot_id", self.snapshot_id.as_ref())
1578 .map(|_| ()),
1579 RestoreApplyOperationKind::ReinstallCode => Ok(()),
1580 RestoreApplyOperationKind::VerifyMember | RestoreApplyOperationKind::VerifyFleet => {
1581 let kind = self.validate_required_field(
1582 "operations[].verification_kind",
1583 self.verification_kind.as_ref(),
1584 )?;
1585 if kind == "status" {
1586 return Ok(());
1587 }
1588 self.validate_required_field(
1589 "operations[].verification_method",
1590 self.verification_method.as_ref(),
1591 )
1592 .map(|_| ())
1593 }
1594 }
1595 }
1596
1597 fn validate_required_field<'a>(
1599 &self,
1600 field: &'static str,
1601 value: Option<&'a String>,
1602 ) -> Result<&'a str, RestoreApplyJournalError> {
1603 let value = value.map(String::as_str).ok_or_else(|| {
1604 RestoreApplyJournalError::OperationMissingField {
1605 sequence: self.sequence,
1606 operation: self.operation.clone(),
1607 field,
1608 }
1609 })?;
1610 if value.trim().is_empty() {
1611 return Err(RestoreApplyJournalError::OperationMissingField {
1612 sequence: self.sequence,
1613 operation: self.operation.clone(),
1614 field,
1615 });
1616 }
1617
1618 Ok(value)
1619 }
1620
1621 const fn can_transition_to(&self, next_state: &RestoreApplyOperationState) -> bool {
1623 match (&self.state, next_state) {
1624 (
1625 RestoreApplyOperationState::Ready | RestoreApplyOperationState::Pending,
1626 RestoreApplyOperationState::Pending,
1627 )
1628 | (RestoreApplyOperationState::Pending, RestoreApplyOperationState::Ready)
1629 | (
1630 RestoreApplyOperationState::Ready
1631 | RestoreApplyOperationState::Pending
1632 | RestoreApplyOperationState::Completed,
1633 RestoreApplyOperationState::Completed,
1634 )
1635 | (
1636 RestoreApplyOperationState::Ready
1637 | RestoreApplyOperationState::Pending
1638 | RestoreApplyOperationState::Failed,
1639 RestoreApplyOperationState::Failed,
1640 ) => true,
1641 (
1642 RestoreApplyOperationState::Blocked
1643 | RestoreApplyOperationState::Completed
1644 | RestoreApplyOperationState::Failed
1645 | RestoreApplyOperationState::Pending
1646 | RestoreApplyOperationState::Ready,
1647 _,
1648 ) => false,
1649 }
1650 }
1651}
1652
1653#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1658#[serde(rename_all = "kebab-case")]
1659pub enum RestoreApplyOperationState {
1660 Pending,
1661 Ready,
1662 Blocked,
1663 Completed,
1664 Failed,
1665}
1666
1667#[derive(Debug, ThisError)]
1672pub enum RestoreApplyJournalError {
1673 #[error("unsupported restore apply journal version {0}")]
1674 UnsupportedVersion(u16),
1675
1676 #[error("restore apply journal field {0} is required")]
1677 MissingField(&'static str),
1678
1679 #[error("restore apply journal count {field} mismatch: reported={reported}, actual={actual}")]
1680 CountMismatch {
1681 field: &'static str,
1682 reported: usize,
1683 actual: usize,
1684 },
1685
1686 #[error("restore apply journal has duplicate operation sequence {0}")]
1687 DuplicateSequence(usize),
1688
1689 #[error("restore apply journal is missing operation sequence {0}")]
1690 MissingSequence(usize),
1691
1692 #[error("ready restore apply journal cannot include blocked reasons or blocked operations")]
1693 ReadyJournalHasBlockingState,
1694
1695 #[error("blocked restore apply journal operation {0} is missing a blocking reason")]
1696 BlockedOperationMissingReason(usize),
1697
1698 #[error("unblocked restore apply journal operation {0} cannot have blocking reasons")]
1699 UnblockedOperationHasReasons(usize),
1700
1701 #[error("restore apply journal operation {sequence} {operation:?} is missing field {field}")]
1702 OperationMissingField {
1703 sequence: usize,
1704 operation: RestoreApplyOperationKind,
1705 field: &'static str,
1706 },
1707
1708 #[error("restore apply journal operation {0} was not found")]
1709 OperationNotFound(usize),
1710
1711 #[error("restore apply journal operation {sequence} cannot transition from {from:?} to {to:?}")]
1712 InvalidOperationTransition {
1713 sequence: usize,
1714 from: RestoreApplyOperationState,
1715 to: RestoreApplyOperationState,
1716 },
1717
1718 #[error("failed restore apply journal operation {0} requires a reason")]
1719 FailureReasonRequired(usize),
1720
1721 #[error("restore apply journal has no operation that can be advanced")]
1722 NoTransitionableOperation,
1723
1724 #[error("restore apply journal has no pending operation to release")]
1725 NoPendingOperation,
1726
1727 #[error("restore apply journal operation {requested} cannot advance before operation {next}")]
1728 OutOfOrderOperationTransition { requested: usize, next: usize },
1729}
1730
1731fn validate_restore_apply_artifacts(
1733 plan: &RestorePlan,
1734 backup_root: &Path,
1735) -> Result<RestoreApplyArtifactValidation, RestoreApplyDryRunError> {
1736 let mut checks = Vec::new();
1737
1738 for member in plan.ordered_members() {
1739 checks.push(validate_restore_apply_artifact(member, backup_root)?);
1740 }
1741
1742 let members_with_expected_checksums = checks
1743 .iter()
1744 .filter(|check| check.checksum_expected.is_some())
1745 .count();
1746 let artifacts_present = checks.iter().all(|check| check.exists);
1747 let checksums_verified = members_with_expected_checksums == plan.member_count
1748 && checks.iter().all(|check| check.checksum_verified);
1749
1750 Ok(RestoreApplyArtifactValidation {
1751 backup_root: backup_root.to_string_lossy().to_string(),
1752 checked_members: checks.len(),
1753 artifacts_present,
1754 checksums_verified,
1755 members_with_expected_checksums,
1756 checks,
1757 })
1758}
1759
1760fn validate_restore_apply_artifact(
1762 member: &RestorePlanMember,
1763 backup_root: &Path,
1764) -> Result<RestoreApplyArtifactCheck, RestoreApplyDryRunError> {
1765 let artifact_path = safe_restore_artifact_path(
1766 &member.source_canister,
1767 &member.source_snapshot.artifact_path,
1768 )?;
1769 let resolved_path = backup_root.join(&artifact_path);
1770
1771 if !resolved_path.exists() {
1772 return Err(RestoreApplyDryRunError::ArtifactMissing {
1773 source_canister: member.source_canister.clone(),
1774 artifact_path: member.source_snapshot.artifact_path.clone(),
1775 resolved_path: resolved_path.to_string_lossy().to_string(),
1776 });
1777 }
1778
1779 let (checksum_actual, checksum_verified) =
1780 if let Some(expected) = &member.source_snapshot.checksum {
1781 let checksum = ArtifactChecksum::from_path(&resolved_path).map_err(|source| {
1782 RestoreApplyDryRunError::ArtifactChecksum {
1783 source_canister: member.source_canister.clone(),
1784 artifact_path: member.source_snapshot.artifact_path.clone(),
1785 source,
1786 }
1787 })?;
1788 checksum.verify(expected).map_err(|source| {
1789 RestoreApplyDryRunError::ArtifactChecksum {
1790 source_canister: member.source_canister.clone(),
1791 artifact_path: member.source_snapshot.artifact_path.clone(),
1792 source,
1793 }
1794 })?;
1795 (Some(checksum.hash), true)
1796 } else {
1797 (None, false)
1798 };
1799
1800 Ok(RestoreApplyArtifactCheck {
1801 source_canister: member.source_canister.clone(),
1802 target_canister: member.target_canister.clone(),
1803 snapshot_id: member.source_snapshot.snapshot_id.clone(),
1804 artifact_path: member.source_snapshot.artifact_path.clone(),
1805 resolved_path: resolved_path.to_string_lossy().to_string(),
1806 exists: true,
1807 checksum_algorithm: member.source_snapshot.checksum_algorithm.clone(),
1808 checksum_expected: member.source_snapshot.checksum.clone(),
1809 checksum_actual,
1810 checksum_verified,
1811 })
1812}
1813
1814fn safe_restore_artifact_path(
1816 source_canister: &str,
1817 artifact_path: &str,
1818) -> Result<PathBuf, RestoreApplyDryRunError> {
1819 let path = Path::new(artifact_path);
1820 let is_safe = path
1821 .components()
1822 .all(|component| matches!(component, Component::Normal(_) | Component::CurDir));
1823
1824 if is_safe {
1825 return Ok(path.to_path_buf());
1826 }
1827
1828 Err(RestoreApplyDryRunError::ArtifactPathEscapesBackup {
1829 source_canister: source_canister.to_string(),
1830 artifact_path: artifact_path.to_string(),
1831 })
1832}
1833
1834fn validate_restore_status_matches_plan(
1836 plan: &RestorePlan,
1837 status: &RestoreStatus,
1838) -> Result<(), RestoreApplyDryRunError> {
1839 validate_status_string_field("backup_id", &plan.backup_id, &status.backup_id)?;
1840 validate_status_string_field(
1841 "source_environment",
1842 &plan.source_environment,
1843 &status.source_environment,
1844 )?;
1845 validate_status_string_field(
1846 "source_root_canister",
1847 &plan.source_root_canister,
1848 &status.source_root_canister,
1849 )?;
1850 validate_status_string_field("topology_hash", &plan.topology_hash, &status.topology_hash)?;
1851 validate_status_usize_field("member_count", plan.member_count, status.member_count)?;
1852 validate_status_usize_field(
1853 "phase_count",
1854 plan.ordering_summary.phase_count,
1855 status.phase_count,
1856 )?;
1857 Ok(())
1858}
1859
1860fn validate_status_string_field(
1862 field: &'static str,
1863 plan: &str,
1864 status: &str,
1865) -> Result<(), RestoreApplyDryRunError> {
1866 if plan == status {
1867 return Ok(());
1868 }
1869
1870 Err(RestoreApplyDryRunError::StatusPlanMismatch {
1871 field,
1872 plan: plan.to_string(),
1873 status: status.to_string(),
1874 })
1875}
1876
1877const fn validate_status_usize_field(
1879 field: &'static str,
1880 plan: usize,
1881 status: usize,
1882) -> Result<(), RestoreApplyDryRunError> {
1883 if plan == status {
1884 return Ok(());
1885 }
1886
1887 Err(RestoreApplyDryRunError::StatusPlanCountMismatch {
1888 field,
1889 plan,
1890 status,
1891 })
1892}
1893
1894#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1899pub struct RestoreApplyArtifactValidation {
1900 pub backup_root: String,
1901 pub checked_members: usize,
1902 pub artifacts_present: bool,
1903 pub checksums_verified: bool,
1904 pub members_with_expected_checksums: usize,
1905 pub checks: Vec<RestoreApplyArtifactCheck>,
1906}
1907
1908#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1913pub struct RestoreApplyArtifactCheck {
1914 pub source_canister: String,
1915 pub target_canister: String,
1916 pub snapshot_id: String,
1917 pub artifact_path: String,
1918 pub resolved_path: String,
1919 pub exists: bool,
1920 pub checksum_algorithm: String,
1921 pub checksum_expected: Option<String>,
1922 pub checksum_actual: Option<String>,
1923 pub checksum_verified: bool,
1924}
1925
1926#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
1931pub struct RestoreApplyDryRunPhase {
1932 pub restore_group: u16,
1933 pub operations: Vec<RestoreApplyDryRunOperation>,
1934}
1935
1936impl RestoreApplyDryRunPhase {
1937 fn from_plan_phase(phase: &RestorePhase, next_sequence: &mut usize) -> Self {
1939 let mut operations = Vec::new();
1940
1941 for member in &phase.members {
1942 push_member_operation(
1943 &mut operations,
1944 next_sequence,
1945 RestoreApplyOperationKind::UploadSnapshot,
1946 member,
1947 None,
1948 );
1949 push_member_operation(
1950 &mut operations,
1951 next_sequence,
1952 RestoreApplyOperationKind::LoadSnapshot,
1953 member,
1954 None,
1955 );
1956 push_member_operation(
1957 &mut operations,
1958 next_sequence,
1959 RestoreApplyOperationKind::ReinstallCode,
1960 member,
1961 None,
1962 );
1963
1964 for check in &member.verification_checks {
1965 push_member_operation(
1966 &mut operations,
1967 next_sequence,
1968 RestoreApplyOperationKind::VerifyMember,
1969 member,
1970 Some(check),
1971 );
1972 }
1973 }
1974
1975 Self {
1976 restore_group: phase.restore_group,
1977 operations,
1978 }
1979 }
1980}
1981
1982fn push_member_operation(
1984 operations: &mut Vec<RestoreApplyDryRunOperation>,
1985 next_sequence: &mut usize,
1986 operation: RestoreApplyOperationKind,
1987 member: &RestorePlanMember,
1988 check: Option<&VerificationCheck>,
1989) {
1990 let sequence = *next_sequence;
1991 *next_sequence += 1;
1992
1993 operations.push(RestoreApplyDryRunOperation {
1994 sequence,
1995 operation,
1996 restore_group: member.restore_group,
1997 phase_order: member.phase_order,
1998 source_canister: member.source_canister.clone(),
1999 target_canister: member.target_canister.clone(),
2000 role: member.role.clone(),
2001 snapshot_id: Some(member.source_snapshot.snapshot_id.clone()),
2002 artifact_path: Some(member.source_snapshot.artifact_path.clone()),
2003 verification_kind: check.map(|check| check.kind.clone()),
2004 verification_method: check.and_then(|check| check.method.clone()),
2005 });
2006}
2007
2008fn append_fleet_verification_operations(
2010 plan: &RestorePlan,
2011 phases: &mut [RestoreApplyDryRunPhase],
2012 next_sequence: &mut usize,
2013) {
2014 if plan.fleet_verification_checks.is_empty() {
2015 return;
2016 }
2017
2018 let Some(phase) = phases.last_mut() else {
2019 return;
2020 };
2021 let root = plan
2022 .phases
2023 .iter()
2024 .flat_map(|phase| phase.members.iter())
2025 .find(|member| member.source_canister == plan.source_root_canister);
2026 let source_canister = root.map_or_else(
2027 || plan.source_root_canister.clone(),
2028 |member| member.source_canister.clone(),
2029 );
2030 let target_canister = root.map_or_else(
2031 || plan.source_root_canister.clone(),
2032 |member| member.target_canister.clone(),
2033 );
2034 let restore_group = phase.restore_group;
2035
2036 for check in &plan.fleet_verification_checks {
2037 push_fleet_operation(
2038 &mut phase.operations,
2039 next_sequence,
2040 restore_group,
2041 &source_canister,
2042 &target_canister,
2043 check,
2044 );
2045 }
2046}
2047
2048fn push_fleet_operation(
2050 operations: &mut Vec<RestoreApplyDryRunOperation>,
2051 next_sequence: &mut usize,
2052 restore_group: u16,
2053 source_canister: &str,
2054 target_canister: &str,
2055 check: &VerificationCheck,
2056) {
2057 let sequence = *next_sequence;
2058 *next_sequence += 1;
2059 let phase_order = operations.len();
2060
2061 operations.push(RestoreApplyDryRunOperation {
2062 sequence,
2063 operation: RestoreApplyOperationKind::VerifyFleet,
2064 restore_group,
2065 phase_order,
2066 source_canister: source_canister.to_string(),
2067 target_canister: target_canister.to_string(),
2068 role: "fleet".to_string(),
2069 snapshot_id: None,
2070 artifact_path: None,
2071 verification_kind: Some(check.kind.clone()),
2072 verification_method: check.method.clone(),
2073 });
2074}
2075
2076#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2081pub struct RestoreApplyDryRunOperation {
2082 pub sequence: usize,
2083 pub operation: RestoreApplyOperationKind,
2084 pub restore_group: u16,
2085 pub phase_order: usize,
2086 pub source_canister: String,
2087 pub target_canister: String,
2088 pub role: String,
2089 pub snapshot_id: Option<String>,
2090 pub artifact_path: Option<String>,
2091 pub verification_kind: Option<String>,
2092 pub verification_method: Option<String>,
2093}
2094
2095#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2100#[serde(rename_all = "kebab-case")]
2101pub enum RestoreApplyOperationKind {
2102 UploadSnapshot,
2103 LoadSnapshot,
2104 ReinstallCode,
2105 VerifyMember,
2106 VerifyFleet,
2107}
2108
2109#[derive(Debug, ThisError)]
2114pub enum RestoreApplyDryRunError {
2115 #[error("restore status field {field} does not match plan: plan={plan}, status={status}")]
2116 StatusPlanMismatch {
2117 field: &'static str,
2118 plan: String,
2119 status: String,
2120 },
2121
2122 #[error("restore status field {field} does not match plan: plan={plan}, status={status}")]
2123 StatusPlanCountMismatch {
2124 field: &'static str,
2125 plan: usize,
2126 status: usize,
2127 },
2128
2129 #[error("restore artifact path for {source_canister} escapes backup root: {artifact_path}")]
2130 ArtifactPathEscapesBackup {
2131 source_canister: String,
2132 artifact_path: String,
2133 },
2134
2135 #[error(
2136 "restore artifact for {source_canister} is missing: {artifact_path} at {resolved_path}"
2137 )]
2138 ArtifactMissing {
2139 source_canister: String,
2140 artifact_path: String,
2141 resolved_path: String,
2142 },
2143
2144 #[error("restore artifact checksum failed for {source_canister} at {artifact_path}: {source}")]
2145 ArtifactChecksum {
2146 source_canister: String,
2147 artifact_path: String,
2148 #[source]
2149 source: ArtifactChecksumError,
2150 },
2151}
2152
2153#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2158pub struct RestoreIdentitySummary {
2159 pub mapping_supplied: bool,
2160 pub all_sources_mapped: bool,
2161 pub fixed_members: usize,
2162 pub relocatable_members: usize,
2163 pub in_place_members: usize,
2164 pub mapped_members: usize,
2165 pub remapped_members: usize,
2166}
2167
2168#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2173#[expect(
2174 clippy::struct_excessive_bools,
2175 reason = "restore summaries intentionally expose machine-readable readiness flags"
2176)]
2177pub struct RestoreSnapshotSummary {
2178 pub all_members_have_module_hash: bool,
2179 pub all_members_have_wasm_hash: bool,
2180 pub all_members_have_code_version: bool,
2181 pub all_members_have_checksum: bool,
2182 pub members_with_module_hash: usize,
2183 pub members_with_wasm_hash: usize,
2184 pub members_with_code_version: usize,
2185 pub members_with_checksum: usize,
2186}
2187
2188#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2193pub struct RestoreVerificationSummary {
2194 pub verification_required: bool,
2195 pub all_members_have_checks: bool,
2196 pub fleet_checks: usize,
2197 pub member_check_groups: usize,
2198 pub member_checks: usize,
2199 pub members_with_checks: usize,
2200 pub total_checks: usize,
2201}
2202
2203#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2208pub struct RestoreReadinessSummary {
2209 pub ready: bool,
2210 pub reasons: Vec<String>,
2211}
2212
2213#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2218pub struct RestoreOperationSummary {
2219 #[serde(default)]
2220 pub planned_snapshot_uploads: usize,
2221 pub planned_snapshot_loads: usize,
2222 pub planned_code_reinstalls: usize,
2223 pub planned_verification_checks: usize,
2224 #[serde(default)]
2225 pub planned_operations: usize,
2226 pub planned_phases: usize,
2227}
2228
2229impl RestoreOperationSummary {
2230 #[must_use]
2232 pub const fn effective_planned_snapshot_uploads(&self, member_count: usize) -> usize {
2233 if self.planned_snapshot_uploads == 0 && member_count > 0 {
2234 return member_count;
2235 }
2236
2237 self.planned_snapshot_uploads
2238 }
2239
2240 #[must_use]
2242 pub const fn effective_planned_operations(&self, member_count: usize) -> usize {
2243 if self.planned_operations == 0 {
2244 return self.effective_planned_snapshot_uploads(member_count)
2245 + self.planned_snapshot_loads
2246 + self.planned_code_reinstalls
2247 + self.planned_verification_checks;
2248 }
2249
2250 self.planned_operations
2251 }
2252}
2253
2254#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2259pub struct RestoreOrderingSummary {
2260 pub phase_count: usize,
2261 pub dependency_free_members: usize,
2262 pub in_group_parent_edges: usize,
2263 pub cross_group_parent_edges: usize,
2264}
2265
2266#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2271pub struct RestorePhase {
2272 pub restore_group: u16,
2273 pub members: Vec<RestorePlanMember>,
2274}
2275
2276#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2281pub struct RestorePlanMember {
2282 pub source_canister: String,
2283 pub target_canister: String,
2284 pub role: String,
2285 pub parent_source_canister: Option<String>,
2286 pub parent_target_canister: Option<String>,
2287 pub ordering_dependency: Option<RestoreOrderingDependency>,
2288 pub phase_order: usize,
2289 pub restore_group: u16,
2290 pub identity_mode: IdentityMode,
2291 pub verification_class: String,
2292 pub verification_checks: Vec<VerificationCheck>,
2293 pub source_snapshot: SourceSnapshot,
2294}
2295
2296#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2301pub struct RestoreOrderingDependency {
2302 pub source_canister: String,
2303 pub target_canister: String,
2304 pub relationship: RestoreOrderingRelationship,
2305}
2306
2307#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
2312#[serde(rename_all = "kebab-case")]
2313pub enum RestoreOrderingRelationship {
2314 ParentInSameGroup,
2315 ParentInEarlierGroup,
2316}
2317
2318pub struct RestorePlanner;
2323
2324impl RestorePlanner {
2325 pub fn plan(
2327 manifest: &FleetBackupManifest,
2328 mapping: Option<&RestoreMapping>,
2329 ) -> Result<RestorePlan, RestorePlanError> {
2330 manifest.validate()?;
2331 if let Some(mapping) = mapping {
2332 validate_mapping(mapping)?;
2333 validate_mapping_sources(manifest, mapping)?;
2334 }
2335
2336 let members = resolve_members(manifest, mapping)?;
2337 let identity_summary = restore_identity_summary(&members, mapping.is_some());
2338 let snapshot_summary = restore_snapshot_summary(&members);
2339 let verification_summary = restore_verification_summary(manifest, &members);
2340 let readiness_summary = restore_readiness_summary(&snapshot_summary, &verification_summary);
2341 validate_restore_group_dependencies(&members)?;
2342 let phases = group_and_order_members(members)?;
2343 let ordering_summary = restore_ordering_summary(&phases);
2344 let operation_summary =
2345 restore_operation_summary(manifest.fleet.members.len(), &verification_summary, &phases);
2346
2347 Ok(RestorePlan {
2348 backup_id: manifest.backup_id.clone(),
2349 source_environment: manifest.source.environment.clone(),
2350 source_root_canister: manifest.source.root_canister.clone(),
2351 topology_hash: manifest.fleet.topology_hash.clone(),
2352 member_count: manifest.fleet.members.len(),
2353 identity_summary,
2354 snapshot_summary,
2355 verification_summary,
2356 readiness_summary,
2357 operation_summary,
2358 ordering_summary,
2359 fleet_verification_checks: manifest.verification.fleet_checks.clone(),
2360 phases,
2361 })
2362 }
2363}
2364
2365#[derive(Debug, ThisError)]
2370pub enum RestorePlanError {
2371 #[error(transparent)]
2372 InvalidManifest(#[from] ManifestValidationError),
2373
2374 #[error("field {field} must be a valid principal: {value}")]
2375 InvalidPrincipal { field: &'static str, value: String },
2376
2377 #[error("mapping contains duplicate source canister {0}")]
2378 DuplicateMappingSource(String),
2379
2380 #[error("mapping contains duplicate target canister {0}")]
2381 DuplicateMappingTarget(String),
2382
2383 #[error("mapping references unknown source canister {0}")]
2384 UnknownMappingSource(String),
2385
2386 #[error("mapping is missing source canister {0}")]
2387 MissingMappingSource(String),
2388
2389 #[error("fixed-identity member {source_canister} cannot be mapped to {target_canister}")]
2390 FixedIdentityRemap {
2391 source_canister: String,
2392 target_canister: String,
2393 },
2394
2395 #[error("restore plan contains duplicate target canister {0}")]
2396 DuplicatePlanTarget(String),
2397
2398 #[error("restore group {0} contains a parent cycle or unresolved dependency")]
2399 RestoreOrderCycle(u16),
2400
2401 #[error(
2402 "restore plan places parent {parent_source_canister} in group {parent_restore_group} after child {child_source_canister} in group {child_restore_group}"
2403 )]
2404 ParentRestoreGroupAfterChild {
2405 child_source_canister: String,
2406 parent_source_canister: String,
2407 child_restore_group: u16,
2408 parent_restore_group: u16,
2409 },
2410}
2411
2412fn validate_mapping(mapping: &RestoreMapping) -> Result<(), RestorePlanError> {
2414 let mut sources = BTreeSet::new();
2415 let mut targets = BTreeSet::new();
2416
2417 for entry in &mapping.members {
2418 validate_principal("mapping.members[].source_canister", &entry.source_canister)?;
2419 validate_principal("mapping.members[].target_canister", &entry.target_canister)?;
2420
2421 if !sources.insert(entry.source_canister.clone()) {
2422 return Err(RestorePlanError::DuplicateMappingSource(
2423 entry.source_canister.clone(),
2424 ));
2425 }
2426
2427 if !targets.insert(entry.target_canister.clone()) {
2428 return Err(RestorePlanError::DuplicateMappingTarget(
2429 entry.target_canister.clone(),
2430 ));
2431 }
2432 }
2433
2434 Ok(())
2435}
2436
2437fn validate_mapping_sources(
2439 manifest: &FleetBackupManifest,
2440 mapping: &RestoreMapping,
2441) -> Result<(), RestorePlanError> {
2442 let sources = manifest
2443 .fleet
2444 .members
2445 .iter()
2446 .map(|member| member.canister_id.as_str())
2447 .collect::<BTreeSet<_>>();
2448
2449 for entry in &mapping.members {
2450 if !sources.contains(entry.source_canister.as_str()) {
2451 return Err(RestorePlanError::UnknownMappingSource(
2452 entry.source_canister.clone(),
2453 ));
2454 }
2455 }
2456
2457 Ok(())
2458}
2459
2460fn resolve_members(
2462 manifest: &FleetBackupManifest,
2463 mapping: Option<&RestoreMapping>,
2464) -> Result<Vec<RestorePlanMember>, RestorePlanError> {
2465 let mut plan_members = Vec::with_capacity(manifest.fleet.members.len());
2466 let mut targets = BTreeSet::new();
2467 let mut source_to_target = BTreeMap::new();
2468
2469 for member in &manifest.fleet.members {
2470 let target = resolve_target(member, mapping)?;
2471 if !targets.insert(target.clone()) {
2472 return Err(RestorePlanError::DuplicatePlanTarget(target));
2473 }
2474
2475 source_to_target.insert(member.canister_id.clone(), target.clone());
2476 plan_members.push(RestorePlanMember {
2477 source_canister: member.canister_id.clone(),
2478 target_canister: target,
2479 role: member.role.clone(),
2480 parent_source_canister: member.parent_canister_id.clone(),
2481 parent_target_canister: None,
2482 ordering_dependency: None,
2483 phase_order: 0,
2484 restore_group: member.restore_group,
2485 identity_mode: member.identity_mode.clone(),
2486 verification_class: member.verification_class.clone(),
2487 verification_checks: concrete_member_verification_checks(
2488 member,
2489 &manifest.verification,
2490 ),
2491 source_snapshot: member.source_snapshot.clone(),
2492 });
2493 }
2494
2495 for member in &mut plan_members {
2496 member.parent_target_canister = member
2497 .parent_source_canister
2498 .as_ref()
2499 .and_then(|parent| source_to_target.get(parent))
2500 .cloned();
2501 }
2502
2503 Ok(plan_members)
2504}
2505
2506fn concrete_member_verification_checks(
2508 member: &FleetMember,
2509 verification: &VerificationPlan,
2510) -> Vec<VerificationCheck> {
2511 let mut checks = member
2512 .verification_checks
2513 .iter()
2514 .filter(|check| verification_check_applies_to_role(check, &member.role))
2515 .cloned()
2516 .collect::<Vec<_>>();
2517
2518 for group in &verification.member_checks {
2519 if group.role != member.role {
2520 continue;
2521 }
2522
2523 checks.extend(
2524 group
2525 .checks
2526 .iter()
2527 .filter(|check| verification_check_applies_to_role(check, &member.role))
2528 .cloned(),
2529 );
2530 }
2531
2532 checks
2533}
2534
2535fn verification_check_applies_to_role(check: &VerificationCheck, role: &str) -> bool {
2537 check.roles.is_empty() || check.roles.iter().any(|check_role| check_role == role)
2538}
2539
2540fn resolve_target(
2542 member: &FleetMember,
2543 mapping: Option<&RestoreMapping>,
2544) -> Result<String, RestorePlanError> {
2545 let target = match mapping {
2546 Some(mapping) => mapping
2547 .target_for(&member.canister_id)
2548 .ok_or_else(|| RestorePlanError::MissingMappingSource(member.canister_id.clone()))?
2549 .to_string(),
2550 None => member.canister_id.clone(),
2551 };
2552
2553 if matches!(member.identity_mode, IdentityMode::Fixed) && target != member.canister_id {
2554 return Err(RestorePlanError::FixedIdentityRemap {
2555 source_canister: member.canister_id.clone(),
2556 target_canister: target,
2557 });
2558 }
2559
2560 Ok(target)
2561}
2562
2563fn restore_identity_summary(
2565 members: &[RestorePlanMember],
2566 mapping_supplied: bool,
2567) -> RestoreIdentitySummary {
2568 let mut summary = RestoreIdentitySummary {
2569 mapping_supplied,
2570 all_sources_mapped: false,
2571 fixed_members: 0,
2572 relocatable_members: 0,
2573 in_place_members: 0,
2574 mapped_members: 0,
2575 remapped_members: 0,
2576 };
2577
2578 for member in members {
2579 match member.identity_mode {
2580 IdentityMode::Fixed => summary.fixed_members += 1,
2581 IdentityMode::Relocatable => summary.relocatable_members += 1,
2582 }
2583
2584 if member.source_canister == member.target_canister {
2585 summary.in_place_members += 1;
2586 } else {
2587 summary.remapped_members += 1;
2588 }
2589 if mapping_supplied {
2590 summary.mapped_members += 1;
2591 }
2592 }
2593
2594 summary.all_sources_mapped = mapping_supplied && summary.mapped_members == members.len();
2595
2596 summary
2597}
2598
2599fn restore_snapshot_summary(members: &[RestorePlanMember]) -> RestoreSnapshotSummary {
2601 let members_with_module_hash = members
2602 .iter()
2603 .filter(|member| member.source_snapshot.module_hash.is_some())
2604 .count();
2605 let members_with_wasm_hash = members
2606 .iter()
2607 .filter(|member| member.source_snapshot.wasm_hash.is_some())
2608 .count();
2609 let members_with_code_version = members
2610 .iter()
2611 .filter(|member| member.source_snapshot.code_version.is_some())
2612 .count();
2613 let members_with_checksum = members
2614 .iter()
2615 .filter(|member| member.source_snapshot.checksum.is_some())
2616 .count();
2617
2618 RestoreSnapshotSummary {
2619 all_members_have_module_hash: members_with_module_hash == members.len(),
2620 all_members_have_wasm_hash: members_with_wasm_hash == members.len(),
2621 all_members_have_code_version: members_with_code_version == members.len(),
2622 all_members_have_checksum: members_with_checksum == members.len(),
2623 members_with_module_hash,
2624 members_with_wasm_hash,
2625 members_with_code_version,
2626 members_with_checksum,
2627 }
2628}
2629
2630fn restore_readiness_summary(
2632 snapshot: &RestoreSnapshotSummary,
2633 verification: &RestoreVerificationSummary,
2634) -> RestoreReadinessSummary {
2635 let mut reasons = Vec::new();
2636
2637 if !snapshot.all_members_have_module_hash {
2638 reasons.push("missing-module-hash".to_string());
2639 }
2640 if !snapshot.all_members_have_wasm_hash {
2641 reasons.push("missing-wasm-hash".to_string());
2642 }
2643 if !snapshot.all_members_have_code_version {
2644 reasons.push("missing-code-version".to_string());
2645 }
2646 if !snapshot.all_members_have_checksum {
2647 reasons.push("missing-snapshot-checksum".to_string());
2648 }
2649 if !verification.all_members_have_checks {
2650 reasons.push("missing-verification-checks".to_string());
2651 }
2652
2653 RestoreReadinessSummary {
2654 ready: reasons.is_empty(),
2655 reasons,
2656 }
2657}
2658
2659fn restore_verification_summary(
2661 manifest: &FleetBackupManifest,
2662 members: &[RestorePlanMember],
2663) -> RestoreVerificationSummary {
2664 let fleet_checks = manifest.verification.fleet_checks.len();
2665 let member_check_groups = manifest.verification.member_checks.len();
2666 let member_checks = members
2667 .iter()
2668 .map(|member| member.verification_checks.len())
2669 .sum::<usize>();
2670 let members_with_checks = members
2671 .iter()
2672 .filter(|member| !member.verification_checks.is_empty())
2673 .count();
2674
2675 RestoreVerificationSummary {
2676 verification_required: true,
2677 all_members_have_checks: members_with_checks == members.len(),
2678 fleet_checks,
2679 member_check_groups,
2680 member_checks,
2681 members_with_checks,
2682 total_checks: fleet_checks + member_checks,
2683 }
2684}
2685
2686const fn restore_operation_summary(
2688 member_count: usize,
2689 verification_summary: &RestoreVerificationSummary,
2690 phases: &[RestorePhase],
2691) -> RestoreOperationSummary {
2692 RestoreOperationSummary {
2693 planned_snapshot_uploads: member_count,
2694 planned_snapshot_loads: member_count,
2695 planned_code_reinstalls: member_count,
2696 planned_verification_checks: verification_summary.total_checks,
2697 planned_operations: member_count
2698 + member_count
2699 + member_count
2700 + verification_summary.total_checks,
2701 planned_phases: phases.len(),
2702 }
2703}
2704
2705fn validate_restore_group_dependencies(
2707 members: &[RestorePlanMember],
2708) -> Result<(), RestorePlanError> {
2709 let groups_by_source = members
2710 .iter()
2711 .map(|member| (member.source_canister.as_str(), member.restore_group))
2712 .collect::<BTreeMap<_, _>>();
2713
2714 for member in members {
2715 let Some(parent) = &member.parent_source_canister else {
2716 continue;
2717 };
2718 let Some(parent_group) = groups_by_source.get(parent.as_str()) else {
2719 continue;
2720 };
2721
2722 if *parent_group > member.restore_group {
2723 return Err(RestorePlanError::ParentRestoreGroupAfterChild {
2724 child_source_canister: member.source_canister.clone(),
2725 parent_source_canister: parent.clone(),
2726 child_restore_group: member.restore_group,
2727 parent_restore_group: *parent_group,
2728 });
2729 }
2730 }
2731
2732 Ok(())
2733}
2734
2735fn group_and_order_members(
2737 members: Vec<RestorePlanMember>,
2738) -> Result<Vec<RestorePhase>, RestorePlanError> {
2739 let mut groups = BTreeMap::<u16, Vec<RestorePlanMember>>::new();
2740 for member in members {
2741 groups.entry(member.restore_group).or_default().push(member);
2742 }
2743
2744 groups
2745 .into_iter()
2746 .map(|(restore_group, members)| {
2747 let members = order_group(restore_group, members)?;
2748 Ok(RestorePhase {
2749 restore_group,
2750 members,
2751 })
2752 })
2753 .collect()
2754}
2755
2756fn order_group(
2758 restore_group: u16,
2759 members: Vec<RestorePlanMember>,
2760) -> Result<Vec<RestorePlanMember>, RestorePlanError> {
2761 let mut remaining = members;
2762 let group_sources = remaining
2763 .iter()
2764 .map(|member| member.source_canister.clone())
2765 .collect::<BTreeSet<_>>();
2766 let mut emitted = BTreeSet::new();
2767 let mut ordered = Vec::with_capacity(remaining.len());
2768
2769 while !remaining.is_empty() {
2770 let Some(index) = remaining
2771 .iter()
2772 .position(|member| parent_satisfied(member, &group_sources, &emitted))
2773 else {
2774 return Err(RestorePlanError::RestoreOrderCycle(restore_group));
2775 };
2776
2777 let mut member = remaining.remove(index);
2778 member.phase_order = ordered.len();
2779 member.ordering_dependency = ordering_dependency(&member, &group_sources);
2780 emitted.insert(member.source_canister.clone());
2781 ordered.push(member);
2782 }
2783
2784 Ok(ordered)
2785}
2786
2787fn ordering_dependency(
2789 member: &RestorePlanMember,
2790 group_sources: &BTreeSet<String>,
2791) -> Option<RestoreOrderingDependency> {
2792 let parent_source = member.parent_source_canister.as_ref()?;
2793 let parent_target = member.parent_target_canister.as_ref()?;
2794 let relationship = if group_sources.contains(parent_source) {
2795 RestoreOrderingRelationship::ParentInSameGroup
2796 } else {
2797 RestoreOrderingRelationship::ParentInEarlierGroup
2798 };
2799
2800 Some(RestoreOrderingDependency {
2801 source_canister: parent_source.clone(),
2802 target_canister: parent_target.clone(),
2803 relationship,
2804 })
2805}
2806
2807fn restore_ordering_summary(phases: &[RestorePhase]) -> RestoreOrderingSummary {
2809 let mut summary = RestoreOrderingSummary {
2810 phase_count: phases.len(),
2811 dependency_free_members: 0,
2812 in_group_parent_edges: 0,
2813 cross_group_parent_edges: 0,
2814 };
2815
2816 for member in phases.iter().flat_map(|phase| phase.members.iter()) {
2817 match &member.ordering_dependency {
2818 Some(dependency)
2819 if dependency.relationship == RestoreOrderingRelationship::ParentInSameGroup =>
2820 {
2821 summary.in_group_parent_edges += 1;
2822 }
2823 Some(dependency)
2824 if dependency.relationship == RestoreOrderingRelationship::ParentInEarlierGroup =>
2825 {
2826 summary.cross_group_parent_edges += 1;
2827 }
2828 Some(_) => {}
2829 None => summary.dependency_free_members += 1,
2830 }
2831 }
2832
2833 summary
2834}
2835
2836fn parent_satisfied(
2838 member: &RestorePlanMember,
2839 group_sources: &BTreeSet<String>,
2840 emitted: &BTreeSet<String>,
2841) -> bool {
2842 match &member.parent_source_canister {
2843 Some(parent) if group_sources.contains(parent) => emitted.contains(parent),
2844 _ => true,
2845 }
2846}
2847
2848fn validate_principal(field: &'static str, value: &str) -> Result<(), RestorePlanError> {
2850 Principal::from_str(value)
2851 .map(|_| ())
2852 .map_err(|_| RestorePlanError::InvalidPrincipal {
2853 field,
2854 value: value.to_string(),
2855 })
2856}
2857
2858#[cfg(test)]
2859mod tests {
2860 use super::*;
2861 use crate::manifest::{
2862 BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetSection,
2863 MemberVerificationChecks, SourceMetadata, SourceSnapshot, ToolMetadata, VerificationCheck,
2864 VerificationPlan,
2865 };
2866 use std::{
2867 env, fs,
2868 path::{Path, PathBuf},
2869 time::{SystemTime, UNIX_EPOCH},
2870 };
2871
2872 const ROOT: &str = "aaaaa-aa";
2873 const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2874 const CHILD_TWO: &str = "r7inp-6aaaa-aaaaa-aaabq-cai";
2875 const TARGET: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2876 const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2877
2878 fn command_preview_journal(
2880 operation: RestoreApplyOperationKind,
2881 verification_kind: Option<&str>,
2882 verification_method: Option<&str>,
2883 ) -> RestoreApplyJournal {
2884 let journal = RestoreApplyJournal {
2885 journal_version: 1,
2886 backup_id: "fbk_test_001".to_string(),
2887 ready: true,
2888 blocked_reasons: Vec::new(),
2889 operation_count: 1,
2890 operation_counts: RestoreApplyOperationKindCounts::default(),
2891 pending_operations: 0,
2892 ready_operations: 1,
2893 blocked_operations: 0,
2894 completed_operations: 0,
2895 failed_operations: 0,
2896 operations: vec![RestoreApplyJournalOperation {
2897 sequence: 0,
2898 operation,
2899 state: RestoreApplyOperationState::Ready,
2900 state_updated_at: None,
2901 blocking_reasons: Vec::new(),
2902 restore_group: 1,
2903 phase_order: 0,
2904 source_canister: ROOT.to_string(),
2905 target_canister: ROOT.to_string(),
2906 role: "root".to_string(),
2907 snapshot_id: Some("snap-root".to_string()),
2908 artifact_path: Some("artifacts/root".to_string()),
2909 verification_kind: verification_kind.map(str::to_string),
2910 verification_method: verification_method.map(str::to_string),
2911 }],
2912 };
2913
2914 journal.validate().expect("journal should validate");
2915 journal
2916 }
2917
2918 fn valid_manifest(identity_mode: IdentityMode) -> FleetBackupManifest {
2920 FleetBackupManifest {
2921 manifest_version: 1,
2922 backup_id: "fbk_test_001".to_string(),
2923 created_at: "2026-04-10T12:00:00Z".to_string(),
2924 tool: ToolMetadata {
2925 name: "canic".to_string(),
2926 version: "v1".to_string(),
2927 },
2928 source: SourceMetadata {
2929 environment: "local".to_string(),
2930 root_canister: ROOT.to_string(),
2931 },
2932 consistency: ConsistencySection {
2933 mode: ConsistencyMode::CrashConsistent,
2934 backup_units: vec![BackupUnit {
2935 unit_id: "whole-fleet".to_string(),
2936 kind: BackupUnitKind::WholeFleet,
2937 roles: vec!["root".to_string(), "app".to_string()],
2938 consistency_reason: None,
2939 dependency_closure: Vec::new(),
2940 topology_validation: "subtree-closed".to_string(),
2941 quiescence_strategy: None,
2942 }],
2943 },
2944 fleet: FleetSection {
2945 topology_hash_algorithm: "sha256".to_string(),
2946 topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
2947 discovery_topology_hash: HASH.to_string(),
2948 pre_snapshot_topology_hash: HASH.to_string(),
2949 topology_hash: HASH.to_string(),
2950 members: vec![
2951 fleet_member("app", CHILD, Some(ROOT), identity_mode, 1),
2952 fleet_member("root", ROOT, None, IdentityMode::Fixed, 1),
2953 ],
2954 },
2955 verification: VerificationPlan {
2956 fleet_checks: Vec::new(),
2957 member_checks: Vec::new(),
2958 },
2959 }
2960 }
2961
2962 fn fleet_member(
2964 role: &str,
2965 canister_id: &str,
2966 parent_canister_id: Option<&str>,
2967 identity_mode: IdentityMode,
2968 restore_group: u16,
2969 ) -> FleetMember {
2970 FleetMember {
2971 role: role.to_string(),
2972 canister_id: canister_id.to_string(),
2973 parent_canister_id: parent_canister_id.map(str::to_string),
2974 subnet_canister_id: None,
2975 controller_hint: Some(ROOT.to_string()),
2976 identity_mode,
2977 restore_group,
2978 verification_class: "basic".to_string(),
2979 verification_checks: vec![VerificationCheck {
2980 kind: "call".to_string(),
2981 method: Some("canic_ready".to_string()),
2982 roles: Vec::new(),
2983 }],
2984 source_snapshot: SourceSnapshot {
2985 snapshot_id: format!("snap-{role}"),
2986 module_hash: Some(HASH.to_string()),
2987 wasm_hash: Some(HASH.to_string()),
2988 code_version: Some("v0.30.0".to_string()),
2989 artifact_path: format!("artifacts/{role}"),
2990 checksum_algorithm: "sha256".to_string(),
2991 checksum: Some(HASH.to_string()),
2992 },
2993 }
2994 }
2995
2996 #[test]
2998 fn in_place_plan_orders_parent_before_child() {
2999 let manifest = valid_manifest(IdentityMode::Relocatable);
3000
3001 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3002 let ordered = plan.ordered_members();
3003
3004 assert_eq!(plan.backup_id, "fbk_test_001");
3005 assert_eq!(plan.source_environment, "local");
3006 assert_eq!(plan.source_root_canister, ROOT);
3007 assert_eq!(plan.topology_hash, HASH);
3008 assert_eq!(plan.member_count, 2);
3009 assert_eq!(plan.identity_summary.fixed_members, 1);
3010 assert_eq!(plan.identity_summary.relocatable_members, 1);
3011 assert_eq!(plan.identity_summary.in_place_members, 2);
3012 assert_eq!(plan.identity_summary.mapped_members, 0);
3013 assert_eq!(plan.identity_summary.remapped_members, 0);
3014 assert!(plan.verification_summary.verification_required);
3015 assert!(plan.verification_summary.all_members_have_checks);
3016 assert!(plan.readiness_summary.ready);
3017 assert!(plan.readiness_summary.reasons.is_empty());
3018 assert_eq!(plan.verification_summary.fleet_checks, 0);
3019 assert_eq!(plan.verification_summary.member_check_groups, 0);
3020 assert_eq!(plan.verification_summary.member_checks, 2);
3021 assert_eq!(plan.verification_summary.members_with_checks, 2);
3022 assert_eq!(plan.verification_summary.total_checks, 2);
3023 assert_eq!(plan.ordering_summary.phase_count, 1);
3024 assert_eq!(plan.ordering_summary.dependency_free_members, 1);
3025 assert_eq!(plan.ordering_summary.in_group_parent_edges, 1);
3026 assert_eq!(plan.ordering_summary.cross_group_parent_edges, 0);
3027 assert_eq!(ordered[0].phase_order, 0);
3028 assert_eq!(ordered[1].phase_order, 1);
3029 assert_eq!(ordered[0].source_canister, ROOT);
3030 assert_eq!(ordered[1].source_canister, CHILD);
3031 assert_eq!(
3032 ordered[1].ordering_dependency,
3033 Some(RestoreOrderingDependency {
3034 source_canister: ROOT.to_string(),
3035 target_canister: ROOT.to_string(),
3036 relationship: RestoreOrderingRelationship::ParentInSameGroup,
3037 })
3038 );
3039 }
3040
3041 #[test]
3043 fn plan_reports_parent_dependency_from_earlier_group() {
3044 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3045 manifest.fleet.members[0].restore_group = 2;
3046 manifest.fleet.members[1].restore_group = 1;
3047
3048 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3049 let ordered = plan.ordered_members();
3050
3051 assert_eq!(plan.phases.len(), 2);
3052 assert_eq!(plan.ordering_summary.phase_count, 2);
3053 assert_eq!(plan.ordering_summary.dependency_free_members, 1);
3054 assert_eq!(plan.ordering_summary.in_group_parent_edges, 0);
3055 assert_eq!(plan.ordering_summary.cross_group_parent_edges, 1);
3056 assert_eq!(ordered[0].source_canister, ROOT);
3057 assert_eq!(ordered[1].source_canister, CHILD);
3058 assert_eq!(
3059 ordered[1].ordering_dependency,
3060 Some(RestoreOrderingDependency {
3061 source_canister: ROOT.to_string(),
3062 target_canister: ROOT.to_string(),
3063 relationship: RestoreOrderingRelationship::ParentInEarlierGroup,
3064 })
3065 );
3066 }
3067
3068 #[test]
3070 fn plan_rejects_parent_in_later_restore_group() {
3071 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3072 manifest.fleet.members[0].restore_group = 1;
3073 manifest.fleet.members[1].restore_group = 2;
3074
3075 let err = RestorePlanner::plan(&manifest, None)
3076 .expect_err("parent-after-child group ordering should fail");
3077
3078 assert!(matches!(
3079 err,
3080 RestorePlanError::ParentRestoreGroupAfterChild { .. }
3081 ));
3082 }
3083
3084 #[test]
3086 fn fixed_identity_member_cannot_be_remapped() {
3087 let manifest = valid_manifest(IdentityMode::Fixed);
3088 let mapping = RestoreMapping {
3089 members: vec![
3090 RestoreMappingEntry {
3091 source_canister: ROOT.to_string(),
3092 target_canister: ROOT.to_string(),
3093 },
3094 RestoreMappingEntry {
3095 source_canister: CHILD.to_string(),
3096 target_canister: TARGET.to_string(),
3097 },
3098 ],
3099 };
3100
3101 let err = RestorePlanner::plan(&manifest, Some(&mapping))
3102 .expect_err("fixed member remap should fail");
3103
3104 assert!(matches!(err, RestorePlanError::FixedIdentityRemap { .. }));
3105 }
3106
3107 #[test]
3109 fn relocatable_member_can_be_mapped() {
3110 let manifest = valid_manifest(IdentityMode::Relocatable);
3111 let mapping = RestoreMapping {
3112 members: vec![
3113 RestoreMappingEntry {
3114 source_canister: ROOT.to_string(),
3115 target_canister: ROOT.to_string(),
3116 },
3117 RestoreMappingEntry {
3118 source_canister: CHILD.to_string(),
3119 target_canister: TARGET.to_string(),
3120 },
3121 ],
3122 };
3123
3124 let plan = RestorePlanner::plan(&manifest, Some(&mapping)).expect("plan should build");
3125 let child = plan
3126 .ordered_members()
3127 .into_iter()
3128 .find(|member| member.source_canister == CHILD)
3129 .expect("child member should be planned");
3130
3131 assert_eq!(plan.identity_summary.fixed_members, 1);
3132 assert_eq!(plan.identity_summary.relocatable_members, 1);
3133 assert_eq!(plan.identity_summary.in_place_members, 1);
3134 assert_eq!(plan.identity_summary.mapped_members, 2);
3135 assert_eq!(plan.identity_summary.remapped_members, 1);
3136 assert_eq!(child.target_canister, TARGET);
3137 assert_eq!(child.parent_target_canister, Some(ROOT.to_string()));
3138 }
3139
3140 #[test]
3142 fn plan_members_include_snapshot_and_verification_metadata() {
3143 let manifest = valid_manifest(IdentityMode::Relocatable);
3144
3145 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3146 let root = plan
3147 .ordered_members()
3148 .into_iter()
3149 .find(|member| member.source_canister == ROOT)
3150 .expect("root member should be planned");
3151
3152 assert_eq!(root.identity_mode, IdentityMode::Fixed);
3153 assert_eq!(root.verification_class, "basic");
3154 assert_eq!(root.verification_checks[0].kind, "call");
3155 assert_eq!(root.source_snapshot.snapshot_id, "snap-root");
3156 assert_eq!(root.source_snapshot.artifact_path, "artifacts/root");
3157 }
3158
3159 #[test]
3161 fn plan_includes_mapping_summary() {
3162 let manifest = valid_manifest(IdentityMode::Relocatable);
3163 let in_place = RestorePlanner::plan(&manifest, None).expect("plan should build");
3164
3165 assert!(!in_place.identity_summary.mapping_supplied);
3166 assert!(!in_place.identity_summary.all_sources_mapped);
3167 assert_eq!(in_place.identity_summary.mapped_members, 0);
3168
3169 let mapping = RestoreMapping {
3170 members: vec![
3171 RestoreMappingEntry {
3172 source_canister: ROOT.to_string(),
3173 target_canister: ROOT.to_string(),
3174 },
3175 RestoreMappingEntry {
3176 source_canister: CHILD.to_string(),
3177 target_canister: TARGET.to_string(),
3178 },
3179 ],
3180 };
3181 let mapped = RestorePlanner::plan(&manifest, Some(&mapping)).expect("plan should build");
3182
3183 assert!(mapped.identity_summary.mapping_supplied);
3184 assert!(mapped.identity_summary.all_sources_mapped);
3185 assert_eq!(mapped.identity_summary.mapped_members, 2);
3186 assert_eq!(mapped.identity_summary.remapped_members, 1);
3187 }
3188
3189 #[test]
3191 fn plan_includes_snapshot_summary() {
3192 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3193 manifest.fleet.members[1].source_snapshot.module_hash = None;
3194 manifest.fleet.members[1].source_snapshot.wasm_hash = None;
3195 manifest.fleet.members[1].source_snapshot.checksum = None;
3196
3197 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3198
3199 assert!(!plan.snapshot_summary.all_members_have_module_hash);
3200 assert!(!plan.snapshot_summary.all_members_have_wasm_hash);
3201 assert!(plan.snapshot_summary.all_members_have_code_version);
3202 assert!(!plan.snapshot_summary.all_members_have_checksum);
3203 assert_eq!(plan.snapshot_summary.members_with_module_hash, 1);
3204 assert_eq!(plan.snapshot_summary.members_with_wasm_hash, 1);
3205 assert_eq!(plan.snapshot_summary.members_with_code_version, 2);
3206 assert_eq!(plan.snapshot_summary.members_with_checksum, 1);
3207 assert!(!plan.readiness_summary.ready);
3208 assert_eq!(
3209 plan.readiness_summary.reasons,
3210 [
3211 "missing-module-hash",
3212 "missing-wasm-hash",
3213 "missing-snapshot-checksum"
3214 ]
3215 );
3216 }
3217
3218 #[test]
3220 fn plan_includes_verification_summary() {
3221 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3222 manifest.verification.fleet_checks.push(VerificationCheck {
3223 kind: "fleet-ready".to_string(),
3224 method: None,
3225 roles: Vec::new(),
3226 });
3227 manifest
3228 .verification
3229 .member_checks
3230 .push(MemberVerificationChecks {
3231 role: "app".to_string(),
3232 checks: vec![VerificationCheck {
3233 kind: "app-ready".to_string(),
3234 method: Some("ready".to_string()),
3235 roles: Vec::new(),
3236 }],
3237 });
3238
3239 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3240
3241 assert!(plan.verification_summary.verification_required);
3242 assert!(plan.verification_summary.all_members_have_checks);
3243 let app = plan
3244 .ordered_members()
3245 .into_iter()
3246 .find(|member| member.role == "app")
3247 .expect("app member should be planned");
3248 assert_eq!(app.verification_checks.len(), 2);
3249 assert_eq!(plan.fleet_verification_checks.len(), 1);
3250 assert_eq!(plan.fleet_verification_checks[0].kind, "fleet-ready");
3251 assert_eq!(plan.verification_summary.fleet_checks, 1);
3252 assert_eq!(plan.verification_summary.member_check_groups, 1);
3253 assert_eq!(plan.verification_summary.member_checks, 3);
3254 assert_eq!(plan.verification_summary.members_with_checks, 2);
3255 assert_eq!(plan.verification_summary.total_checks, 4);
3256 }
3257
3258 #[test]
3260 fn plan_includes_operation_summary() {
3261 let manifest = valid_manifest(IdentityMode::Relocatable);
3262
3263 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3264
3265 assert_eq!(plan.operation_summary.planned_snapshot_uploads, 2);
3266 assert_eq!(plan.operation_summary.planned_snapshot_loads, 2);
3267 assert_eq!(plan.operation_summary.planned_code_reinstalls, 2);
3268 assert_eq!(plan.operation_summary.planned_verification_checks, 2);
3269 assert_eq!(plan.operation_summary.planned_operations, 8);
3270 assert_eq!(plan.operation_summary.planned_phases, 1);
3271 }
3272
3273 #[test]
3275 fn restore_plan_defaults_missing_newer_restore_fields() {
3276 let manifest = valid_manifest(IdentityMode::Relocatable);
3277 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3278 let mut value = serde_json::to_value(&plan).expect("serialize plan");
3279 value
3280 .as_object_mut()
3281 .expect("plan should serialize as an object")
3282 .remove("fleet_verification_checks");
3283 let operation_summary = value
3284 .get_mut("operation_summary")
3285 .and_then(serde_json::Value::as_object_mut)
3286 .expect("operation summary should serialize as an object");
3287 operation_summary.remove("planned_snapshot_uploads");
3288 operation_summary.remove("planned_operations");
3289
3290 let decoded: RestorePlan = serde_json::from_value(value).expect("decode old plan shape");
3291 let status = RestoreStatus::from_plan(&decoded);
3292 let dry_run =
3293 RestoreApplyDryRun::try_from_plan(&decoded, None).expect("old plan should dry-run");
3294
3295 assert!(decoded.fleet_verification_checks.is_empty());
3296 assert_eq!(decoded.operation_summary.planned_snapshot_uploads, 0);
3297 assert_eq!(decoded.operation_summary.planned_operations, 0);
3298 assert_eq!(status.planned_snapshot_uploads, 2);
3299 assert_eq!(status.planned_operations, 8);
3300 assert_eq!(dry_run.planned_snapshot_uploads, 2);
3301 assert_eq!(dry_run.planned_operations, 8);
3302 assert_eq!(decoded.backup_id, plan.backup_id);
3303 assert_eq!(decoded.member_count, plan.member_count);
3304 }
3305
3306 #[test]
3308 fn restore_status_starts_all_members_as_planned() {
3309 let manifest = valid_manifest(IdentityMode::Relocatable);
3310
3311 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3312 let status = RestoreStatus::from_plan(&plan);
3313
3314 assert_eq!(status.status_version, 1);
3315 assert_eq!(status.backup_id.as_str(), plan.backup_id.as_str());
3316 assert_eq!(
3317 status.source_environment.as_str(),
3318 plan.source_environment.as_str()
3319 );
3320 assert_eq!(
3321 status.source_root_canister.as_str(),
3322 plan.source_root_canister.as_str()
3323 );
3324 assert_eq!(status.topology_hash.as_str(), plan.topology_hash.as_str());
3325 assert!(status.ready);
3326 assert!(status.readiness_reasons.is_empty());
3327 assert!(status.verification_required);
3328 assert_eq!(status.member_count, 2);
3329 assert_eq!(status.phase_count, 1);
3330 assert_eq!(status.planned_snapshot_uploads, 2);
3331 assert_eq!(status.planned_snapshot_loads, 2);
3332 assert_eq!(status.planned_code_reinstalls, 2);
3333 assert_eq!(status.planned_verification_checks, 2);
3334 assert_eq!(status.planned_operations, 8);
3335 assert_eq!(status.phases.len(), 1);
3336 assert_eq!(status.phases[0].restore_group, 1);
3337 assert_eq!(status.phases[0].members.len(), 2);
3338 assert_eq!(
3339 status.phases[0].members[0].state,
3340 RestoreMemberState::Planned
3341 );
3342 assert_eq!(status.phases[0].members[0].source_canister, ROOT);
3343 assert_eq!(status.phases[0].members[0].target_canister, ROOT);
3344 assert_eq!(status.phases[0].members[0].snapshot_id, "snap-root");
3345 assert_eq!(status.phases[0].members[0].artifact_path, "artifacts/root");
3346 assert_eq!(
3347 status.phases[0].members[1].state,
3348 RestoreMemberState::Planned
3349 );
3350 assert_eq!(status.phases[0].members[1].source_canister, CHILD);
3351 }
3352
3353 #[test]
3355 fn apply_dry_run_renders_ordered_member_operations() {
3356 let manifest = valid_manifest(IdentityMode::Relocatable);
3357
3358 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3359 let status = RestoreStatus::from_plan(&plan);
3360 let dry_run =
3361 RestoreApplyDryRun::try_from_plan(&plan, Some(&status)).expect("dry-run should build");
3362
3363 assert_eq!(dry_run.dry_run_version, 1);
3364 assert_eq!(dry_run.backup_id.as_str(), "fbk_test_001");
3365 assert!(dry_run.ready);
3366 assert!(dry_run.status_supplied);
3367 assert_eq!(dry_run.member_count, 2);
3368 assert_eq!(dry_run.phase_count, 1);
3369 assert_eq!(dry_run.planned_snapshot_uploads, 2);
3370 assert_eq!(dry_run.planned_snapshot_loads, 2);
3371 assert_eq!(dry_run.planned_code_reinstalls, 2);
3372 assert_eq!(dry_run.planned_verification_checks, 2);
3373 assert_eq!(dry_run.planned_operations, 8);
3374 assert_eq!(dry_run.rendered_operations, 8);
3375 assert_eq!(dry_run.operation_counts.snapshot_uploads, 2);
3376 assert_eq!(dry_run.operation_counts.snapshot_loads, 2);
3377 assert_eq!(dry_run.operation_counts.code_reinstalls, 2);
3378 assert_eq!(dry_run.operation_counts.member_verifications, 2);
3379 assert_eq!(dry_run.operation_counts.fleet_verifications, 0);
3380 assert_eq!(dry_run.operation_counts.verification_operations, 2);
3381 assert_eq!(dry_run.phases.len(), 1);
3382
3383 let operations = &dry_run.phases[0].operations;
3384 assert_eq!(operations[0].sequence, 0);
3385 assert_eq!(
3386 operations[0].operation,
3387 RestoreApplyOperationKind::UploadSnapshot
3388 );
3389 assert_eq!(operations[0].source_canister, ROOT);
3390 assert_eq!(operations[0].target_canister, ROOT);
3391 assert_eq!(operations[0].snapshot_id, Some("snap-root".to_string()));
3392 assert_eq!(
3393 operations[0].artifact_path,
3394 Some("artifacts/root".to_string())
3395 );
3396 assert_eq!(
3397 operations[1].operation,
3398 RestoreApplyOperationKind::LoadSnapshot
3399 );
3400 assert_eq!(
3401 operations[2].operation,
3402 RestoreApplyOperationKind::ReinstallCode
3403 );
3404 assert_eq!(
3405 operations[3].operation,
3406 RestoreApplyOperationKind::VerifyMember
3407 );
3408 assert_eq!(operations[3].verification_kind, Some("call".to_string()));
3409 assert_eq!(
3410 operations[3].verification_method,
3411 Some("canic_ready".to_string())
3412 );
3413 assert_eq!(operations[4].source_canister, CHILD);
3414 assert_eq!(
3415 operations[7].operation,
3416 RestoreApplyOperationKind::VerifyMember
3417 );
3418 }
3419
3420 #[test]
3422 fn apply_dry_run_renders_fleet_verification_operations() {
3423 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3424 manifest.verification.fleet_checks.push(VerificationCheck {
3425 kind: "fleet-ready".to_string(),
3426 method: Some("canic_fleet_ready".to_string()),
3427 roles: Vec::new(),
3428 });
3429
3430 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3431 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3432
3433 assert_eq!(plan.operation_summary.planned_verification_checks, 3);
3434 assert_eq!(dry_run.rendered_operations, 9);
3435 let operation = dry_run.phases[0]
3436 .operations
3437 .last()
3438 .expect("fleet verification operation should be rendered");
3439 assert_eq!(operation.sequence, 8);
3440 assert_eq!(operation.operation, RestoreApplyOperationKind::VerifyFleet);
3441 assert_eq!(operation.source_canister, ROOT);
3442 assert_eq!(operation.target_canister, ROOT);
3443 assert_eq!(operation.role, "fleet");
3444 assert_eq!(operation.snapshot_id, None);
3445 assert_eq!(operation.artifact_path, None);
3446 assert_eq!(operation.verification_kind, Some("fleet-ready".to_string()));
3447 assert_eq!(
3448 operation.verification_method,
3449 Some("canic_fleet_ready".to_string())
3450 );
3451 }
3452
3453 #[test]
3455 fn apply_dry_run_sequences_operations_across_phases() {
3456 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3457 manifest.fleet.members[0].restore_group = 2;
3458
3459 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3460 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3461
3462 assert_eq!(dry_run.phases.len(), 2);
3463 assert_eq!(dry_run.rendered_operations, 8);
3464 assert_eq!(dry_run.phases[0].operations[0].sequence, 0);
3465 assert_eq!(dry_run.phases[0].operations[3].sequence, 3);
3466 assert_eq!(dry_run.phases[1].operations[0].sequence, 4);
3467 assert_eq!(dry_run.phases[1].operations[3].sequence, 7);
3468 }
3469
3470 #[test]
3472 fn apply_dry_run_validates_artifacts_under_backup_root() {
3473 let root = temp_dir("canic-restore-apply-artifacts-ok");
3474 fs::create_dir_all(&root).expect("create temp root");
3475 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3476 set_member_artifact(
3477 &mut manifest,
3478 CHILD,
3479 &root,
3480 "artifacts/child",
3481 b"child-snapshot",
3482 );
3483 set_member_artifact(
3484 &mut manifest,
3485 ROOT,
3486 &root,
3487 "artifacts/root",
3488 b"root-snapshot",
3489 );
3490
3491 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3492 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3493 .expect("dry-run should validate artifacts");
3494
3495 let validation = dry_run
3496 .artifact_validation
3497 .expect("artifact validation should be present");
3498 assert_eq!(validation.checked_members, 2);
3499 assert!(validation.artifacts_present);
3500 assert!(validation.checksums_verified);
3501 assert_eq!(validation.members_with_expected_checksums, 2);
3502 assert_eq!(validation.checks[0].source_canister, ROOT);
3503 assert!(validation.checks[0].checksum_verified);
3504
3505 fs::remove_dir_all(root).expect("remove temp root");
3506 }
3507
3508 #[test]
3510 fn apply_journal_marks_validated_operations_ready() {
3511 let root = temp_dir("canic-restore-apply-journal-ready");
3512 fs::create_dir_all(&root).expect("create temp root");
3513 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3514 set_member_artifact(
3515 &mut manifest,
3516 CHILD,
3517 &root,
3518 "artifacts/child",
3519 b"child-snapshot",
3520 );
3521 set_member_artifact(
3522 &mut manifest,
3523 ROOT,
3524 &root,
3525 "artifacts/root",
3526 b"root-snapshot",
3527 );
3528
3529 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3530 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3531 .expect("dry-run should validate artifacts");
3532 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3533
3534 fs::remove_dir_all(root).expect("remove temp root");
3535 assert_eq!(journal.journal_version, 1);
3536 assert_eq!(journal.backup_id.as_str(), "fbk_test_001");
3537 assert!(journal.ready);
3538 assert!(journal.blocked_reasons.is_empty());
3539 assert_eq!(journal.operation_count, 8);
3540 assert_eq!(journal.ready_operations, 8);
3541 assert_eq!(journal.blocked_operations, 0);
3542 assert_eq!(journal.operations[0].sequence, 0);
3543 assert_eq!(
3544 journal.operations[0].state,
3545 RestoreApplyOperationState::Ready
3546 );
3547 assert!(journal.operations[0].blocking_reasons.is_empty());
3548 }
3549
3550 #[test]
3552 fn apply_journal_blocks_without_artifact_validation() {
3553 let manifest = valid_manifest(IdentityMode::Relocatable);
3554
3555 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3556 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3557 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3558
3559 assert!(!journal.ready);
3560 assert_eq!(journal.ready_operations, 0);
3561 assert_eq!(journal.blocked_operations, 8);
3562 assert!(
3563 journal
3564 .blocked_reasons
3565 .contains(&"missing-artifact-validation".to_string())
3566 );
3567 assert!(
3568 journal.operations[0]
3569 .blocking_reasons
3570 .contains(&"missing-artifact-validation".to_string())
3571 );
3572 }
3573
3574 #[test]
3576 fn apply_journal_status_reports_next_ready_operation() {
3577 let root = temp_dir("canic-restore-apply-journal-status");
3578 fs::create_dir_all(&root).expect("create temp root");
3579 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3580 set_member_artifact(
3581 &mut manifest,
3582 CHILD,
3583 &root,
3584 "artifacts/child",
3585 b"child-snapshot",
3586 );
3587 set_member_artifact(
3588 &mut manifest,
3589 ROOT,
3590 &root,
3591 "artifacts/root",
3592 b"root-snapshot",
3593 );
3594
3595 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3596 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3597 .expect("dry-run should validate artifacts");
3598 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3599 let status = journal.status();
3600 let report = journal.report();
3601
3602 fs::remove_dir_all(root).expect("remove temp root");
3603 assert_eq!(status.status_version, 1);
3604 assert_eq!(status.backup_id.as_str(), "fbk_test_001");
3605 assert!(status.ready);
3606 assert!(!status.complete);
3607 assert_eq!(status.operation_count, 8);
3608 assert_eq!(status.operation_counts.snapshot_uploads, 2);
3609 assert_eq!(status.operation_counts.snapshot_loads, 2);
3610 assert_eq!(status.operation_counts.code_reinstalls, 2);
3611 assert_eq!(status.operation_counts.member_verifications, 2);
3612 assert_eq!(status.operation_counts.fleet_verifications, 0);
3613 assert_eq!(status.operation_counts.verification_operations, 2);
3614 assert!(status.operation_counts_supplied);
3615 assert_eq!(journal.operation_counts, status.operation_counts);
3616 assert_eq!(report.operation_counts, status.operation_counts);
3617 assert!(report.operation_counts_supplied);
3618 assert_eq!(status.progress.operation_count, 8);
3619 assert_eq!(status.progress.completed_operations, 0);
3620 assert_eq!(status.progress.remaining_operations, 8);
3621 assert_eq!(status.progress.transitionable_operations, 8);
3622 assert_eq!(status.progress.attention_operations, 0);
3623 assert_eq!(status.progress.completion_basis_points, 0);
3624 assert_eq!(report.progress, status.progress);
3625 assert_eq!(status.pending_summary.pending_operations, 0);
3626 assert!(!status.pending_summary.pending_operation_available);
3627 assert_eq!(status.pending_summary.pending_sequence, None);
3628 assert_eq!(status.pending_summary.pending_operation, None);
3629 assert_eq!(status.pending_summary.pending_updated_at, None);
3630 assert!(!status.pending_summary.pending_updated_at_known);
3631 assert_eq!(report.pending_summary, status.pending_summary);
3632 assert_eq!(status.ready_operations, 8);
3633 assert_eq!(status.next_ready_sequence, Some(0));
3634 assert_eq!(
3635 status.next_ready_operation,
3636 Some(RestoreApplyOperationKind::UploadSnapshot)
3637 );
3638 assert_eq!(status.next_transition_sequence, Some(0));
3639 assert_eq!(
3640 status.next_transition_state,
3641 Some(RestoreApplyOperationState::Ready)
3642 );
3643 assert_eq!(
3644 status.next_transition_operation,
3645 Some(RestoreApplyOperationKind::UploadSnapshot)
3646 );
3647 }
3648
3649 #[test]
3651 fn apply_journal_next_operation_reports_full_ready_row() {
3652 let root = temp_dir("canic-restore-apply-journal-next");
3653 fs::create_dir_all(&root).expect("create temp root");
3654 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3655 set_member_artifact(
3656 &mut manifest,
3657 CHILD,
3658 &root,
3659 "artifacts/child",
3660 b"child-snapshot",
3661 );
3662 set_member_artifact(
3663 &mut manifest,
3664 ROOT,
3665 &root,
3666 "artifacts/root",
3667 b"root-snapshot",
3668 );
3669
3670 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3671 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3672 .expect("dry-run should validate artifacts");
3673 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3674 journal
3675 .mark_operation_completed(0)
3676 .expect("mark operation completed");
3677 let next = journal.next_operation();
3678
3679 fs::remove_dir_all(root).expect("remove temp root");
3680 assert!(next.ready);
3681 assert!(!next.complete);
3682 assert!(next.operation_available);
3683 let operation = next.operation.expect("next operation");
3684 assert_eq!(operation.sequence, 1);
3685 assert_eq!(operation.state, RestoreApplyOperationState::Ready);
3686 assert_eq!(operation.operation, RestoreApplyOperationKind::LoadSnapshot);
3687 assert_eq!(operation.source_canister, ROOT);
3688 }
3689
3690 #[test]
3692 fn apply_journal_next_operation_reports_blocked_state() {
3693 let manifest = valid_manifest(IdentityMode::Relocatable);
3694
3695 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3696 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
3697 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3698 let next = journal.next_operation();
3699
3700 assert!(!next.ready);
3701 assert!(!next.operation_available);
3702 assert!(next.operation.is_none());
3703 assert!(
3704 next.blocked_reasons
3705 .contains(&"missing-artifact-validation".to_string())
3706 );
3707 }
3708
3709 #[test]
3711 fn apply_journal_command_preview_reports_upload_command() {
3712 let root = temp_dir("canic-restore-apply-command-upload");
3713 fs::create_dir_all(&root).expect("create temp root");
3714 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3715 set_member_artifact(
3716 &mut manifest,
3717 CHILD,
3718 &root,
3719 "artifacts/child",
3720 b"child-snapshot",
3721 );
3722 set_member_artifact(
3723 &mut manifest,
3724 ROOT,
3725 &root,
3726 "artifacts/root",
3727 b"root-snapshot",
3728 );
3729
3730 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3731 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3732 .expect("dry-run should validate artifacts");
3733 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3734 let preview = journal.next_command_preview();
3735
3736 fs::remove_dir_all(root).expect("remove temp root");
3737 assert!(preview.ready);
3738 assert!(preview.operation_available);
3739 assert!(preview.command_available);
3740 let command = preview.command.expect("command preview");
3741 assert_eq!(command.program, "dfx");
3742 assert_eq!(
3743 command.args,
3744 vec![
3745 "canister".to_string(),
3746 "snapshot".to_string(),
3747 "upload".to_string(),
3748 "--dir".to_string(),
3749 "artifacts/root".to_string(),
3750 ROOT.to_string(),
3751 ]
3752 );
3753 assert!(command.mutates);
3754 assert!(!command.requires_stopped_canister);
3755 }
3756
3757 #[test]
3759 fn apply_journal_command_preview_honors_command_config() {
3760 let root = temp_dir("canic-restore-apply-command-config");
3761 fs::create_dir_all(&root).expect("create temp root");
3762 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3763 set_member_artifact(
3764 &mut manifest,
3765 CHILD,
3766 &root,
3767 "artifacts/child",
3768 b"child-snapshot",
3769 );
3770 set_member_artifact(
3771 &mut manifest,
3772 ROOT,
3773 &root,
3774 "artifacts/root",
3775 b"root-snapshot",
3776 );
3777
3778 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3779 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3780 .expect("dry-run should validate artifacts");
3781 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3782 let preview = journal.next_command_preview_with_config(&RestoreApplyCommandConfig {
3783 program: "/tmp/dfx".to_string(),
3784 network: Some("local".to_string()),
3785 });
3786
3787 fs::remove_dir_all(root).expect("remove temp root");
3788 let command = preview.command.expect("command preview");
3789 assert_eq!(command.program, "/tmp/dfx");
3790 assert_eq!(
3791 command.args,
3792 vec![
3793 "canister".to_string(),
3794 "--network".to_string(),
3795 "local".to_string(),
3796 "snapshot".to_string(),
3797 "upload".to_string(),
3798 "--dir".to_string(),
3799 "artifacts/root".to_string(),
3800 ROOT.to_string(),
3801 ]
3802 );
3803 }
3804
3805 #[test]
3807 fn apply_journal_command_preview_reports_load_command() {
3808 let root = temp_dir("canic-restore-apply-command-load");
3809 fs::create_dir_all(&root).expect("create temp root");
3810 let mut manifest = valid_manifest(IdentityMode::Relocatable);
3811 set_member_artifact(
3812 &mut manifest,
3813 CHILD,
3814 &root,
3815 "artifacts/child",
3816 b"child-snapshot",
3817 );
3818 set_member_artifact(
3819 &mut manifest,
3820 ROOT,
3821 &root,
3822 "artifacts/root",
3823 b"root-snapshot",
3824 );
3825
3826 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
3827 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
3828 .expect("dry-run should validate artifacts");
3829 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3830 journal
3831 .mark_operation_completed(0)
3832 .expect("mark upload completed");
3833 let preview = journal.next_command_preview();
3834
3835 fs::remove_dir_all(root).expect("remove temp root");
3836 let command = preview.command.expect("command preview");
3837 assert_eq!(
3838 command.args,
3839 vec![
3840 "canister".to_string(),
3841 "snapshot".to_string(),
3842 "load".to_string(),
3843 ROOT.to_string(),
3844 "snap-root".to_string(),
3845 ]
3846 );
3847 assert!(command.mutates);
3848 assert!(command.requires_stopped_canister);
3849 }
3850
3851 #[test]
3853 fn apply_journal_command_preview_reports_reinstall_command() {
3854 let journal = command_preview_journal(RestoreApplyOperationKind::ReinstallCode, None, None);
3855 let preview = journal.next_command_preview_with_config(&RestoreApplyCommandConfig {
3856 program: "dfx".to_string(),
3857 network: Some("local".to_string()),
3858 });
3859
3860 assert!(preview.command_available);
3861 let command = preview.command.expect("command preview");
3862 assert_eq!(
3863 command.args,
3864 vec![
3865 "canister".to_string(),
3866 "--network".to_string(),
3867 "local".to_string(),
3868 "install".to_string(),
3869 "--mode".to_string(),
3870 "reinstall".to_string(),
3871 "--yes".to_string(),
3872 ROOT.to_string(),
3873 ]
3874 );
3875 assert!(command.mutates);
3876 assert!(!command.requires_stopped_canister);
3877 }
3878
3879 #[test]
3881 fn apply_journal_command_preview_reports_status_verification_command() {
3882 let journal = command_preview_journal(
3883 RestoreApplyOperationKind::VerifyMember,
3884 Some("status"),
3885 None,
3886 );
3887 let preview = journal.next_command_preview();
3888
3889 assert!(preview.command_available);
3890 let command = preview.command.expect("command preview");
3891 assert_eq!(
3892 command.args,
3893 vec![
3894 "canister".to_string(),
3895 "status".to_string(),
3896 ROOT.to_string()
3897 ]
3898 );
3899 assert!(!command.mutates);
3900 assert!(!command.requires_stopped_canister);
3901 }
3902
3903 #[test]
3905 fn apply_journal_command_preview_reports_method_verification_command() {
3906 let journal = command_preview_journal(
3907 RestoreApplyOperationKind::VerifyMember,
3908 Some("query"),
3909 Some("health"),
3910 );
3911 let preview = journal.next_command_preview();
3912
3913 assert!(preview.command_available);
3914 let command = preview.command.expect("command preview");
3915 assert_eq!(
3916 command.args,
3917 vec![
3918 "canister".to_string(),
3919 "call".to_string(),
3920 ROOT.to_string(),
3921 "health".to_string(),
3922 ]
3923 );
3924 assert!(!command.mutates);
3925 assert!(!command.requires_stopped_canister);
3926 }
3927
3928 #[test]
3930 fn apply_journal_command_preview_reports_fleet_verification_command() {
3931 let journal = command_preview_journal(
3932 RestoreApplyOperationKind::VerifyFleet,
3933 Some("fleet-ready"),
3934 Some("canic_fleet_ready"),
3935 );
3936 let preview = journal.next_command_preview();
3937
3938 assert!(preview.command_available);
3939 let command = preview.command.expect("command preview");
3940 assert_eq!(
3941 command.args,
3942 vec![
3943 "canister".to_string(),
3944 "call".to_string(),
3945 ROOT.to_string(),
3946 "canic_fleet_ready".to_string(),
3947 ]
3948 );
3949 assert!(!command.mutates);
3950 assert!(!command.requires_stopped_canister);
3951 assert_eq!(command.note, "calls the declared fleet verification method");
3952 }
3953
3954 #[test]
3956 fn apply_journal_validation_rejects_method_verification_without_method() {
3957 let journal = RestoreApplyJournal {
3958 journal_version: 1,
3959 backup_id: "fbk_test_001".to_string(),
3960 ready: true,
3961 blocked_reasons: Vec::new(),
3962 operation_count: 1,
3963 operation_counts: RestoreApplyOperationKindCounts::default(),
3964 pending_operations: 0,
3965 ready_operations: 1,
3966 blocked_operations: 0,
3967 completed_operations: 0,
3968 failed_operations: 0,
3969 operations: vec![RestoreApplyJournalOperation {
3970 sequence: 0,
3971 operation: RestoreApplyOperationKind::VerifyMember,
3972 state: RestoreApplyOperationState::Ready,
3973 state_updated_at: None,
3974 blocking_reasons: Vec::new(),
3975 restore_group: 1,
3976 phase_order: 0,
3977 source_canister: ROOT.to_string(),
3978 target_canister: ROOT.to_string(),
3979 role: "root".to_string(),
3980 snapshot_id: Some("snap-root".to_string()),
3981 artifact_path: Some("artifacts/root".to_string()),
3982 verification_kind: Some("query".to_string()),
3983 verification_method: None,
3984 }],
3985 };
3986
3987 let err = journal
3988 .validate()
3989 .expect_err("method verification without method should fail");
3990
3991 assert!(matches!(
3992 err,
3993 RestoreApplyJournalError::OperationMissingField {
3994 sequence: 0,
3995 operation: RestoreApplyOperationKind::VerifyMember,
3996 field: "operations[].verification_method",
3997 }
3998 ));
3999 }
4000
4001 #[test]
4003 fn apply_journal_validation_rejects_count_mismatch() {
4004 let manifest = valid_manifest(IdentityMode::Relocatable);
4005
4006 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4007 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4008 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4009 journal.blocked_operations = 0;
4010
4011 let err = journal.validate().expect_err("count mismatch should fail");
4012
4013 assert!(matches!(
4014 err,
4015 RestoreApplyJournalError::CountMismatch {
4016 field: "blocked_operations",
4017 ..
4018 }
4019 ));
4020 }
4021
4022 #[test]
4024 fn apply_journal_validation_rejects_operation_kind_count_mismatch() {
4025 let mut journal =
4026 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4027 journal.operation_counts = RestoreApplyOperationKindCounts {
4028 snapshot_uploads: 0,
4029 snapshot_loads: 1,
4030 code_reinstalls: 0,
4031 member_verifications: 0,
4032 fleet_verifications: 0,
4033 verification_operations: 0,
4034 };
4035
4036 let err = journal
4037 .validate()
4038 .expect_err("operation-kind count mismatch should fail");
4039
4040 assert!(matches!(
4041 err,
4042 RestoreApplyJournalError::CountMismatch {
4043 field: "operation_counts.snapshot_uploads",
4044 reported: 0,
4045 actual: 1,
4046 }
4047 ));
4048 }
4049
4050 #[test]
4052 fn apply_journal_defaults_missing_operation_kind_counts() {
4053 let mut journal =
4054 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4055 journal.operation_counts =
4056 RestoreApplyOperationKindCounts::from_operations(&journal.operations);
4057 let mut value = serde_json::to_value(&journal).expect("serialize journal");
4058 value
4059 .as_object_mut()
4060 .expect("journal should serialize as an object")
4061 .remove("operation_counts");
4062
4063 let decoded: RestoreApplyJournal =
4064 serde_json::from_value(value).expect("decode old journal shape");
4065 decoded.validate().expect("old journal should validate");
4066 let status = decoded.status();
4067
4068 assert_eq!(
4069 decoded.operation_counts,
4070 RestoreApplyOperationKindCounts::default()
4071 );
4072 assert_eq!(status.operation_counts.snapshot_uploads, 1);
4073 assert_eq!(status.operation_counts.snapshot_loads, 0);
4074 assert!(!status.operation_counts_supplied);
4075 }
4076
4077 #[test]
4079 fn apply_journal_validation_rejects_duplicate_sequences() {
4080 let manifest = valid_manifest(IdentityMode::Relocatable);
4081
4082 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4083 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4084 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4085 journal.operations[1].sequence = journal.operations[0].sequence;
4086
4087 let err = journal
4088 .validate()
4089 .expect_err("duplicate sequence should fail");
4090
4091 assert!(matches!(
4092 err,
4093 RestoreApplyJournalError::DuplicateSequence(0)
4094 ));
4095 }
4096
4097 #[test]
4099 fn apply_journal_validation_rejects_failed_without_reason() {
4100 let manifest = valid_manifest(IdentityMode::Relocatable);
4101
4102 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4103 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4104 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4105 journal.operations[0].state = RestoreApplyOperationState::Failed;
4106 journal.operations[0].blocking_reasons = Vec::new();
4107 journal.blocked_operations -= 1;
4108 journal.failed_operations = 1;
4109
4110 let err = journal
4111 .validate()
4112 .expect_err("failed operation without reason should fail");
4113
4114 assert!(matches!(
4115 err,
4116 RestoreApplyJournalError::FailureReasonRequired(0)
4117 ));
4118 }
4119
4120 #[test]
4122 fn apply_journal_mark_next_operation_pending_claims_first_operation() {
4123 let mut journal =
4124 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4125
4126 journal
4127 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4128 .expect("mark operation pending");
4129 let status = journal.status();
4130 let report = journal.report();
4131 let next = journal.next_operation();
4132 let preview = journal.next_command_preview();
4133
4134 assert_eq!(journal.pending_operations, 1);
4135 assert_eq!(journal.ready_operations, 0);
4136 assert_eq!(
4137 journal.operations[0].state,
4138 RestoreApplyOperationState::Pending
4139 );
4140 assert_eq!(
4141 journal.operations[0].state_updated_at.as_deref(),
4142 Some("2026-05-04T12:00:00Z")
4143 );
4144 assert_eq!(status.next_ready_sequence, None);
4145 assert_eq!(status.next_transition_sequence, Some(0));
4146 assert_eq!(
4147 status.next_transition_state,
4148 Some(RestoreApplyOperationState::Pending)
4149 );
4150 assert_eq!(
4151 status.next_transition_updated_at.as_deref(),
4152 Some("2026-05-04T12:00:00Z")
4153 );
4154 assert_eq!(status.pending_summary.pending_operations, 1);
4155 assert!(status.pending_summary.pending_operation_available);
4156 assert_eq!(status.pending_summary.pending_sequence, Some(0));
4157 assert_eq!(
4158 status.pending_summary.pending_operation,
4159 Some(RestoreApplyOperationKind::UploadSnapshot)
4160 );
4161 assert_eq!(
4162 status.pending_summary.pending_updated_at.as_deref(),
4163 Some("2026-05-04T12:00:00Z")
4164 );
4165 assert!(status.pending_summary.pending_updated_at_known);
4166 assert_eq!(report.pending_summary, status.pending_summary);
4167 assert!(next.operation_available);
4168 assert_eq!(
4169 next.operation.expect("next operation").state,
4170 RestoreApplyOperationState::Pending
4171 );
4172 assert!(preview.operation_available);
4173 assert!(preview.command_available);
4174 assert_eq!(
4175 preview.operation.expect("preview operation").state,
4176 RestoreApplyOperationState::Pending
4177 );
4178 }
4179
4180 #[test]
4182 fn apply_journal_mark_next_operation_ready_unclaims_pending_operation() {
4183 let mut journal =
4184 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4185
4186 journal
4187 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4188 .expect("mark operation pending");
4189 journal
4190 .mark_next_operation_ready_at(Some("2026-05-04T12:01:00Z".to_string()))
4191 .expect("mark operation ready");
4192 let status = journal.status();
4193 let next = journal.next_operation();
4194
4195 assert_eq!(journal.pending_operations, 0);
4196 assert_eq!(journal.ready_operations, 1);
4197 assert_eq!(
4198 journal.operations[0].state,
4199 RestoreApplyOperationState::Ready
4200 );
4201 assert_eq!(
4202 journal.operations[0].state_updated_at.as_deref(),
4203 Some("2026-05-04T12:01:00Z")
4204 );
4205 assert_eq!(status.next_ready_sequence, Some(0));
4206 assert_eq!(status.next_transition_sequence, Some(0));
4207 assert_eq!(
4208 status.next_transition_state,
4209 Some(RestoreApplyOperationState::Ready)
4210 );
4211 assert_eq!(
4212 status.next_transition_updated_at.as_deref(),
4213 Some("2026-05-04T12:01:00Z")
4214 );
4215 assert_eq!(
4216 next.operation.expect("next operation").state,
4217 RestoreApplyOperationState::Ready
4218 );
4219 }
4220
4221 #[test]
4223 fn apply_journal_validation_rejects_empty_state_updated_at() {
4224 let mut journal =
4225 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4226
4227 journal.operations[0].state_updated_at = Some(String::new());
4228 let err = journal
4229 .validate()
4230 .expect_err("empty state update marker should fail");
4231
4232 assert!(matches!(
4233 err,
4234 RestoreApplyJournalError::MissingField("operations[].state_updated_at")
4235 ));
4236 }
4237
4238 #[test]
4240 fn apply_journal_validation_rejects_missing_operation_fields() {
4241 let mut upload =
4242 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4243 upload.operations[0].artifact_path = None;
4244 let err = upload
4245 .validate()
4246 .expect_err("upload without artifact path should fail");
4247 assert!(matches!(
4248 err,
4249 RestoreApplyJournalError::OperationMissingField {
4250 sequence: 0,
4251 operation: RestoreApplyOperationKind::UploadSnapshot,
4252 field: "operations[].artifact_path",
4253 }
4254 ));
4255
4256 let mut load = command_preview_journal(RestoreApplyOperationKind::LoadSnapshot, None, None);
4257 load.operations[0].snapshot_id = None;
4258 let err = load
4259 .validate()
4260 .expect_err("load without snapshot id should fail");
4261 assert!(matches!(
4262 err,
4263 RestoreApplyJournalError::OperationMissingField {
4264 sequence: 0,
4265 operation: RestoreApplyOperationKind::LoadSnapshot,
4266 field: "operations[].snapshot_id",
4267 }
4268 ));
4269
4270 let mut verify = command_preview_journal(
4271 RestoreApplyOperationKind::VerifyMember,
4272 Some("query"),
4273 Some("health"),
4274 );
4275 verify.operations[0].verification_method = None;
4276 let err = verify
4277 .validate()
4278 .expect_err("method verification without method should fail");
4279 assert!(matches!(
4280 err,
4281 RestoreApplyJournalError::OperationMissingField {
4282 sequence: 0,
4283 operation: RestoreApplyOperationKind::VerifyMember,
4284 field: "operations[].verification_method",
4285 }
4286 ));
4287 }
4288
4289 #[test]
4291 fn apply_journal_mark_next_operation_ready_rejects_without_pending_operation() {
4292 let mut journal =
4293 command_preview_journal(RestoreApplyOperationKind::UploadSnapshot, None, None);
4294
4295 let err = journal
4296 .mark_next_operation_ready()
4297 .expect_err("ready operation should not unclaim");
4298
4299 assert!(matches!(err, RestoreApplyJournalError::NoPendingOperation));
4300 assert_eq!(journal.ready_operations, 1);
4301 assert_eq!(journal.pending_operations, 0);
4302 }
4303
4304 #[test]
4306 fn apply_journal_mark_pending_rejects_out_of_order_operation() {
4307 let root = temp_dir("canic-restore-apply-journal-pending-out-of-order");
4308 fs::create_dir_all(&root).expect("create temp root");
4309 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4310 set_member_artifact(
4311 &mut manifest,
4312 CHILD,
4313 &root,
4314 "artifacts/child",
4315 b"child-snapshot",
4316 );
4317 set_member_artifact(
4318 &mut manifest,
4319 ROOT,
4320 &root,
4321 "artifacts/root",
4322 b"root-snapshot",
4323 );
4324
4325 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4326 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4327 .expect("dry-run should validate artifacts");
4328 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4329
4330 let err = journal
4331 .mark_operation_pending(1)
4332 .expect_err("out-of-order pending claim should fail");
4333
4334 fs::remove_dir_all(root).expect("remove temp root");
4335 assert!(matches!(
4336 err,
4337 RestoreApplyJournalError::OutOfOrderOperationTransition {
4338 requested: 1,
4339 next: 0
4340 }
4341 ));
4342 assert_eq!(journal.pending_operations, 0);
4343 assert_eq!(journal.ready_operations, 8);
4344 }
4345
4346 #[test]
4348 fn apply_journal_mark_completed_advances_next_ready_operation() {
4349 let root = temp_dir("canic-restore-apply-journal-completed");
4350 fs::create_dir_all(&root).expect("create temp root");
4351 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4352 set_member_artifact(
4353 &mut manifest,
4354 CHILD,
4355 &root,
4356 "artifacts/child",
4357 b"child-snapshot",
4358 );
4359 set_member_artifact(
4360 &mut manifest,
4361 ROOT,
4362 &root,
4363 "artifacts/root",
4364 b"root-snapshot",
4365 );
4366
4367 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4368 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4369 .expect("dry-run should validate artifacts");
4370 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4371
4372 journal
4373 .mark_operation_completed(0)
4374 .expect("mark operation completed");
4375 let status = journal.status();
4376
4377 fs::remove_dir_all(root).expect("remove temp root");
4378 assert_eq!(
4379 journal.operations[0].state,
4380 RestoreApplyOperationState::Completed
4381 );
4382 assert_eq!(journal.completed_operations, 1);
4383 assert_eq!(journal.ready_operations, 7);
4384 assert_eq!(status.next_ready_sequence, Some(1));
4385 assert_eq!(status.progress.completed_operations, 1);
4386 assert_eq!(status.progress.remaining_operations, 7);
4387 assert_eq!(status.progress.transitionable_operations, 7);
4388 assert_eq!(status.progress.attention_operations, 0);
4389 assert_eq!(status.progress.completion_basis_points, 1250);
4390 }
4391
4392 #[test]
4394 fn apply_journal_mark_completed_rejects_out_of_order_operation() {
4395 let root = temp_dir("canic-restore-apply-journal-out-of-order");
4396 fs::create_dir_all(&root).expect("create temp root");
4397 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4398 set_member_artifact(
4399 &mut manifest,
4400 CHILD,
4401 &root,
4402 "artifacts/child",
4403 b"child-snapshot",
4404 );
4405 set_member_artifact(
4406 &mut manifest,
4407 ROOT,
4408 &root,
4409 "artifacts/root",
4410 b"root-snapshot",
4411 );
4412
4413 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4414 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4415 .expect("dry-run should validate artifacts");
4416 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4417
4418 let err = journal
4419 .mark_operation_completed(1)
4420 .expect_err("out-of-order operation should fail");
4421
4422 fs::remove_dir_all(root).expect("remove temp root");
4423 assert!(matches!(
4424 err,
4425 RestoreApplyJournalError::OutOfOrderOperationTransition {
4426 requested: 1,
4427 next: 0
4428 }
4429 ));
4430 assert_eq!(journal.completed_operations, 0);
4431 assert_eq!(journal.ready_operations, 8);
4432 }
4433
4434 #[test]
4436 fn apply_journal_mark_failed_records_reason() {
4437 let root = temp_dir("canic-restore-apply-journal-failed");
4438 fs::create_dir_all(&root).expect("create temp root");
4439 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4440 set_member_artifact(
4441 &mut manifest,
4442 CHILD,
4443 &root,
4444 "artifacts/child",
4445 b"child-snapshot",
4446 );
4447 set_member_artifact(
4448 &mut manifest,
4449 ROOT,
4450 &root,
4451 "artifacts/root",
4452 b"root-snapshot",
4453 );
4454
4455 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4456 let dry_run = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4457 .expect("dry-run should validate artifacts");
4458 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4459
4460 journal
4461 .mark_operation_failed(0, "dfx-load-failed".to_string())
4462 .expect("mark operation failed");
4463
4464 fs::remove_dir_all(root).expect("remove temp root");
4465 assert_eq!(
4466 journal.operations[0].state,
4467 RestoreApplyOperationState::Failed
4468 );
4469 assert_eq!(
4470 journal.operations[0].blocking_reasons,
4471 vec!["dfx-load-failed".to_string()]
4472 );
4473 assert_eq!(journal.failed_operations, 1);
4474 assert_eq!(journal.ready_operations, 7);
4475 }
4476
4477 #[test]
4479 fn apply_journal_rejects_blocked_operation_completion() {
4480 let manifest = valid_manifest(IdentityMode::Relocatable);
4481
4482 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4483 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4484 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4485
4486 let err = journal
4487 .mark_operation_completed(0)
4488 .expect_err("blocked operation should not complete");
4489
4490 assert!(matches!(
4491 err,
4492 RestoreApplyJournalError::InvalidOperationTransition { sequence: 0, .. }
4493 ));
4494 }
4495
4496 #[test]
4498 fn apply_dry_run_rejects_missing_artifacts() {
4499 let root = temp_dir("canic-restore-apply-artifacts-missing");
4500 fs::create_dir_all(&root).expect("create temp root");
4501 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4502 manifest.fleet.members[0].source_snapshot.artifact_path = "missing-child".to_string();
4503
4504 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4505 let err = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4506 .expect_err("missing artifact should fail");
4507
4508 fs::remove_dir_all(root).expect("remove temp root");
4509 assert!(matches!(
4510 err,
4511 RestoreApplyDryRunError::ArtifactMissing { .. }
4512 ));
4513 }
4514
4515 #[test]
4517 fn apply_dry_run_rejects_artifact_path_traversal() {
4518 let root = temp_dir("canic-restore-apply-artifacts-traversal");
4519 fs::create_dir_all(&root).expect("create temp root");
4520 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4521 manifest.fleet.members[1].source_snapshot.artifact_path = "../outside".to_string();
4522
4523 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4524 let err = RestoreApplyDryRun::try_from_plan_with_artifacts(&plan, None, &root)
4525 .expect_err("path traversal should fail");
4526
4527 fs::remove_dir_all(root).expect("remove temp root");
4528 assert!(matches!(
4529 err,
4530 RestoreApplyDryRunError::ArtifactPathEscapesBackup { .. }
4531 ));
4532 }
4533
4534 #[test]
4536 fn apply_dry_run_rejects_mismatched_status() {
4537 let manifest = valid_manifest(IdentityMode::Relocatable);
4538
4539 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4540 let mut status = RestoreStatus::from_plan(&plan);
4541 status.backup_id = "other-backup".to_string();
4542
4543 let err = RestoreApplyDryRun::try_from_plan(&plan, Some(&status))
4544 .expect_err("mismatched status should fail");
4545
4546 assert!(matches!(
4547 err,
4548 RestoreApplyDryRunError::StatusPlanMismatch {
4549 field: "backup_id",
4550 ..
4551 }
4552 ));
4553 }
4554
4555 #[test]
4557 fn plan_expands_role_verification_checks_per_matching_member() {
4558 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4559 manifest.fleet.members.push(fleet_member(
4560 "app",
4561 CHILD_TWO,
4562 Some(ROOT),
4563 IdentityMode::Relocatable,
4564 1,
4565 ));
4566 manifest
4567 .verification
4568 .member_checks
4569 .push(MemberVerificationChecks {
4570 role: "app".to_string(),
4571 checks: vec![VerificationCheck {
4572 kind: "app-ready".to_string(),
4573 method: Some("ready".to_string()),
4574 roles: Vec::new(),
4575 }],
4576 });
4577
4578 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4579
4580 assert_eq!(plan.verification_summary.fleet_checks, 0);
4581 assert_eq!(plan.verification_summary.member_check_groups, 1);
4582 assert_eq!(plan.verification_summary.member_checks, 5);
4583 assert_eq!(plan.verification_summary.members_with_checks, 3);
4584 assert_eq!(plan.verification_summary.total_checks, 5);
4585 }
4586
4587 #[test]
4589 fn plan_applies_member_verification_role_filters() {
4590 let mut manifest = valid_manifest(IdentityMode::Relocatable);
4591 manifest.fleet.members[0]
4592 .verification_checks
4593 .push(VerificationCheck {
4594 kind: "root-only-inline".to_string(),
4595 method: Some("wrong_member".to_string()),
4596 roles: vec!["root".to_string()],
4597 });
4598 manifest
4599 .verification
4600 .member_checks
4601 .push(MemberVerificationChecks {
4602 role: "app".to_string(),
4603 checks: vec![
4604 VerificationCheck {
4605 kind: "app-role-check".to_string(),
4606 method: Some("app_ready".to_string()),
4607 roles: vec!["app".to_string()],
4608 },
4609 VerificationCheck {
4610 kind: "root-filtered-check".to_string(),
4611 method: Some("wrong_role".to_string()),
4612 roles: vec!["root".to_string()],
4613 },
4614 ],
4615 });
4616
4617 let plan = RestorePlanner::plan(&manifest, None).expect("plan should build");
4618 let app = plan
4619 .ordered_members()
4620 .into_iter()
4621 .find(|member| member.role == "app")
4622 .expect("app member should be planned");
4623 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run should build");
4624 let app_verification_methods = dry_run.phases[0]
4625 .operations
4626 .iter()
4627 .filter(|operation| {
4628 operation.source_canister == CHILD
4629 && operation.operation == RestoreApplyOperationKind::VerifyMember
4630 })
4631 .filter_map(|operation| operation.verification_method.as_deref())
4632 .collect::<Vec<_>>();
4633
4634 assert_eq!(app.verification_checks.len(), 2);
4635 assert_eq!(
4636 app.verification_checks
4637 .iter()
4638 .map(|check| check.kind.as_str())
4639 .collect::<Vec<_>>(),
4640 ["call", "app-role-check"]
4641 );
4642 assert_eq!(plan.verification_summary.member_checks, 3);
4643 assert_eq!(plan.verification_summary.total_checks, 3);
4644 assert_eq!(dry_run.rendered_operations, 9);
4645 assert_eq!(app_verification_methods, ["canic_ready", "app_ready"]);
4646 }
4647
4648 #[test]
4650 fn mapped_restore_requires_complete_mapping() {
4651 let manifest = valid_manifest(IdentityMode::Relocatable);
4652 let mapping = RestoreMapping {
4653 members: vec![RestoreMappingEntry {
4654 source_canister: ROOT.to_string(),
4655 target_canister: ROOT.to_string(),
4656 }],
4657 };
4658
4659 let err = RestorePlanner::plan(&manifest, Some(&mapping))
4660 .expect_err("incomplete mapping should fail");
4661
4662 assert!(matches!(err, RestorePlanError::MissingMappingSource(_)));
4663 }
4664
4665 #[test]
4667 fn mapped_restore_rejects_unknown_mapping_sources() {
4668 let manifest = valid_manifest(IdentityMode::Relocatable);
4669 let unknown = "rdmx6-jaaaa-aaaaa-aaadq-cai";
4670 let mapping = RestoreMapping {
4671 members: vec![
4672 RestoreMappingEntry {
4673 source_canister: ROOT.to_string(),
4674 target_canister: ROOT.to_string(),
4675 },
4676 RestoreMappingEntry {
4677 source_canister: CHILD.to_string(),
4678 target_canister: TARGET.to_string(),
4679 },
4680 RestoreMappingEntry {
4681 source_canister: unknown.to_string(),
4682 target_canister: unknown.to_string(),
4683 },
4684 ],
4685 };
4686
4687 let err = RestorePlanner::plan(&manifest, Some(&mapping))
4688 .expect_err("unknown mapping source should fail");
4689
4690 assert!(matches!(err, RestorePlanError::UnknownMappingSource(_)));
4691 }
4692
4693 #[test]
4695 fn duplicate_mapping_targets_fail_validation() {
4696 let manifest = valid_manifest(IdentityMode::Relocatable);
4697 let mapping = RestoreMapping {
4698 members: vec![
4699 RestoreMappingEntry {
4700 source_canister: ROOT.to_string(),
4701 target_canister: ROOT.to_string(),
4702 },
4703 RestoreMappingEntry {
4704 source_canister: CHILD.to_string(),
4705 target_canister: ROOT.to_string(),
4706 },
4707 ],
4708 };
4709
4710 let err = RestorePlanner::plan(&manifest, Some(&mapping))
4711 .expect_err("duplicate targets should fail");
4712
4713 assert!(matches!(err, RestorePlanError::DuplicateMappingTarget(_)));
4714 }
4715
4716 fn set_member_artifact(
4718 manifest: &mut FleetBackupManifest,
4719 canister_id: &str,
4720 root: &Path,
4721 artifact_path: &str,
4722 bytes: &[u8],
4723 ) {
4724 let full_path = root.join(artifact_path);
4725 fs::create_dir_all(full_path.parent().expect("artifact parent")).expect("create parent");
4726 fs::write(&full_path, bytes).expect("write artifact");
4727 let checksum = ArtifactChecksum::from_bytes(bytes);
4728 let member = manifest
4729 .fleet
4730 .members
4731 .iter_mut()
4732 .find(|member| member.canister_id == canister_id)
4733 .expect("member should exist");
4734 member.source_snapshot.artifact_path = artifact_path.to_string();
4735 member.source_snapshot.checksum = Some(checksum.hash);
4736 }
4737
4738 fn temp_dir(name: &str) -> PathBuf {
4740 let nanos = SystemTime::now()
4741 .duration_since(UNIX_EPOCH)
4742 .expect("system time should be after epoch")
4743 .as_nanos();
4744 env::temp_dir().join(format!("{name}-{nanos}"))
4745 }
4746}