1use canic_backup::{
2 manifest::FleetBackupManifest,
3 persistence::{BackupLayout, PersistenceError},
4 restore::{
5 RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6 RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7 RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8 RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationKindCounts,
9 RestoreApplyOperationState, RestoreApplyReportOperation, RestoreApplyReportOutcome,
10 RestoreApplyRunnerCommand, RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner,
11 RestoreStatus,
12 },
13};
14use serde::Serialize;
15use std::{
16 ffi::OsString,
17 fs,
18 io::{self, Write},
19 path::PathBuf,
20 process::Command,
21};
22use thiserror::Error as ThisError;
23
24#[derive(Debug, ThisError)]
29pub enum RestoreCommandError {
30 #[error("{0}")]
31 Usage(&'static str),
32
33 #[error("missing required option {0}")]
34 MissingOption(&'static str),
35
36 #[error("use either --manifest or --backup-dir, not both")]
37 ConflictingManifestSources,
38
39 #[error("--require-verified requires --backup-dir")]
40 RequireVerifiedNeedsBackupDir,
41
42 #[error("restore apply currently requires --dry-run")]
43 ApplyRequiresDryRun,
44
45 #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
46 RestoreRunRequiresMode,
47
48 #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
49 RestoreRunConflictingModes,
50
51 #[error("restore run command failed for operation {sequence}: status={status}")]
52 RestoreRunCommandFailed { sequence: usize, status: String },
53
54 #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
55 RestoreRunModeMismatch {
56 backup_id: String,
57 expected: String,
58 actual: String,
59 },
60
61 #[error(
62 "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
63 )]
64 RestoreRunStoppedReasonMismatch {
65 backup_id: String,
66 expected: String,
67 actual: String,
68 },
69
70 #[error(
71 "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
72 )]
73 RestoreRunNextActionMismatch {
74 backup_id: String,
75 expected: String,
76 actual: String,
77 },
78
79 #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
80 RestoreRunExecutedCountMismatch {
81 backup_id: String,
82 expected: usize,
83 actual: usize,
84 },
85
86 #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
87 RestoreNotReady {
88 backup_id: String,
89 reasons: Vec<String>,
90 },
91
92 #[error(
93 "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
94 )]
95 RestoreApplyPending {
96 backup_id: String,
97 pending_operations: usize,
98 next_transition_sequence: Option<usize>,
99 },
100
101 #[error(
102 "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
103 )]
104 RestoreApplyIncomplete {
105 backup_id: String,
106 completed_operations: usize,
107 operation_count: usize,
108 },
109
110 #[error(
111 "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
112 )]
113 RestoreApplyFailed {
114 backup_id: String,
115 failed_operations: usize,
116 },
117
118 #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
119 RestoreApplyNotReady {
120 backup_id: String,
121 reasons: Vec<String>,
122 },
123
124 #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
125 RestoreApplyReportNeedsAttention {
126 backup_id: String,
127 outcome: canic_backup::restore::RestoreApplyReportOutcome,
128 },
129
130 #[error(
131 "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
132 )]
133 RestoreApplyCommandUnavailable {
134 backup_id: String,
135 operation_available: bool,
136 complete: bool,
137 blocked_reasons: Vec<String>,
138 },
139
140 #[error(
141 "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
142 )]
143 RestoreApplyMarkRequiresPending {
144 sequence: usize,
145 state: RestoreApplyOperationState,
146 },
147
148 #[error(
149 "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
150 )]
151 RestoreApplyClaimSequenceMismatch {
152 expected: usize,
153 actual: Option<usize>,
154 },
155
156 #[error(
157 "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
158 )]
159 RestoreApplyUnclaimSequenceMismatch {
160 expected: usize,
161 actual: Option<usize>,
162 },
163
164 #[error("unknown option {0}")]
165 UnknownOption(String),
166
167 #[error("option {0} requires a value")]
168 MissingValue(&'static str),
169
170 #[error("option --sequence requires a non-negative integer value")]
171 InvalidSequence,
172
173 #[error("unsupported apply-mark state {0}; use completed or failed")]
174 InvalidApplyMarkState(String),
175
176 #[error(transparent)]
177 Io(#[from] std::io::Error),
178
179 #[error(transparent)]
180 Json(#[from] serde_json::Error),
181
182 #[error(transparent)]
183 Persistence(#[from] PersistenceError),
184
185 #[error(transparent)]
186 RestorePlan(#[from] RestorePlanError),
187
188 #[error(transparent)]
189 RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
190
191 #[error(transparent)]
192 RestoreApplyJournal(#[from] RestoreApplyJournalError),
193}
194
195#[derive(Clone, Debug, Eq, PartialEq)]
200pub struct RestorePlanOptions {
201 pub manifest: Option<PathBuf>,
202 pub backup_dir: Option<PathBuf>,
203 pub mapping: Option<PathBuf>,
204 pub out: Option<PathBuf>,
205 pub require_verified: bool,
206 pub require_restore_ready: bool,
207}
208
209impl RestorePlanOptions {
210 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
212 where
213 I: IntoIterator<Item = OsString>,
214 {
215 let mut manifest = None;
216 let mut backup_dir = None;
217 let mut mapping = None;
218 let mut out = None;
219 let mut require_verified = false;
220 let mut require_restore_ready = false;
221
222 let mut args = args.into_iter();
223 while let Some(arg) = args.next() {
224 let arg = arg
225 .into_string()
226 .map_err(|_| RestoreCommandError::Usage(usage()))?;
227 match arg.as_str() {
228 "--manifest" => {
229 manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
230 }
231 "--backup-dir" => {
232 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
233 }
234 "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
235 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
236 "--require-verified" => require_verified = true,
237 "--require-restore-ready" => require_restore_ready = true,
238 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
239 _ => return Err(RestoreCommandError::UnknownOption(arg)),
240 }
241 }
242
243 if manifest.is_some() && backup_dir.is_some() {
244 return Err(RestoreCommandError::ConflictingManifestSources);
245 }
246
247 if manifest.is_none() && backup_dir.is_none() {
248 return Err(RestoreCommandError::MissingOption(
249 "--manifest or --backup-dir",
250 ));
251 }
252
253 if require_verified && backup_dir.is_none() {
254 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
255 }
256
257 Ok(Self {
258 manifest,
259 backup_dir,
260 mapping,
261 out,
262 require_verified,
263 require_restore_ready,
264 })
265 }
266}
267
268#[derive(Clone, Debug, Eq, PartialEq)]
273pub struct RestoreStatusOptions {
274 pub plan: PathBuf,
275 pub out: Option<PathBuf>,
276}
277
278impl RestoreStatusOptions {
279 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
281 where
282 I: IntoIterator<Item = OsString>,
283 {
284 let mut plan = None;
285 let mut out = None;
286
287 let mut args = args.into_iter();
288 while let Some(arg) = args.next() {
289 let arg = arg
290 .into_string()
291 .map_err(|_| RestoreCommandError::Usage(usage()))?;
292 match arg.as_str() {
293 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
294 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
295 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
296 _ => return Err(RestoreCommandError::UnknownOption(arg)),
297 }
298 }
299
300 Ok(Self {
301 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
302 out,
303 })
304 }
305}
306
307#[derive(Clone, Debug, Eq, PartialEq)]
312pub struct RestoreApplyOptions {
313 pub plan: PathBuf,
314 pub status: Option<PathBuf>,
315 pub backup_dir: Option<PathBuf>,
316 pub out: Option<PathBuf>,
317 pub journal_out: Option<PathBuf>,
318 pub dry_run: bool,
319}
320
321impl RestoreApplyOptions {
322 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
324 where
325 I: IntoIterator<Item = OsString>,
326 {
327 let mut plan = None;
328 let mut status = None;
329 let mut backup_dir = None;
330 let mut out = None;
331 let mut journal_out = None;
332 let mut dry_run = false;
333
334 let mut args = args.into_iter();
335 while let Some(arg) = args.next() {
336 let arg = arg
337 .into_string()
338 .map_err(|_| RestoreCommandError::Usage(usage()))?;
339 match arg.as_str() {
340 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
341 "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
342 "--backup-dir" => {
343 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
344 }
345 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
346 "--journal-out" => {
347 journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
348 }
349 "--dry-run" => dry_run = true,
350 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
351 _ => return Err(RestoreCommandError::UnknownOption(arg)),
352 }
353 }
354
355 if !dry_run {
356 return Err(RestoreCommandError::ApplyRequiresDryRun);
357 }
358
359 Ok(Self {
360 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
361 status,
362 backup_dir,
363 out,
364 journal_out,
365 dry_run,
366 })
367 }
368}
369
370#[derive(Clone, Debug, Eq, PartialEq)]
375#[expect(
376 clippy::struct_excessive_bools,
377 reason = "CLI status options mirror independent fail-closed guard flags"
378)]
379pub struct RestoreApplyStatusOptions {
380 pub journal: PathBuf,
381 pub require_ready: bool,
382 pub require_no_pending: bool,
383 pub require_no_failed: bool,
384 pub require_complete: bool,
385 pub out: Option<PathBuf>,
386}
387
388impl RestoreApplyStatusOptions {
389 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
391 where
392 I: IntoIterator<Item = OsString>,
393 {
394 let mut journal = None;
395 let mut require_ready = false;
396 let mut require_no_pending = false;
397 let mut require_no_failed = false;
398 let mut require_complete = false;
399 let mut out = None;
400
401 let mut args = args.into_iter();
402 while let Some(arg) = args.next() {
403 let arg = arg
404 .into_string()
405 .map_err(|_| RestoreCommandError::Usage(usage()))?;
406 match arg.as_str() {
407 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
408 "--require-ready" => require_ready = true,
409 "--require-no-pending" => require_no_pending = true,
410 "--require-no-failed" => require_no_failed = true,
411 "--require-complete" => require_complete = true,
412 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
413 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
414 _ => return Err(RestoreCommandError::UnknownOption(arg)),
415 }
416 }
417
418 Ok(Self {
419 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
420 require_ready,
421 require_no_pending,
422 require_no_failed,
423 require_complete,
424 out,
425 })
426 }
427}
428
429#[derive(Clone, Debug, Eq, PartialEq)]
434pub struct RestoreApplyReportOptions {
435 pub journal: PathBuf,
436 pub require_no_attention: bool,
437 pub out: Option<PathBuf>,
438}
439
440impl RestoreApplyReportOptions {
441 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
443 where
444 I: IntoIterator<Item = OsString>,
445 {
446 let mut journal = None;
447 let mut require_no_attention = false;
448 let mut out = None;
449
450 let mut args = args.into_iter();
451 while let Some(arg) = args.next() {
452 let arg = arg
453 .into_string()
454 .map_err(|_| RestoreCommandError::Usage(usage()))?;
455 match arg.as_str() {
456 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
457 "--require-no-attention" => require_no_attention = true,
458 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
459 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
460 _ => return Err(RestoreCommandError::UnknownOption(arg)),
461 }
462 }
463
464 Ok(Self {
465 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
466 require_no_attention,
467 out,
468 })
469 }
470}
471
472#[derive(Clone, Debug, Eq, PartialEq)]
477#[expect(
478 clippy::struct_excessive_bools,
479 reason = "CLI runner options mirror independent mode and fail-closed guard flags"
480)]
481pub struct RestoreRunOptions {
482 pub journal: PathBuf,
483 pub dfx: String,
484 pub network: Option<String>,
485 pub out: Option<PathBuf>,
486 pub dry_run: bool,
487 pub execute: bool,
488 pub unclaim_pending: bool,
489 pub max_steps: Option<usize>,
490 pub require_complete: bool,
491 pub require_no_attention: bool,
492 pub require_run_mode: Option<String>,
493 pub require_stopped_reason: Option<String>,
494 pub require_next_action: Option<String>,
495 pub require_executed_count: Option<usize>,
496}
497
498impl RestoreRunOptions {
499 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
501 where
502 I: IntoIterator<Item = OsString>,
503 {
504 let mut journal = None;
505 let mut dfx = "dfx".to_string();
506 let mut network = None;
507 let mut out = None;
508 let mut dry_run = false;
509 let mut execute = false;
510 let mut unclaim_pending = false;
511 let mut max_steps = None;
512 let mut require_complete = false;
513 let mut require_no_attention = false;
514 let mut require_run_mode = None;
515 let mut require_stopped_reason = None;
516 let mut require_next_action = None;
517 let mut require_executed_count = None;
518
519 let mut args = args.into_iter();
520 while let Some(arg) = args.next() {
521 let arg = arg
522 .into_string()
523 .map_err(|_| RestoreCommandError::Usage(usage()))?;
524 match arg.as_str() {
525 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
526 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
527 "--network" => network = Some(next_value(&mut args, "--network")?),
528 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
529 "--dry-run" => dry_run = true,
530 "--execute" => execute = true,
531 "--unclaim-pending" => unclaim_pending = true,
532 "--max-steps" => {
533 max_steps = Some(parse_sequence(next_value(&mut args, "--max-steps")?)?);
534 }
535 "--require-complete" => require_complete = true,
536 "--require-no-attention" => require_no_attention = true,
537 "--require-run-mode" => {
538 require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
539 }
540 "--require-stopped-reason" => {
541 require_stopped_reason =
542 Some(next_value(&mut args, "--require-stopped-reason")?);
543 }
544 "--require-next-action" => {
545 require_next_action = Some(next_value(&mut args, "--require-next-action")?);
546 }
547 "--require-executed-count" => {
548 require_executed_count = Some(parse_sequence(next_value(
549 &mut args,
550 "--require-executed-count",
551 )?)?);
552 }
553 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
554 _ => return Err(RestoreCommandError::UnknownOption(arg)),
555 }
556 }
557
558 let mode_count = [dry_run, execute, unclaim_pending]
559 .into_iter()
560 .filter(|enabled| *enabled)
561 .count();
562 if mode_count > 1 {
563 return Err(RestoreCommandError::RestoreRunConflictingModes);
564 }
565
566 if mode_count == 0 {
567 return Err(RestoreCommandError::RestoreRunRequiresMode);
568 }
569
570 Ok(Self {
571 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
572 dfx,
573 network,
574 out,
575 dry_run,
576 execute,
577 unclaim_pending,
578 max_steps,
579 require_complete,
580 require_no_attention,
581 require_run_mode,
582 require_stopped_reason,
583 require_next_action,
584 require_executed_count,
585 })
586 }
587}
588
589struct RestoreRunResult {
594 response: RestoreRunResponse,
595 error: Option<RestoreCommandError>,
596}
597
598impl RestoreRunResult {
599 const fn ok(response: RestoreRunResponse) -> Self {
601 Self {
602 response,
603 error: None,
604 }
605 }
606}
607
608const RESTORE_RUN_MODE_DRY_RUN: &str = "dry-run";
609const RESTORE_RUN_MODE_EXECUTE: &str = "execute";
610const RESTORE_RUN_MODE_UNCLAIM_PENDING: &str = "unclaim-pending";
611
612const RESTORE_RUN_STOPPED_BLOCKED: &str = "blocked";
613const RESTORE_RUN_STOPPED_COMMAND_FAILED: &str = "command-failed";
614const RESTORE_RUN_STOPPED_COMPLETE: &str = "complete";
615const RESTORE_RUN_STOPPED_MAX_STEPS: &str = "max-steps-reached";
616const RESTORE_RUN_STOPPED_PENDING: &str = "pending";
617const RESTORE_RUN_STOPPED_PREVIEW: &str = "preview";
618const RESTORE_RUN_STOPPED_READY: &str = "ready";
619const RESTORE_RUN_STOPPED_RECOVERED_PENDING: &str = "recovered-pending";
620
621const RESTORE_RUN_ACTION_DONE: &str = "done";
622const RESTORE_RUN_ACTION_FIX_BLOCKED: &str = "fix-blocked-journal";
623const RESTORE_RUN_ACTION_INSPECT_FAILED: &str = "inspect-failed-operation";
624const RESTORE_RUN_ACTION_RERUN: &str = "rerun";
625const RESTORE_RUN_ACTION_UNCLAIM_PENDING: &str = "unclaim-pending";
626
627const RESTORE_RUN_EXECUTED_COMPLETED: &str = "completed";
628const RESTORE_RUN_EXECUTED_FAILED: &str = "failed";
629const RESTORE_RUN_COMMAND_EXIT_PREFIX: &str = "runner-command-exit";
630const RESTORE_RUN_RESPONSE_VERSION: u16 = 1;
631
632#[derive(Clone, Debug, Serialize)]
637#[expect(
638 clippy::struct_excessive_bools,
639 reason = "Runner response exposes stable JSON status flags for operators and CI"
640)]
641pub struct RestoreRunResponse {
642 run_version: u16,
643 backup_id: String,
644 run_mode: &'static str,
645 dry_run: bool,
646 execute: bool,
647 unclaim_pending: bool,
648 stopped_reason: &'static str,
649 next_action: &'static str,
650 #[serde(skip_serializing_if = "Option::is_none")]
651 max_steps_reached: Option<bool>,
652 #[serde(default, skip_serializing_if = "Vec::is_empty")]
653 executed_operations: Vec<RestoreRunExecutedOperation>,
654 #[serde(skip_serializing_if = "Option::is_none")]
655 executed_operation_count: Option<usize>,
656 #[serde(skip_serializing_if = "Option::is_none")]
657 recovered_operation: Option<RestoreApplyJournalOperation>,
658 ready: bool,
659 complete: bool,
660 attention_required: bool,
661 outcome: RestoreApplyReportOutcome,
662 operation_count: usize,
663 operation_counts: RestoreApplyOperationKindCounts,
664 operation_counts_supplied: bool,
665 pending_operations: usize,
666 ready_operations: usize,
667 blocked_operations: usize,
668 completed_operations: usize,
669 failed_operations: usize,
670 blocked_reasons: Vec<String>,
671 next_transition: Option<RestoreApplyReportOperation>,
672 #[serde(skip_serializing_if = "Option::is_none")]
673 operation_available: Option<bool>,
674 #[serde(skip_serializing_if = "Option::is_none")]
675 command_available: Option<bool>,
676 #[serde(skip_serializing_if = "Option::is_none")]
677 command: Option<RestoreApplyRunnerCommand>,
678}
679
680impl RestoreRunResponse {
681 fn from_report(
683 backup_id: String,
684 report: RestoreApplyJournalReport,
685 mode: RestoreRunResponseMode,
686 ) -> Self {
687 Self {
688 run_version: RESTORE_RUN_RESPONSE_VERSION,
689 backup_id,
690 run_mode: mode.run_mode,
691 dry_run: mode.dry_run,
692 execute: mode.execute,
693 unclaim_pending: mode.unclaim_pending,
694 stopped_reason: mode.stopped_reason,
695 next_action: mode.next_action,
696 max_steps_reached: None,
697 executed_operations: Vec::new(),
698 executed_operation_count: None,
699 recovered_operation: None,
700 ready: report.ready,
701 complete: report.complete,
702 attention_required: report.attention_required,
703 outcome: report.outcome,
704 operation_count: report.operation_count,
705 operation_counts: report.operation_counts,
706 operation_counts_supplied: report.operation_counts_supplied,
707 pending_operations: report.pending_operations,
708 ready_operations: report.ready_operations,
709 blocked_operations: report.blocked_operations,
710 completed_operations: report.completed_operations,
711 failed_operations: report.failed_operations,
712 blocked_reasons: report.blocked_reasons,
713 next_transition: report.next_transition,
714 operation_available: None,
715 command_available: None,
716 command: None,
717 }
718 }
719}
720
721#[derive(Clone, Debug, Serialize)]
726struct RestoreRunExecutedOperation {
727 sequence: usize,
728 operation: RestoreApplyOperationKind,
729 target_canister: String,
730 command: RestoreApplyRunnerCommand,
731 status: String,
732 state: &'static str,
733}
734
735impl RestoreRunExecutedOperation {
736 fn completed(
738 operation: RestoreApplyJournalOperation,
739 command: RestoreApplyRunnerCommand,
740 status: String,
741 ) -> Self {
742 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_COMPLETED)
743 }
744
745 fn failed(
747 operation: RestoreApplyJournalOperation,
748 command: RestoreApplyRunnerCommand,
749 status: String,
750 ) -> Self {
751 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_FAILED)
752 }
753
754 fn from_operation(
756 operation: RestoreApplyJournalOperation,
757 command: RestoreApplyRunnerCommand,
758 status: String,
759 state: &'static str,
760 ) -> Self {
761 Self {
762 sequence: operation.sequence,
763 operation: operation.operation,
764 target_canister: operation.target_canister,
765 command,
766 status,
767 state,
768 }
769 }
770}
771
772struct RestoreRunResponseMode {
777 run_mode: &'static str,
778 dry_run: bool,
779 execute: bool,
780 unclaim_pending: bool,
781 stopped_reason: &'static str,
782 next_action: &'static str,
783}
784
785impl RestoreRunResponseMode {
786 const fn new(
788 run_mode: &'static str,
789 dry_run: bool,
790 execute: bool,
791 unclaim_pending: bool,
792 stopped_reason: &'static str,
793 next_action: &'static str,
794 ) -> Self {
795 Self {
796 run_mode,
797 dry_run,
798 execute,
799 unclaim_pending,
800 stopped_reason,
801 next_action,
802 }
803 }
804
805 const fn dry_run(stopped_reason: &'static str, next_action: &'static str) -> Self {
807 Self::new(
808 RESTORE_RUN_MODE_DRY_RUN,
809 true,
810 false,
811 false,
812 stopped_reason,
813 next_action,
814 )
815 }
816
817 const fn execute(stopped_reason: &'static str, next_action: &'static str) -> Self {
819 Self::new(
820 RESTORE_RUN_MODE_EXECUTE,
821 false,
822 true,
823 false,
824 stopped_reason,
825 next_action,
826 )
827 }
828
829 const fn unclaim_pending(next_action: &'static str) -> Self {
831 Self::new(
832 RESTORE_RUN_MODE_UNCLAIM_PENDING,
833 false,
834 false,
835 true,
836 RESTORE_RUN_STOPPED_RECOVERED_PENDING,
837 next_action,
838 )
839 }
840}
841
842#[derive(Clone, Debug, Eq, PartialEq)]
847pub struct RestoreApplyNextOptions {
848 pub journal: PathBuf,
849 pub out: Option<PathBuf>,
850}
851
852impl RestoreApplyNextOptions {
853 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
855 where
856 I: IntoIterator<Item = OsString>,
857 {
858 let mut journal = None;
859 let mut out = None;
860
861 let mut args = args.into_iter();
862 while let Some(arg) = args.next() {
863 let arg = arg
864 .into_string()
865 .map_err(|_| RestoreCommandError::Usage(usage()))?;
866 match arg.as_str() {
867 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
868 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
869 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
870 _ => return Err(RestoreCommandError::UnknownOption(arg)),
871 }
872 }
873
874 Ok(Self {
875 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
876 out,
877 })
878 }
879}
880
881#[derive(Clone, Debug, Eq, PartialEq)]
886pub struct RestoreApplyCommandOptions {
887 pub journal: PathBuf,
888 pub dfx: String,
889 pub network: Option<String>,
890 pub out: Option<PathBuf>,
891 pub require_command: bool,
892}
893
894impl RestoreApplyCommandOptions {
895 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
897 where
898 I: IntoIterator<Item = OsString>,
899 {
900 let mut journal = None;
901 let mut dfx = "dfx".to_string();
902 let mut network = None;
903 let mut out = None;
904 let mut require_command = false;
905
906 let mut args = args.into_iter();
907 while let Some(arg) = args.next() {
908 let arg = arg
909 .into_string()
910 .map_err(|_| RestoreCommandError::Usage(usage()))?;
911 match arg.as_str() {
912 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
913 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
914 "--network" => network = Some(next_value(&mut args, "--network")?),
915 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
916 "--require-command" => require_command = true,
917 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
918 _ => return Err(RestoreCommandError::UnknownOption(arg)),
919 }
920 }
921
922 Ok(Self {
923 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
924 dfx,
925 network,
926 out,
927 require_command,
928 })
929 }
930}
931
932#[derive(Clone, Debug, Eq, PartialEq)]
937pub struct RestoreApplyClaimOptions {
938 pub journal: PathBuf,
939 pub sequence: Option<usize>,
940 pub updated_at: Option<String>,
941 pub out: Option<PathBuf>,
942}
943
944impl RestoreApplyClaimOptions {
945 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
947 where
948 I: IntoIterator<Item = OsString>,
949 {
950 let mut journal = None;
951 let mut sequence = None;
952 let mut updated_at = None;
953 let mut out = None;
954
955 let mut args = args.into_iter();
956 while let Some(arg) = args.next() {
957 let arg = arg
958 .into_string()
959 .map_err(|_| RestoreCommandError::Usage(usage()))?;
960 match arg.as_str() {
961 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
962 "--sequence" => {
963 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
964 }
965 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
966 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
967 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
968 _ => return Err(RestoreCommandError::UnknownOption(arg)),
969 }
970 }
971
972 Ok(Self {
973 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
974 sequence,
975 updated_at,
976 out,
977 })
978 }
979}
980
981#[derive(Clone, Debug, Eq, PartialEq)]
986pub struct RestoreApplyUnclaimOptions {
987 pub journal: PathBuf,
988 pub sequence: Option<usize>,
989 pub updated_at: Option<String>,
990 pub out: Option<PathBuf>,
991}
992
993impl RestoreApplyUnclaimOptions {
994 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
996 where
997 I: IntoIterator<Item = OsString>,
998 {
999 let mut journal = None;
1000 let mut sequence = None;
1001 let mut updated_at = None;
1002 let mut out = None;
1003
1004 let mut args = args.into_iter();
1005 while let Some(arg) = args.next() {
1006 let arg = arg
1007 .into_string()
1008 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1009 match arg.as_str() {
1010 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1011 "--sequence" => {
1012 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1013 }
1014 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1015 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1016 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1017 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1018 }
1019 }
1020
1021 Ok(Self {
1022 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1023 sequence,
1024 updated_at,
1025 out,
1026 })
1027 }
1028}
1029
1030#[derive(Clone, Debug, Eq, PartialEq)]
1035pub struct RestoreApplyMarkOptions {
1036 pub journal: PathBuf,
1037 pub sequence: usize,
1038 pub state: RestoreApplyMarkState,
1039 pub reason: Option<String>,
1040 pub updated_at: Option<String>,
1041 pub out: Option<PathBuf>,
1042 pub require_pending: bool,
1043}
1044
1045impl RestoreApplyMarkOptions {
1046 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1048 where
1049 I: IntoIterator<Item = OsString>,
1050 {
1051 let mut journal = None;
1052 let mut sequence = None;
1053 let mut state = None;
1054 let mut reason = None;
1055 let mut updated_at = None;
1056 let mut out = None;
1057 let mut require_pending = false;
1058
1059 let mut args = args.into_iter();
1060 while let Some(arg) = args.next() {
1061 let arg = arg
1062 .into_string()
1063 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1064 match arg.as_str() {
1065 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1066 "--sequence" => {
1067 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1068 }
1069 "--state" => {
1070 state = Some(RestoreApplyMarkState::parse(next_value(
1071 &mut args, "--state",
1072 )?)?);
1073 }
1074 "--reason" => reason = Some(next_value(&mut args, "--reason")?),
1075 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1076 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1077 "--require-pending" => require_pending = true,
1078 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1079 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1080 }
1081 }
1082
1083 Ok(Self {
1084 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1085 sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
1086 state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
1087 reason,
1088 updated_at,
1089 out,
1090 require_pending,
1091 })
1092 }
1093}
1094
1095#[derive(Clone, Debug, Eq, PartialEq)]
1100pub enum RestoreApplyMarkState {
1101 Completed,
1102 Failed,
1103}
1104
1105impl RestoreApplyMarkState {
1106 fn parse(value: String) -> Result<Self, RestoreCommandError> {
1108 match value.as_str() {
1109 "completed" => Ok(Self::Completed),
1110 "failed" => Ok(Self::Failed),
1111 _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
1112 }
1113 }
1114}
1115
1116pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
1118where
1119 I: IntoIterator<Item = OsString>,
1120{
1121 let mut args = args.into_iter();
1122 let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
1123 return Err(RestoreCommandError::Usage(usage()));
1124 };
1125
1126 match command.as_str() {
1127 "plan" => {
1128 let options = RestorePlanOptions::parse(args)?;
1129 let plan = plan_restore(&options)?;
1130 write_plan(&options, &plan)?;
1131 enforce_restore_plan_requirements(&options, &plan)?;
1132 Ok(())
1133 }
1134 "status" => {
1135 let options = RestoreStatusOptions::parse(args)?;
1136 let status = restore_status(&options)?;
1137 write_status(&options, &status)?;
1138 Ok(())
1139 }
1140 "apply" => {
1141 let options = RestoreApplyOptions::parse(args)?;
1142 let dry_run = restore_apply_dry_run(&options)?;
1143 write_apply_dry_run(&options, &dry_run)?;
1144 write_apply_journal_if_requested(&options, &dry_run)?;
1145 Ok(())
1146 }
1147 "apply-status" => {
1148 let options = RestoreApplyStatusOptions::parse(args)?;
1149 let status = restore_apply_status(&options)?;
1150 write_apply_status(&options, &status)?;
1151 enforce_apply_status_requirements(&options, &status)?;
1152 Ok(())
1153 }
1154 "apply-report" => {
1155 let options = RestoreApplyReportOptions::parse(args)?;
1156 let report = restore_apply_report(&options)?;
1157 write_apply_report(&options, &report)?;
1158 enforce_apply_report_requirements(&options, &report)?;
1159 Ok(())
1160 }
1161 "run" => {
1162 let options = RestoreRunOptions::parse(args)?;
1163 let run = if options.execute {
1164 restore_run_execute_result(&options)?
1165 } else if options.unclaim_pending {
1166 RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1167 } else {
1168 RestoreRunResult::ok(restore_run_dry_run(&options)?)
1169 };
1170 write_restore_run(&options, &run.response)?;
1171 if let Some(error) = run.error {
1172 return Err(error);
1173 }
1174 enforce_restore_run_requirements(&options, &run.response)?;
1175 Ok(())
1176 }
1177 "apply-next" => {
1178 let options = RestoreApplyNextOptions::parse(args)?;
1179 let next = restore_apply_next(&options)?;
1180 write_apply_next(&options, &next)?;
1181 Ok(())
1182 }
1183 "apply-command" => {
1184 let options = RestoreApplyCommandOptions::parse(args)?;
1185 let preview = restore_apply_command(&options)?;
1186 write_apply_command(&options, &preview)?;
1187 enforce_apply_command_requirements(&options, &preview)?;
1188 Ok(())
1189 }
1190 "apply-claim" => {
1191 let options = RestoreApplyClaimOptions::parse(args)?;
1192 let journal = restore_apply_claim(&options)?;
1193 write_apply_claim(&options, &journal)?;
1194 Ok(())
1195 }
1196 "apply-unclaim" => {
1197 let options = RestoreApplyUnclaimOptions::parse(args)?;
1198 let journal = restore_apply_unclaim(&options)?;
1199 write_apply_unclaim(&options, &journal)?;
1200 Ok(())
1201 }
1202 "apply-mark" => {
1203 let options = RestoreApplyMarkOptions::parse(args)?;
1204 let journal = restore_apply_mark(&options)?;
1205 write_apply_mark(&options, &journal)?;
1206 Ok(())
1207 }
1208 "help" | "--help" | "-h" => Err(RestoreCommandError::Usage(usage())),
1209 _ => Err(RestoreCommandError::UnknownOption(command)),
1210 }
1211}
1212
1213pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1215 verify_backup_layout_if_required(options)?;
1216
1217 let manifest = read_manifest_source(options)?;
1218 let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1219
1220 RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1221}
1222
1223pub fn restore_status(
1225 options: &RestoreStatusOptions,
1226) -> Result<RestoreStatus, RestoreCommandError> {
1227 let plan = read_plan(&options.plan)?;
1228 Ok(RestoreStatus::from_plan(&plan))
1229}
1230
1231pub fn restore_apply_dry_run(
1233 options: &RestoreApplyOptions,
1234) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1235 let plan = read_plan(&options.plan)?;
1236 let status = options.status.as_ref().map(read_status).transpose()?;
1237 if let Some(backup_dir) = &options.backup_dir {
1238 return RestoreApplyDryRun::try_from_plan_with_artifacts(
1239 &plan,
1240 status.as_ref(),
1241 backup_dir,
1242 )
1243 .map_err(RestoreCommandError::from);
1244 }
1245
1246 RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1247}
1248
1249pub fn restore_apply_status(
1251 options: &RestoreApplyStatusOptions,
1252) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1253 let journal = read_apply_journal(&options.journal)?;
1254 Ok(journal.status())
1255}
1256
1257pub fn restore_apply_report(
1259 options: &RestoreApplyReportOptions,
1260) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1261 let journal = read_apply_journal(&options.journal)?;
1262 Ok(journal.report())
1263}
1264
1265pub fn restore_run_dry_run(
1267 options: &RestoreRunOptions,
1268) -> Result<RestoreRunResponse, RestoreCommandError> {
1269 let journal = read_apply_journal(&options.journal)?;
1270 let report = journal.report();
1271 let preview = journal.next_command_preview_with_config(&restore_run_command_config(options));
1272 let stopped_reason = restore_run_stopped_reason(&report, false, false);
1273 let next_action = restore_run_next_action(&report, false);
1274
1275 let mut response = RestoreRunResponse::from_report(
1276 journal.backup_id,
1277 report,
1278 RestoreRunResponseMode::dry_run(stopped_reason, next_action),
1279 );
1280 response.operation_available = Some(preview.operation_available);
1281 response.command_available = Some(preview.command_available);
1282 response.command = preview.command;
1283 Ok(response)
1284}
1285
1286pub fn restore_run_unclaim_pending(
1288 options: &RestoreRunOptions,
1289) -> Result<RestoreRunResponse, RestoreCommandError> {
1290 let mut journal = read_apply_journal(&options.journal)?;
1291 let recovered_operation = journal
1292 .next_transition_operation()
1293 .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1294 .cloned()
1295 .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1296
1297 journal.mark_next_operation_ready_at(Some(timestamp_placeholder()))?;
1298 write_apply_journal_file(&options.journal, &journal)?;
1299
1300 let report = journal.report();
1301 let next_action = restore_run_next_action(&report, true);
1302 let mut response = RestoreRunResponse::from_report(
1303 journal.backup_id,
1304 report,
1305 RestoreRunResponseMode::unclaim_pending(next_action),
1306 );
1307 response.recovered_operation = Some(recovered_operation);
1308 Ok(response)
1309}
1310
1311pub fn restore_run_execute(
1313 options: &RestoreRunOptions,
1314) -> Result<RestoreRunResponse, RestoreCommandError> {
1315 let run = restore_run_execute_result(options)?;
1316 if let Some(error) = run.error {
1317 return Err(error);
1318 }
1319
1320 Ok(run.response)
1321}
1322
1323fn restore_run_execute_result(
1325 options: &RestoreRunOptions,
1326) -> Result<RestoreRunResult, RestoreCommandError> {
1327 let mut journal = read_apply_journal(&options.journal)?;
1328 let mut executed_operations = Vec::new();
1329 let config = restore_run_command_config(options);
1330
1331 loop {
1332 let report = journal.report();
1333 let max_steps_reached =
1334 restore_run_max_steps_reached(options, executed_operations.len(), &report);
1335 if report.complete || max_steps_reached {
1336 return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1337 &journal,
1338 executed_operations,
1339 max_steps_reached,
1340 )));
1341 }
1342
1343 enforce_restore_run_executable(&journal, &report)?;
1344 let preview = journal.next_command_preview_with_config(&config);
1345 enforce_restore_run_command_available(&preview)?;
1346
1347 let operation = preview
1348 .operation
1349 .clone()
1350 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1351 let command = preview
1352 .command
1353 .clone()
1354 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1355 let sequence = operation.sequence;
1356
1357 enforce_apply_claim_sequence(sequence, &journal)?;
1358 journal.mark_operation_pending_at(sequence, Some(timestamp_placeholder()))?;
1359 write_apply_journal_file(&options.journal, &journal)?;
1360
1361 let status = Command::new(&command.program)
1362 .args(&command.args)
1363 .status()?;
1364 let status_label = exit_status_label(status);
1365 if status.success() {
1366 journal.mark_operation_completed_at(sequence, Some(timestamp_placeholder()))?;
1367 write_apply_journal_file(&options.journal, &journal)?;
1368 executed_operations.push(RestoreRunExecutedOperation::completed(
1369 operation,
1370 command,
1371 status_label,
1372 ));
1373 continue;
1374 }
1375
1376 journal.mark_operation_failed_at(
1377 sequence,
1378 format!("{RESTORE_RUN_COMMAND_EXIT_PREFIX}-{status_label}"),
1379 Some(timestamp_placeholder()),
1380 )?;
1381 write_apply_journal_file(&options.journal, &journal)?;
1382 executed_operations.push(RestoreRunExecutedOperation::failed(
1383 operation,
1384 command,
1385 status_label.clone(),
1386 ));
1387 let response = restore_run_execute_summary(&journal, executed_operations, false);
1388 return Ok(RestoreRunResult {
1389 response,
1390 error: Some(RestoreCommandError::RestoreRunCommandFailed {
1391 sequence,
1392 status: status_label,
1393 }),
1394 });
1395 }
1396}
1397
1398fn restore_run_command_config(options: &RestoreRunOptions) -> RestoreApplyCommandConfig {
1400 restore_command_config(&options.dfx, options.network.as_deref())
1401}
1402
1403fn restore_apply_command_config(options: &RestoreApplyCommandOptions) -> RestoreApplyCommandConfig {
1405 restore_command_config(&options.dfx, options.network.as_deref())
1406}
1407
1408fn restore_command_config(program: &str, network: Option<&str>) -> RestoreApplyCommandConfig {
1410 RestoreApplyCommandConfig {
1411 program: program.to_string(),
1412 network: network.map(str::to_string),
1413 }
1414}
1415
1416fn restore_run_max_steps_reached(
1418 options: &RestoreRunOptions,
1419 executed_operation_count: usize,
1420 report: &RestoreApplyJournalReport,
1421) -> bool {
1422 options.max_steps == Some(executed_operation_count) && !report.complete
1423}
1424
1425fn restore_run_execute_summary(
1427 journal: &RestoreApplyJournal,
1428 executed_operations: Vec<RestoreRunExecutedOperation>,
1429 max_steps_reached: bool,
1430) -> RestoreRunResponse {
1431 let report = journal.report();
1432 let executed_operation_count = executed_operations.len();
1433 let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
1434 let next_action = restore_run_next_action(&report, false);
1435
1436 let mut response = RestoreRunResponse::from_report(
1437 journal.backup_id.clone(),
1438 report,
1439 RestoreRunResponseMode::execute(stopped_reason, next_action),
1440 );
1441 response.max_steps_reached = Some(max_steps_reached);
1442 response.executed_operation_count = Some(executed_operation_count);
1443 response.executed_operations = executed_operations;
1444 response
1445}
1446
1447const fn restore_run_stopped_reason(
1449 report: &RestoreApplyJournalReport,
1450 max_steps_reached: bool,
1451 executed: bool,
1452) -> &'static str {
1453 if report.complete {
1454 return RESTORE_RUN_STOPPED_COMPLETE;
1455 }
1456 if report.failed_operations > 0 {
1457 return RESTORE_RUN_STOPPED_COMMAND_FAILED;
1458 }
1459 if report.pending_operations > 0 {
1460 return RESTORE_RUN_STOPPED_PENDING;
1461 }
1462 if !report.ready || report.blocked_operations > 0 {
1463 return RESTORE_RUN_STOPPED_BLOCKED;
1464 }
1465 if max_steps_reached {
1466 return RESTORE_RUN_STOPPED_MAX_STEPS;
1467 }
1468 if executed {
1469 return RESTORE_RUN_STOPPED_READY;
1470 }
1471 RESTORE_RUN_STOPPED_PREVIEW
1472}
1473
1474const fn restore_run_next_action(
1476 report: &RestoreApplyJournalReport,
1477 recovered_pending: bool,
1478) -> &'static str {
1479 if report.complete {
1480 return RESTORE_RUN_ACTION_DONE;
1481 }
1482 if report.failed_operations > 0 {
1483 return RESTORE_RUN_ACTION_INSPECT_FAILED;
1484 }
1485 if report.pending_operations > 0 {
1486 return RESTORE_RUN_ACTION_UNCLAIM_PENDING;
1487 }
1488 if !report.ready || report.blocked_operations > 0 {
1489 return RESTORE_RUN_ACTION_FIX_BLOCKED;
1490 }
1491 if recovered_pending {
1492 return RESTORE_RUN_ACTION_RERUN;
1493 }
1494 RESTORE_RUN_ACTION_RERUN
1495}
1496
1497fn enforce_restore_run_executable(
1499 journal: &RestoreApplyJournal,
1500 report: &RestoreApplyJournalReport,
1501) -> Result<(), RestoreCommandError> {
1502 if report.pending_operations > 0 {
1503 return Err(RestoreCommandError::RestoreApplyPending {
1504 backup_id: report.backup_id.clone(),
1505 pending_operations: report.pending_operations,
1506 next_transition_sequence: report
1507 .next_transition
1508 .as_ref()
1509 .map(|operation| operation.sequence),
1510 });
1511 }
1512
1513 if report.failed_operations > 0 {
1514 return Err(RestoreCommandError::RestoreApplyFailed {
1515 backup_id: report.backup_id.clone(),
1516 failed_operations: report.failed_operations,
1517 });
1518 }
1519
1520 if report.ready {
1521 return Ok(());
1522 }
1523
1524 Err(RestoreCommandError::RestoreApplyNotReady {
1525 backup_id: journal.backup_id.clone(),
1526 reasons: report.blocked_reasons.clone(),
1527 })
1528}
1529
1530fn enforce_restore_run_command_available(
1532 preview: &RestoreApplyCommandPreview,
1533) -> Result<(), RestoreCommandError> {
1534 if preview.command_available {
1535 return Ok(());
1536 }
1537
1538 Err(restore_command_unavailable_error(preview))
1539}
1540
1541fn restore_command_unavailable_error(preview: &RestoreApplyCommandPreview) -> RestoreCommandError {
1543 RestoreCommandError::RestoreApplyCommandUnavailable {
1544 backup_id: preview.backup_id.clone(),
1545 operation_available: preview.operation_available,
1546 complete: preview.complete,
1547 blocked_reasons: preview.blocked_reasons.clone(),
1548 }
1549}
1550
1551fn exit_status_label(status: std::process::ExitStatus) -> String {
1553 status
1554 .code()
1555 .map_or_else(|| "signal".to_string(), |code| code.to_string())
1556}
1557
1558fn enforce_restore_run_requirements(
1560 options: &RestoreRunOptions,
1561 run: &RestoreRunResponse,
1562) -> Result<(), RestoreCommandError> {
1563 if options.require_complete && !run.complete {
1564 return Err(RestoreCommandError::RestoreApplyIncomplete {
1565 backup_id: run.backup_id.clone(),
1566 completed_operations: run.completed_operations,
1567 operation_count: run.operation_count,
1568 });
1569 }
1570
1571 if options.require_no_attention && run.attention_required {
1572 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1573 backup_id: run.backup_id.clone(),
1574 outcome: run.outcome.clone(),
1575 });
1576 }
1577
1578 if let Some(expected) = &options.require_run_mode
1579 && run.run_mode != expected
1580 {
1581 return Err(RestoreCommandError::RestoreRunModeMismatch {
1582 backup_id: run.backup_id.clone(),
1583 expected: expected.clone(),
1584 actual: run.run_mode.to_string(),
1585 });
1586 }
1587
1588 if let Some(expected) = &options.require_stopped_reason
1589 && run.stopped_reason != expected
1590 {
1591 return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
1592 backup_id: run.backup_id.clone(),
1593 expected: expected.clone(),
1594 actual: run.stopped_reason.to_string(),
1595 });
1596 }
1597
1598 if let Some(expected) = &options.require_next_action
1599 && run.next_action != expected
1600 {
1601 return Err(RestoreCommandError::RestoreRunNextActionMismatch {
1602 backup_id: run.backup_id.clone(),
1603 expected: expected.clone(),
1604 actual: run.next_action.to_string(),
1605 });
1606 }
1607
1608 if let Some(expected) = options.require_executed_count {
1609 let actual = run.executed_operation_count.unwrap_or(0);
1610 if actual != expected {
1611 return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
1612 backup_id: run.backup_id.clone(),
1613 expected,
1614 actual,
1615 });
1616 }
1617 }
1618
1619 Ok(())
1620}
1621
1622fn enforce_apply_report_requirements(
1624 options: &RestoreApplyReportOptions,
1625 report: &RestoreApplyJournalReport,
1626) -> Result<(), RestoreCommandError> {
1627 if !options.require_no_attention || !report.attention_required {
1628 return Ok(());
1629 }
1630
1631 Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1632 backup_id: report.backup_id.clone(),
1633 outcome: report.outcome.clone(),
1634 })
1635}
1636
1637fn enforce_apply_status_requirements(
1639 options: &RestoreApplyStatusOptions,
1640 status: &RestoreApplyJournalStatus,
1641) -> Result<(), RestoreCommandError> {
1642 if options.require_ready && !status.ready {
1643 return Err(RestoreCommandError::RestoreApplyNotReady {
1644 backup_id: status.backup_id.clone(),
1645 reasons: status.blocked_reasons.clone(),
1646 });
1647 }
1648
1649 if options.require_no_pending && status.pending_operations > 0 {
1650 return Err(RestoreCommandError::RestoreApplyPending {
1651 backup_id: status.backup_id.clone(),
1652 pending_operations: status.pending_operations,
1653 next_transition_sequence: status.next_transition_sequence,
1654 });
1655 }
1656
1657 if options.require_no_failed && status.failed_operations > 0 {
1658 return Err(RestoreCommandError::RestoreApplyFailed {
1659 backup_id: status.backup_id.clone(),
1660 failed_operations: status.failed_operations,
1661 });
1662 }
1663
1664 if options.require_complete && !status.complete {
1665 return Err(RestoreCommandError::RestoreApplyIncomplete {
1666 backup_id: status.backup_id.clone(),
1667 completed_operations: status.completed_operations,
1668 operation_count: status.operation_count,
1669 });
1670 }
1671
1672 Ok(())
1673}
1674
1675pub fn restore_apply_next(
1677 options: &RestoreApplyNextOptions,
1678) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
1679 let journal = read_apply_journal(&options.journal)?;
1680 Ok(journal.next_operation())
1681}
1682
1683pub fn restore_apply_command(
1685 options: &RestoreApplyCommandOptions,
1686) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
1687 let journal = read_apply_journal(&options.journal)?;
1688 Ok(journal.next_command_preview_with_config(&restore_apply_command_config(options)))
1689}
1690
1691fn enforce_apply_command_requirements(
1693 options: &RestoreApplyCommandOptions,
1694 preview: &RestoreApplyCommandPreview,
1695) -> Result<(), RestoreCommandError> {
1696 if !options.require_command || preview.command_available {
1697 return Ok(());
1698 }
1699
1700 Err(restore_command_unavailable_error(preview))
1701}
1702
1703pub fn restore_apply_claim(
1705 options: &RestoreApplyClaimOptions,
1706) -> Result<RestoreApplyJournal, RestoreCommandError> {
1707 let mut journal = read_apply_journal(&options.journal)?;
1708 let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
1709
1710 if let Some(sequence) = options.sequence {
1711 enforce_apply_claim_sequence(sequence, &journal)?;
1712 journal.mark_operation_pending_at(sequence, updated_at)?;
1713 return Ok(journal);
1714 }
1715
1716 journal.mark_next_operation_pending_at(updated_at)?;
1717 Ok(journal)
1718}
1719
1720fn enforce_apply_claim_sequence(
1722 expected: usize,
1723 journal: &RestoreApplyJournal,
1724) -> Result<(), RestoreCommandError> {
1725 let actual = journal
1726 .next_transition_operation()
1727 .map(|operation| operation.sequence);
1728
1729 if actual == Some(expected) {
1730 return Ok(());
1731 }
1732
1733 Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
1734}
1735
1736pub fn restore_apply_unclaim(
1738 options: &RestoreApplyUnclaimOptions,
1739) -> Result<RestoreApplyJournal, RestoreCommandError> {
1740 let mut journal = read_apply_journal(&options.journal)?;
1741 if let Some(sequence) = options.sequence {
1742 enforce_apply_unclaim_sequence(sequence, &journal)?;
1743 }
1744
1745 journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
1746 Ok(journal)
1747}
1748
1749fn enforce_apply_unclaim_sequence(
1751 expected: usize,
1752 journal: &RestoreApplyJournal,
1753) -> Result<(), RestoreCommandError> {
1754 let actual = journal
1755 .next_transition_operation()
1756 .map(|operation| operation.sequence);
1757
1758 if actual == Some(expected) {
1759 return Ok(());
1760 }
1761
1762 Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
1763}
1764
1765pub fn restore_apply_mark(
1767 options: &RestoreApplyMarkOptions,
1768) -> Result<RestoreApplyJournal, RestoreCommandError> {
1769 let mut journal = read_apply_journal(&options.journal)?;
1770 enforce_apply_mark_pending_requirement(options, &journal)?;
1771
1772 match options.state {
1773 RestoreApplyMarkState::Completed => {
1774 journal.mark_operation_completed_at(
1775 options.sequence,
1776 Some(state_updated_at(options.updated_at.as_ref())),
1777 )?;
1778 }
1779 RestoreApplyMarkState::Failed => {
1780 let reason =
1781 options
1782 .reason
1783 .clone()
1784 .ok_or(RestoreApplyJournalError::FailureReasonRequired(
1785 options.sequence,
1786 ))?;
1787 journal.mark_operation_failed_at(
1788 options.sequence,
1789 reason,
1790 Some(state_updated_at(options.updated_at.as_ref())),
1791 )?;
1792 }
1793 }
1794
1795 Ok(journal)
1796}
1797
1798fn enforce_apply_mark_pending_requirement(
1800 options: &RestoreApplyMarkOptions,
1801 journal: &RestoreApplyJournal,
1802) -> Result<(), RestoreCommandError> {
1803 if !options.require_pending {
1804 return Ok(());
1805 }
1806
1807 let state = journal
1808 .operations
1809 .iter()
1810 .find(|operation| operation.sequence == options.sequence)
1811 .map(|operation| operation.state.clone())
1812 .ok_or(RestoreApplyJournalError::OperationNotFound(
1813 options.sequence,
1814 ))?;
1815
1816 if state == RestoreApplyOperationState::Pending {
1817 return Ok(());
1818 }
1819
1820 Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
1821 sequence: options.sequence,
1822 state,
1823 })
1824}
1825
1826fn enforce_restore_plan_requirements(
1828 options: &RestorePlanOptions,
1829 plan: &RestorePlan,
1830) -> Result<(), RestoreCommandError> {
1831 if !options.require_restore_ready || plan.readiness_summary.ready {
1832 return Ok(());
1833 }
1834
1835 Err(RestoreCommandError::RestoreNotReady {
1836 backup_id: plan.backup_id.clone(),
1837 reasons: plan.readiness_summary.reasons.clone(),
1838 })
1839}
1840
1841fn verify_backup_layout_if_required(
1843 options: &RestorePlanOptions,
1844) -> Result<(), RestoreCommandError> {
1845 if !options.require_verified {
1846 return Ok(());
1847 }
1848
1849 let Some(dir) = &options.backup_dir else {
1850 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
1851 };
1852
1853 BackupLayout::new(dir.clone()).verify_integrity()?;
1854 Ok(())
1855}
1856
1857fn read_manifest_source(
1859 options: &RestorePlanOptions,
1860) -> Result<FleetBackupManifest, RestoreCommandError> {
1861 if let Some(path) = &options.manifest {
1862 return read_manifest(path);
1863 }
1864
1865 let Some(dir) = &options.backup_dir else {
1866 return Err(RestoreCommandError::MissingOption(
1867 "--manifest or --backup-dir",
1868 ));
1869 };
1870
1871 BackupLayout::new(dir.clone())
1872 .read_manifest()
1873 .map_err(RestoreCommandError::from)
1874}
1875
1876fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
1878 let data = fs::read_to_string(path)?;
1879 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1880}
1881
1882fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
1884 let data = fs::read_to_string(path)?;
1885 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1886}
1887
1888fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
1890 let data = fs::read_to_string(path)?;
1891 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1892}
1893
1894fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
1896 let data = fs::read_to_string(path)?;
1897 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1898}
1899
1900fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
1902 let data = fs::read_to_string(path)?;
1903 let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
1904 journal.validate()?;
1905 Ok(journal)
1906}
1907
1908fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
1910 value
1911 .parse::<usize>()
1912 .map_err(|_| RestoreCommandError::InvalidSequence)
1913}
1914
1915fn state_updated_at(updated_at: Option<&String>) -> String {
1917 updated_at.cloned().unwrap_or_else(timestamp_placeholder)
1918}
1919
1920fn timestamp_placeholder() -> String {
1922 "unknown".to_string()
1923}
1924
1925fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
1927 if let Some(path) = &options.out {
1928 let data = serde_json::to_vec_pretty(plan)?;
1929 fs::write(path, data)?;
1930 return Ok(());
1931 }
1932
1933 let stdout = io::stdout();
1934 let mut handle = stdout.lock();
1935 serde_json::to_writer_pretty(&mut handle, plan)?;
1936 writeln!(handle)?;
1937 Ok(())
1938}
1939
1940fn write_status(
1942 options: &RestoreStatusOptions,
1943 status: &RestoreStatus,
1944) -> Result<(), RestoreCommandError> {
1945 if let Some(path) = &options.out {
1946 let data = serde_json::to_vec_pretty(status)?;
1947 fs::write(path, data)?;
1948 return Ok(());
1949 }
1950
1951 let stdout = io::stdout();
1952 let mut handle = stdout.lock();
1953 serde_json::to_writer_pretty(&mut handle, status)?;
1954 writeln!(handle)?;
1955 Ok(())
1956}
1957
1958fn write_apply_dry_run(
1960 options: &RestoreApplyOptions,
1961 dry_run: &RestoreApplyDryRun,
1962) -> Result<(), RestoreCommandError> {
1963 if let Some(path) = &options.out {
1964 let data = serde_json::to_vec_pretty(dry_run)?;
1965 fs::write(path, data)?;
1966 return Ok(());
1967 }
1968
1969 let stdout = io::stdout();
1970 let mut handle = stdout.lock();
1971 serde_json::to_writer_pretty(&mut handle, dry_run)?;
1972 writeln!(handle)?;
1973 Ok(())
1974}
1975
1976fn write_apply_journal_if_requested(
1978 options: &RestoreApplyOptions,
1979 dry_run: &RestoreApplyDryRun,
1980) -> Result<(), RestoreCommandError> {
1981 let Some(path) = &options.journal_out else {
1982 return Ok(());
1983 };
1984
1985 let journal = RestoreApplyJournal::from_dry_run(dry_run);
1986 let data = serde_json::to_vec_pretty(&journal)?;
1987 fs::write(path, data)?;
1988 Ok(())
1989}
1990
1991fn write_apply_status(
1993 options: &RestoreApplyStatusOptions,
1994 status: &RestoreApplyJournalStatus,
1995) -> Result<(), RestoreCommandError> {
1996 if let Some(path) = &options.out {
1997 let data = serde_json::to_vec_pretty(status)?;
1998 fs::write(path, data)?;
1999 return Ok(());
2000 }
2001
2002 let stdout = io::stdout();
2003 let mut handle = stdout.lock();
2004 serde_json::to_writer_pretty(&mut handle, status)?;
2005 writeln!(handle)?;
2006 Ok(())
2007}
2008
2009fn write_apply_report(
2011 options: &RestoreApplyReportOptions,
2012 report: &RestoreApplyJournalReport,
2013) -> Result<(), RestoreCommandError> {
2014 if let Some(path) = &options.out {
2015 let data = serde_json::to_vec_pretty(report)?;
2016 fs::write(path, data)?;
2017 return Ok(());
2018 }
2019
2020 let stdout = io::stdout();
2021 let mut handle = stdout.lock();
2022 serde_json::to_writer_pretty(&mut handle, report)?;
2023 writeln!(handle)?;
2024 Ok(())
2025}
2026
2027fn write_restore_run(
2029 options: &RestoreRunOptions,
2030 run: &RestoreRunResponse,
2031) -> Result<(), RestoreCommandError> {
2032 if let Some(path) = &options.out {
2033 let data = serde_json::to_vec_pretty(run)?;
2034 fs::write(path, data)?;
2035 return Ok(());
2036 }
2037
2038 let stdout = io::stdout();
2039 let mut handle = stdout.lock();
2040 serde_json::to_writer_pretty(&mut handle, run)?;
2041 writeln!(handle)?;
2042 Ok(())
2043}
2044
2045fn write_apply_journal_file(
2047 path: &PathBuf,
2048 journal: &RestoreApplyJournal,
2049) -> Result<(), RestoreCommandError> {
2050 let data = serde_json::to_vec_pretty(journal)?;
2051 fs::write(path, data)?;
2052 Ok(())
2053}
2054
2055fn write_apply_next(
2057 options: &RestoreApplyNextOptions,
2058 next: &RestoreApplyNextOperation,
2059) -> Result<(), RestoreCommandError> {
2060 if let Some(path) = &options.out {
2061 let data = serde_json::to_vec_pretty(next)?;
2062 fs::write(path, data)?;
2063 return Ok(());
2064 }
2065
2066 let stdout = io::stdout();
2067 let mut handle = stdout.lock();
2068 serde_json::to_writer_pretty(&mut handle, next)?;
2069 writeln!(handle)?;
2070 Ok(())
2071}
2072
2073fn write_apply_command(
2075 options: &RestoreApplyCommandOptions,
2076 preview: &RestoreApplyCommandPreview,
2077) -> Result<(), RestoreCommandError> {
2078 if let Some(path) = &options.out {
2079 let data = serde_json::to_vec_pretty(preview)?;
2080 fs::write(path, data)?;
2081 return Ok(());
2082 }
2083
2084 let stdout = io::stdout();
2085 let mut handle = stdout.lock();
2086 serde_json::to_writer_pretty(&mut handle, preview)?;
2087 writeln!(handle)?;
2088 Ok(())
2089}
2090
2091fn write_apply_claim(
2093 options: &RestoreApplyClaimOptions,
2094 journal: &RestoreApplyJournal,
2095) -> Result<(), RestoreCommandError> {
2096 if let Some(path) = &options.out {
2097 let data = serde_json::to_vec_pretty(journal)?;
2098 fs::write(path, data)?;
2099 return Ok(());
2100 }
2101
2102 let stdout = io::stdout();
2103 let mut handle = stdout.lock();
2104 serde_json::to_writer_pretty(&mut handle, journal)?;
2105 writeln!(handle)?;
2106 Ok(())
2107}
2108
2109fn write_apply_unclaim(
2111 options: &RestoreApplyUnclaimOptions,
2112 journal: &RestoreApplyJournal,
2113) -> Result<(), RestoreCommandError> {
2114 if let Some(path) = &options.out {
2115 let data = serde_json::to_vec_pretty(journal)?;
2116 fs::write(path, data)?;
2117 return Ok(());
2118 }
2119
2120 let stdout = io::stdout();
2121 let mut handle = stdout.lock();
2122 serde_json::to_writer_pretty(&mut handle, journal)?;
2123 writeln!(handle)?;
2124 Ok(())
2125}
2126
2127fn write_apply_mark(
2129 options: &RestoreApplyMarkOptions,
2130 journal: &RestoreApplyJournal,
2131) -> Result<(), RestoreCommandError> {
2132 if let Some(path) = &options.out {
2133 let data = serde_json::to_vec_pretty(journal)?;
2134 fs::write(path, data)?;
2135 return Ok(());
2136 }
2137
2138 let stdout = io::stdout();
2139 let mut handle = stdout.lock();
2140 serde_json::to_writer_pretty(&mut handle, journal)?;
2141 writeln!(handle)?;
2142 Ok(())
2143}
2144
2145fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
2147where
2148 I: Iterator<Item = OsString>,
2149{
2150 args.next()
2151 .and_then(|value| value.into_string().ok())
2152 .ok_or(RestoreCommandError::MissingValue(option))
2153}
2154
2155const fn usage() -> &'static str {
2157 "usage: canic restore plan (--manifest <file> | --backup-dir <dir>) [--mapping <file>] [--out <file>] [--require-verified] [--require-restore-ready]\n canic restore status --plan <file> [--out <file>]\n canic restore apply --plan <file> [--status <file>] [--backup-dir <dir>] --dry-run [--out <file>] [--journal-out <file>]\n canic restore apply-status --journal <file> [--out <file>] [--require-ready] [--require-no-pending] [--require-no-failed] [--require-complete]\n canic restore apply-report --journal <file> [--out <file>] [--require-no-attention]\n canic restore run --journal <file> (--dry-run | --execute | --unclaim-pending) [--dfx <path>] [--network <name>] [--max-steps <n>] [--out <file>] [--require-complete] [--require-no-attention] [--require-run-mode <text>] [--require-stopped-reason <text>] [--require-next-action <text>] [--require-executed-count <n>]\n canic restore apply-next --journal <file> [--out <file>]\n canic restore apply-command --journal <file> [--dfx <path>] [--network <name>] [--out <file>] [--require-command]\n canic restore apply-claim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-unclaim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-mark --journal <file> --sequence <n> --state completed|failed [--reason <text>] [--updated-at <text>] [--out <file>] [--require-pending]"
2158}
2159
2160#[cfg(test)]
2161mod tests {
2162 use super::*;
2163 use canic_backup::restore::RestoreApplyOperationState;
2164 use canic_backup::{
2165 artifacts::ArtifactChecksum,
2166 journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
2167 manifest::{
2168 BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
2169 FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
2170 VerificationCheck, VerificationPlan,
2171 },
2172 };
2173 use serde_json::json;
2174 use std::{
2175 path::Path,
2176 time::{SystemTime, UNIX_EPOCH},
2177 };
2178
2179 const ROOT: &str = "aaaaa-aa";
2180 const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2181 const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2182 const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2183
2184 #[test]
2186 fn parses_restore_plan_options() {
2187 let options = RestorePlanOptions::parse([
2188 OsString::from("--manifest"),
2189 OsString::from("manifest.json"),
2190 OsString::from("--mapping"),
2191 OsString::from("mapping.json"),
2192 OsString::from("--out"),
2193 OsString::from("plan.json"),
2194 OsString::from("--require-restore-ready"),
2195 ])
2196 .expect("parse options");
2197
2198 assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
2199 assert_eq!(options.backup_dir, None);
2200 assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
2201 assert_eq!(options.out, Some(PathBuf::from("plan.json")));
2202 assert!(!options.require_verified);
2203 assert!(options.require_restore_ready);
2204 }
2205
2206 #[test]
2208 fn parses_verified_restore_plan_options() {
2209 let options = RestorePlanOptions::parse([
2210 OsString::from("--backup-dir"),
2211 OsString::from("backups/run"),
2212 OsString::from("--require-verified"),
2213 ])
2214 .expect("parse verified options");
2215
2216 assert_eq!(options.manifest, None);
2217 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2218 assert_eq!(options.mapping, None);
2219 assert_eq!(options.out, None);
2220 assert!(options.require_verified);
2221 assert!(!options.require_restore_ready);
2222 }
2223
2224 #[test]
2226 fn parses_restore_status_options() {
2227 let options = RestoreStatusOptions::parse([
2228 OsString::from("--plan"),
2229 OsString::from("restore-plan.json"),
2230 OsString::from("--out"),
2231 OsString::from("restore-status.json"),
2232 ])
2233 .expect("parse status options");
2234
2235 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2236 assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
2237 }
2238
2239 #[test]
2241 fn parses_restore_apply_dry_run_options() {
2242 let options = RestoreApplyOptions::parse([
2243 OsString::from("--plan"),
2244 OsString::from("restore-plan.json"),
2245 OsString::from("--status"),
2246 OsString::from("restore-status.json"),
2247 OsString::from("--backup-dir"),
2248 OsString::from("backups/run"),
2249 OsString::from("--dry-run"),
2250 OsString::from("--out"),
2251 OsString::from("restore-apply-dry-run.json"),
2252 OsString::from("--journal-out"),
2253 OsString::from("restore-apply-journal.json"),
2254 ])
2255 .expect("parse apply options");
2256
2257 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2258 assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
2259 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2260 assert_eq!(
2261 options.out,
2262 Some(PathBuf::from("restore-apply-dry-run.json"))
2263 );
2264 assert_eq!(
2265 options.journal_out,
2266 Some(PathBuf::from("restore-apply-journal.json"))
2267 );
2268 assert!(options.dry_run);
2269 }
2270
2271 #[test]
2273 fn parses_restore_apply_status_options() {
2274 let options = RestoreApplyStatusOptions::parse([
2275 OsString::from("--journal"),
2276 OsString::from("restore-apply-journal.json"),
2277 OsString::from("--out"),
2278 OsString::from("restore-apply-status.json"),
2279 OsString::from("--require-ready"),
2280 OsString::from("--require-no-pending"),
2281 OsString::from("--require-no-failed"),
2282 OsString::from("--require-complete"),
2283 ])
2284 .expect("parse apply-status options");
2285
2286 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2287 assert!(options.require_ready);
2288 assert!(options.require_no_pending);
2289 assert!(options.require_no_failed);
2290 assert!(options.require_complete);
2291 assert_eq!(
2292 options.out,
2293 Some(PathBuf::from("restore-apply-status.json"))
2294 );
2295 }
2296
2297 #[test]
2299 fn parses_restore_apply_report_options() {
2300 let options = RestoreApplyReportOptions::parse([
2301 OsString::from("--journal"),
2302 OsString::from("restore-apply-journal.json"),
2303 OsString::from("--out"),
2304 OsString::from("restore-apply-report.json"),
2305 OsString::from("--require-no-attention"),
2306 ])
2307 .expect("parse apply-report options");
2308
2309 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2310 assert!(options.require_no_attention);
2311 assert_eq!(
2312 options.out,
2313 Some(PathBuf::from("restore-apply-report.json"))
2314 );
2315 }
2316
2317 #[test]
2319 fn parses_restore_run_dry_run_options() {
2320 let options = RestoreRunOptions::parse([
2321 OsString::from("--journal"),
2322 OsString::from("restore-apply-journal.json"),
2323 OsString::from("--dry-run"),
2324 OsString::from("--dfx"),
2325 OsString::from("/tmp/dfx"),
2326 OsString::from("--network"),
2327 OsString::from("local"),
2328 OsString::from("--out"),
2329 OsString::from("restore-run-dry-run.json"),
2330 OsString::from("--max-steps"),
2331 OsString::from("1"),
2332 OsString::from("--require-complete"),
2333 OsString::from("--require-no-attention"),
2334 OsString::from("--require-run-mode"),
2335 OsString::from("dry-run"),
2336 OsString::from("--require-stopped-reason"),
2337 OsString::from("preview"),
2338 OsString::from("--require-next-action"),
2339 OsString::from("rerun"),
2340 OsString::from("--require-executed-count"),
2341 OsString::from("0"),
2342 ])
2343 .expect("parse restore run options");
2344
2345 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2346 assert_eq!(options.dfx, "/tmp/dfx");
2347 assert_eq!(options.network.as_deref(), Some("local"));
2348 assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
2349 assert!(options.dry_run);
2350 assert!(!options.execute);
2351 assert!(!options.unclaim_pending);
2352 assert_eq!(options.max_steps, Some(1));
2353 assert!(options.require_complete);
2354 assert!(options.require_no_attention);
2355 assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
2356 assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
2357 assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
2358 assert_eq!(options.require_executed_count, Some(0));
2359 }
2360
2361 #[test]
2363 fn parses_restore_run_execute_options() {
2364 let options = RestoreRunOptions::parse([
2365 OsString::from("--journal"),
2366 OsString::from("restore-apply-journal.json"),
2367 OsString::from("--execute"),
2368 OsString::from("--dfx"),
2369 OsString::from("/bin/true"),
2370 OsString::from("--max-steps"),
2371 OsString::from("4"),
2372 ])
2373 .expect("parse restore run execute options");
2374
2375 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2376 assert_eq!(options.dfx, "/bin/true");
2377 assert_eq!(options.network, None);
2378 assert_eq!(options.out, None);
2379 assert!(!options.dry_run);
2380 assert!(options.execute);
2381 assert!(!options.unclaim_pending);
2382 assert_eq!(options.max_steps, Some(4));
2383 assert!(!options.require_complete);
2384 assert!(!options.require_no_attention);
2385 assert_eq!(options.require_run_mode, None);
2386 assert_eq!(options.require_stopped_reason, None);
2387 assert_eq!(options.require_next_action, None);
2388 assert_eq!(options.require_executed_count, None);
2389 }
2390
2391 #[test]
2393 fn parses_restore_run_unclaim_pending_options() {
2394 let options = RestoreRunOptions::parse([
2395 OsString::from("--journal"),
2396 OsString::from("restore-apply-journal.json"),
2397 OsString::from("--unclaim-pending"),
2398 OsString::from("--out"),
2399 OsString::from("restore-run.json"),
2400 ])
2401 .expect("parse restore run unclaim options");
2402
2403 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2404 assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
2405 assert!(!options.dry_run);
2406 assert!(!options.execute);
2407 assert!(options.unclaim_pending);
2408 }
2409
2410 #[test]
2412 fn parses_restore_apply_next_options() {
2413 let options = RestoreApplyNextOptions::parse([
2414 OsString::from("--journal"),
2415 OsString::from("restore-apply-journal.json"),
2416 OsString::from("--out"),
2417 OsString::from("restore-apply-next.json"),
2418 ])
2419 .expect("parse apply-next options");
2420
2421 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2422 assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
2423 }
2424
2425 #[test]
2427 fn parses_restore_apply_command_options() {
2428 let options = RestoreApplyCommandOptions::parse([
2429 OsString::from("--journal"),
2430 OsString::from("restore-apply-journal.json"),
2431 OsString::from("--dfx"),
2432 OsString::from("/tmp/dfx"),
2433 OsString::from("--network"),
2434 OsString::from("local"),
2435 OsString::from("--out"),
2436 OsString::from("restore-apply-command.json"),
2437 OsString::from("--require-command"),
2438 ])
2439 .expect("parse apply-command options");
2440
2441 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2442 assert_eq!(options.dfx, "/tmp/dfx");
2443 assert_eq!(options.network.as_deref(), Some("local"));
2444 assert!(options.require_command);
2445 assert_eq!(
2446 options.out,
2447 Some(PathBuf::from("restore-apply-command.json"))
2448 );
2449 }
2450
2451 #[test]
2453 fn parses_restore_apply_claim_options() {
2454 let options = RestoreApplyClaimOptions::parse([
2455 OsString::from("--journal"),
2456 OsString::from("restore-apply-journal.json"),
2457 OsString::from("--sequence"),
2458 OsString::from("0"),
2459 OsString::from("--updated-at"),
2460 OsString::from("2026-05-04T12:00:00Z"),
2461 OsString::from("--out"),
2462 OsString::from("restore-apply-journal.claimed.json"),
2463 ])
2464 .expect("parse apply-claim options");
2465
2466 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2467 assert_eq!(options.sequence, Some(0));
2468 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
2469 assert_eq!(
2470 options.out,
2471 Some(PathBuf::from("restore-apply-journal.claimed.json"))
2472 );
2473 }
2474
2475 #[test]
2477 fn parses_restore_apply_unclaim_options() {
2478 let options = RestoreApplyUnclaimOptions::parse([
2479 OsString::from("--journal"),
2480 OsString::from("restore-apply-journal.json"),
2481 OsString::from("--sequence"),
2482 OsString::from("0"),
2483 OsString::from("--updated-at"),
2484 OsString::from("2026-05-04T12:01:00Z"),
2485 OsString::from("--out"),
2486 OsString::from("restore-apply-journal.unclaimed.json"),
2487 ])
2488 .expect("parse apply-unclaim options");
2489
2490 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2491 assert_eq!(options.sequence, Some(0));
2492 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
2493 assert_eq!(
2494 options.out,
2495 Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
2496 );
2497 }
2498
2499 #[test]
2501 fn parses_restore_apply_mark_options() {
2502 let options = RestoreApplyMarkOptions::parse([
2503 OsString::from("--journal"),
2504 OsString::from("restore-apply-journal.json"),
2505 OsString::from("--sequence"),
2506 OsString::from("4"),
2507 OsString::from("--state"),
2508 OsString::from("failed"),
2509 OsString::from("--reason"),
2510 OsString::from("dfx-load-failed"),
2511 OsString::from("--updated-at"),
2512 OsString::from("2026-05-04T12:02:00Z"),
2513 OsString::from("--out"),
2514 OsString::from("restore-apply-journal.updated.json"),
2515 OsString::from("--require-pending"),
2516 ])
2517 .expect("parse apply-mark options");
2518
2519 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2520 assert_eq!(options.sequence, 4);
2521 assert_eq!(options.state, RestoreApplyMarkState::Failed);
2522 assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
2523 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
2524 assert!(options.require_pending);
2525 assert_eq!(
2526 options.out,
2527 Some(PathBuf::from("restore-apply-journal.updated.json"))
2528 );
2529 }
2530
2531 #[test]
2533 fn restore_apply_requires_dry_run() {
2534 let err = RestoreApplyOptions::parse([
2535 OsString::from("--plan"),
2536 OsString::from("restore-plan.json"),
2537 ])
2538 .expect_err("apply without dry-run should fail");
2539
2540 assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
2541 }
2542
2543 #[test]
2545 fn restore_run_requires_mode() {
2546 let err = RestoreRunOptions::parse([
2547 OsString::from("--journal"),
2548 OsString::from("restore-apply-journal.json"),
2549 ])
2550 .expect_err("restore run without dry-run should fail");
2551
2552 assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
2553 }
2554
2555 #[test]
2557 fn restore_run_rejects_conflicting_modes() {
2558 let err = RestoreRunOptions::parse([
2559 OsString::from("--journal"),
2560 OsString::from("restore-apply-journal.json"),
2561 OsString::from("--dry-run"),
2562 OsString::from("--execute"),
2563 OsString::from("--unclaim-pending"),
2564 ])
2565 .expect_err("restore run should reject conflicting modes");
2566
2567 assert!(matches!(
2568 err,
2569 RestoreCommandError::RestoreRunConflictingModes
2570 ));
2571 }
2572
2573 #[test]
2575 fn plan_restore_reads_manifest_from_backup_dir() {
2576 let root = temp_dir("canic-cli-restore-plan-layout");
2577 let layout = BackupLayout::new(root.clone());
2578 layout
2579 .write_manifest(&valid_manifest())
2580 .expect("write manifest");
2581
2582 let options = RestorePlanOptions {
2583 manifest: None,
2584 backup_dir: Some(root.clone()),
2585 mapping: None,
2586 out: None,
2587 require_verified: false,
2588 require_restore_ready: false,
2589 };
2590
2591 let plan = plan_restore(&options).expect("plan restore");
2592
2593 fs::remove_dir_all(root).expect("remove temp root");
2594 assert_eq!(plan.backup_id, "backup-test");
2595 assert_eq!(plan.member_count, 2);
2596 }
2597
2598 #[test]
2600 fn parse_rejects_conflicting_manifest_sources() {
2601 let err = RestorePlanOptions::parse([
2602 OsString::from("--manifest"),
2603 OsString::from("manifest.json"),
2604 OsString::from("--backup-dir"),
2605 OsString::from("backups/run"),
2606 ])
2607 .expect_err("conflicting sources should fail");
2608
2609 assert!(matches!(
2610 err,
2611 RestoreCommandError::ConflictingManifestSources
2612 ));
2613 }
2614
2615 #[test]
2617 fn parse_rejects_require_verified_with_manifest_source() {
2618 let err = RestorePlanOptions::parse([
2619 OsString::from("--manifest"),
2620 OsString::from("manifest.json"),
2621 OsString::from("--require-verified"),
2622 ])
2623 .expect_err("verification should require a backup layout");
2624
2625 assert!(matches!(
2626 err,
2627 RestoreCommandError::RequireVerifiedNeedsBackupDir
2628 ));
2629 }
2630
2631 #[test]
2633 fn plan_restore_requires_verified_backup_layout() {
2634 let root = temp_dir("canic-cli-restore-plan-verified");
2635 let layout = BackupLayout::new(root.clone());
2636 let manifest = valid_manifest();
2637 write_verified_layout(&root, &layout, &manifest);
2638
2639 let options = RestorePlanOptions {
2640 manifest: None,
2641 backup_dir: Some(root.clone()),
2642 mapping: None,
2643 out: None,
2644 require_verified: true,
2645 require_restore_ready: false,
2646 };
2647
2648 let plan = plan_restore(&options).expect("plan verified restore");
2649
2650 fs::remove_dir_all(root).expect("remove temp root");
2651 assert_eq!(plan.backup_id, "backup-test");
2652 assert_eq!(plan.member_count, 2);
2653 }
2654
2655 #[test]
2657 fn plan_restore_rejects_unverified_backup_layout() {
2658 let root = temp_dir("canic-cli-restore-plan-unverified");
2659 let layout = BackupLayout::new(root.clone());
2660 layout
2661 .write_manifest(&valid_manifest())
2662 .expect("write manifest");
2663
2664 let options = RestorePlanOptions {
2665 manifest: None,
2666 backup_dir: Some(root.clone()),
2667 mapping: None,
2668 out: None,
2669 require_verified: true,
2670 require_restore_ready: false,
2671 };
2672
2673 let err = plan_restore(&options).expect_err("missing journal should fail");
2674
2675 fs::remove_dir_all(root).expect("remove temp root");
2676 assert!(matches!(err, RestoreCommandError::Persistence(_)));
2677 }
2678
2679 #[test]
2681 fn plan_restore_reads_manifest_and_mapping() {
2682 let root = temp_dir("canic-cli-restore-plan");
2683 fs::create_dir_all(&root).expect("create temp root");
2684 let manifest_path = root.join("manifest.json");
2685 let mapping_path = root.join("mapping.json");
2686
2687 fs::write(
2688 &manifest_path,
2689 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2690 )
2691 .expect("write manifest");
2692 fs::write(
2693 &mapping_path,
2694 json!({
2695 "members": [
2696 {"source_canister": ROOT, "target_canister": ROOT},
2697 {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
2698 ]
2699 })
2700 .to_string(),
2701 )
2702 .expect("write mapping");
2703
2704 let options = RestorePlanOptions {
2705 manifest: Some(manifest_path),
2706 backup_dir: None,
2707 mapping: Some(mapping_path),
2708 out: None,
2709 require_verified: false,
2710 require_restore_ready: false,
2711 };
2712
2713 let plan = plan_restore(&options).expect("plan restore");
2714
2715 fs::remove_dir_all(root).expect("remove temp root");
2716 let members = plan.ordered_members();
2717 assert_eq!(members.len(), 2);
2718 assert_eq!(members[0].source_canister, ROOT);
2719 assert_eq!(members[1].target_canister, MAPPED_CHILD);
2720 }
2721
2722 #[test]
2724 fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
2725 let root = temp_dir("canic-cli-restore-plan-require-ready");
2726 fs::create_dir_all(&root).expect("create temp root");
2727 let manifest_path = root.join("manifest.json");
2728 let out_path = root.join("plan.json");
2729
2730 fs::write(
2731 &manifest_path,
2732 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2733 )
2734 .expect("write manifest");
2735
2736 let err = run([
2737 OsString::from("plan"),
2738 OsString::from("--manifest"),
2739 OsString::from(manifest_path.as_os_str()),
2740 OsString::from("--out"),
2741 OsString::from(out_path.as_os_str()),
2742 OsString::from("--require-restore-ready"),
2743 ])
2744 .expect_err("restore readiness should be enforced");
2745
2746 assert!(out_path.exists());
2747 let plan: RestorePlan =
2748 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2749
2750 fs::remove_dir_all(root).expect("remove temp root");
2751 assert!(!plan.readiness_summary.ready);
2752 assert!(matches!(
2753 err,
2754 RestoreCommandError::RestoreNotReady {
2755 reasons,
2756 ..
2757 } if reasons == [
2758 "missing-module-hash",
2759 "missing-wasm-hash",
2760 "missing-snapshot-checksum"
2761 ]
2762 ));
2763 }
2764
2765 #[test]
2767 fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
2768 let root = temp_dir("canic-cli-restore-plan-ready");
2769 fs::create_dir_all(&root).expect("create temp root");
2770 let manifest_path = root.join("manifest.json");
2771 let out_path = root.join("plan.json");
2772
2773 fs::write(
2774 &manifest_path,
2775 serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
2776 )
2777 .expect("write manifest");
2778
2779 run([
2780 OsString::from("plan"),
2781 OsString::from("--manifest"),
2782 OsString::from(manifest_path.as_os_str()),
2783 OsString::from("--out"),
2784 OsString::from(out_path.as_os_str()),
2785 OsString::from("--require-restore-ready"),
2786 ])
2787 .expect("restore-ready plan should pass");
2788
2789 let plan: RestorePlan =
2790 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2791
2792 fs::remove_dir_all(root).expect("remove temp root");
2793 assert!(plan.readiness_summary.ready);
2794 assert!(plan.readiness_summary.reasons.is_empty());
2795 }
2796
2797 #[test]
2799 fn run_restore_status_writes_planned_status() {
2800 let root = temp_dir("canic-cli-restore-status");
2801 fs::create_dir_all(&root).expect("create temp root");
2802 let plan_path = root.join("restore-plan.json");
2803 let out_path = root.join("restore-status.json");
2804 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2805
2806 fs::write(
2807 &plan_path,
2808 serde_json::to_vec(&plan).expect("serialize plan"),
2809 )
2810 .expect("write plan");
2811
2812 run([
2813 OsString::from("status"),
2814 OsString::from("--plan"),
2815 OsString::from(plan_path.as_os_str()),
2816 OsString::from("--out"),
2817 OsString::from(out_path.as_os_str()),
2818 ])
2819 .expect("write restore status");
2820
2821 let status: RestoreStatus =
2822 serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
2823 .expect("decode restore status");
2824 let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
2825
2826 fs::remove_dir_all(root).expect("remove temp root");
2827 assert_eq!(status.status_version, 1);
2828 assert_eq!(status.backup_id.as_str(), "backup-test");
2829 assert!(status.ready);
2830 assert!(status.readiness_reasons.is_empty());
2831 assert_eq!(status.member_count, 2);
2832 assert_eq!(status.phase_count, 1);
2833 assert_eq!(status.planned_snapshot_uploads, 2);
2834 assert_eq!(status.planned_snapshot_loads, 2);
2835 assert_eq!(status.planned_code_reinstalls, 2);
2836 assert_eq!(status.planned_verification_checks, 2);
2837 assert_eq!(status.planned_operations, 8);
2838 assert_eq!(status.phases[0].members[0].source_canister, ROOT);
2839 assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
2840 }
2841
2842 #[test]
2844 fn run_restore_apply_dry_run_writes_operations() {
2845 let root = temp_dir("canic-cli-restore-apply-dry-run");
2846 fs::create_dir_all(&root).expect("create temp root");
2847 let plan_path = root.join("restore-plan.json");
2848 let status_path = root.join("restore-status.json");
2849 let out_path = root.join("restore-apply-dry-run.json");
2850 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2851 let status = RestoreStatus::from_plan(&plan);
2852
2853 fs::write(
2854 &plan_path,
2855 serde_json::to_vec(&plan).expect("serialize plan"),
2856 )
2857 .expect("write plan");
2858 fs::write(
2859 &status_path,
2860 serde_json::to_vec(&status).expect("serialize status"),
2861 )
2862 .expect("write status");
2863
2864 run([
2865 OsString::from("apply"),
2866 OsString::from("--plan"),
2867 OsString::from(plan_path.as_os_str()),
2868 OsString::from("--status"),
2869 OsString::from(status_path.as_os_str()),
2870 OsString::from("--dry-run"),
2871 OsString::from("--out"),
2872 OsString::from(out_path.as_os_str()),
2873 ])
2874 .expect("write apply dry-run");
2875
2876 let dry_run: RestoreApplyDryRun =
2877 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2878 .expect("decode dry-run");
2879 let dry_run_json: serde_json::Value =
2880 serde_json::to_value(&dry_run).expect("encode dry-run");
2881
2882 fs::remove_dir_all(root).expect("remove temp root");
2883 assert_eq!(dry_run.dry_run_version, 1);
2884 assert_eq!(dry_run.backup_id.as_str(), "backup-test");
2885 assert!(dry_run.ready);
2886 assert!(dry_run.status_supplied);
2887 assert_eq!(dry_run.member_count, 2);
2888 assert_eq!(dry_run.phase_count, 1);
2889 assert_eq!(dry_run.planned_snapshot_uploads, 2);
2890 assert_eq!(dry_run.planned_operations, 8);
2891 assert_eq!(dry_run.rendered_operations, 8);
2892 assert_eq!(dry_run_json["operation_counts"]["snapshot_uploads"], 2);
2893 assert_eq!(dry_run_json["operation_counts"]["snapshot_loads"], 2);
2894 assert_eq!(dry_run_json["operation_counts"]["code_reinstalls"], 2);
2895 assert_eq!(dry_run_json["operation_counts"]["member_verifications"], 2);
2896 assert_eq!(dry_run_json["operation_counts"]["fleet_verifications"], 0);
2897 assert_eq!(
2898 dry_run_json["operation_counts"]["verification_operations"],
2899 2
2900 );
2901 assert_eq!(
2902 dry_run_json["phases"][0]["operations"][0]["operation"],
2903 "upload-snapshot"
2904 );
2905 assert_eq!(
2906 dry_run_json["phases"][0]["operations"][3]["operation"],
2907 "verify-member"
2908 );
2909 assert_eq!(
2910 dry_run_json["phases"][0]["operations"][3]["verification_kind"],
2911 "status"
2912 );
2913 assert_eq!(
2914 dry_run_json["phases"][0]["operations"][3]["verification_method"],
2915 serde_json::Value::Null
2916 );
2917 }
2918
2919 #[test]
2921 fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
2922 let root = temp_dir("canic-cli-restore-apply-artifacts");
2923 fs::create_dir_all(&root).expect("create temp root");
2924 let plan_path = root.join("restore-plan.json");
2925 let out_path = root.join("restore-apply-dry-run.json");
2926 let journal_path = root.join("restore-apply-journal.json");
2927 let status_path = root.join("restore-apply-status.json");
2928 let mut manifest = restore_ready_manifest();
2929 write_manifest_artifacts(&root, &mut manifest);
2930 let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
2931
2932 fs::write(
2933 &plan_path,
2934 serde_json::to_vec(&plan).expect("serialize plan"),
2935 )
2936 .expect("write plan");
2937
2938 run([
2939 OsString::from("apply"),
2940 OsString::from("--plan"),
2941 OsString::from(plan_path.as_os_str()),
2942 OsString::from("--backup-dir"),
2943 OsString::from(root.as_os_str()),
2944 OsString::from("--dry-run"),
2945 OsString::from("--out"),
2946 OsString::from(out_path.as_os_str()),
2947 OsString::from("--journal-out"),
2948 OsString::from(journal_path.as_os_str()),
2949 ])
2950 .expect("write apply dry-run");
2951 run([
2952 OsString::from("apply-status"),
2953 OsString::from("--journal"),
2954 OsString::from(journal_path.as_os_str()),
2955 OsString::from("--out"),
2956 OsString::from(status_path.as_os_str()),
2957 ])
2958 .expect("write apply status");
2959
2960 let dry_run: RestoreApplyDryRun =
2961 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2962 .expect("decode dry-run");
2963 let validation = dry_run
2964 .artifact_validation
2965 .expect("artifact validation should be present");
2966 let journal_json: serde_json::Value =
2967 serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
2968 .expect("decode journal");
2969 let status_json: serde_json::Value =
2970 serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
2971 .expect("decode apply status");
2972
2973 fs::remove_dir_all(root).expect("remove temp root");
2974 assert_eq!(validation.checked_members, 2);
2975 assert!(validation.artifacts_present);
2976 assert!(validation.checksums_verified);
2977 assert_eq!(validation.members_with_expected_checksums, 2);
2978 assert_eq!(journal_json["ready"], true);
2979 assert_eq!(journal_json["operation_count"], 8);
2980 assert_eq!(journal_json["operation_counts"]["snapshot_uploads"], 2);
2981 assert_eq!(journal_json["operation_counts"]["snapshot_loads"], 2);
2982 assert_eq!(journal_json["operation_counts"]["code_reinstalls"], 2);
2983 assert_eq!(journal_json["operation_counts"]["member_verifications"], 2);
2984 assert_eq!(journal_json["operation_counts"]["fleet_verifications"], 0);
2985 assert_eq!(
2986 journal_json["operation_counts"]["verification_operations"],
2987 2
2988 );
2989 assert_eq!(journal_json["ready_operations"], 8);
2990 assert_eq!(journal_json["blocked_operations"], 0);
2991 assert_eq!(journal_json["operations"][0]["state"], "ready");
2992 assert_eq!(status_json["ready"], true);
2993 assert_eq!(status_json["operation_count"], 8);
2994 assert_eq!(status_json["operation_counts"]["snapshot_uploads"], 2);
2995 assert_eq!(status_json["operation_counts"]["snapshot_loads"], 2);
2996 assert_eq!(status_json["operation_counts"]["code_reinstalls"], 2);
2997 assert_eq!(status_json["operation_counts"]["member_verifications"], 2);
2998 assert_eq!(status_json["operation_counts"]["fleet_verifications"], 0);
2999 assert_eq!(
3000 status_json["operation_counts"]["verification_operations"],
3001 2
3002 );
3003 assert_eq!(status_json["operation_counts_supplied"], true);
3004 assert_eq!(status_json["next_ready_sequence"], 0);
3005 assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
3006 }
3007
3008 #[test]
3010 fn run_restore_apply_status_rejects_invalid_journal() {
3011 let root = temp_dir("canic-cli-restore-apply-status-invalid");
3012 fs::create_dir_all(&root).expect("create temp root");
3013 let journal_path = root.join("restore-apply-journal.json");
3014 let out_path = root.join("restore-apply-status.json");
3015 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3016 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3017 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3018 journal.operation_count += 1;
3019
3020 fs::write(
3021 &journal_path,
3022 serde_json::to_vec(&journal).expect("serialize journal"),
3023 )
3024 .expect("write journal");
3025
3026 let err = run([
3027 OsString::from("apply-status"),
3028 OsString::from("--journal"),
3029 OsString::from(journal_path.as_os_str()),
3030 OsString::from("--out"),
3031 OsString::from(out_path.as_os_str()),
3032 ])
3033 .expect_err("invalid journal should fail");
3034
3035 assert!(!out_path.exists());
3036 fs::remove_dir_all(root).expect("remove temp root");
3037 assert!(matches!(
3038 err,
3039 RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
3040 field: "operation_count",
3041 ..
3042 })
3043 ));
3044 }
3045
3046 #[test]
3048 fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
3049 let root = temp_dir("canic-cli-restore-apply-status-pending");
3050 fs::create_dir_all(&root).expect("create temp root");
3051 let journal_path = root.join("restore-apply-journal.json");
3052 let out_path = root.join("restore-apply-status.json");
3053 let mut journal = ready_apply_journal();
3054 journal
3055 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
3056 .expect("claim operation");
3057
3058 fs::write(
3059 &journal_path,
3060 serde_json::to_vec(&journal).expect("serialize journal"),
3061 )
3062 .expect("write journal");
3063
3064 let err = run([
3065 OsString::from("apply-status"),
3066 OsString::from("--journal"),
3067 OsString::from(journal_path.as_os_str()),
3068 OsString::from("--out"),
3069 OsString::from(out_path.as_os_str()),
3070 OsString::from("--require-no-pending"),
3071 ])
3072 .expect_err("pending operation should fail requirement");
3073
3074 assert!(out_path.exists());
3075 let status: RestoreApplyJournalStatus =
3076 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3077 .expect("decode apply status");
3078
3079 fs::remove_dir_all(root).expect("remove temp root");
3080 assert_eq!(status.pending_operations, 1);
3081 assert_eq!(status.next_transition_sequence, Some(0));
3082 assert_eq!(
3083 status.next_transition_updated_at.as_deref(),
3084 Some("2026-05-04T12:00:00Z")
3085 );
3086 assert!(matches!(
3087 err,
3088 RestoreCommandError::RestoreApplyPending {
3089 pending_operations: 1,
3090 next_transition_sequence: Some(0),
3091 ..
3092 }
3093 ));
3094 }
3095
3096 #[test]
3098 fn run_restore_apply_status_require_ready_writes_status_then_fails() {
3099 let root = temp_dir("canic-cli-restore-apply-status-ready");
3100 fs::create_dir_all(&root).expect("create temp root");
3101 let journal_path = root.join("restore-apply-journal.json");
3102 let out_path = root.join("restore-apply-status.json");
3103 let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
3104 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3105 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3106
3107 fs::write(
3108 &journal_path,
3109 serde_json::to_vec(&journal).expect("serialize journal"),
3110 )
3111 .expect("write journal");
3112
3113 let err = run([
3114 OsString::from("apply-status"),
3115 OsString::from("--journal"),
3116 OsString::from(journal_path.as_os_str()),
3117 OsString::from("--out"),
3118 OsString::from(out_path.as_os_str()),
3119 OsString::from("--require-ready"),
3120 ])
3121 .expect_err("unready journal should fail requirement");
3122
3123 let status: RestoreApplyJournalStatus =
3124 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3125 .expect("decode apply status");
3126
3127 fs::remove_dir_all(root).expect("remove temp root");
3128 assert!(!status.ready);
3129 assert_eq!(status.blocked_operations, status.operation_count);
3130 assert!(
3131 status
3132 .blocked_reasons
3133 .contains(&"missing-snapshot-checksum".to_string())
3134 );
3135 assert!(matches!(
3136 err,
3137 RestoreCommandError::RestoreApplyNotReady { reasons, .. }
3138 if reasons.contains(&"missing-snapshot-checksum".to_string())
3139 ));
3140 }
3141
3142 #[test]
3144 fn run_restore_apply_report_writes_attention_summary() {
3145 let root = temp_dir("canic-cli-restore-apply-report");
3146 fs::create_dir_all(&root).expect("create temp root");
3147 let journal_path = root.join("restore-apply-journal.json");
3148 let out_path = root.join("restore-apply-report.json");
3149 let mut journal = ready_apply_journal();
3150 journal
3151 .mark_operation_failed_at(
3152 0,
3153 "dfx-upload-failed".to_string(),
3154 Some("2026-05-05T12:00:00Z".to_string()),
3155 )
3156 .expect("mark failed operation");
3157 journal
3158 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3159 .expect("mark pending operation");
3160
3161 fs::write(
3162 &journal_path,
3163 serde_json::to_vec(&journal).expect("serialize journal"),
3164 )
3165 .expect("write journal");
3166
3167 run([
3168 OsString::from("apply-report"),
3169 OsString::from("--journal"),
3170 OsString::from(journal_path.as_os_str()),
3171 OsString::from("--out"),
3172 OsString::from(out_path.as_os_str()),
3173 ])
3174 .expect("write apply report");
3175
3176 let report: RestoreApplyJournalReport =
3177 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3178 .expect("decode apply report");
3179 let report_json: serde_json::Value =
3180 serde_json::to_value(&report).expect("encode apply report");
3181
3182 fs::remove_dir_all(root).expect("remove temp root");
3183 assert_eq!(report.backup_id, "backup-test");
3184 assert!(report.attention_required);
3185 assert_eq!(report.failed_operations, 1);
3186 assert_eq!(report.pending_operations, 1);
3187 assert_eq!(report.operation_counts.snapshot_uploads, 2);
3188 assert_eq!(report.operation_counts.snapshot_loads, 2);
3189 assert_eq!(report.operation_counts.code_reinstalls, 2);
3190 assert_eq!(report.operation_counts.member_verifications, 2);
3191 assert_eq!(report.operation_counts.fleet_verifications, 0);
3192 assert_eq!(report.operation_counts.verification_operations, 2);
3193 assert!(report.operation_counts_supplied);
3194 assert_eq!(report.failed.len(), 1);
3195 assert_eq!(report.pending.len(), 1);
3196 assert_eq!(report.failed[0].sequence, 0);
3197 assert_eq!(report.pending[0].sequence, 1);
3198 assert_eq!(
3199 report.next_transition.as_ref().map(|op| op.sequence),
3200 Some(1)
3201 );
3202 assert_eq!(report_json["outcome"], "failed");
3203 assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
3204 }
3205
3206 #[test]
3208 fn run_restore_run_dry_run_writes_native_runner_preview() {
3209 let root = temp_dir("canic-cli-restore-run-dry-run");
3210 fs::create_dir_all(&root).expect("create temp root");
3211 let journal_path = root.join("restore-apply-journal.json");
3212 let out_path = root.join("restore-run-dry-run.json");
3213 let journal = ready_apply_journal();
3214
3215 fs::write(
3216 &journal_path,
3217 serde_json::to_vec(&journal).expect("serialize journal"),
3218 )
3219 .expect("write journal");
3220
3221 run([
3222 OsString::from("run"),
3223 OsString::from("--journal"),
3224 OsString::from(journal_path.as_os_str()),
3225 OsString::from("--dry-run"),
3226 OsString::from("--dfx"),
3227 OsString::from("/tmp/dfx"),
3228 OsString::from("--network"),
3229 OsString::from("local"),
3230 OsString::from("--out"),
3231 OsString::from(out_path.as_os_str()),
3232 ])
3233 .expect("write restore run dry-run");
3234
3235 let dry_run: serde_json::Value =
3236 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3237 .expect("decode dry-run");
3238
3239 fs::remove_dir_all(root).expect("remove temp root");
3240 assert_eq!(dry_run["run_version"], 1);
3241 assert_eq!(dry_run["backup_id"], "backup-test");
3242 assert_eq!(dry_run["run_mode"], "dry-run");
3243 assert_eq!(dry_run["dry_run"], true);
3244 assert_eq!(dry_run["ready"], true);
3245 assert_eq!(dry_run["complete"], false);
3246 assert_eq!(dry_run["attention_required"], false);
3247 assert_eq!(dry_run["operation_counts"]["snapshot_uploads"], 2);
3248 assert_eq!(dry_run["operation_counts"]["snapshot_loads"], 2);
3249 assert_eq!(dry_run["operation_counts"]["code_reinstalls"], 2);
3250 assert_eq!(dry_run["operation_counts"]["member_verifications"], 2);
3251 assert_eq!(dry_run["operation_counts"]["fleet_verifications"], 0);
3252 assert_eq!(dry_run["operation_counts"]["verification_operations"], 2);
3253 assert_eq!(dry_run["operation_counts_supplied"], true);
3254 assert_eq!(dry_run["stopped_reason"], "preview");
3255 assert_eq!(dry_run["next_action"], "rerun");
3256 assert_eq!(dry_run["operation_available"], true);
3257 assert_eq!(dry_run["command_available"], true);
3258 assert_eq!(dry_run["next_transition"]["sequence"], 0);
3259 assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
3260 assert_eq!(
3261 dry_run["command"]["args"],
3262 json!([
3263 "canister",
3264 "--network",
3265 "local",
3266 "snapshot",
3267 "upload",
3268 "--dir",
3269 "artifacts/root",
3270 ROOT
3271 ])
3272 );
3273 assert_eq!(dry_run["command"]["mutates"], true);
3274 }
3275
3276 #[test]
3278 fn run_restore_run_unclaim_pending_marks_operation_ready() {
3279 let root = temp_dir("canic-cli-restore-run-unclaim-pending");
3280 fs::create_dir_all(&root).expect("create temp root");
3281 let journal_path = root.join("restore-apply-journal.json");
3282 let out_path = root.join("restore-run.json");
3283 let mut journal = ready_apply_journal();
3284 journal
3285 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3286 .expect("mark pending operation");
3287
3288 fs::write(
3289 &journal_path,
3290 serde_json::to_vec(&journal).expect("serialize journal"),
3291 )
3292 .expect("write journal");
3293
3294 run([
3295 OsString::from("run"),
3296 OsString::from("--journal"),
3297 OsString::from(journal_path.as_os_str()),
3298 OsString::from("--unclaim-pending"),
3299 OsString::from("--out"),
3300 OsString::from(out_path.as_os_str()),
3301 ])
3302 .expect("unclaim pending operation");
3303
3304 let run_summary: serde_json::Value =
3305 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3306 .expect("decode run summary");
3307 let updated: RestoreApplyJournal =
3308 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3309 .expect("decode updated journal");
3310
3311 fs::remove_dir_all(root).expect("remove temp root");
3312 assert_eq!(run_summary["run_mode"], "unclaim-pending");
3313 assert_eq!(run_summary["unclaim_pending"], true);
3314 assert_eq!(run_summary["stopped_reason"], "recovered-pending");
3315 assert_eq!(run_summary["next_action"], "rerun");
3316 assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
3317 assert_eq!(run_summary["recovered_operation"]["state"], "pending");
3318 assert_eq!(run_summary["pending_operations"], 0);
3319 assert_eq!(run_summary["ready_operations"], 8);
3320 assert_eq!(run_summary["attention_required"], false);
3321 assert_eq!(updated.pending_operations, 0);
3322 assert_eq!(updated.ready_operations, 8);
3323 assert_eq!(
3324 updated.operations[0].state,
3325 RestoreApplyOperationState::Ready
3326 );
3327 }
3328
3329 #[test]
3331 fn run_restore_run_execute_marks_completed_operation() {
3332 let root = temp_dir("canic-cli-restore-run-execute");
3333 fs::create_dir_all(&root).expect("create temp root");
3334 let journal_path = root.join("restore-apply-journal.json");
3335 let out_path = root.join("restore-run.json");
3336 let journal = ready_apply_journal();
3337
3338 fs::write(
3339 &journal_path,
3340 serde_json::to_vec(&journal).expect("serialize journal"),
3341 )
3342 .expect("write journal");
3343
3344 run([
3345 OsString::from("run"),
3346 OsString::from("--journal"),
3347 OsString::from(journal_path.as_os_str()),
3348 OsString::from("--execute"),
3349 OsString::from("--dfx"),
3350 OsString::from("/bin/true"),
3351 OsString::from("--max-steps"),
3352 OsString::from("1"),
3353 OsString::from("--out"),
3354 OsString::from(out_path.as_os_str()),
3355 ])
3356 .expect("execute one restore run step");
3357
3358 let run_summary: serde_json::Value =
3359 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3360 .expect("decode run summary");
3361 let updated: RestoreApplyJournal =
3362 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3363 .expect("decode updated journal");
3364
3365 fs::remove_dir_all(root).expect("remove temp root");
3366 assert_eq!(run_summary["run_mode"], "execute");
3367 assert_eq!(run_summary["execute"], true);
3368 assert_eq!(run_summary["dry_run"], false);
3369 assert_eq!(run_summary["max_steps_reached"], true);
3370 assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
3371 assert_eq!(run_summary["next_action"], "rerun");
3372 assert_eq!(run_summary["executed_operation_count"], 1);
3373 assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
3374 assert_eq!(
3375 run_summary["executed_operations"][0]["command"]["program"],
3376 "/bin/true"
3377 );
3378 assert_eq!(updated.completed_operations, 1);
3379 assert_eq!(updated.pending_operations, 0);
3380 assert_eq!(updated.failed_operations, 0);
3381 assert_eq!(
3382 updated.operations[0].state,
3383 RestoreApplyOperationState::Completed
3384 );
3385 }
3386
3387 #[test]
3389 fn run_restore_run_require_complete_writes_summary_then_fails() {
3390 let root = temp_dir("canic-cli-restore-run-require-complete");
3391 fs::create_dir_all(&root).expect("create temp root");
3392 let journal_path = root.join("restore-apply-journal.json");
3393 let out_path = root.join("restore-run.json");
3394 let journal = ready_apply_journal();
3395
3396 fs::write(
3397 &journal_path,
3398 serde_json::to_vec(&journal).expect("serialize journal"),
3399 )
3400 .expect("write journal");
3401
3402 let err = run([
3403 OsString::from("run"),
3404 OsString::from("--journal"),
3405 OsString::from(journal_path.as_os_str()),
3406 OsString::from("--execute"),
3407 OsString::from("--dfx"),
3408 OsString::from("/bin/true"),
3409 OsString::from("--max-steps"),
3410 OsString::from("1"),
3411 OsString::from("--out"),
3412 OsString::from(out_path.as_os_str()),
3413 OsString::from("--require-complete"),
3414 ])
3415 .expect_err("incomplete run should fail requirement");
3416
3417 let run_summary: serde_json::Value =
3418 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3419 .expect("decode run summary");
3420
3421 fs::remove_dir_all(root).expect("remove temp root");
3422 assert_eq!(run_summary["executed_operation_count"], 1);
3423 assert_eq!(run_summary["complete"], false);
3424 assert!(matches!(
3425 err,
3426 RestoreCommandError::RestoreApplyIncomplete {
3427 completed_operations: 1,
3428 operation_count: 8,
3429 ..
3430 }
3431 ));
3432 }
3433
3434 #[test]
3436 fn run_restore_run_execute_marks_failed_operation() {
3437 let root = temp_dir("canic-cli-restore-run-execute-failed");
3438 fs::create_dir_all(&root).expect("create temp root");
3439 let journal_path = root.join("restore-apply-journal.json");
3440 let out_path = root.join("restore-run.json");
3441 let journal = ready_apply_journal();
3442
3443 fs::write(
3444 &journal_path,
3445 serde_json::to_vec(&journal).expect("serialize journal"),
3446 )
3447 .expect("write journal");
3448
3449 let err = run([
3450 OsString::from("run"),
3451 OsString::from("--journal"),
3452 OsString::from(journal_path.as_os_str()),
3453 OsString::from("--execute"),
3454 OsString::from("--dfx"),
3455 OsString::from("/bin/false"),
3456 OsString::from("--max-steps"),
3457 OsString::from("1"),
3458 OsString::from("--out"),
3459 OsString::from(out_path.as_os_str()),
3460 ])
3461 .expect_err("failing runner command should fail");
3462
3463 let run_summary: serde_json::Value =
3464 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3465 .expect("decode run summary");
3466 let updated: RestoreApplyJournal =
3467 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3468 .expect("decode updated journal");
3469
3470 fs::remove_dir_all(root).expect("remove temp root");
3471 assert!(matches!(
3472 err,
3473 RestoreCommandError::RestoreRunCommandFailed {
3474 sequence: 0,
3475 status,
3476 } if status == "1"
3477 ));
3478 assert_eq!(updated.failed_operations, 1);
3479 assert_eq!(updated.pending_operations, 0);
3480 assert_eq!(
3481 updated.operations[0].state,
3482 RestoreApplyOperationState::Failed
3483 );
3484 assert_eq!(run_summary["execute"], true);
3485 assert_eq!(run_summary["attention_required"], true);
3486 assert_eq!(run_summary["outcome"], "failed");
3487 assert_eq!(run_summary["stopped_reason"], "command-failed");
3488 assert_eq!(run_summary["next_action"], "inspect-failed-operation");
3489 assert_eq!(run_summary["executed_operation_count"], 1);
3490 assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
3491 assert_eq!(run_summary["executed_operations"][0]["status"], "1");
3492 assert_eq!(
3493 updated.operations[0].blocking_reasons,
3494 vec!["runner-command-exit-1".to_string()]
3495 );
3496 }
3497
3498 #[test]
3500 fn run_restore_run_require_no_attention_writes_summary_then_fails() {
3501 let root = temp_dir("canic-cli-restore-run-require-attention");
3502 fs::create_dir_all(&root).expect("create temp root");
3503 let journal_path = root.join("restore-apply-journal.json");
3504 let out_path = root.join("restore-run.json");
3505 let mut journal = ready_apply_journal();
3506 journal
3507 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3508 .expect("mark pending operation");
3509
3510 fs::write(
3511 &journal_path,
3512 serde_json::to_vec(&journal).expect("serialize journal"),
3513 )
3514 .expect("write journal");
3515
3516 let err = run([
3517 OsString::from("run"),
3518 OsString::from("--journal"),
3519 OsString::from(journal_path.as_os_str()),
3520 OsString::from("--dry-run"),
3521 OsString::from("--out"),
3522 OsString::from(out_path.as_os_str()),
3523 OsString::from("--require-no-attention"),
3524 ])
3525 .expect_err("attention run should fail requirement");
3526
3527 let run_summary: serde_json::Value =
3528 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3529 .expect("decode run summary");
3530
3531 fs::remove_dir_all(root).expect("remove temp root");
3532 assert_eq!(run_summary["attention_required"], true);
3533 assert_eq!(run_summary["outcome"], "pending");
3534 assert_eq!(run_summary["stopped_reason"], "pending");
3535 assert_eq!(run_summary["next_action"], "unclaim-pending");
3536 assert!(matches!(
3537 err,
3538 RestoreCommandError::RestoreApplyReportNeedsAttention {
3539 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3540 ..
3541 }
3542 ));
3543 }
3544
3545 #[test]
3547 fn run_restore_run_require_run_mode_writes_summary_then_fails() {
3548 let root = temp_dir("canic-cli-restore-run-require-run-mode");
3549 fs::create_dir_all(&root).expect("create temp root");
3550 let journal_path = root.join("restore-apply-journal.json");
3551 let out_path = root.join("restore-run.json");
3552 let journal = ready_apply_journal();
3553
3554 fs::write(
3555 &journal_path,
3556 serde_json::to_vec(&journal).expect("serialize journal"),
3557 )
3558 .expect("write journal");
3559
3560 let err = run([
3561 OsString::from("run"),
3562 OsString::from("--journal"),
3563 OsString::from(journal_path.as_os_str()),
3564 OsString::from("--dry-run"),
3565 OsString::from("--out"),
3566 OsString::from(out_path.as_os_str()),
3567 OsString::from("--require-run-mode"),
3568 OsString::from("execute"),
3569 ])
3570 .expect_err("run mode mismatch should fail requirement");
3571
3572 let run_summary: serde_json::Value =
3573 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3574 .expect("decode run summary");
3575
3576 fs::remove_dir_all(root).expect("remove temp root");
3577 assert_eq!(run_summary["run_mode"], "dry-run");
3578 assert!(matches!(
3579 err,
3580 RestoreCommandError::RestoreRunModeMismatch {
3581 expected,
3582 actual,
3583 ..
3584 } if expected == "execute" && actual == "dry-run"
3585 ));
3586 }
3587
3588 #[test]
3590 fn run_restore_run_require_executed_count_writes_summary_then_fails() {
3591 let root = temp_dir("canic-cli-restore-run-require-executed-count");
3592 fs::create_dir_all(&root).expect("create temp root");
3593 let journal_path = root.join("restore-apply-journal.json");
3594 let out_path = root.join("restore-run.json");
3595 let journal = ready_apply_journal();
3596
3597 fs::write(
3598 &journal_path,
3599 serde_json::to_vec(&journal).expect("serialize journal"),
3600 )
3601 .expect("write journal");
3602
3603 let err = run([
3604 OsString::from("run"),
3605 OsString::from("--journal"),
3606 OsString::from(journal_path.as_os_str()),
3607 OsString::from("--execute"),
3608 OsString::from("--dfx"),
3609 OsString::from("/bin/true"),
3610 OsString::from("--max-steps"),
3611 OsString::from("1"),
3612 OsString::from("--out"),
3613 OsString::from(out_path.as_os_str()),
3614 OsString::from("--require-executed-count"),
3615 OsString::from("2"),
3616 ])
3617 .expect_err("executed count mismatch should fail requirement");
3618
3619 let run_summary: serde_json::Value =
3620 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3621 .expect("decode run summary");
3622
3623 fs::remove_dir_all(root).expect("remove temp root");
3624 assert_eq!(run_summary["executed_operation_count"], 1);
3625 assert!(matches!(
3626 err,
3627 RestoreCommandError::RestoreRunExecutedCountMismatch {
3628 expected: 2,
3629 actual: 1,
3630 ..
3631 }
3632 ));
3633 }
3634
3635 #[test]
3637 fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
3638 let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
3639 fs::create_dir_all(&root).expect("create temp root");
3640 let journal_path = root.join("restore-apply-journal.json");
3641 let out_path = root.join("restore-run.json");
3642 let journal = ready_apply_journal();
3643
3644 fs::write(
3645 &journal_path,
3646 serde_json::to_vec(&journal).expect("serialize journal"),
3647 )
3648 .expect("write journal");
3649
3650 let err = run([
3651 OsString::from("run"),
3652 OsString::from("--journal"),
3653 OsString::from(journal_path.as_os_str()),
3654 OsString::from("--dry-run"),
3655 OsString::from("--out"),
3656 OsString::from(out_path.as_os_str()),
3657 OsString::from("--require-stopped-reason"),
3658 OsString::from("complete"),
3659 ])
3660 .expect_err("stopped reason mismatch should fail requirement");
3661
3662 let run_summary: serde_json::Value =
3663 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3664 .expect("decode run summary");
3665
3666 fs::remove_dir_all(root).expect("remove temp root");
3667 assert_eq!(run_summary["stopped_reason"], "preview");
3668 assert!(matches!(
3669 err,
3670 RestoreCommandError::RestoreRunStoppedReasonMismatch {
3671 expected,
3672 actual,
3673 ..
3674 } if expected == "complete" && actual == "preview"
3675 ));
3676 }
3677
3678 #[test]
3680 fn run_restore_run_require_next_action_writes_summary_then_fails() {
3681 let root = temp_dir("canic-cli-restore-run-require-next-action");
3682 fs::create_dir_all(&root).expect("create temp root");
3683 let journal_path = root.join("restore-apply-journal.json");
3684 let out_path = root.join("restore-run.json");
3685 let journal = ready_apply_journal();
3686
3687 fs::write(
3688 &journal_path,
3689 serde_json::to_vec(&journal).expect("serialize journal"),
3690 )
3691 .expect("write journal");
3692
3693 let err = run([
3694 OsString::from("run"),
3695 OsString::from("--journal"),
3696 OsString::from(journal_path.as_os_str()),
3697 OsString::from("--dry-run"),
3698 OsString::from("--out"),
3699 OsString::from(out_path.as_os_str()),
3700 OsString::from("--require-next-action"),
3701 OsString::from("done"),
3702 ])
3703 .expect_err("next action mismatch should fail requirement");
3704
3705 let run_summary: serde_json::Value =
3706 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3707 .expect("decode run summary");
3708
3709 fs::remove_dir_all(root).expect("remove temp root");
3710 assert_eq!(run_summary["next_action"], "rerun");
3711 assert!(matches!(
3712 err,
3713 RestoreCommandError::RestoreRunNextActionMismatch {
3714 expected,
3715 actual,
3716 ..
3717 } if expected == "done" && actual == "rerun"
3718 ));
3719 }
3720
3721 #[test]
3723 fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
3724 let root = temp_dir("canic-cli-restore-apply-report-attention");
3725 fs::create_dir_all(&root).expect("create temp root");
3726 let journal_path = root.join("restore-apply-journal.json");
3727 let out_path = root.join("restore-apply-report.json");
3728 let mut journal = ready_apply_journal();
3729 journal
3730 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3731 .expect("mark pending operation");
3732
3733 fs::write(
3734 &journal_path,
3735 serde_json::to_vec(&journal).expect("serialize journal"),
3736 )
3737 .expect("write journal");
3738
3739 let err = run([
3740 OsString::from("apply-report"),
3741 OsString::from("--journal"),
3742 OsString::from(journal_path.as_os_str()),
3743 OsString::from("--out"),
3744 OsString::from(out_path.as_os_str()),
3745 OsString::from("--require-no-attention"),
3746 ])
3747 .expect_err("attention report should fail requirement");
3748
3749 let report: RestoreApplyJournalReport =
3750 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3751 .expect("decode apply report");
3752
3753 fs::remove_dir_all(root).expect("remove temp root");
3754 assert!(report.attention_required);
3755 assert_eq!(report.pending_operations, 1);
3756 assert!(matches!(
3757 err,
3758 RestoreCommandError::RestoreApplyReportNeedsAttention {
3759 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3760 ..
3761 }
3762 ));
3763 }
3764
3765 #[test]
3767 fn run_restore_apply_status_require_complete_writes_status_then_fails() {
3768 let root = temp_dir("canic-cli-restore-apply-status-incomplete");
3769 fs::create_dir_all(&root).expect("create temp root");
3770 let journal_path = root.join("restore-apply-journal.json");
3771 let out_path = root.join("restore-apply-status.json");
3772 let journal = ready_apply_journal();
3773
3774 fs::write(
3775 &journal_path,
3776 serde_json::to_vec(&journal).expect("serialize journal"),
3777 )
3778 .expect("write journal");
3779
3780 let err = run([
3781 OsString::from("apply-status"),
3782 OsString::from("--journal"),
3783 OsString::from(journal_path.as_os_str()),
3784 OsString::from("--out"),
3785 OsString::from(out_path.as_os_str()),
3786 OsString::from("--require-complete"),
3787 ])
3788 .expect_err("incomplete journal should fail requirement");
3789
3790 assert!(out_path.exists());
3791 let status: RestoreApplyJournalStatus =
3792 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3793 .expect("decode apply status");
3794
3795 fs::remove_dir_all(root).expect("remove temp root");
3796 assert!(!status.complete);
3797 assert_eq!(status.completed_operations, 0);
3798 assert_eq!(status.operation_count, 8);
3799 assert!(matches!(
3800 err,
3801 RestoreCommandError::RestoreApplyIncomplete {
3802 completed_operations: 0,
3803 operation_count: 8,
3804 ..
3805 }
3806 ));
3807 }
3808
3809 #[test]
3811 fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
3812 let root = temp_dir("canic-cli-restore-apply-status-failed");
3813 fs::create_dir_all(&root).expect("create temp root");
3814 let journal_path = root.join("restore-apply-journal.json");
3815 let out_path = root.join("restore-apply-status.json");
3816 let mut journal = ready_apply_journal();
3817 journal
3818 .mark_operation_failed(0, "dfx-load-failed".to_string())
3819 .expect("mark failed operation");
3820
3821 fs::write(
3822 &journal_path,
3823 serde_json::to_vec(&journal).expect("serialize journal"),
3824 )
3825 .expect("write journal");
3826
3827 let err = run([
3828 OsString::from("apply-status"),
3829 OsString::from("--journal"),
3830 OsString::from(journal_path.as_os_str()),
3831 OsString::from("--out"),
3832 OsString::from(out_path.as_os_str()),
3833 OsString::from("--require-no-failed"),
3834 ])
3835 .expect_err("failed operation should fail requirement");
3836
3837 assert!(out_path.exists());
3838 let status: RestoreApplyJournalStatus =
3839 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3840 .expect("decode apply status");
3841
3842 fs::remove_dir_all(root).expect("remove temp root");
3843 assert_eq!(status.failed_operations, 1);
3844 assert!(matches!(
3845 err,
3846 RestoreCommandError::RestoreApplyFailed {
3847 failed_operations: 1,
3848 ..
3849 }
3850 ));
3851 }
3852
3853 #[test]
3855 fn run_restore_apply_status_require_complete_accepts_complete_journal() {
3856 let root = temp_dir("canic-cli-restore-apply-status-complete");
3857 fs::create_dir_all(&root).expect("create temp root");
3858 let journal_path = root.join("restore-apply-journal.json");
3859 let out_path = root.join("restore-apply-status.json");
3860 let mut journal = ready_apply_journal();
3861 for sequence in 0..journal.operation_count {
3862 journal
3863 .mark_operation_completed(sequence)
3864 .expect("complete operation");
3865 }
3866
3867 fs::write(
3868 &journal_path,
3869 serde_json::to_vec(&journal).expect("serialize journal"),
3870 )
3871 .expect("write journal");
3872
3873 run([
3874 OsString::from("apply-status"),
3875 OsString::from("--journal"),
3876 OsString::from(journal_path.as_os_str()),
3877 OsString::from("--out"),
3878 OsString::from(out_path.as_os_str()),
3879 OsString::from("--require-complete"),
3880 ])
3881 .expect("complete journal should pass requirement");
3882
3883 let status: RestoreApplyJournalStatus =
3884 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3885 .expect("decode apply status");
3886
3887 fs::remove_dir_all(root).expect("remove temp root");
3888 assert!(status.complete);
3889 assert_eq!(status.completed_operations, 8);
3890 assert_eq!(status.operation_count, 8);
3891 }
3892
3893 #[test]
3895 fn run_restore_apply_next_writes_next_ready_operation() {
3896 let root = temp_dir("canic-cli-restore-apply-next");
3897 fs::create_dir_all(&root).expect("create temp root");
3898 let journal_path = root.join("restore-apply-journal.json");
3899 let out_path = root.join("restore-apply-next.json");
3900 let mut journal = ready_apply_journal();
3901 journal
3902 .mark_operation_completed(0)
3903 .expect("mark first operation complete");
3904
3905 fs::write(
3906 &journal_path,
3907 serde_json::to_vec(&journal).expect("serialize journal"),
3908 )
3909 .expect("write journal");
3910
3911 run([
3912 OsString::from("apply-next"),
3913 OsString::from("--journal"),
3914 OsString::from(journal_path.as_os_str()),
3915 OsString::from("--out"),
3916 OsString::from(out_path.as_os_str()),
3917 ])
3918 .expect("write apply next");
3919
3920 let next: RestoreApplyNextOperation =
3921 serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
3922 .expect("decode next operation");
3923 let operation = next.operation.expect("operation should be available");
3924
3925 fs::remove_dir_all(root).expect("remove temp root");
3926 assert!(next.ready);
3927 assert!(next.operation_available);
3928 assert_eq!(operation.sequence, 1);
3929 assert_eq!(
3930 operation.operation,
3931 canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
3932 );
3933 }
3934
3935 #[test]
3937 fn run_restore_apply_command_writes_next_command_preview() {
3938 let root = temp_dir("canic-cli-restore-apply-command");
3939 fs::create_dir_all(&root).expect("create temp root");
3940 let journal_path = root.join("restore-apply-journal.json");
3941 let out_path = root.join("restore-apply-command.json");
3942 let journal = ready_apply_journal();
3943
3944 fs::write(
3945 &journal_path,
3946 serde_json::to_vec(&journal).expect("serialize journal"),
3947 )
3948 .expect("write journal");
3949
3950 run([
3951 OsString::from("apply-command"),
3952 OsString::from("--journal"),
3953 OsString::from(journal_path.as_os_str()),
3954 OsString::from("--dfx"),
3955 OsString::from("/tmp/dfx"),
3956 OsString::from("--network"),
3957 OsString::from("local"),
3958 OsString::from("--out"),
3959 OsString::from(out_path.as_os_str()),
3960 ])
3961 .expect("write command preview");
3962
3963 let preview: RestoreApplyCommandPreview =
3964 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
3965 .expect("decode command preview");
3966 let command = preview.command.expect("command should be available");
3967
3968 fs::remove_dir_all(root).expect("remove temp root");
3969 assert!(preview.ready);
3970 assert!(preview.command_available);
3971 assert_eq!(command.program, "/tmp/dfx");
3972 assert_eq!(
3973 command.args,
3974 vec![
3975 "canister".to_string(),
3976 "--network".to_string(),
3977 "local".to_string(),
3978 "snapshot".to_string(),
3979 "upload".to_string(),
3980 "--dir".to_string(),
3981 "artifacts/root".to_string(),
3982 ROOT.to_string(),
3983 ]
3984 );
3985 assert!(command.mutates);
3986 }
3987
3988 #[test]
3990 fn run_restore_apply_command_require_command_writes_preview_then_fails() {
3991 let root = temp_dir("canic-cli-restore-apply-command-require");
3992 fs::create_dir_all(&root).expect("create temp root");
3993 let journal_path = root.join("restore-apply-journal.json");
3994 let out_path = root.join("restore-apply-command.json");
3995 let mut journal = ready_apply_journal();
3996
3997 for sequence in 0..journal.operation_count {
3998 journal
3999 .mark_operation_completed(sequence)
4000 .expect("mark operation completed");
4001 }
4002
4003 fs::write(
4004 &journal_path,
4005 serde_json::to_vec(&journal).expect("serialize journal"),
4006 )
4007 .expect("write journal");
4008
4009 let err = run([
4010 OsString::from("apply-command"),
4011 OsString::from("--journal"),
4012 OsString::from(journal_path.as_os_str()),
4013 OsString::from("--out"),
4014 OsString::from(out_path.as_os_str()),
4015 OsString::from("--require-command"),
4016 ])
4017 .expect_err("missing command should fail");
4018
4019 let preview: RestoreApplyCommandPreview =
4020 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
4021 .expect("decode command preview");
4022
4023 fs::remove_dir_all(root).expect("remove temp root");
4024 assert!(preview.complete);
4025 assert!(!preview.operation_available);
4026 assert!(!preview.command_available);
4027 assert!(matches!(
4028 err,
4029 RestoreCommandError::RestoreApplyCommandUnavailable {
4030 operation_available: false,
4031 complete: true,
4032 ..
4033 }
4034 ));
4035 }
4036
4037 #[test]
4039 fn run_restore_apply_claim_marks_next_operation_pending() {
4040 let root = temp_dir("canic-cli-restore-apply-claim");
4041 fs::create_dir_all(&root).expect("create temp root");
4042 let journal_path = root.join("restore-apply-journal.json");
4043 let claimed_path = root.join("restore-apply-journal.claimed.json");
4044 let journal = ready_apply_journal();
4045
4046 fs::write(
4047 &journal_path,
4048 serde_json::to_vec(&journal).expect("serialize journal"),
4049 )
4050 .expect("write journal");
4051
4052 run([
4053 OsString::from("apply-claim"),
4054 OsString::from("--journal"),
4055 OsString::from(journal_path.as_os_str()),
4056 OsString::from("--sequence"),
4057 OsString::from("0"),
4058 OsString::from("--updated-at"),
4059 OsString::from("2026-05-04T12:00:00Z"),
4060 OsString::from("--out"),
4061 OsString::from(claimed_path.as_os_str()),
4062 ])
4063 .expect("claim operation");
4064
4065 let claimed: RestoreApplyJournal =
4066 serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
4067 .expect("decode claimed journal");
4068 let status = claimed.status();
4069 let next = claimed.next_operation();
4070
4071 fs::remove_dir_all(root).expect("remove temp root");
4072 assert_eq!(claimed.pending_operations, 1);
4073 assert_eq!(claimed.ready_operations, 7);
4074 assert_eq!(
4075 claimed.operations[0].state,
4076 RestoreApplyOperationState::Pending
4077 );
4078 assert_eq!(
4079 claimed.operations[0].state_updated_at.as_deref(),
4080 Some("2026-05-04T12:00:00Z")
4081 );
4082 assert_eq!(status.next_transition_sequence, Some(0));
4083 assert_eq!(
4084 status.next_transition_state,
4085 Some(RestoreApplyOperationState::Pending)
4086 );
4087 assert_eq!(
4088 status.next_transition_updated_at.as_deref(),
4089 Some("2026-05-04T12:00:00Z")
4090 );
4091 assert_eq!(
4092 next.operation.expect("next operation").state,
4093 RestoreApplyOperationState::Pending
4094 );
4095 }
4096
4097 #[test]
4099 fn run_restore_apply_claim_rejects_sequence_mismatch() {
4100 let root = temp_dir("canic-cli-restore-apply-claim-sequence");
4101 fs::create_dir_all(&root).expect("create temp root");
4102 let journal_path = root.join("restore-apply-journal.json");
4103 let claimed_path = root.join("restore-apply-journal.claimed.json");
4104 let journal = ready_apply_journal();
4105
4106 fs::write(
4107 &journal_path,
4108 serde_json::to_vec(&journal).expect("serialize journal"),
4109 )
4110 .expect("write journal");
4111
4112 let err = run([
4113 OsString::from("apply-claim"),
4114 OsString::from("--journal"),
4115 OsString::from(journal_path.as_os_str()),
4116 OsString::from("--sequence"),
4117 OsString::from("1"),
4118 OsString::from("--out"),
4119 OsString::from(claimed_path.as_os_str()),
4120 ])
4121 .expect_err("stale sequence should fail claim");
4122
4123 assert!(!claimed_path.exists());
4124 fs::remove_dir_all(root).expect("remove temp root");
4125 assert!(matches!(
4126 err,
4127 RestoreCommandError::RestoreApplyClaimSequenceMismatch {
4128 expected: 1,
4129 actual: Some(0),
4130 }
4131 ));
4132 }
4133
4134 #[test]
4136 fn run_restore_apply_unclaim_marks_pending_operation_ready() {
4137 let root = temp_dir("canic-cli-restore-apply-unclaim");
4138 fs::create_dir_all(&root).expect("create temp root");
4139 let journal_path = root.join("restore-apply-journal.json");
4140 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
4141 let mut journal = ready_apply_journal();
4142 journal
4143 .mark_next_operation_pending()
4144 .expect("claim operation");
4145
4146 fs::write(
4147 &journal_path,
4148 serde_json::to_vec(&journal).expect("serialize journal"),
4149 )
4150 .expect("write journal");
4151
4152 run([
4153 OsString::from("apply-unclaim"),
4154 OsString::from("--journal"),
4155 OsString::from(journal_path.as_os_str()),
4156 OsString::from("--sequence"),
4157 OsString::from("0"),
4158 OsString::from("--updated-at"),
4159 OsString::from("2026-05-04T12:01:00Z"),
4160 OsString::from("--out"),
4161 OsString::from(unclaimed_path.as_os_str()),
4162 ])
4163 .expect("unclaim operation");
4164
4165 let unclaimed: RestoreApplyJournal =
4166 serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
4167 .expect("decode unclaimed journal");
4168 let status = unclaimed.status();
4169
4170 fs::remove_dir_all(root).expect("remove temp root");
4171 assert_eq!(unclaimed.pending_operations, 0);
4172 assert_eq!(unclaimed.ready_operations, 8);
4173 assert_eq!(
4174 unclaimed.operations[0].state,
4175 RestoreApplyOperationState::Ready
4176 );
4177 assert_eq!(
4178 unclaimed.operations[0].state_updated_at.as_deref(),
4179 Some("2026-05-04T12:01:00Z")
4180 );
4181 assert_eq!(status.next_ready_sequence, Some(0));
4182 assert_eq!(
4183 status.next_transition_state,
4184 Some(RestoreApplyOperationState::Ready)
4185 );
4186 assert_eq!(
4187 status.next_transition_updated_at.as_deref(),
4188 Some("2026-05-04T12:01:00Z")
4189 );
4190 }
4191
4192 #[test]
4194 fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
4195 let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
4196 fs::create_dir_all(&root).expect("create temp root");
4197 let journal_path = root.join("restore-apply-journal.json");
4198 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
4199 let mut journal = ready_apply_journal();
4200 journal
4201 .mark_next_operation_pending()
4202 .expect("claim operation");
4203
4204 fs::write(
4205 &journal_path,
4206 serde_json::to_vec(&journal).expect("serialize journal"),
4207 )
4208 .expect("write journal");
4209
4210 let err = run([
4211 OsString::from("apply-unclaim"),
4212 OsString::from("--journal"),
4213 OsString::from(journal_path.as_os_str()),
4214 OsString::from("--sequence"),
4215 OsString::from("1"),
4216 OsString::from("--out"),
4217 OsString::from(unclaimed_path.as_os_str()),
4218 ])
4219 .expect_err("stale sequence should fail unclaim");
4220
4221 assert!(!unclaimed_path.exists());
4222 fs::remove_dir_all(root).expect("remove temp root");
4223 assert!(matches!(
4224 err,
4225 RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
4226 expected: 1,
4227 actual: Some(0),
4228 }
4229 ));
4230 }
4231
4232 #[test]
4234 fn run_restore_apply_mark_completes_operation() {
4235 let root = temp_dir("canic-cli-restore-apply-mark-complete");
4236 fs::create_dir_all(&root).expect("create temp root");
4237 let journal_path = root.join("restore-apply-journal.json");
4238 let updated_path = root.join("restore-apply-journal.updated.json");
4239 let journal = ready_apply_journal();
4240
4241 fs::write(
4242 &journal_path,
4243 serde_json::to_vec(&journal).expect("serialize journal"),
4244 )
4245 .expect("write journal");
4246
4247 run([
4248 OsString::from("apply-mark"),
4249 OsString::from("--journal"),
4250 OsString::from(journal_path.as_os_str()),
4251 OsString::from("--sequence"),
4252 OsString::from("0"),
4253 OsString::from("--state"),
4254 OsString::from("completed"),
4255 OsString::from("--updated-at"),
4256 OsString::from("2026-05-04T12:02:00Z"),
4257 OsString::from("--out"),
4258 OsString::from(updated_path.as_os_str()),
4259 ])
4260 .expect("mark operation completed");
4261
4262 let updated: RestoreApplyJournal =
4263 serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
4264 .expect("decode updated journal");
4265 let status = updated.status();
4266
4267 fs::remove_dir_all(root).expect("remove temp root");
4268 assert_eq!(updated.completed_operations, 1);
4269 assert_eq!(updated.ready_operations, 7);
4270 assert_eq!(
4271 updated.operations[0].state_updated_at.as_deref(),
4272 Some("2026-05-04T12:02:00Z")
4273 );
4274 assert_eq!(status.next_ready_sequence, Some(1));
4275 }
4276
4277 #[test]
4279 fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
4280 let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
4281 fs::create_dir_all(&root).expect("create temp root");
4282 let journal_path = root.join("restore-apply-journal.json");
4283 let updated_path = root.join("restore-apply-journal.updated.json");
4284 let journal = ready_apply_journal();
4285
4286 fs::write(
4287 &journal_path,
4288 serde_json::to_vec(&journal).expect("serialize journal"),
4289 )
4290 .expect("write journal");
4291
4292 let err = run([
4293 OsString::from("apply-mark"),
4294 OsString::from("--journal"),
4295 OsString::from(journal_path.as_os_str()),
4296 OsString::from("--sequence"),
4297 OsString::from("0"),
4298 OsString::from("--state"),
4299 OsString::from("completed"),
4300 OsString::from("--out"),
4301 OsString::from(updated_path.as_os_str()),
4302 OsString::from("--require-pending"),
4303 ])
4304 .expect_err("ready operation should fail pending requirement");
4305
4306 assert!(!updated_path.exists());
4307 fs::remove_dir_all(root).expect("remove temp root");
4308 assert!(matches!(
4309 err,
4310 RestoreCommandError::RestoreApplyMarkRequiresPending {
4311 sequence: 0,
4312 state: RestoreApplyOperationState::Ready,
4313 }
4314 ));
4315 }
4316
4317 #[test]
4319 fn run_restore_apply_mark_rejects_out_of_order_operation() {
4320 let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
4321 fs::create_dir_all(&root).expect("create temp root");
4322 let journal_path = root.join("restore-apply-journal.json");
4323 let updated_path = root.join("restore-apply-journal.updated.json");
4324 let journal = ready_apply_journal();
4325
4326 fs::write(
4327 &journal_path,
4328 serde_json::to_vec(&journal).expect("serialize journal"),
4329 )
4330 .expect("write journal");
4331
4332 let err = run([
4333 OsString::from("apply-mark"),
4334 OsString::from("--journal"),
4335 OsString::from(journal_path.as_os_str()),
4336 OsString::from("--sequence"),
4337 OsString::from("1"),
4338 OsString::from("--state"),
4339 OsString::from("completed"),
4340 OsString::from("--out"),
4341 OsString::from(updated_path.as_os_str()),
4342 ])
4343 .expect_err("out-of-order operation should fail");
4344
4345 assert!(!updated_path.exists());
4346 fs::remove_dir_all(root).expect("remove temp root");
4347 assert!(matches!(
4348 err,
4349 RestoreCommandError::RestoreApplyJournal(
4350 RestoreApplyJournalError::OutOfOrderOperationTransition {
4351 requested: 1,
4352 next: 0
4353 }
4354 )
4355 ));
4356 }
4357
4358 #[test]
4360 fn run_restore_apply_mark_failed_requires_reason() {
4361 let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
4362 fs::create_dir_all(&root).expect("create temp root");
4363 let journal_path = root.join("restore-apply-journal.json");
4364 let journal = ready_apply_journal();
4365
4366 fs::write(
4367 &journal_path,
4368 serde_json::to_vec(&journal).expect("serialize journal"),
4369 )
4370 .expect("write journal");
4371
4372 let err = run([
4373 OsString::from("apply-mark"),
4374 OsString::from("--journal"),
4375 OsString::from(journal_path.as_os_str()),
4376 OsString::from("--sequence"),
4377 OsString::from("0"),
4378 OsString::from("--state"),
4379 OsString::from("failed"),
4380 ])
4381 .expect_err("failed state should require reason");
4382
4383 fs::remove_dir_all(root).expect("remove temp root");
4384 assert!(matches!(
4385 err,
4386 RestoreCommandError::RestoreApplyJournal(
4387 RestoreApplyJournalError::FailureReasonRequired(0)
4388 )
4389 ));
4390 }
4391
4392 #[test]
4394 fn run_restore_apply_dry_run_rejects_mismatched_status() {
4395 let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
4396 fs::create_dir_all(&root).expect("create temp root");
4397 let plan_path = root.join("restore-plan.json");
4398 let status_path = root.join("restore-status.json");
4399 let out_path = root.join("restore-apply-dry-run.json");
4400 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4401 let mut status = RestoreStatus::from_plan(&plan);
4402 status.backup_id = "other-backup".to_string();
4403
4404 fs::write(
4405 &plan_path,
4406 serde_json::to_vec(&plan).expect("serialize plan"),
4407 )
4408 .expect("write plan");
4409 fs::write(
4410 &status_path,
4411 serde_json::to_vec(&status).expect("serialize status"),
4412 )
4413 .expect("write status");
4414
4415 let err = run([
4416 OsString::from("apply"),
4417 OsString::from("--plan"),
4418 OsString::from(plan_path.as_os_str()),
4419 OsString::from("--status"),
4420 OsString::from(status_path.as_os_str()),
4421 OsString::from("--dry-run"),
4422 OsString::from("--out"),
4423 OsString::from(out_path.as_os_str()),
4424 ])
4425 .expect_err("mismatched status should fail");
4426
4427 assert!(!out_path.exists());
4428 fs::remove_dir_all(root).expect("remove temp root");
4429 assert!(matches!(
4430 err,
4431 RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
4432 field: "backup_id",
4433 ..
4434 })
4435 ));
4436 }
4437
4438 fn ready_apply_journal() -> RestoreApplyJournal {
4440 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4441 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4442 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4443
4444 journal.ready = true;
4445 journal.blocked_reasons = Vec::new();
4446 for operation in &mut journal.operations {
4447 operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
4448 operation.blocking_reasons = Vec::new();
4449 }
4450 journal.blocked_operations = 0;
4451 journal.ready_operations = journal.operation_count;
4452 journal.validate().expect("journal should validate");
4453 journal
4454 }
4455
4456 fn valid_manifest() -> FleetBackupManifest {
4458 FleetBackupManifest {
4459 manifest_version: 1,
4460 backup_id: "backup-test".to_string(),
4461 created_at: "2026-05-03T00:00:00Z".to_string(),
4462 tool: ToolMetadata {
4463 name: "canic".to_string(),
4464 version: "0.30.1".to_string(),
4465 },
4466 source: SourceMetadata {
4467 environment: "local".to_string(),
4468 root_canister: ROOT.to_string(),
4469 },
4470 consistency: ConsistencySection {
4471 mode: ConsistencyMode::CrashConsistent,
4472 backup_units: vec![BackupUnit {
4473 unit_id: "fleet".to_string(),
4474 kind: BackupUnitKind::SubtreeRooted,
4475 roles: vec!["root".to_string(), "app".to_string()],
4476 consistency_reason: None,
4477 dependency_closure: Vec::new(),
4478 topology_validation: "subtree-closed".to_string(),
4479 quiescence_strategy: None,
4480 }],
4481 },
4482 fleet: FleetSection {
4483 topology_hash_algorithm: "sha256".to_string(),
4484 topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
4485 discovery_topology_hash: HASH.to_string(),
4486 pre_snapshot_topology_hash: HASH.to_string(),
4487 topology_hash: HASH.to_string(),
4488 members: vec![
4489 fleet_member("root", ROOT, None, IdentityMode::Fixed),
4490 fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
4491 ],
4492 },
4493 verification: VerificationPlan::default(),
4494 }
4495 }
4496
4497 fn restore_ready_manifest() -> FleetBackupManifest {
4499 let mut manifest = valid_manifest();
4500 for member in &mut manifest.fleet.members {
4501 member.source_snapshot.module_hash = Some(HASH.to_string());
4502 member.source_snapshot.wasm_hash = Some(HASH.to_string());
4503 member.source_snapshot.checksum = Some(HASH.to_string());
4504 }
4505 manifest
4506 }
4507
4508 fn fleet_member(
4510 role: &str,
4511 canister_id: &str,
4512 parent_canister_id: Option<&str>,
4513 identity_mode: IdentityMode,
4514 ) -> FleetMember {
4515 FleetMember {
4516 role: role.to_string(),
4517 canister_id: canister_id.to_string(),
4518 parent_canister_id: parent_canister_id.map(str::to_string),
4519 subnet_canister_id: Some(ROOT.to_string()),
4520 controller_hint: None,
4521 identity_mode,
4522 restore_group: 1,
4523 verification_class: "basic".to_string(),
4524 verification_checks: vec![VerificationCheck {
4525 kind: "status".to_string(),
4526 method: None,
4527 roles: vec![role.to_string()],
4528 }],
4529 source_snapshot: SourceSnapshot {
4530 snapshot_id: format!("{role}-snapshot"),
4531 module_hash: None,
4532 wasm_hash: None,
4533 code_version: Some("v0.30.1".to_string()),
4534 artifact_path: format!("artifacts/{role}"),
4535 checksum_algorithm: "sha256".to_string(),
4536 checksum: None,
4537 },
4538 }
4539 }
4540
4541 fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
4543 layout.write_manifest(manifest).expect("write manifest");
4544
4545 let artifacts = manifest
4546 .fleet
4547 .members
4548 .iter()
4549 .map(|member| {
4550 let bytes = format!("{} artifact", member.role);
4551 let artifact_path = root.join(&member.source_snapshot.artifact_path);
4552 if let Some(parent) = artifact_path.parent() {
4553 fs::create_dir_all(parent).expect("create artifact parent");
4554 }
4555 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4556 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4557
4558 ArtifactJournalEntry {
4559 canister_id: member.canister_id.clone(),
4560 snapshot_id: member.source_snapshot.snapshot_id.clone(),
4561 state: ArtifactState::Durable,
4562 temp_path: None,
4563 artifact_path: member.source_snapshot.artifact_path.clone(),
4564 checksum_algorithm: checksum.algorithm,
4565 checksum: Some(checksum.hash),
4566 updated_at: "2026-05-03T00:00:00Z".to_string(),
4567 }
4568 })
4569 .collect();
4570
4571 layout
4572 .write_journal(&DownloadJournal {
4573 journal_version: 1,
4574 backup_id: manifest.backup_id.clone(),
4575 discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
4576 pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
4577 operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
4578 artifacts,
4579 })
4580 .expect("write journal");
4581 }
4582
4583 fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
4585 for member in &mut manifest.fleet.members {
4586 let bytes = format!("{} apply artifact", member.role);
4587 let artifact_path = root.join(&member.source_snapshot.artifact_path);
4588 if let Some(parent) = artifact_path.parent() {
4589 fs::create_dir_all(parent).expect("create artifact parent");
4590 }
4591 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4592 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4593 member.source_snapshot.checksum = Some(checksum.hash);
4594 }
4595 }
4596
4597 fn temp_dir(prefix: &str) -> PathBuf {
4599 let nanos = SystemTime::now()
4600 .duration_since(UNIX_EPOCH)
4601 .expect("system time after epoch")
4602 .as_nanos();
4603 std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
4604 }
4605}