1use canic_backup::{
2 manifest::FleetBackupManifest,
3 persistence::{BackupLayout, PersistenceError},
4 restore::{
5 RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6 RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7 RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8 RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationKindCounts,
9 RestoreApplyOperationState, RestoreApplyReportOperation, RestoreApplyReportOutcome,
10 RestoreApplyRunnerCommand, RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner,
11 RestoreStatus,
12 },
13};
14use serde::Serialize;
15use std::{
16 ffi::OsString,
17 fs,
18 io::{self, Write},
19 path::PathBuf,
20 process::Command,
21};
22use thiserror::Error as ThisError;
23
24#[derive(Debug, ThisError)]
29pub enum RestoreCommandError {
30 #[error("{0}")]
31 Usage(&'static str),
32
33 #[error("missing required option {0}")]
34 MissingOption(&'static str),
35
36 #[error("use either --manifest or --backup-dir, not both")]
37 ConflictingManifestSources,
38
39 #[error("--require-verified requires --backup-dir")]
40 RequireVerifiedNeedsBackupDir,
41
42 #[error("restore apply currently requires --dry-run")]
43 ApplyRequiresDryRun,
44
45 #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
46 RestoreRunRequiresMode,
47
48 #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
49 RestoreRunConflictingModes,
50
51 #[error("restore run command failed for operation {sequence}: status={status}")]
52 RestoreRunCommandFailed { sequence: usize, status: String },
53
54 #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
55 RestoreRunModeMismatch {
56 backup_id: String,
57 expected: String,
58 actual: String,
59 },
60
61 #[error(
62 "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
63 )]
64 RestoreRunStoppedReasonMismatch {
65 backup_id: String,
66 expected: String,
67 actual: String,
68 },
69
70 #[error(
71 "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
72 )]
73 RestoreRunNextActionMismatch {
74 backup_id: String,
75 expected: String,
76 actual: String,
77 },
78
79 #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
80 RestoreRunExecutedCountMismatch {
81 backup_id: String,
82 expected: usize,
83 actual: usize,
84 },
85
86 #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
87 RestoreNotReady {
88 backup_id: String,
89 reasons: Vec<String>,
90 },
91
92 #[error(
93 "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
94 )]
95 RestoreApplyPending {
96 backup_id: String,
97 pending_operations: usize,
98 next_transition_sequence: Option<usize>,
99 },
100
101 #[error(
102 "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
103 )]
104 RestoreApplyIncomplete {
105 backup_id: String,
106 completed_operations: usize,
107 operation_count: usize,
108 },
109
110 #[error(
111 "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
112 )]
113 RestoreApplyFailed {
114 backup_id: String,
115 failed_operations: usize,
116 },
117
118 #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
119 RestoreApplyNotReady {
120 backup_id: String,
121 reasons: Vec<String>,
122 },
123
124 #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
125 RestoreApplyReportNeedsAttention {
126 backup_id: String,
127 outcome: canic_backup::restore::RestoreApplyReportOutcome,
128 },
129
130 #[error(
131 "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
132 )]
133 RestoreApplyCommandUnavailable {
134 backup_id: String,
135 operation_available: bool,
136 complete: bool,
137 blocked_reasons: Vec<String>,
138 },
139
140 #[error(
141 "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
142 )]
143 RestoreApplyMarkRequiresPending {
144 sequence: usize,
145 state: RestoreApplyOperationState,
146 },
147
148 #[error(
149 "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
150 )]
151 RestoreApplyClaimSequenceMismatch {
152 expected: usize,
153 actual: Option<usize>,
154 },
155
156 #[error(
157 "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
158 )]
159 RestoreApplyUnclaimSequenceMismatch {
160 expected: usize,
161 actual: Option<usize>,
162 },
163
164 #[error("unknown option {0}")]
165 UnknownOption(String),
166
167 #[error("option {0} requires a value")]
168 MissingValue(&'static str),
169
170 #[error("option --sequence requires a non-negative integer value")]
171 InvalidSequence,
172
173 #[error("unsupported apply-mark state {0}; use completed or failed")]
174 InvalidApplyMarkState(String),
175
176 #[error(transparent)]
177 Io(#[from] std::io::Error),
178
179 #[error(transparent)]
180 Json(#[from] serde_json::Error),
181
182 #[error(transparent)]
183 Persistence(#[from] PersistenceError),
184
185 #[error(transparent)]
186 RestorePlan(#[from] RestorePlanError),
187
188 #[error(transparent)]
189 RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
190
191 #[error(transparent)]
192 RestoreApplyJournal(#[from] RestoreApplyJournalError),
193}
194
195#[derive(Clone, Debug, Eq, PartialEq)]
200pub struct RestorePlanOptions {
201 pub manifest: Option<PathBuf>,
202 pub backup_dir: Option<PathBuf>,
203 pub mapping: Option<PathBuf>,
204 pub out: Option<PathBuf>,
205 pub require_verified: bool,
206 pub require_restore_ready: bool,
207}
208
209impl RestorePlanOptions {
210 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
212 where
213 I: IntoIterator<Item = OsString>,
214 {
215 let mut manifest = None;
216 let mut backup_dir = None;
217 let mut mapping = None;
218 let mut out = None;
219 let mut require_verified = false;
220 let mut require_restore_ready = false;
221
222 let mut args = args.into_iter();
223 while let Some(arg) = args.next() {
224 let arg = arg
225 .into_string()
226 .map_err(|_| RestoreCommandError::Usage(usage()))?;
227 match arg.as_str() {
228 "--manifest" => {
229 manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
230 }
231 "--backup-dir" => {
232 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
233 }
234 "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
235 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
236 "--require-verified" => require_verified = true,
237 "--require-restore-ready" => require_restore_ready = true,
238 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
239 _ => return Err(RestoreCommandError::UnknownOption(arg)),
240 }
241 }
242
243 if manifest.is_some() && backup_dir.is_some() {
244 return Err(RestoreCommandError::ConflictingManifestSources);
245 }
246
247 if manifest.is_none() && backup_dir.is_none() {
248 return Err(RestoreCommandError::MissingOption(
249 "--manifest or --backup-dir",
250 ));
251 }
252
253 if require_verified && backup_dir.is_none() {
254 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
255 }
256
257 Ok(Self {
258 manifest,
259 backup_dir,
260 mapping,
261 out,
262 require_verified,
263 require_restore_ready,
264 })
265 }
266}
267
268#[derive(Clone, Debug, Eq, PartialEq)]
273pub struct RestoreStatusOptions {
274 pub plan: PathBuf,
275 pub out: Option<PathBuf>,
276}
277
278impl RestoreStatusOptions {
279 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
281 where
282 I: IntoIterator<Item = OsString>,
283 {
284 let mut plan = None;
285 let mut out = None;
286
287 let mut args = args.into_iter();
288 while let Some(arg) = args.next() {
289 let arg = arg
290 .into_string()
291 .map_err(|_| RestoreCommandError::Usage(usage()))?;
292 match arg.as_str() {
293 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
294 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
295 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
296 _ => return Err(RestoreCommandError::UnknownOption(arg)),
297 }
298 }
299
300 Ok(Self {
301 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
302 out,
303 })
304 }
305}
306
307#[derive(Clone, Debug, Eq, PartialEq)]
312pub struct RestoreApplyOptions {
313 pub plan: PathBuf,
314 pub status: Option<PathBuf>,
315 pub backup_dir: Option<PathBuf>,
316 pub out: Option<PathBuf>,
317 pub journal_out: Option<PathBuf>,
318 pub dry_run: bool,
319}
320
321impl RestoreApplyOptions {
322 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
324 where
325 I: IntoIterator<Item = OsString>,
326 {
327 let mut plan = None;
328 let mut status = None;
329 let mut backup_dir = None;
330 let mut out = None;
331 let mut journal_out = None;
332 let mut dry_run = false;
333
334 let mut args = args.into_iter();
335 while let Some(arg) = args.next() {
336 let arg = arg
337 .into_string()
338 .map_err(|_| RestoreCommandError::Usage(usage()))?;
339 match arg.as_str() {
340 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
341 "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
342 "--backup-dir" => {
343 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
344 }
345 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
346 "--journal-out" => {
347 journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
348 }
349 "--dry-run" => dry_run = true,
350 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
351 _ => return Err(RestoreCommandError::UnknownOption(arg)),
352 }
353 }
354
355 if !dry_run {
356 return Err(RestoreCommandError::ApplyRequiresDryRun);
357 }
358
359 Ok(Self {
360 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
361 status,
362 backup_dir,
363 out,
364 journal_out,
365 dry_run,
366 })
367 }
368}
369
370#[derive(Clone, Debug, Eq, PartialEq)]
375#[expect(
376 clippy::struct_excessive_bools,
377 reason = "CLI status options mirror independent fail-closed guard flags"
378)]
379pub struct RestoreApplyStatusOptions {
380 pub journal: PathBuf,
381 pub require_ready: bool,
382 pub require_no_pending: bool,
383 pub require_no_failed: bool,
384 pub require_complete: bool,
385 pub out: Option<PathBuf>,
386}
387
388impl RestoreApplyStatusOptions {
389 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
391 where
392 I: IntoIterator<Item = OsString>,
393 {
394 let mut journal = None;
395 let mut require_ready = false;
396 let mut require_no_pending = false;
397 let mut require_no_failed = false;
398 let mut require_complete = false;
399 let mut out = None;
400
401 let mut args = args.into_iter();
402 while let Some(arg) = args.next() {
403 let arg = arg
404 .into_string()
405 .map_err(|_| RestoreCommandError::Usage(usage()))?;
406 match arg.as_str() {
407 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
408 "--require-ready" => require_ready = true,
409 "--require-no-pending" => require_no_pending = true,
410 "--require-no-failed" => require_no_failed = true,
411 "--require-complete" => require_complete = true,
412 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
413 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
414 _ => return Err(RestoreCommandError::UnknownOption(arg)),
415 }
416 }
417
418 Ok(Self {
419 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
420 require_ready,
421 require_no_pending,
422 require_no_failed,
423 require_complete,
424 out,
425 })
426 }
427}
428
429#[derive(Clone, Debug, Eq, PartialEq)]
434pub struct RestoreApplyReportOptions {
435 pub journal: PathBuf,
436 pub require_no_attention: bool,
437 pub out: Option<PathBuf>,
438}
439
440impl RestoreApplyReportOptions {
441 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
443 where
444 I: IntoIterator<Item = OsString>,
445 {
446 let mut journal = None;
447 let mut require_no_attention = false;
448 let mut out = None;
449
450 let mut args = args.into_iter();
451 while let Some(arg) = args.next() {
452 let arg = arg
453 .into_string()
454 .map_err(|_| RestoreCommandError::Usage(usage()))?;
455 match arg.as_str() {
456 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
457 "--require-no-attention" => require_no_attention = true,
458 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
459 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
460 _ => return Err(RestoreCommandError::UnknownOption(arg)),
461 }
462 }
463
464 Ok(Self {
465 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
466 require_no_attention,
467 out,
468 })
469 }
470}
471
472#[derive(Clone, Debug, Eq, PartialEq)]
477#[expect(
478 clippy::struct_excessive_bools,
479 reason = "CLI runner options mirror independent mode and fail-closed guard flags"
480)]
481pub struct RestoreRunOptions {
482 pub journal: PathBuf,
483 pub dfx: String,
484 pub network: Option<String>,
485 pub out: Option<PathBuf>,
486 pub dry_run: bool,
487 pub execute: bool,
488 pub unclaim_pending: bool,
489 pub max_steps: Option<usize>,
490 pub require_complete: bool,
491 pub require_no_attention: bool,
492 pub require_run_mode: Option<String>,
493 pub require_stopped_reason: Option<String>,
494 pub require_next_action: Option<String>,
495 pub require_executed_count: Option<usize>,
496}
497
498impl RestoreRunOptions {
499 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
501 where
502 I: IntoIterator<Item = OsString>,
503 {
504 let mut journal = None;
505 let mut dfx = "dfx".to_string();
506 let mut network = None;
507 let mut out = None;
508 let mut dry_run = false;
509 let mut execute = false;
510 let mut unclaim_pending = false;
511 let mut max_steps = None;
512 let mut require_complete = false;
513 let mut require_no_attention = false;
514 let mut require_run_mode = None;
515 let mut require_stopped_reason = None;
516 let mut require_next_action = None;
517 let mut require_executed_count = None;
518
519 let mut args = args.into_iter();
520 while let Some(arg) = args.next() {
521 let arg = arg
522 .into_string()
523 .map_err(|_| RestoreCommandError::Usage(usage()))?;
524 match arg.as_str() {
525 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
526 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
527 "--network" => network = Some(next_value(&mut args, "--network")?),
528 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
529 "--dry-run" => dry_run = true,
530 "--execute" => execute = true,
531 "--unclaim-pending" => unclaim_pending = true,
532 "--max-steps" => {
533 max_steps = Some(parse_sequence(next_value(&mut args, "--max-steps")?)?);
534 }
535 "--require-complete" => require_complete = true,
536 "--require-no-attention" => require_no_attention = true,
537 "--require-run-mode" => {
538 require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
539 }
540 "--require-stopped-reason" => {
541 require_stopped_reason =
542 Some(next_value(&mut args, "--require-stopped-reason")?);
543 }
544 "--require-next-action" => {
545 require_next_action = Some(next_value(&mut args, "--require-next-action")?);
546 }
547 "--require-executed-count" => {
548 require_executed_count = Some(parse_sequence(next_value(
549 &mut args,
550 "--require-executed-count",
551 )?)?);
552 }
553 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
554 _ => return Err(RestoreCommandError::UnknownOption(arg)),
555 }
556 }
557
558 let mode_count = [dry_run, execute, unclaim_pending]
559 .into_iter()
560 .filter(|enabled| *enabled)
561 .count();
562 if mode_count > 1 {
563 return Err(RestoreCommandError::RestoreRunConflictingModes);
564 }
565
566 if mode_count == 0 {
567 return Err(RestoreCommandError::RestoreRunRequiresMode);
568 }
569
570 Ok(Self {
571 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
572 dfx,
573 network,
574 out,
575 dry_run,
576 execute,
577 unclaim_pending,
578 max_steps,
579 require_complete,
580 require_no_attention,
581 require_run_mode,
582 require_stopped_reason,
583 require_next_action,
584 require_executed_count,
585 })
586 }
587}
588
589struct RestoreRunResult {
594 response: RestoreRunResponse,
595 error: Option<RestoreCommandError>,
596}
597
598impl RestoreRunResult {
599 const fn ok(response: RestoreRunResponse) -> Self {
601 Self {
602 response,
603 error: None,
604 }
605 }
606}
607
608const RESTORE_RUN_MODE_DRY_RUN: &str = "dry-run";
609const RESTORE_RUN_MODE_EXECUTE: &str = "execute";
610const RESTORE_RUN_MODE_UNCLAIM_PENDING: &str = "unclaim-pending";
611
612const RESTORE_RUN_STOPPED_BLOCKED: &str = "blocked";
613const RESTORE_RUN_STOPPED_COMMAND_FAILED: &str = "command-failed";
614const RESTORE_RUN_STOPPED_COMPLETE: &str = "complete";
615const RESTORE_RUN_STOPPED_MAX_STEPS: &str = "max-steps-reached";
616const RESTORE_RUN_STOPPED_PENDING: &str = "pending";
617const RESTORE_RUN_STOPPED_PREVIEW: &str = "preview";
618const RESTORE_RUN_STOPPED_READY: &str = "ready";
619const RESTORE_RUN_STOPPED_RECOVERED_PENDING: &str = "recovered-pending";
620
621const RESTORE_RUN_ACTION_DONE: &str = "done";
622const RESTORE_RUN_ACTION_FIX_BLOCKED: &str = "fix-blocked-journal";
623const RESTORE_RUN_ACTION_INSPECT_FAILED: &str = "inspect-failed-operation";
624const RESTORE_RUN_ACTION_RERUN: &str = "rerun";
625const RESTORE_RUN_ACTION_UNCLAIM_PENDING: &str = "unclaim-pending";
626
627const RESTORE_RUN_EXECUTED_COMPLETED: &str = "completed";
628const RESTORE_RUN_EXECUTED_FAILED: &str = "failed";
629const RESTORE_RUN_COMMAND_EXIT_PREFIX: &str = "runner-command-exit";
630const RESTORE_RUN_RESPONSE_VERSION: u16 = 1;
631
632#[derive(Clone, Debug, Serialize)]
637#[expect(
638 clippy::struct_excessive_bools,
639 reason = "Runner response exposes stable JSON status flags for operators and CI"
640)]
641pub struct RestoreRunResponse {
642 run_version: u16,
643 backup_id: String,
644 run_mode: &'static str,
645 dry_run: bool,
646 execute: bool,
647 unclaim_pending: bool,
648 stopped_reason: &'static str,
649 next_action: &'static str,
650 #[serde(skip_serializing_if = "Option::is_none")]
651 max_steps_reached: Option<bool>,
652 #[serde(default, skip_serializing_if = "Vec::is_empty")]
653 executed_operations: Vec<RestoreRunExecutedOperation>,
654 #[serde(skip_serializing_if = "Option::is_none")]
655 executed_operation_count: Option<usize>,
656 #[serde(skip_serializing_if = "Option::is_none")]
657 recovered_operation: Option<RestoreApplyJournalOperation>,
658 ready: bool,
659 complete: bool,
660 attention_required: bool,
661 outcome: RestoreApplyReportOutcome,
662 operation_count: usize,
663 operation_counts: RestoreApplyOperationKindCounts,
664 pending_operations: usize,
665 ready_operations: usize,
666 blocked_operations: usize,
667 completed_operations: usize,
668 failed_operations: usize,
669 blocked_reasons: Vec<String>,
670 next_transition: Option<RestoreApplyReportOperation>,
671 #[serde(skip_serializing_if = "Option::is_none")]
672 operation_available: Option<bool>,
673 #[serde(skip_serializing_if = "Option::is_none")]
674 command_available: Option<bool>,
675 #[serde(skip_serializing_if = "Option::is_none")]
676 command: Option<RestoreApplyRunnerCommand>,
677}
678
679impl RestoreRunResponse {
680 fn from_report(
682 backup_id: String,
683 report: RestoreApplyJournalReport,
684 mode: RestoreRunResponseMode,
685 ) -> Self {
686 Self {
687 run_version: RESTORE_RUN_RESPONSE_VERSION,
688 backup_id,
689 run_mode: mode.run_mode,
690 dry_run: mode.dry_run,
691 execute: mode.execute,
692 unclaim_pending: mode.unclaim_pending,
693 stopped_reason: mode.stopped_reason,
694 next_action: mode.next_action,
695 max_steps_reached: None,
696 executed_operations: Vec::new(),
697 executed_operation_count: None,
698 recovered_operation: None,
699 ready: report.ready,
700 complete: report.complete,
701 attention_required: report.attention_required,
702 outcome: report.outcome,
703 operation_count: report.operation_count,
704 operation_counts: report.operation_counts,
705 pending_operations: report.pending_operations,
706 ready_operations: report.ready_operations,
707 blocked_operations: report.blocked_operations,
708 completed_operations: report.completed_operations,
709 failed_operations: report.failed_operations,
710 blocked_reasons: report.blocked_reasons,
711 next_transition: report.next_transition,
712 operation_available: None,
713 command_available: None,
714 command: None,
715 }
716 }
717}
718
719#[derive(Clone, Debug, Serialize)]
724struct RestoreRunExecutedOperation {
725 sequence: usize,
726 operation: RestoreApplyOperationKind,
727 target_canister: String,
728 command: RestoreApplyRunnerCommand,
729 status: String,
730 state: &'static str,
731}
732
733impl RestoreRunExecutedOperation {
734 fn completed(
736 operation: RestoreApplyJournalOperation,
737 command: RestoreApplyRunnerCommand,
738 status: String,
739 ) -> Self {
740 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_COMPLETED)
741 }
742
743 fn failed(
745 operation: RestoreApplyJournalOperation,
746 command: RestoreApplyRunnerCommand,
747 status: String,
748 ) -> Self {
749 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_FAILED)
750 }
751
752 fn from_operation(
754 operation: RestoreApplyJournalOperation,
755 command: RestoreApplyRunnerCommand,
756 status: String,
757 state: &'static str,
758 ) -> Self {
759 Self {
760 sequence: operation.sequence,
761 operation: operation.operation,
762 target_canister: operation.target_canister,
763 command,
764 status,
765 state,
766 }
767 }
768}
769
770struct RestoreRunResponseMode {
775 run_mode: &'static str,
776 dry_run: bool,
777 execute: bool,
778 unclaim_pending: bool,
779 stopped_reason: &'static str,
780 next_action: &'static str,
781}
782
783impl RestoreRunResponseMode {
784 const fn new(
786 run_mode: &'static str,
787 dry_run: bool,
788 execute: bool,
789 unclaim_pending: bool,
790 stopped_reason: &'static str,
791 next_action: &'static str,
792 ) -> Self {
793 Self {
794 run_mode,
795 dry_run,
796 execute,
797 unclaim_pending,
798 stopped_reason,
799 next_action,
800 }
801 }
802
803 const fn dry_run(stopped_reason: &'static str, next_action: &'static str) -> Self {
805 Self::new(
806 RESTORE_RUN_MODE_DRY_RUN,
807 true,
808 false,
809 false,
810 stopped_reason,
811 next_action,
812 )
813 }
814
815 const fn execute(stopped_reason: &'static str, next_action: &'static str) -> Self {
817 Self::new(
818 RESTORE_RUN_MODE_EXECUTE,
819 false,
820 true,
821 false,
822 stopped_reason,
823 next_action,
824 )
825 }
826
827 const fn unclaim_pending(next_action: &'static str) -> Self {
829 Self::new(
830 RESTORE_RUN_MODE_UNCLAIM_PENDING,
831 false,
832 false,
833 true,
834 RESTORE_RUN_STOPPED_RECOVERED_PENDING,
835 next_action,
836 )
837 }
838}
839
840#[derive(Clone, Debug, Eq, PartialEq)]
845pub struct RestoreApplyNextOptions {
846 pub journal: PathBuf,
847 pub out: Option<PathBuf>,
848}
849
850impl RestoreApplyNextOptions {
851 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
853 where
854 I: IntoIterator<Item = OsString>,
855 {
856 let mut journal = None;
857 let mut out = None;
858
859 let mut args = args.into_iter();
860 while let Some(arg) = args.next() {
861 let arg = arg
862 .into_string()
863 .map_err(|_| RestoreCommandError::Usage(usage()))?;
864 match arg.as_str() {
865 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
866 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
867 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
868 _ => return Err(RestoreCommandError::UnknownOption(arg)),
869 }
870 }
871
872 Ok(Self {
873 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
874 out,
875 })
876 }
877}
878
879#[derive(Clone, Debug, Eq, PartialEq)]
884pub struct RestoreApplyCommandOptions {
885 pub journal: PathBuf,
886 pub dfx: String,
887 pub network: Option<String>,
888 pub out: Option<PathBuf>,
889 pub require_command: bool,
890}
891
892impl RestoreApplyCommandOptions {
893 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
895 where
896 I: IntoIterator<Item = OsString>,
897 {
898 let mut journal = None;
899 let mut dfx = "dfx".to_string();
900 let mut network = None;
901 let mut out = None;
902 let mut require_command = false;
903
904 let mut args = args.into_iter();
905 while let Some(arg) = args.next() {
906 let arg = arg
907 .into_string()
908 .map_err(|_| RestoreCommandError::Usage(usage()))?;
909 match arg.as_str() {
910 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
911 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
912 "--network" => network = Some(next_value(&mut args, "--network")?),
913 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
914 "--require-command" => require_command = true,
915 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
916 _ => return Err(RestoreCommandError::UnknownOption(arg)),
917 }
918 }
919
920 Ok(Self {
921 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
922 dfx,
923 network,
924 out,
925 require_command,
926 })
927 }
928}
929
930#[derive(Clone, Debug, Eq, PartialEq)]
935pub struct RestoreApplyClaimOptions {
936 pub journal: PathBuf,
937 pub sequence: Option<usize>,
938 pub updated_at: Option<String>,
939 pub out: Option<PathBuf>,
940}
941
942impl RestoreApplyClaimOptions {
943 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
945 where
946 I: IntoIterator<Item = OsString>,
947 {
948 let mut journal = None;
949 let mut sequence = None;
950 let mut updated_at = None;
951 let mut out = None;
952
953 let mut args = args.into_iter();
954 while let Some(arg) = args.next() {
955 let arg = arg
956 .into_string()
957 .map_err(|_| RestoreCommandError::Usage(usage()))?;
958 match arg.as_str() {
959 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
960 "--sequence" => {
961 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
962 }
963 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
964 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
965 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
966 _ => return Err(RestoreCommandError::UnknownOption(arg)),
967 }
968 }
969
970 Ok(Self {
971 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
972 sequence,
973 updated_at,
974 out,
975 })
976 }
977}
978
979#[derive(Clone, Debug, Eq, PartialEq)]
984pub struct RestoreApplyUnclaimOptions {
985 pub journal: PathBuf,
986 pub sequence: Option<usize>,
987 pub updated_at: Option<String>,
988 pub out: Option<PathBuf>,
989}
990
991impl RestoreApplyUnclaimOptions {
992 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
994 where
995 I: IntoIterator<Item = OsString>,
996 {
997 let mut journal = None;
998 let mut sequence = None;
999 let mut updated_at = None;
1000 let mut out = None;
1001
1002 let mut args = args.into_iter();
1003 while let Some(arg) = args.next() {
1004 let arg = arg
1005 .into_string()
1006 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1007 match arg.as_str() {
1008 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1009 "--sequence" => {
1010 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1011 }
1012 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1013 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1014 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1015 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1016 }
1017 }
1018
1019 Ok(Self {
1020 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1021 sequence,
1022 updated_at,
1023 out,
1024 })
1025 }
1026}
1027
1028#[derive(Clone, Debug, Eq, PartialEq)]
1033pub struct RestoreApplyMarkOptions {
1034 pub journal: PathBuf,
1035 pub sequence: usize,
1036 pub state: RestoreApplyMarkState,
1037 pub reason: Option<String>,
1038 pub updated_at: Option<String>,
1039 pub out: Option<PathBuf>,
1040 pub require_pending: bool,
1041}
1042
1043impl RestoreApplyMarkOptions {
1044 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1046 where
1047 I: IntoIterator<Item = OsString>,
1048 {
1049 let mut journal = None;
1050 let mut sequence = None;
1051 let mut state = None;
1052 let mut reason = None;
1053 let mut updated_at = None;
1054 let mut out = None;
1055 let mut require_pending = false;
1056
1057 let mut args = args.into_iter();
1058 while let Some(arg) = args.next() {
1059 let arg = arg
1060 .into_string()
1061 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1062 match arg.as_str() {
1063 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1064 "--sequence" => {
1065 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1066 }
1067 "--state" => {
1068 state = Some(RestoreApplyMarkState::parse(next_value(
1069 &mut args, "--state",
1070 )?)?);
1071 }
1072 "--reason" => reason = Some(next_value(&mut args, "--reason")?),
1073 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1074 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1075 "--require-pending" => require_pending = true,
1076 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1077 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1078 }
1079 }
1080
1081 Ok(Self {
1082 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1083 sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
1084 state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
1085 reason,
1086 updated_at,
1087 out,
1088 require_pending,
1089 })
1090 }
1091}
1092
1093#[derive(Clone, Debug, Eq, PartialEq)]
1098pub enum RestoreApplyMarkState {
1099 Completed,
1100 Failed,
1101}
1102
1103impl RestoreApplyMarkState {
1104 fn parse(value: String) -> Result<Self, RestoreCommandError> {
1106 match value.as_str() {
1107 "completed" => Ok(Self::Completed),
1108 "failed" => Ok(Self::Failed),
1109 _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
1110 }
1111 }
1112}
1113
1114pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
1116where
1117 I: IntoIterator<Item = OsString>,
1118{
1119 let mut args = args.into_iter();
1120 let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
1121 return Err(RestoreCommandError::Usage(usage()));
1122 };
1123
1124 match command.as_str() {
1125 "plan" => {
1126 let options = RestorePlanOptions::parse(args)?;
1127 let plan = plan_restore(&options)?;
1128 write_plan(&options, &plan)?;
1129 enforce_restore_plan_requirements(&options, &plan)?;
1130 Ok(())
1131 }
1132 "status" => {
1133 let options = RestoreStatusOptions::parse(args)?;
1134 let status = restore_status(&options)?;
1135 write_status(&options, &status)?;
1136 Ok(())
1137 }
1138 "apply" => {
1139 let options = RestoreApplyOptions::parse(args)?;
1140 let dry_run = restore_apply_dry_run(&options)?;
1141 write_apply_dry_run(&options, &dry_run)?;
1142 write_apply_journal_if_requested(&options, &dry_run)?;
1143 Ok(())
1144 }
1145 "apply-status" => {
1146 let options = RestoreApplyStatusOptions::parse(args)?;
1147 let status = restore_apply_status(&options)?;
1148 write_apply_status(&options, &status)?;
1149 enforce_apply_status_requirements(&options, &status)?;
1150 Ok(())
1151 }
1152 "apply-report" => {
1153 let options = RestoreApplyReportOptions::parse(args)?;
1154 let report = restore_apply_report(&options)?;
1155 write_apply_report(&options, &report)?;
1156 enforce_apply_report_requirements(&options, &report)?;
1157 Ok(())
1158 }
1159 "run" => {
1160 let options = RestoreRunOptions::parse(args)?;
1161 let run = if options.execute {
1162 restore_run_execute_result(&options)?
1163 } else if options.unclaim_pending {
1164 RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1165 } else {
1166 RestoreRunResult::ok(restore_run_dry_run(&options)?)
1167 };
1168 write_restore_run(&options, &run.response)?;
1169 if let Some(error) = run.error {
1170 return Err(error);
1171 }
1172 enforce_restore_run_requirements(&options, &run.response)?;
1173 Ok(())
1174 }
1175 "apply-next" => {
1176 let options = RestoreApplyNextOptions::parse(args)?;
1177 let next = restore_apply_next(&options)?;
1178 write_apply_next(&options, &next)?;
1179 Ok(())
1180 }
1181 "apply-command" => {
1182 let options = RestoreApplyCommandOptions::parse(args)?;
1183 let preview = restore_apply_command(&options)?;
1184 write_apply_command(&options, &preview)?;
1185 enforce_apply_command_requirements(&options, &preview)?;
1186 Ok(())
1187 }
1188 "apply-claim" => {
1189 let options = RestoreApplyClaimOptions::parse(args)?;
1190 let journal = restore_apply_claim(&options)?;
1191 write_apply_claim(&options, &journal)?;
1192 Ok(())
1193 }
1194 "apply-unclaim" => {
1195 let options = RestoreApplyUnclaimOptions::parse(args)?;
1196 let journal = restore_apply_unclaim(&options)?;
1197 write_apply_unclaim(&options, &journal)?;
1198 Ok(())
1199 }
1200 "apply-mark" => {
1201 let options = RestoreApplyMarkOptions::parse(args)?;
1202 let journal = restore_apply_mark(&options)?;
1203 write_apply_mark(&options, &journal)?;
1204 Ok(())
1205 }
1206 "help" | "--help" | "-h" => Err(RestoreCommandError::Usage(usage())),
1207 _ => Err(RestoreCommandError::UnknownOption(command)),
1208 }
1209}
1210
1211pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1213 verify_backup_layout_if_required(options)?;
1214
1215 let manifest = read_manifest_source(options)?;
1216 let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1217
1218 RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1219}
1220
1221pub fn restore_status(
1223 options: &RestoreStatusOptions,
1224) -> Result<RestoreStatus, RestoreCommandError> {
1225 let plan = read_plan(&options.plan)?;
1226 Ok(RestoreStatus::from_plan(&plan))
1227}
1228
1229pub fn restore_apply_dry_run(
1231 options: &RestoreApplyOptions,
1232) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1233 let plan = read_plan(&options.plan)?;
1234 let status = options.status.as_ref().map(read_status).transpose()?;
1235 if let Some(backup_dir) = &options.backup_dir {
1236 return RestoreApplyDryRun::try_from_plan_with_artifacts(
1237 &plan,
1238 status.as_ref(),
1239 backup_dir,
1240 )
1241 .map_err(RestoreCommandError::from);
1242 }
1243
1244 RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1245}
1246
1247pub fn restore_apply_status(
1249 options: &RestoreApplyStatusOptions,
1250) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1251 let journal = read_apply_journal(&options.journal)?;
1252 Ok(journal.status())
1253}
1254
1255pub fn restore_apply_report(
1257 options: &RestoreApplyReportOptions,
1258) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1259 let journal = read_apply_journal(&options.journal)?;
1260 Ok(journal.report())
1261}
1262
1263pub fn restore_run_dry_run(
1265 options: &RestoreRunOptions,
1266) -> Result<RestoreRunResponse, RestoreCommandError> {
1267 let journal = read_apply_journal(&options.journal)?;
1268 let report = journal.report();
1269 let preview = journal.next_command_preview_with_config(&restore_run_command_config(options));
1270 let stopped_reason = restore_run_stopped_reason(&report, false, false);
1271 let next_action = restore_run_next_action(&report, false);
1272
1273 let mut response = RestoreRunResponse::from_report(
1274 journal.backup_id,
1275 report,
1276 RestoreRunResponseMode::dry_run(stopped_reason, next_action),
1277 );
1278 response.operation_available = Some(preview.operation_available);
1279 response.command_available = Some(preview.command_available);
1280 response.command = preview.command;
1281 Ok(response)
1282}
1283
1284pub fn restore_run_unclaim_pending(
1286 options: &RestoreRunOptions,
1287) -> Result<RestoreRunResponse, RestoreCommandError> {
1288 let mut journal = read_apply_journal(&options.journal)?;
1289 let recovered_operation = journal
1290 .next_transition_operation()
1291 .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1292 .cloned()
1293 .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1294
1295 journal.mark_next_operation_ready_at(Some(timestamp_placeholder()))?;
1296 write_apply_journal_file(&options.journal, &journal)?;
1297
1298 let report = journal.report();
1299 let next_action = restore_run_next_action(&report, true);
1300 let mut response = RestoreRunResponse::from_report(
1301 journal.backup_id,
1302 report,
1303 RestoreRunResponseMode::unclaim_pending(next_action),
1304 );
1305 response.recovered_operation = Some(recovered_operation);
1306 Ok(response)
1307}
1308
1309pub fn restore_run_execute(
1311 options: &RestoreRunOptions,
1312) -> Result<RestoreRunResponse, RestoreCommandError> {
1313 let run = restore_run_execute_result(options)?;
1314 if let Some(error) = run.error {
1315 return Err(error);
1316 }
1317
1318 Ok(run.response)
1319}
1320
1321fn restore_run_execute_result(
1323 options: &RestoreRunOptions,
1324) -> Result<RestoreRunResult, RestoreCommandError> {
1325 let mut journal = read_apply_journal(&options.journal)?;
1326 let mut executed_operations = Vec::new();
1327 let config = restore_run_command_config(options);
1328
1329 loop {
1330 let report = journal.report();
1331 let max_steps_reached =
1332 restore_run_max_steps_reached(options, executed_operations.len(), &report);
1333 if report.complete || max_steps_reached {
1334 return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1335 &journal,
1336 executed_operations,
1337 max_steps_reached,
1338 )));
1339 }
1340
1341 enforce_restore_run_executable(&journal, &report)?;
1342 let preview = journal.next_command_preview_with_config(&config);
1343 enforce_restore_run_command_available(&preview)?;
1344
1345 let operation = preview
1346 .operation
1347 .clone()
1348 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1349 let command = preview
1350 .command
1351 .clone()
1352 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1353 let sequence = operation.sequence;
1354
1355 enforce_apply_claim_sequence(sequence, &journal)?;
1356 journal.mark_operation_pending_at(sequence, Some(timestamp_placeholder()))?;
1357 write_apply_journal_file(&options.journal, &journal)?;
1358
1359 let status = Command::new(&command.program)
1360 .args(&command.args)
1361 .status()?;
1362 let status_label = exit_status_label(status);
1363 if status.success() {
1364 journal.mark_operation_completed_at(sequence, Some(timestamp_placeholder()))?;
1365 write_apply_journal_file(&options.journal, &journal)?;
1366 executed_operations.push(RestoreRunExecutedOperation::completed(
1367 operation,
1368 command,
1369 status_label,
1370 ));
1371 continue;
1372 }
1373
1374 journal.mark_operation_failed_at(
1375 sequence,
1376 format!("{RESTORE_RUN_COMMAND_EXIT_PREFIX}-{status_label}"),
1377 Some(timestamp_placeholder()),
1378 )?;
1379 write_apply_journal_file(&options.journal, &journal)?;
1380 executed_operations.push(RestoreRunExecutedOperation::failed(
1381 operation,
1382 command,
1383 status_label.clone(),
1384 ));
1385 let response = restore_run_execute_summary(&journal, executed_operations, false);
1386 return Ok(RestoreRunResult {
1387 response,
1388 error: Some(RestoreCommandError::RestoreRunCommandFailed {
1389 sequence,
1390 status: status_label,
1391 }),
1392 });
1393 }
1394}
1395
1396fn restore_run_command_config(options: &RestoreRunOptions) -> RestoreApplyCommandConfig {
1398 restore_command_config(&options.dfx, options.network.as_deref())
1399}
1400
1401fn restore_apply_command_config(options: &RestoreApplyCommandOptions) -> RestoreApplyCommandConfig {
1403 restore_command_config(&options.dfx, options.network.as_deref())
1404}
1405
1406fn restore_command_config(program: &str, network: Option<&str>) -> RestoreApplyCommandConfig {
1408 RestoreApplyCommandConfig {
1409 program: program.to_string(),
1410 network: network.map(str::to_string),
1411 }
1412}
1413
1414fn restore_run_max_steps_reached(
1416 options: &RestoreRunOptions,
1417 executed_operation_count: usize,
1418 report: &RestoreApplyJournalReport,
1419) -> bool {
1420 options.max_steps == Some(executed_operation_count) && !report.complete
1421}
1422
1423fn restore_run_execute_summary(
1425 journal: &RestoreApplyJournal,
1426 executed_operations: Vec<RestoreRunExecutedOperation>,
1427 max_steps_reached: bool,
1428) -> RestoreRunResponse {
1429 let report = journal.report();
1430 let executed_operation_count = executed_operations.len();
1431 let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
1432 let next_action = restore_run_next_action(&report, false);
1433
1434 let mut response = RestoreRunResponse::from_report(
1435 journal.backup_id.clone(),
1436 report,
1437 RestoreRunResponseMode::execute(stopped_reason, next_action),
1438 );
1439 response.max_steps_reached = Some(max_steps_reached);
1440 response.executed_operation_count = Some(executed_operation_count);
1441 response.executed_operations = executed_operations;
1442 response
1443}
1444
1445const fn restore_run_stopped_reason(
1447 report: &RestoreApplyJournalReport,
1448 max_steps_reached: bool,
1449 executed: bool,
1450) -> &'static str {
1451 if report.complete {
1452 return RESTORE_RUN_STOPPED_COMPLETE;
1453 }
1454 if report.failed_operations > 0 {
1455 return RESTORE_RUN_STOPPED_COMMAND_FAILED;
1456 }
1457 if report.pending_operations > 0 {
1458 return RESTORE_RUN_STOPPED_PENDING;
1459 }
1460 if !report.ready || report.blocked_operations > 0 {
1461 return RESTORE_RUN_STOPPED_BLOCKED;
1462 }
1463 if max_steps_reached {
1464 return RESTORE_RUN_STOPPED_MAX_STEPS;
1465 }
1466 if executed {
1467 return RESTORE_RUN_STOPPED_READY;
1468 }
1469 RESTORE_RUN_STOPPED_PREVIEW
1470}
1471
1472const fn restore_run_next_action(
1474 report: &RestoreApplyJournalReport,
1475 recovered_pending: bool,
1476) -> &'static str {
1477 if report.complete {
1478 return RESTORE_RUN_ACTION_DONE;
1479 }
1480 if report.failed_operations > 0 {
1481 return RESTORE_RUN_ACTION_INSPECT_FAILED;
1482 }
1483 if report.pending_operations > 0 {
1484 return RESTORE_RUN_ACTION_UNCLAIM_PENDING;
1485 }
1486 if !report.ready || report.blocked_operations > 0 {
1487 return RESTORE_RUN_ACTION_FIX_BLOCKED;
1488 }
1489 if recovered_pending {
1490 return RESTORE_RUN_ACTION_RERUN;
1491 }
1492 RESTORE_RUN_ACTION_RERUN
1493}
1494
1495fn enforce_restore_run_executable(
1497 journal: &RestoreApplyJournal,
1498 report: &RestoreApplyJournalReport,
1499) -> Result<(), RestoreCommandError> {
1500 if report.pending_operations > 0 {
1501 return Err(RestoreCommandError::RestoreApplyPending {
1502 backup_id: report.backup_id.clone(),
1503 pending_operations: report.pending_operations,
1504 next_transition_sequence: report
1505 .next_transition
1506 .as_ref()
1507 .map(|operation| operation.sequence),
1508 });
1509 }
1510
1511 if report.failed_operations > 0 {
1512 return Err(RestoreCommandError::RestoreApplyFailed {
1513 backup_id: report.backup_id.clone(),
1514 failed_operations: report.failed_operations,
1515 });
1516 }
1517
1518 if report.ready {
1519 return Ok(());
1520 }
1521
1522 Err(RestoreCommandError::RestoreApplyNotReady {
1523 backup_id: journal.backup_id.clone(),
1524 reasons: report.blocked_reasons.clone(),
1525 })
1526}
1527
1528fn enforce_restore_run_command_available(
1530 preview: &RestoreApplyCommandPreview,
1531) -> Result<(), RestoreCommandError> {
1532 if preview.command_available {
1533 return Ok(());
1534 }
1535
1536 Err(restore_command_unavailable_error(preview))
1537}
1538
1539fn restore_command_unavailable_error(preview: &RestoreApplyCommandPreview) -> RestoreCommandError {
1541 RestoreCommandError::RestoreApplyCommandUnavailable {
1542 backup_id: preview.backup_id.clone(),
1543 operation_available: preview.operation_available,
1544 complete: preview.complete,
1545 blocked_reasons: preview.blocked_reasons.clone(),
1546 }
1547}
1548
1549fn exit_status_label(status: std::process::ExitStatus) -> String {
1551 status
1552 .code()
1553 .map_or_else(|| "signal".to_string(), |code| code.to_string())
1554}
1555
1556fn enforce_restore_run_requirements(
1558 options: &RestoreRunOptions,
1559 run: &RestoreRunResponse,
1560) -> Result<(), RestoreCommandError> {
1561 if options.require_complete && !run.complete {
1562 return Err(RestoreCommandError::RestoreApplyIncomplete {
1563 backup_id: run.backup_id.clone(),
1564 completed_operations: run.completed_operations,
1565 operation_count: run.operation_count,
1566 });
1567 }
1568
1569 if options.require_no_attention && run.attention_required {
1570 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1571 backup_id: run.backup_id.clone(),
1572 outcome: run.outcome.clone(),
1573 });
1574 }
1575
1576 if let Some(expected) = &options.require_run_mode
1577 && run.run_mode != expected
1578 {
1579 return Err(RestoreCommandError::RestoreRunModeMismatch {
1580 backup_id: run.backup_id.clone(),
1581 expected: expected.clone(),
1582 actual: run.run_mode.to_string(),
1583 });
1584 }
1585
1586 if let Some(expected) = &options.require_stopped_reason
1587 && run.stopped_reason != expected
1588 {
1589 return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
1590 backup_id: run.backup_id.clone(),
1591 expected: expected.clone(),
1592 actual: run.stopped_reason.to_string(),
1593 });
1594 }
1595
1596 if let Some(expected) = &options.require_next_action
1597 && run.next_action != expected
1598 {
1599 return Err(RestoreCommandError::RestoreRunNextActionMismatch {
1600 backup_id: run.backup_id.clone(),
1601 expected: expected.clone(),
1602 actual: run.next_action.to_string(),
1603 });
1604 }
1605
1606 if let Some(expected) = options.require_executed_count {
1607 let actual = run.executed_operation_count.unwrap_or(0);
1608 if actual != expected {
1609 return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
1610 backup_id: run.backup_id.clone(),
1611 expected,
1612 actual,
1613 });
1614 }
1615 }
1616
1617 Ok(())
1618}
1619
1620fn enforce_apply_report_requirements(
1622 options: &RestoreApplyReportOptions,
1623 report: &RestoreApplyJournalReport,
1624) -> Result<(), RestoreCommandError> {
1625 if !options.require_no_attention || !report.attention_required {
1626 return Ok(());
1627 }
1628
1629 Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1630 backup_id: report.backup_id.clone(),
1631 outcome: report.outcome.clone(),
1632 })
1633}
1634
1635fn enforce_apply_status_requirements(
1637 options: &RestoreApplyStatusOptions,
1638 status: &RestoreApplyJournalStatus,
1639) -> Result<(), RestoreCommandError> {
1640 if options.require_ready && !status.ready {
1641 return Err(RestoreCommandError::RestoreApplyNotReady {
1642 backup_id: status.backup_id.clone(),
1643 reasons: status.blocked_reasons.clone(),
1644 });
1645 }
1646
1647 if options.require_no_pending && status.pending_operations > 0 {
1648 return Err(RestoreCommandError::RestoreApplyPending {
1649 backup_id: status.backup_id.clone(),
1650 pending_operations: status.pending_operations,
1651 next_transition_sequence: status.next_transition_sequence,
1652 });
1653 }
1654
1655 if options.require_no_failed && status.failed_operations > 0 {
1656 return Err(RestoreCommandError::RestoreApplyFailed {
1657 backup_id: status.backup_id.clone(),
1658 failed_operations: status.failed_operations,
1659 });
1660 }
1661
1662 if options.require_complete && !status.complete {
1663 return Err(RestoreCommandError::RestoreApplyIncomplete {
1664 backup_id: status.backup_id.clone(),
1665 completed_operations: status.completed_operations,
1666 operation_count: status.operation_count,
1667 });
1668 }
1669
1670 Ok(())
1671}
1672
1673pub fn restore_apply_next(
1675 options: &RestoreApplyNextOptions,
1676) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
1677 let journal = read_apply_journal(&options.journal)?;
1678 Ok(journal.next_operation())
1679}
1680
1681pub fn restore_apply_command(
1683 options: &RestoreApplyCommandOptions,
1684) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
1685 let journal = read_apply_journal(&options.journal)?;
1686 Ok(journal.next_command_preview_with_config(&restore_apply_command_config(options)))
1687}
1688
1689fn enforce_apply_command_requirements(
1691 options: &RestoreApplyCommandOptions,
1692 preview: &RestoreApplyCommandPreview,
1693) -> Result<(), RestoreCommandError> {
1694 if !options.require_command || preview.command_available {
1695 return Ok(());
1696 }
1697
1698 Err(restore_command_unavailable_error(preview))
1699}
1700
1701pub fn restore_apply_claim(
1703 options: &RestoreApplyClaimOptions,
1704) -> Result<RestoreApplyJournal, RestoreCommandError> {
1705 let mut journal = read_apply_journal(&options.journal)?;
1706 let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
1707
1708 if let Some(sequence) = options.sequence {
1709 enforce_apply_claim_sequence(sequence, &journal)?;
1710 journal.mark_operation_pending_at(sequence, updated_at)?;
1711 return Ok(journal);
1712 }
1713
1714 journal.mark_next_operation_pending_at(updated_at)?;
1715 Ok(journal)
1716}
1717
1718fn enforce_apply_claim_sequence(
1720 expected: usize,
1721 journal: &RestoreApplyJournal,
1722) -> Result<(), RestoreCommandError> {
1723 let actual = journal
1724 .next_transition_operation()
1725 .map(|operation| operation.sequence);
1726
1727 if actual == Some(expected) {
1728 return Ok(());
1729 }
1730
1731 Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
1732}
1733
1734pub fn restore_apply_unclaim(
1736 options: &RestoreApplyUnclaimOptions,
1737) -> Result<RestoreApplyJournal, RestoreCommandError> {
1738 let mut journal = read_apply_journal(&options.journal)?;
1739 if let Some(sequence) = options.sequence {
1740 enforce_apply_unclaim_sequence(sequence, &journal)?;
1741 }
1742
1743 journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
1744 Ok(journal)
1745}
1746
1747fn enforce_apply_unclaim_sequence(
1749 expected: usize,
1750 journal: &RestoreApplyJournal,
1751) -> Result<(), RestoreCommandError> {
1752 let actual = journal
1753 .next_transition_operation()
1754 .map(|operation| operation.sequence);
1755
1756 if actual == Some(expected) {
1757 return Ok(());
1758 }
1759
1760 Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
1761}
1762
1763pub fn restore_apply_mark(
1765 options: &RestoreApplyMarkOptions,
1766) -> Result<RestoreApplyJournal, RestoreCommandError> {
1767 let mut journal = read_apply_journal(&options.journal)?;
1768 enforce_apply_mark_pending_requirement(options, &journal)?;
1769
1770 match options.state {
1771 RestoreApplyMarkState::Completed => {
1772 journal.mark_operation_completed_at(
1773 options.sequence,
1774 Some(state_updated_at(options.updated_at.as_ref())),
1775 )?;
1776 }
1777 RestoreApplyMarkState::Failed => {
1778 let reason =
1779 options
1780 .reason
1781 .clone()
1782 .ok_or(RestoreApplyJournalError::FailureReasonRequired(
1783 options.sequence,
1784 ))?;
1785 journal.mark_operation_failed_at(
1786 options.sequence,
1787 reason,
1788 Some(state_updated_at(options.updated_at.as_ref())),
1789 )?;
1790 }
1791 }
1792
1793 Ok(journal)
1794}
1795
1796fn enforce_apply_mark_pending_requirement(
1798 options: &RestoreApplyMarkOptions,
1799 journal: &RestoreApplyJournal,
1800) -> Result<(), RestoreCommandError> {
1801 if !options.require_pending {
1802 return Ok(());
1803 }
1804
1805 let state = journal
1806 .operations
1807 .iter()
1808 .find(|operation| operation.sequence == options.sequence)
1809 .map(|operation| operation.state.clone())
1810 .ok_or(RestoreApplyJournalError::OperationNotFound(
1811 options.sequence,
1812 ))?;
1813
1814 if state == RestoreApplyOperationState::Pending {
1815 return Ok(());
1816 }
1817
1818 Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
1819 sequence: options.sequence,
1820 state,
1821 })
1822}
1823
1824fn enforce_restore_plan_requirements(
1826 options: &RestorePlanOptions,
1827 plan: &RestorePlan,
1828) -> Result<(), RestoreCommandError> {
1829 if !options.require_restore_ready || plan.readiness_summary.ready {
1830 return Ok(());
1831 }
1832
1833 Err(RestoreCommandError::RestoreNotReady {
1834 backup_id: plan.backup_id.clone(),
1835 reasons: plan.readiness_summary.reasons.clone(),
1836 })
1837}
1838
1839fn verify_backup_layout_if_required(
1841 options: &RestorePlanOptions,
1842) -> Result<(), RestoreCommandError> {
1843 if !options.require_verified {
1844 return Ok(());
1845 }
1846
1847 let Some(dir) = &options.backup_dir else {
1848 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
1849 };
1850
1851 BackupLayout::new(dir.clone()).verify_integrity()?;
1852 Ok(())
1853}
1854
1855fn read_manifest_source(
1857 options: &RestorePlanOptions,
1858) -> Result<FleetBackupManifest, RestoreCommandError> {
1859 if let Some(path) = &options.manifest {
1860 return read_manifest(path);
1861 }
1862
1863 let Some(dir) = &options.backup_dir else {
1864 return Err(RestoreCommandError::MissingOption(
1865 "--manifest or --backup-dir",
1866 ));
1867 };
1868
1869 BackupLayout::new(dir.clone())
1870 .read_manifest()
1871 .map_err(RestoreCommandError::from)
1872}
1873
1874fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
1876 let data = fs::read_to_string(path)?;
1877 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1878}
1879
1880fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
1882 let data = fs::read_to_string(path)?;
1883 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1884}
1885
1886fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
1888 let data = fs::read_to_string(path)?;
1889 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1890}
1891
1892fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
1894 let data = fs::read_to_string(path)?;
1895 serde_json::from_str(&data).map_err(RestoreCommandError::from)
1896}
1897
1898fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
1900 let data = fs::read_to_string(path)?;
1901 let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
1902 journal.validate()?;
1903 Ok(journal)
1904}
1905
1906fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
1908 value
1909 .parse::<usize>()
1910 .map_err(|_| RestoreCommandError::InvalidSequence)
1911}
1912
1913fn state_updated_at(updated_at: Option<&String>) -> String {
1915 updated_at.cloned().unwrap_or_else(timestamp_placeholder)
1916}
1917
1918fn timestamp_placeholder() -> String {
1920 "unknown".to_string()
1921}
1922
1923fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
1925 if let Some(path) = &options.out {
1926 let data = serde_json::to_vec_pretty(plan)?;
1927 fs::write(path, data)?;
1928 return Ok(());
1929 }
1930
1931 let stdout = io::stdout();
1932 let mut handle = stdout.lock();
1933 serde_json::to_writer_pretty(&mut handle, plan)?;
1934 writeln!(handle)?;
1935 Ok(())
1936}
1937
1938fn write_status(
1940 options: &RestoreStatusOptions,
1941 status: &RestoreStatus,
1942) -> Result<(), RestoreCommandError> {
1943 if let Some(path) = &options.out {
1944 let data = serde_json::to_vec_pretty(status)?;
1945 fs::write(path, data)?;
1946 return Ok(());
1947 }
1948
1949 let stdout = io::stdout();
1950 let mut handle = stdout.lock();
1951 serde_json::to_writer_pretty(&mut handle, status)?;
1952 writeln!(handle)?;
1953 Ok(())
1954}
1955
1956fn write_apply_dry_run(
1958 options: &RestoreApplyOptions,
1959 dry_run: &RestoreApplyDryRun,
1960) -> Result<(), RestoreCommandError> {
1961 if let Some(path) = &options.out {
1962 let data = serde_json::to_vec_pretty(dry_run)?;
1963 fs::write(path, data)?;
1964 return Ok(());
1965 }
1966
1967 let stdout = io::stdout();
1968 let mut handle = stdout.lock();
1969 serde_json::to_writer_pretty(&mut handle, dry_run)?;
1970 writeln!(handle)?;
1971 Ok(())
1972}
1973
1974fn write_apply_journal_if_requested(
1976 options: &RestoreApplyOptions,
1977 dry_run: &RestoreApplyDryRun,
1978) -> Result<(), RestoreCommandError> {
1979 let Some(path) = &options.journal_out else {
1980 return Ok(());
1981 };
1982
1983 let journal = RestoreApplyJournal::from_dry_run(dry_run);
1984 let data = serde_json::to_vec_pretty(&journal)?;
1985 fs::write(path, data)?;
1986 Ok(())
1987}
1988
1989fn write_apply_status(
1991 options: &RestoreApplyStatusOptions,
1992 status: &RestoreApplyJournalStatus,
1993) -> Result<(), RestoreCommandError> {
1994 if let Some(path) = &options.out {
1995 let data = serde_json::to_vec_pretty(status)?;
1996 fs::write(path, data)?;
1997 return Ok(());
1998 }
1999
2000 let stdout = io::stdout();
2001 let mut handle = stdout.lock();
2002 serde_json::to_writer_pretty(&mut handle, status)?;
2003 writeln!(handle)?;
2004 Ok(())
2005}
2006
2007fn write_apply_report(
2009 options: &RestoreApplyReportOptions,
2010 report: &RestoreApplyJournalReport,
2011) -> Result<(), RestoreCommandError> {
2012 if let Some(path) = &options.out {
2013 let data = serde_json::to_vec_pretty(report)?;
2014 fs::write(path, data)?;
2015 return Ok(());
2016 }
2017
2018 let stdout = io::stdout();
2019 let mut handle = stdout.lock();
2020 serde_json::to_writer_pretty(&mut handle, report)?;
2021 writeln!(handle)?;
2022 Ok(())
2023}
2024
2025fn write_restore_run(
2027 options: &RestoreRunOptions,
2028 run: &RestoreRunResponse,
2029) -> Result<(), RestoreCommandError> {
2030 if let Some(path) = &options.out {
2031 let data = serde_json::to_vec_pretty(run)?;
2032 fs::write(path, data)?;
2033 return Ok(());
2034 }
2035
2036 let stdout = io::stdout();
2037 let mut handle = stdout.lock();
2038 serde_json::to_writer_pretty(&mut handle, run)?;
2039 writeln!(handle)?;
2040 Ok(())
2041}
2042
2043fn write_apply_journal_file(
2045 path: &PathBuf,
2046 journal: &RestoreApplyJournal,
2047) -> Result<(), RestoreCommandError> {
2048 let data = serde_json::to_vec_pretty(journal)?;
2049 fs::write(path, data)?;
2050 Ok(())
2051}
2052
2053fn write_apply_next(
2055 options: &RestoreApplyNextOptions,
2056 next: &RestoreApplyNextOperation,
2057) -> Result<(), RestoreCommandError> {
2058 if let Some(path) = &options.out {
2059 let data = serde_json::to_vec_pretty(next)?;
2060 fs::write(path, data)?;
2061 return Ok(());
2062 }
2063
2064 let stdout = io::stdout();
2065 let mut handle = stdout.lock();
2066 serde_json::to_writer_pretty(&mut handle, next)?;
2067 writeln!(handle)?;
2068 Ok(())
2069}
2070
2071fn write_apply_command(
2073 options: &RestoreApplyCommandOptions,
2074 preview: &RestoreApplyCommandPreview,
2075) -> Result<(), RestoreCommandError> {
2076 if let Some(path) = &options.out {
2077 let data = serde_json::to_vec_pretty(preview)?;
2078 fs::write(path, data)?;
2079 return Ok(());
2080 }
2081
2082 let stdout = io::stdout();
2083 let mut handle = stdout.lock();
2084 serde_json::to_writer_pretty(&mut handle, preview)?;
2085 writeln!(handle)?;
2086 Ok(())
2087}
2088
2089fn write_apply_claim(
2091 options: &RestoreApplyClaimOptions,
2092 journal: &RestoreApplyJournal,
2093) -> Result<(), RestoreCommandError> {
2094 if let Some(path) = &options.out {
2095 let data = serde_json::to_vec_pretty(journal)?;
2096 fs::write(path, data)?;
2097 return Ok(());
2098 }
2099
2100 let stdout = io::stdout();
2101 let mut handle = stdout.lock();
2102 serde_json::to_writer_pretty(&mut handle, journal)?;
2103 writeln!(handle)?;
2104 Ok(())
2105}
2106
2107fn write_apply_unclaim(
2109 options: &RestoreApplyUnclaimOptions,
2110 journal: &RestoreApplyJournal,
2111) -> Result<(), RestoreCommandError> {
2112 if let Some(path) = &options.out {
2113 let data = serde_json::to_vec_pretty(journal)?;
2114 fs::write(path, data)?;
2115 return Ok(());
2116 }
2117
2118 let stdout = io::stdout();
2119 let mut handle = stdout.lock();
2120 serde_json::to_writer_pretty(&mut handle, journal)?;
2121 writeln!(handle)?;
2122 Ok(())
2123}
2124
2125fn write_apply_mark(
2127 options: &RestoreApplyMarkOptions,
2128 journal: &RestoreApplyJournal,
2129) -> Result<(), RestoreCommandError> {
2130 if let Some(path) = &options.out {
2131 let data = serde_json::to_vec_pretty(journal)?;
2132 fs::write(path, data)?;
2133 return Ok(());
2134 }
2135
2136 let stdout = io::stdout();
2137 let mut handle = stdout.lock();
2138 serde_json::to_writer_pretty(&mut handle, journal)?;
2139 writeln!(handle)?;
2140 Ok(())
2141}
2142
2143fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
2145where
2146 I: Iterator<Item = OsString>,
2147{
2148 args.next()
2149 .and_then(|value| value.into_string().ok())
2150 .ok_or(RestoreCommandError::MissingValue(option))
2151}
2152
2153const fn usage() -> &'static str {
2155 "usage: canic restore plan (--manifest <file> | --backup-dir <dir>) [--mapping <file>] [--out <file>] [--require-verified] [--require-restore-ready]\n canic restore status --plan <file> [--out <file>]\n canic restore apply --plan <file> [--status <file>] [--backup-dir <dir>] --dry-run [--out <file>] [--journal-out <file>]\n canic restore apply-status --journal <file> [--out <file>] [--require-ready] [--require-no-pending] [--require-no-failed] [--require-complete]\n canic restore apply-report --journal <file> [--out <file>] [--require-no-attention]\n canic restore run --journal <file> (--dry-run | --execute | --unclaim-pending) [--dfx <path>] [--network <name>] [--max-steps <n>] [--out <file>] [--require-complete] [--require-no-attention] [--require-run-mode <text>] [--require-stopped-reason <text>] [--require-next-action <text>] [--require-executed-count <n>]\n canic restore apply-next --journal <file> [--out <file>]\n canic restore apply-command --journal <file> [--dfx <path>] [--network <name>] [--out <file>] [--require-command]\n canic restore apply-claim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-unclaim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-mark --journal <file> --sequence <n> --state completed|failed [--reason <text>] [--updated-at <text>] [--out <file>] [--require-pending]"
2156}
2157
2158#[cfg(test)]
2159mod tests {
2160 use super::*;
2161 use canic_backup::restore::RestoreApplyOperationState;
2162 use canic_backup::{
2163 artifacts::ArtifactChecksum,
2164 journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
2165 manifest::{
2166 BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
2167 FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
2168 VerificationCheck, VerificationPlan,
2169 },
2170 };
2171 use serde_json::json;
2172 use std::{
2173 path::Path,
2174 time::{SystemTime, UNIX_EPOCH},
2175 };
2176
2177 const ROOT: &str = "aaaaa-aa";
2178 const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2179 const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2180 const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2181
2182 #[test]
2184 fn parses_restore_plan_options() {
2185 let options = RestorePlanOptions::parse([
2186 OsString::from("--manifest"),
2187 OsString::from("manifest.json"),
2188 OsString::from("--mapping"),
2189 OsString::from("mapping.json"),
2190 OsString::from("--out"),
2191 OsString::from("plan.json"),
2192 OsString::from("--require-restore-ready"),
2193 ])
2194 .expect("parse options");
2195
2196 assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
2197 assert_eq!(options.backup_dir, None);
2198 assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
2199 assert_eq!(options.out, Some(PathBuf::from("plan.json")));
2200 assert!(!options.require_verified);
2201 assert!(options.require_restore_ready);
2202 }
2203
2204 #[test]
2206 fn parses_verified_restore_plan_options() {
2207 let options = RestorePlanOptions::parse([
2208 OsString::from("--backup-dir"),
2209 OsString::from("backups/run"),
2210 OsString::from("--require-verified"),
2211 ])
2212 .expect("parse verified options");
2213
2214 assert_eq!(options.manifest, None);
2215 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2216 assert_eq!(options.mapping, None);
2217 assert_eq!(options.out, None);
2218 assert!(options.require_verified);
2219 assert!(!options.require_restore_ready);
2220 }
2221
2222 #[test]
2224 fn parses_restore_status_options() {
2225 let options = RestoreStatusOptions::parse([
2226 OsString::from("--plan"),
2227 OsString::from("restore-plan.json"),
2228 OsString::from("--out"),
2229 OsString::from("restore-status.json"),
2230 ])
2231 .expect("parse status options");
2232
2233 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2234 assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
2235 }
2236
2237 #[test]
2239 fn parses_restore_apply_dry_run_options() {
2240 let options = RestoreApplyOptions::parse([
2241 OsString::from("--plan"),
2242 OsString::from("restore-plan.json"),
2243 OsString::from("--status"),
2244 OsString::from("restore-status.json"),
2245 OsString::from("--backup-dir"),
2246 OsString::from("backups/run"),
2247 OsString::from("--dry-run"),
2248 OsString::from("--out"),
2249 OsString::from("restore-apply-dry-run.json"),
2250 OsString::from("--journal-out"),
2251 OsString::from("restore-apply-journal.json"),
2252 ])
2253 .expect("parse apply options");
2254
2255 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2256 assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
2257 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2258 assert_eq!(
2259 options.out,
2260 Some(PathBuf::from("restore-apply-dry-run.json"))
2261 );
2262 assert_eq!(
2263 options.journal_out,
2264 Some(PathBuf::from("restore-apply-journal.json"))
2265 );
2266 assert!(options.dry_run);
2267 }
2268
2269 #[test]
2271 fn parses_restore_apply_status_options() {
2272 let options = RestoreApplyStatusOptions::parse([
2273 OsString::from("--journal"),
2274 OsString::from("restore-apply-journal.json"),
2275 OsString::from("--out"),
2276 OsString::from("restore-apply-status.json"),
2277 OsString::from("--require-ready"),
2278 OsString::from("--require-no-pending"),
2279 OsString::from("--require-no-failed"),
2280 OsString::from("--require-complete"),
2281 ])
2282 .expect("parse apply-status options");
2283
2284 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2285 assert!(options.require_ready);
2286 assert!(options.require_no_pending);
2287 assert!(options.require_no_failed);
2288 assert!(options.require_complete);
2289 assert_eq!(
2290 options.out,
2291 Some(PathBuf::from("restore-apply-status.json"))
2292 );
2293 }
2294
2295 #[test]
2297 fn parses_restore_apply_report_options() {
2298 let options = RestoreApplyReportOptions::parse([
2299 OsString::from("--journal"),
2300 OsString::from("restore-apply-journal.json"),
2301 OsString::from("--out"),
2302 OsString::from("restore-apply-report.json"),
2303 OsString::from("--require-no-attention"),
2304 ])
2305 .expect("parse apply-report options");
2306
2307 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2308 assert!(options.require_no_attention);
2309 assert_eq!(
2310 options.out,
2311 Some(PathBuf::from("restore-apply-report.json"))
2312 );
2313 }
2314
2315 #[test]
2317 fn parses_restore_run_dry_run_options() {
2318 let options = RestoreRunOptions::parse([
2319 OsString::from("--journal"),
2320 OsString::from("restore-apply-journal.json"),
2321 OsString::from("--dry-run"),
2322 OsString::from("--dfx"),
2323 OsString::from("/tmp/dfx"),
2324 OsString::from("--network"),
2325 OsString::from("local"),
2326 OsString::from("--out"),
2327 OsString::from("restore-run-dry-run.json"),
2328 OsString::from("--max-steps"),
2329 OsString::from("1"),
2330 OsString::from("--require-complete"),
2331 OsString::from("--require-no-attention"),
2332 OsString::from("--require-run-mode"),
2333 OsString::from("dry-run"),
2334 OsString::from("--require-stopped-reason"),
2335 OsString::from("preview"),
2336 OsString::from("--require-next-action"),
2337 OsString::from("rerun"),
2338 OsString::from("--require-executed-count"),
2339 OsString::from("0"),
2340 ])
2341 .expect("parse restore run options");
2342
2343 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2344 assert_eq!(options.dfx, "/tmp/dfx");
2345 assert_eq!(options.network.as_deref(), Some("local"));
2346 assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
2347 assert!(options.dry_run);
2348 assert!(!options.execute);
2349 assert!(!options.unclaim_pending);
2350 assert_eq!(options.max_steps, Some(1));
2351 assert!(options.require_complete);
2352 assert!(options.require_no_attention);
2353 assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
2354 assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
2355 assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
2356 assert_eq!(options.require_executed_count, Some(0));
2357 }
2358
2359 #[test]
2361 fn parses_restore_run_execute_options() {
2362 let options = RestoreRunOptions::parse([
2363 OsString::from("--journal"),
2364 OsString::from("restore-apply-journal.json"),
2365 OsString::from("--execute"),
2366 OsString::from("--dfx"),
2367 OsString::from("/bin/true"),
2368 OsString::from("--max-steps"),
2369 OsString::from("4"),
2370 ])
2371 .expect("parse restore run execute options");
2372
2373 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2374 assert_eq!(options.dfx, "/bin/true");
2375 assert_eq!(options.network, None);
2376 assert_eq!(options.out, None);
2377 assert!(!options.dry_run);
2378 assert!(options.execute);
2379 assert!(!options.unclaim_pending);
2380 assert_eq!(options.max_steps, Some(4));
2381 assert!(!options.require_complete);
2382 assert!(!options.require_no_attention);
2383 assert_eq!(options.require_run_mode, None);
2384 assert_eq!(options.require_stopped_reason, None);
2385 assert_eq!(options.require_next_action, None);
2386 assert_eq!(options.require_executed_count, None);
2387 }
2388
2389 #[test]
2391 fn parses_restore_run_unclaim_pending_options() {
2392 let options = RestoreRunOptions::parse([
2393 OsString::from("--journal"),
2394 OsString::from("restore-apply-journal.json"),
2395 OsString::from("--unclaim-pending"),
2396 OsString::from("--out"),
2397 OsString::from("restore-run.json"),
2398 ])
2399 .expect("parse restore run unclaim options");
2400
2401 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2402 assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
2403 assert!(!options.dry_run);
2404 assert!(!options.execute);
2405 assert!(options.unclaim_pending);
2406 }
2407
2408 #[test]
2410 fn parses_restore_apply_next_options() {
2411 let options = RestoreApplyNextOptions::parse([
2412 OsString::from("--journal"),
2413 OsString::from("restore-apply-journal.json"),
2414 OsString::from("--out"),
2415 OsString::from("restore-apply-next.json"),
2416 ])
2417 .expect("parse apply-next options");
2418
2419 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2420 assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
2421 }
2422
2423 #[test]
2425 fn parses_restore_apply_command_options() {
2426 let options = RestoreApplyCommandOptions::parse([
2427 OsString::from("--journal"),
2428 OsString::from("restore-apply-journal.json"),
2429 OsString::from("--dfx"),
2430 OsString::from("/tmp/dfx"),
2431 OsString::from("--network"),
2432 OsString::from("local"),
2433 OsString::from("--out"),
2434 OsString::from("restore-apply-command.json"),
2435 OsString::from("--require-command"),
2436 ])
2437 .expect("parse apply-command options");
2438
2439 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2440 assert_eq!(options.dfx, "/tmp/dfx");
2441 assert_eq!(options.network.as_deref(), Some("local"));
2442 assert!(options.require_command);
2443 assert_eq!(
2444 options.out,
2445 Some(PathBuf::from("restore-apply-command.json"))
2446 );
2447 }
2448
2449 #[test]
2451 fn parses_restore_apply_claim_options() {
2452 let options = RestoreApplyClaimOptions::parse([
2453 OsString::from("--journal"),
2454 OsString::from("restore-apply-journal.json"),
2455 OsString::from("--sequence"),
2456 OsString::from("0"),
2457 OsString::from("--updated-at"),
2458 OsString::from("2026-05-04T12:00:00Z"),
2459 OsString::from("--out"),
2460 OsString::from("restore-apply-journal.claimed.json"),
2461 ])
2462 .expect("parse apply-claim options");
2463
2464 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2465 assert_eq!(options.sequence, Some(0));
2466 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
2467 assert_eq!(
2468 options.out,
2469 Some(PathBuf::from("restore-apply-journal.claimed.json"))
2470 );
2471 }
2472
2473 #[test]
2475 fn parses_restore_apply_unclaim_options() {
2476 let options = RestoreApplyUnclaimOptions::parse([
2477 OsString::from("--journal"),
2478 OsString::from("restore-apply-journal.json"),
2479 OsString::from("--sequence"),
2480 OsString::from("0"),
2481 OsString::from("--updated-at"),
2482 OsString::from("2026-05-04T12:01:00Z"),
2483 OsString::from("--out"),
2484 OsString::from("restore-apply-journal.unclaimed.json"),
2485 ])
2486 .expect("parse apply-unclaim options");
2487
2488 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2489 assert_eq!(options.sequence, Some(0));
2490 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
2491 assert_eq!(
2492 options.out,
2493 Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
2494 );
2495 }
2496
2497 #[test]
2499 fn parses_restore_apply_mark_options() {
2500 let options = RestoreApplyMarkOptions::parse([
2501 OsString::from("--journal"),
2502 OsString::from("restore-apply-journal.json"),
2503 OsString::from("--sequence"),
2504 OsString::from("4"),
2505 OsString::from("--state"),
2506 OsString::from("failed"),
2507 OsString::from("--reason"),
2508 OsString::from("dfx-load-failed"),
2509 OsString::from("--updated-at"),
2510 OsString::from("2026-05-04T12:02:00Z"),
2511 OsString::from("--out"),
2512 OsString::from("restore-apply-journal.updated.json"),
2513 OsString::from("--require-pending"),
2514 ])
2515 .expect("parse apply-mark options");
2516
2517 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2518 assert_eq!(options.sequence, 4);
2519 assert_eq!(options.state, RestoreApplyMarkState::Failed);
2520 assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
2521 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
2522 assert!(options.require_pending);
2523 assert_eq!(
2524 options.out,
2525 Some(PathBuf::from("restore-apply-journal.updated.json"))
2526 );
2527 }
2528
2529 #[test]
2531 fn restore_apply_requires_dry_run() {
2532 let err = RestoreApplyOptions::parse([
2533 OsString::from("--plan"),
2534 OsString::from("restore-plan.json"),
2535 ])
2536 .expect_err("apply without dry-run should fail");
2537
2538 assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
2539 }
2540
2541 #[test]
2543 fn restore_run_requires_mode() {
2544 let err = RestoreRunOptions::parse([
2545 OsString::from("--journal"),
2546 OsString::from("restore-apply-journal.json"),
2547 ])
2548 .expect_err("restore run without dry-run should fail");
2549
2550 assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
2551 }
2552
2553 #[test]
2555 fn restore_run_rejects_conflicting_modes() {
2556 let err = RestoreRunOptions::parse([
2557 OsString::from("--journal"),
2558 OsString::from("restore-apply-journal.json"),
2559 OsString::from("--dry-run"),
2560 OsString::from("--execute"),
2561 OsString::from("--unclaim-pending"),
2562 ])
2563 .expect_err("restore run should reject conflicting modes");
2564
2565 assert!(matches!(
2566 err,
2567 RestoreCommandError::RestoreRunConflictingModes
2568 ));
2569 }
2570
2571 #[test]
2573 fn plan_restore_reads_manifest_from_backup_dir() {
2574 let root = temp_dir("canic-cli-restore-plan-layout");
2575 let layout = BackupLayout::new(root.clone());
2576 layout
2577 .write_manifest(&valid_manifest())
2578 .expect("write manifest");
2579
2580 let options = RestorePlanOptions {
2581 manifest: None,
2582 backup_dir: Some(root.clone()),
2583 mapping: None,
2584 out: None,
2585 require_verified: false,
2586 require_restore_ready: false,
2587 };
2588
2589 let plan = plan_restore(&options).expect("plan restore");
2590
2591 fs::remove_dir_all(root).expect("remove temp root");
2592 assert_eq!(plan.backup_id, "backup-test");
2593 assert_eq!(plan.member_count, 2);
2594 }
2595
2596 #[test]
2598 fn parse_rejects_conflicting_manifest_sources() {
2599 let err = RestorePlanOptions::parse([
2600 OsString::from("--manifest"),
2601 OsString::from("manifest.json"),
2602 OsString::from("--backup-dir"),
2603 OsString::from("backups/run"),
2604 ])
2605 .expect_err("conflicting sources should fail");
2606
2607 assert!(matches!(
2608 err,
2609 RestoreCommandError::ConflictingManifestSources
2610 ));
2611 }
2612
2613 #[test]
2615 fn parse_rejects_require_verified_with_manifest_source() {
2616 let err = RestorePlanOptions::parse([
2617 OsString::from("--manifest"),
2618 OsString::from("manifest.json"),
2619 OsString::from("--require-verified"),
2620 ])
2621 .expect_err("verification should require a backup layout");
2622
2623 assert!(matches!(
2624 err,
2625 RestoreCommandError::RequireVerifiedNeedsBackupDir
2626 ));
2627 }
2628
2629 #[test]
2631 fn plan_restore_requires_verified_backup_layout() {
2632 let root = temp_dir("canic-cli-restore-plan-verified");
2633 let layout = BackupLayout::new(root.clone());
2634 let manifest = valid_manifest();
2635 write_verified_layout(&root, &layout, &manifest);
2636
2637 let options = RestorePlanOptions {
2638 manifest: None,
2639 backup_dir: Some(root.clone()),
2640 mapping: None,
2641 out: None,
2642 require_verified: true,
2643 require_restore_ready: false,
2644 };
2645
2646 let plan = plan_restore(&options).expect("plan verified restore");
2647
2648 fs::remove_dir_all(root).expect("remove temp root");
2649 assert_eq!(plan.backup_id, "backup-test");
2650 assert_eq!(plan.member_count, 2);
2651 }
2652
2653 #[test]
2655 fn plan_restore_rejects_unverified_backup_layout() {
2656 let root = temp_dir("canic-cli-restore-plan-unverified");
2657 let layout = BackupLayout::new(root.clone());
2658 layout
2659 .write_manifest(&valid_manifest())
2660 .expect("write manifest");
2661
2662 let options = RestorePlanOptions {
2663 manifest: None,
2664 backup_dir: Some(root.clone()),
2665 mapping: None,
2666 out: None,
2667 require_verified: true,
2668 require_restore_ready: false,
2669 };
2670
2671 let err = plan_restore(&options).expect_err("missing journal should fail");
2672
2673 fs::remove_dir_all(root).expect("remove temp root");
2674 assert!(matches!(err, RestoreCommandError::Persistence(_)));
2675 }
2676
2677 #[test]
2679 fn plan_restore_reads_manifest_and_mapping() {
2680 let root = temp_dir("canic-cli-restore-plan");
2681 fs::create_dir_all(&root).expect("create temp root");
2682 let manifest_path = root.join("manifest.json");
2683 let mapping_path = root.join("mapping.json");
2684
2685 fs::write(
2686 &manifest_path,
2687 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2688 )
2689 .expect("write manifest");
2690 fs::write(
2691 &mapping_path,
2692 json!({
2693 "members": [
2694 {"source_canister": ROOT, "target_canister": ROOT},
2695 {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
2696 ]
2697 })
2698 .to_string(),
2699 )
2700 .expect("write mapping");
2701
2702 let options = RestorePlanOptions {
2703 manifest: Some(manifest_path),
2704 backup_dir: None,
2705 mapping: Some(mapping_path),
2706 out: None,
2707 require_verified: false,
2708 require_restore_ready: false,
2709 };
2710
2711 let plan = plan_restore(&options).expect("plan restore");
2712
2713 fs::remove_dir_all(root).expect("remove temp root");
2714 let members = plan.ordered_members();
2715 assert_eq!(members.len(), 2);
2716 assert_eq!(members[0].source_canister, ROOT);
2717 assert_eq!(members[1].target_canister, MAPPED_CHILD);
2718 }
2719
2720 #[test]
2722 fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
2723 let root = temp_dir("canic-cli-restore-plan-require-ready");
2724 fs::create_dir_all(&root).expect("create temp root");
2725 let manifest_path = root.join("manifest.json");
2726 let out_path = root.join("plan.json");
2727
2728 fs::write(
2729 &manifest_path,
2730 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2731 )
2732 .expect("write manifest");
2733
2734 let err = run([
2735 OsString::from("plan"),
2736 OsString::from("--manifest"),
2737 OsString::from(manifest_path.as_os_str()),
2738 OsString::from("--out"),
2739 OsString::from(out_path.as_os_str()),
2740 OsString::from("--require-restore-ready"),
2741 ])
2742 .expect_err("restore readiness should be enforced");
2743
2744 assert!(out_path.exists());
2745 let plan: RestorePlan =
2746 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2747
2748 fs::remove_dir_all(root).expect("remove temp root");
2749 assert!(!plan.readiness_summary.ready);
2750 assert!(matches!(
2751 err,
2752 RestoreCommandError::RestoreNotReady {
2753 reasons,
2754 ..
2755 } if reasons == [
2756 "missing-module-hash",
2757 "missing-wasm-hash",
2758 "missing-snapshot-checksum"
2759 ]
2760 ));
2761 }
2762
2763 #[test]
2765 fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
2766 let root = temp_dir("canic-cli-restore-plan-ready");
2767 fs::create_dir_all(&root).expect("create temp root");
2768 let manifest_path = root.join("manifest.json");
2769 let out_path = root.join("plan.json");
2770
2771 fs::write(
2772 &manifest_path,
2773 serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
2774 )
2775 .expect("write manifest");
2776
2777 run([
2778 OsString::from("plan"),
2779 OsString::from("--manifest"),
2780 OsString::from(manifest_path.as_os_str()),
2781 OsString::from("--out"),
2782 OsString::from(out_path.as_os_str()),
2783 OsString::from("--require-restore-ready"),
2784 ])
2785 .expect("restore-ready plan should pass");
2786
2787 let plan: RestorePlan =
2788 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2789
2790 fs::remove_dir_all(root).expect("remove temp root");
2791 assert!(plan.readiness_summary.ready);
2792 assert!(plan.readiness_summary.reasons.is_empty());
2793 }
2794
2795 #[test]
2797 fn run_restore_status_writes_planned_status() {
2798 let root = temp_dir("canic-cli-restore-status");
2799 fs::create_dir_all(&root).expect("create temp root");
2800 let plan_path = root.join("restore-plan.json");
2801 let out_path = root.join("restore-status.json");
2802 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2803
2804 fs::write(
2805 &plan_path,
2806 serde_json::to_vec(&plan).expect("serialize plan"),
2807 )
2808 .expect("write plan");
2809
2810 run([
2811 OsString::from("status"),
2812 OsString::from("--plan"),
2813 OsString::from(plan_path.as_os_str()),
2814 OsString::from("--out"),
2815 OsString::from(out_path.as_os_str()),
2816 ])
2817 .expect("write restore status");
2818
2819 let status: RestoreStatus =
2820 serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
2821 .expect("decode restore status");
2822 let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
2823
2824 fs::remove_dir_all(root).expect("remove temp root");
2825 assert_eq!(status.status_version, 1);
2826 assert_eq!(status.backup_id.as_str(), "backup-test");
2827 assert!(status.ready);
2828 assert!(status.readiness_reasons.is_empty());
2829 assert_eq!(status.member_count, 2);
2830 assert_eq!(status.phase_count, 1);
2831 assert_eq!(status.planned_snapshot_uploads, 2);
2832 assert_eq!(status.planned_snapshot_loads, 2);
2833 assert_eq!(status.planned_code_reinstalls, 2);
2834 assert_eq!(status.planned_verification_checks, 2);
2835 assert_eq!(status.planned_operations, 8);
2836 assert_eq!(status.phases[0].members[0].source_canister, ROOT);
2837 assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
2838 }
2839
2840 #[test]
2842 fn run_restore_apply_dry_run_writes_operations() {
2843 let root = temp_dir("canic-cli-restore-apply-dry-run");
2844 fs::create_dir_all(&root).expect("create temp root");
2845 let plan_path = root.join("restore-plan.json");
2846 let status_path = root.join("restore-status.json");
2847 let out_path = root.join("restore-apply-dry-run.json");
2848 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2849 let status = RestoreStatus::from_plan(&plan);
2850
2851 fs::write(
2852 &plan_path,
2853 serde_json::to_vec(&plan).expect("serialize plan"),
2854 )
2855 .expect("write plan");
2856 fs::write(
2857 &status_path,
2858 serde_json::to_vec(&status).expect("serialize status"),
2859 )
2860 .expect("write status");
2861
2862 run([
2863 OsString::from("apply"),
2864 OsString::from("--plan"),
2865 OsString::from(plan_path.as_os_str()),
2866 OsString::from("--status"),
2867 OsString::from(status_path.as_os_str()),
2868 OsString::from("--dry-run"),
2869 OsString::from("--out"),
2870 OsString::from(out_path.as_os_str()),
2871 ])
2872 .expect("write apply dry-run");
2873
2874 let dry_run: RestoreApplyDryRun =
2875 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2876 .expect("decode dry-run");
2877 let dry_run_json: serde_json::Value =
2878 serde_json::to_value(&dry_run).expect("encode dry-run");
2879
2880 fs::remove_dir_all(root).expect("remove temp root");
2881 assert_eq!(dry_run.dry_run_version, 1);
2882 assert_eq!(dry_run.backup_id.as_str(), "backup-test");
2883 assert!(dry_run.ready);
2884 assert!(dry_run.status_supplied);
2885 assert_eq!(dry_run.member_count, 2);
2886 assert_eq!(dry_run.phase_count, 1);
2887 assert_eq!(dry_run.planned_snapshot_uploads, 2);
2888 assert_eq!(dry_run.planned_operations, 8);
2889 assert_eq!(dry_run.rendered_operations, 8);
2890 assert_eq!(dry_run_json["operation_counts"]["snapshot_uploads"], 2);
2891 assert_eq!(dry_run_json["operation_counts"]["snapshot_loads"], 2);
2892 assert_eq!(dry_run_json["operation_counts"]["code_reinstalls"], 2);
2893 assert_eq!(dry_run_json["operation_counts"]["member_verifications"], 2);
2894 assert_eq!(dry_run_json["operation_counts"]["fleet_verifications"], 0);
2895 assert_eq!(
2896 dry_run_json["operation_counts"]["verification_operations"],
2897 2
2898 );
2899 assert_eq!(
2900 dry_run_json["phases"][0]["operations"][0]["operation"],
2901 "upload-snapshot"
2902 );
2903 assert_eq!(
2904 dry_run_json["phases"][0]["operations"][3]["operation"],
2905 "verify-member"
2906 );
2907 assert_eq!(
2908 dry_run_json["phases"][0]["operations"][3]["verification_kind"],
2909 "status"
2910 );
2911 assert_eq!(
2912 dry_run_json["phases"][0]["operations"][3]["verification_method"],
2913 serde_json::Value::Null
2914 );
2915 }
2916
2917 #[test]
2919 fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
2920 let root = temp_dir("canic-cli-restore-apply-artifacts");
2921 fs::create_dir_all(&root).expect("create temp root");
2922 let plan_path = root.join("restore-plan.json");
2923 let out_path = root.join("restore-apply-dry-run.json");
2924 let journal_path = root.join("restore-apply-journal.json");
2925 let status_path = root.join("restore-apply-status.json");
2926 let mut manifest = restore_ready_manifest();
2927 write_manifest_artifacts(&root, &mut manifest);
2928 let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
2929
2930 fs::write(
2931 &plan_path,
2932 serde_json::to_vec(&plan).expect("serialize plan"),
2933 )
2934 .expect("write plan");
2935
2936 run([
2937 OsString::from("apply"),
2938 OsString::from("--plan"),
2939 OsString::from(plan_path.as_os_str()),
2940 OsString::from("--backup-dir"),
2941 OsString::from(root.as_os_str()),
2942 OsString::from("--dry-run"),
2943 OsString::from("--out"),
2944 OsString::from(out_path.as_os_str()),
2945 OsString::from("--journal-out"),
2946 OsString::from(journal_path.as_os_str()),
2947 ])
2948 .expect("write apply dry-run");
2949 run([
2950 OsString::from("apply-status"),
2951 OsString::from("--journal"),
2952 OsString::from(journal_path.as_os_str()),
2953 OsString::from("--out"),
2954 OsString::from(status_path.as_os_str()),
2955 ])
2956 .expect("write apply status");
2957
2958 let dry_run: RestoreApplyDryRun =
2959 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2960 .expect("decode dry-run");
2961 let validation = dry_run
2962 .artifact_validation
2963 .expect("artifact validation should be present");
2964 let journal_json: serde_json::Value =
2965 serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
2966 .expect("decode journal");
2967 let status_json: serde_json::Value =
2968 serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
2969 .expect("decode apply status");
2970
2971 fs::remove_dir_all(root).expect("remove temp root");
2972 assert_eq!(validation.checked_members, 2);
2973 assert!(validation.artifacts_present);
2974 assert!(validation.checksums_verified);
2975 assert_eq!(validation.members_with_expected_checksums, 2);
2976 assert_eq!(journal_json["ready"], true);
2977 assert_eq!(journal_json["operation_count"], 8);
2978 assert_eq!(journal_json["ready_operations"], 8);
2979 assert_eq!(journal_json["blocked_operations"], 0);
2980 assert_eq!(journal_json["operations"][0]["state"], "ready");
2981 assert_eq!(status_json["ready"], true);
2982 assert_eq!(status_json["operation_count"], 8);
2983 assert_eq!(status_json["operation_counts"]["snapshot_uploads"], 2);
2984 assert_eq!(status_json["operation_counts"]["snapshot_loads"], 2);
2985 assert_eq!(status_json["operation_counts"]["code_reinstalls"], 2);
2986 assert_eq!(status_json["operation_counts"]["member_verifications"], 2);
2987 assert_eq!(status_json["operation_counts"]["fleet_verifications"], 0);
2988 assert_eq!(
2989 status_json["operation_counts"]["verification_operations"],
2990 2
2991 );
2992 assert_eq!(status_json["next_ready_sequence"], 0);
2993 assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
2994 }
2995
2996 #[test]
2998 fn run_restore_apply_status_rejects_invalid_journal() {
2999 let root = temp_dir("canic-cli-restore-apply-status-invalid");
3000 fs::create_dir_all(&root).expect("create temp root");
3001 let journal_path = root.join("restore-apply-journal.json");
3002 let out_path = root.join("restore-apply-status.json");
3003 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3004 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3005 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3006 journal.operation_count += 1;
3007
3008 fs::write(
3009 &journal_path,
3010 serde_json::to_vec(&journal).expect("serialize journal"),
3011 )
3012 .expect("write journal");
3013
3014 let err = run([
3015 OsString::from("apply-status"),
3016 OsString::from("--journal"),
3017 OsString::from(journal_path.as_os_str()),
3018 OsString::from("--out"),
3019 OsString::from(out_path.as_os_str()),
3020 ])
3021 .expect_err("invalid journal should fail");
3022
3023 assert!(!out_path.exists());
3024 fs::remove_dir_all(root).expect("remove temp root");
3025 assert!(matches!(
3026 err,
3027 RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
3028 field: "operation_count",
3029 ..
3030 })
3031 ));
3032 }
3033
3034 #[test]
3036 fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
3037 let root = temp_dir("canic-cli-restore-apply-status-pending");
3038 fs::create_dir_all(&root).expect("create temp root");
3039 let journal_path = root.join("restore-apply-journal.json");
3040 let out_path = root.join("restore-apply-status.json");
3041 let mut journal = ready_apply_journal();
3042 journal
3043 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
3044 .expect("claim operation");
3045
3046 fs::write(
3047 &journal_path,
3048 serde_json::to_vec(&journal).expect("serialize journal"),
3049 )
3050 .expect("write journal");
3051
3052 let err = run([
3053 OsString::from("apply-status"),
3054 OsString::from("--journal"),
3055 OsString::from(journal_path.as_os_str()),
3056 OsString::from("--out"),
3057 OsString::from(out_path.as_os_str()),
3058 OsString::from("--require-no-pending"),
3059 ])
3060 .expect_err("pending operation should fail requirement");
3061
3062 assert!(out_path.exists());
3063 let status: RestoreApplyJournalStatus =
3064 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3065 .expect("decode apply status");
3066
3067 fs::remove_dir_all(root).expect("remove temp root");
3068 assert_eq!(status.pending_operations, 1);
3069 assert_eq!(status.next_transition_sequence, Some(0));
3070 assert_eq!(
3071 status.next_transition_updated_at.as_deref(),
3072 Some("2026-05-04T12:00:00Z")
3073 );
3074 assert!(matches!(
3075 err,
3076 RestoreCommandError::RestoreApplyPending {
3077 pending_operations: 1,
3078 next_transition_sequence: Some(0),
3079 ..
3080 }
3081 ));
3082 }
3083
3084 #[test]
3086 fn run_restore_apply_status_require_ready_writes_status_then_fails() {
3087 let root = temp_dir("canic-cli-restore-apply-status-ready");
3088 fs::create_dir_all(&root).expect("create temp root");
3089 let journal_path = root.join("restore-apply-journal.json");
3090 let out_path = root.join("restore-apply-status.json");
3091 let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
3092 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3093 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3094
3095 fs::write(
3096 &journal_path,
3097 serde_json::to_vec(&journal).expect("serialize journal"),
3098 )
3099 .expect("write journal");
3100
3101 let err = run([
3102 OsString::from("apply-status"),
3103 OsString::from("--journal"),
3104 OsString::from(journal_path.as_os_str()),
3105 OsString::from("--out"),
3106 OsString::from(out_path.as_os_str()),
3107 OsString::from("--require-ready"),
3108 ])
3109 .expect_err("unready journal should fail requirement");
3110
3111 let status: RestoreApplyJournalStatus =
3112 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3113 .expect("decode apply status");
3114
3115 fs::remove_dir_all(root).expect("remove temp root");
3116 assert!(!status.ready);
3117 assert_eq!(status.blocked_operations, status.operation_count);
3118 assert!(
3119 status
3120 .blocked_reasons
3121 .contains(&"missing-snapshot-checksum".to_string())
3122 );
3123 assert!(matches!(
3124 err,
3125 RestoreCommandError::RestoreApplyNotReady { reasons, .. }
3126 if reasons.contains(&"missing-snapshot-checksum".to_string())
3127 ));
3128 }
3129
3130 #[test]
3132 fn run_restore_apply_report_writes_attention_summary() {
3133 let root = temp_dir("canic-cli-restore-apply-report");
3134 fs::create_dir_all(&root).expect("create temp root");
3135 let journal_path = root.join("restore-apply-journal.json");
3136 let out_path = root.join("restore-apply-report.json");
3137 let mut journal = ready_apply_journal();
3138 journal
3139 .mark_operation_failed_at(
3140 0,
3141 "dfx-upload-failed".to_string(),
3142 Some("2026-05-05T12:00:00Z".to_string()),
3143 )
3144 .expect("mark failed operation");
3145 journal
3146 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3147 .expect("mark pending operation");
3148
3149 fs::write(
3150 &journal_path,
3151 serde_json::to_vec(&journal).expect("serialize journal"),
3152 )
3153 .expect("write journal");
3154
3155 run([
3156 OsString::from("apply-report"),
3157 OsString::from("--journal"),
3158 OsString::from(journal_path.as_os_str()),
3159 OsString::from("--out"),
3160 OsString::from(out_path.as_os_str()),
3161 ])
3162 .expect("write apply report");
3163
3164 let report: RestoreApplyJournalReport =
3165 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3166 .expect("decode apply report");
3167 let report_json: serde_json::Value =
3168 serde_json::to_value(&report).expect("encode apply report");
3169
3170 fs::remove_dir_all(root).expect("remove temp root");
3171 assert_eq!(report.backup_id, "backup-test");
3172 assert!(report.attention_required);
3173 assert_eq!(report.failed_operations, 1);
3174 assert_eq!(report.pending_operations, 1);
3175 assert_eq!(report.operation_counts.snapshot_uploads, 2);
3176 assert_eq!(report.operation_counts.snapshot_loads, 2);
3177 assert_eq!(report.operation_counts.code_reinstalls, 2);
3178 assert_eq!(report.operation_counts.member_verifications, 2);
3179 assert_eq!(report.operation_counts.fleet_verifications, 0);
3180 assert_eq!(report.operation_counts.verification_operations, 2);
3181 assert_eq!(report.failed.len(), 1);
3182 assert_eq!(report.pending.len(), 1);
3183 assert_eq!(report.failed[0].sequence, 0);
3184 assert_eq!(report.pending[0].sequence, 1);
3185 assert_eq!(
3186 report.next_transition.as_ref().map(|op| op.sequence),
3187 Some(1)
3188 );
3189 assert_eq!(report_json["outcome"], "failed");
3190 assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
3191 }
3192
3193 #[test]
3195 fn run_restore_run_dry_run_writes_native_runner_preview() {
3196 let root = temp_dir("canic-cli-restore-run-dry-run");
3197 fs::create_dir_all(&root).expect("create temp root");
3198 let journal_path = root.join("restore-apply-journal.json");
3199 let out_path = root.join("restore-run-dry-run.json");
3200 let journal = ready_apply_journal();
3201
3202 fs::write(
3203 &journal_path,
3204 serde_json::to_vec(&journal).expect("serialize journal"),
3205 )
3206 .expect("write journal");
3207
3208 run([
3209 OsString::from("run"),
3210 OsString::from("--journal"),
3211 OsString::from(journal_path.as_os_str()),
3212 OsString::from("--dry-run"),
3213 OsString::from("--dfx"),
3214 OsString::from("/tmp/dfx"),
3215 OsString::from("--network"),
3216 OsString::from("local"),
3217 OsString::from("--out"),
3218 OsString::from(out_path.as_os_str()),
3219 ])
3220 .expect("write restore run dry-run");
3221
3222 let dry_run: serde_json::Value =
3223 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3224 .expect("decode dry-run");
3225
3226 fs::remove_dir_all(root).expect("remove temp root");
3227 assert_eq!(dry_run["run_version"], 1);
3228 assert_eq!(dry_run["backup_id"], "backup-test");
3229 assert_eq!(dry_run["run_mode"], "dry-run");
3230 assert_eq!(dry_run["dry_run"], true);
3231 assert_eq!(dry_run["ready"], true);
3232 assert_eq!(dry_run["complete"], false);
3233 assert_eq!(dry_run["attention_required"], false);
3234 assert_eq!(dry_run["operation_counts"]["snapshot_uploads"], 2);
3235 assert_eq!(dry_run["operation_counts"]["snapshot_loads"], 2);
3236 assert_eq!(dry_run["operation_counts"]["code_reinstalls"], 2);
3237 assert_eq!(dry_run["operation_counts"]["member_verifications"], 2);
3238 assert_eq!(dry_run["operation_counts"]["fleet_verifications"], 0);
3239 assert_eq!(dry_run["operation_counts"]["verification_operations"], 2);
3240 assert_eq!(dry_run["stopped_reason"], "preview");
3241 assert_eq!(dry_run["next_action"], "rerun");
3242 assert_eq!(dry_run["operation_available"], true);
3243 assert_eq!(dry_run["command_available"], true);
3244 assert_eq!(dry_run["next_transition"]["sequence"], 0);
3245 assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
3246 assert_eq!(
3247 dry_run["command"]["args"],
3248 json!([
3249 "canister",
3250 "--network",
3251 "local",
3252 "snapshot",
3253 "upload",
3254 "--dir",
3255 "artifacts/root",
3256 ROOT
3257 ])
3258 );
3259 assert_eq!(dry_run["command"]["mutates"], true);
3260 }
3261
3262 #[test]
3264 fn run_restore_run_unclaim_pending_marks_operation_ready() {
3265 let root = temp_dir("canic-cli-restore-run-unclaim-pending");
3266 fs::create_dir_all(&root).expect("create temp root");
3267 let journal_path = root.join("restore-apply-journal.json");
3268 let out_path = root.join("restore-run.json");
3269 let mut journal = ready_apply_journal();
3270 journal
3271 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3272 .expect("mark pending operation");
3273
3274 fs::write(
3275 &journal_path,
3276 serde_json::to_vec(&journal).expect("serialize journal"),
3277 )
3278 .expect("write journal");
3279
3280 run([
3281 OsString::from("run"),
3282 OsString::from("--journal"),
3283 OsString::from(journal_path.as_os_str()),
3284 OsString::from("--unclaim-pending"),
3285 OsString::from("--out"),
3286 OsString::from(out_path.as_os_str()),
3287 ])
3288 .expect("unclaim pending operation");
3289
3290 let run_summary: serde_json::Value =
3291 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3292 .expect("decode run summary");
3293 let updated: RestoreApplyJournal =
3294 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3295 .expect("decode updated journal");
3296
3297 fs::remove_dir_all(root).expect("remove temp root");
3298 assert_eq!(run_summary["run_mode"], "unclaim-pending");
3299 assert_eq!(run_summary["unclaim_pending"], true);
3300 assert_eq!(run_summary["stopped_reason"], "recovered-pending");
3301 assert_eq!(run_summary["next_action"], "rerun");
3302 assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
3303 assert_eq!(run_summary["recovered_operation"]["state"], "pending");
3304 assert_eq!(run_summary["pending_operations"], 0);
3305 assert_eq!(run_summary["ready_operations"], 8);
3306 assert_eq!(run_summary["attention_required"], false);
3307 assert_eq!(updated.pending_operations, 0);
3308 assert_eq!(updated.ready_operations, 8);
3309 assert_eq!(
3310 updated.operations[0].state,
3311 RestoreApplyOperationState::Ready
3312 );
3313 }
3314
3315 #[test]
3317 fn run_restore_run_execute_marks_completed_operation() {
3318 let root = temp_dir("canic-cli-restore-run-execute");
3319 fs::create_dir_all(&root).expect("create temp root");
3320 let journal_path = root.join("restore-apply-journal.json");
3321 let out_path = root.join("restore-run.json");
3322 let journal = ready_apply_journal();
3323
3324 fs::write(
3325 &journal_path,
3326 serde_json::to_vec(&journal).expect("serialize journal"),
3327 )
3328 .expect("write journal");
3329
3330 run([
3331 OsString::from("run"),
3332 OsString::from("--journal"),
3333 OsString::from(journal_path.as_os_str()),
3334 OsString::from("--execute"),
3335 OsString::from("--dfx"),
3336 OsString::from("/bin/true"),
3337 OsString::from("--max-steps"),
3338 OsString::from("1"),
3339 OsString::from("--out"),
3340 OsString::from(out_path.as_os_str()),
3341 ])
3342 .expect("execute one restore run step");
3343
3344 let run_summary: serde_json::Value =
3345 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3346 .expect("decode run summary");
3347 let updated: RestoreApplyJournal =
3348 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3349 .expect("decode updated journal");
3350
3351 fs::remove_dir_all(root).expect("remove temp root");
3352 assert_eq!(run_summary["run_mode"], "execute");
3353 assert_eq!(run_summary["execute"], true);
3354 assert_eq!(run_summary["dry_run"], false);
3355 assert_eq!(run_summary["max_steps_reached"], true);
3356 assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
3357 assert_eq!(run_summary["next_action"], "rerun");
3358 assert_eq!(run_summary["executed_operation_count"], 1);
3359 assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
3360 assert_eq!(
3361 run_summary["executed_operations"][0]["command"]["program"],
3362 "/bin/true"
3363 );
3364 assert_eq!(updated.completed_operations, 1);
3365 assert_eq!(updated.pending_operations, 0);
3366 assert_eq!(updated.failed_operations, 0);
3367 assert_eq!(
3368 updated.operations[0].state,
3369 RestoreApplyOperationState::Completed
3370 );
3371 }
3372
3373 #[test]
3375 fn run_restore_run_require_complete_writes_summary_then_fails() {
3376 let root = temp_dir("canic-cli-restore-run-require-complete");
3377 fs::create_dir_all(&root).expect("create temp root");
3378 let journal_path = root.join("restore-apply-journal.json");
3379 let out_path = root.join("restore-run.json");
3380 let journal = ready_apply_journal();
3381
3382 fs::write(
3383 &journal_path,
3384 serde_json::to_vec(&journal).expect("serialize journal"),
3385 )
3386 .expect("write journal");
3387
3388 let err = run([
3389 OsString::from("run"),
3390 OsString::from("--journal"),
3391 OsString::from(journal_path.as_os_str()),
3392 OsString::from("--execute"),
3393 OsString::from("--dfx"),
3394 OsString::from("/bin/true"),
3395 OsString::from("--max-steps"),
3396 OsString::from("1"),
3397 OsString::from("--out"),
3398 OsString::from(out_path.as_os_str()),
3399 OsString::from("--require-complete"),
3400 ])
3401 .expect_err("incomplete run should fail requirement");
3402
3403 let run_summary: serde_json::Value =
3404 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3405 .expect("decode run summary");
3406
3407 fs::remove_dir_all(root).expect("remove temp root");
3408 assert_eq!(run_summary["executed_operation_count"], 1);
3409 assert_eq!(run_summary["complete"], false);
3410 assert!(matches!(
3411 err,
3412 RestoreCommandError::RestoreApplyIncomplete {
3413 completed_operations: 1,
3414 operation_count: 8,
3415 ..
3416 }
3417 ));
3418 }
3419
3420 #[test]
3422 fn run_restore_run_execute_marks_failed_operation() {
3423 let root = temp_dir("canic-cli-restore-run-execute-failed");
3424 fs::create_dir_all(&root).expect("create temp root");
3425 let journal_path = root.join("restore-apply-journal.json");
3426 let out_path = root.join("restore-run.json");
3427 let journal = ready_apply_journal();
3428
3429 fs::write(
3430 &journal_path,
3431 serde_json::to_vec(&journal).expect("serialize journal"),
3432 )
3433 .expect("write journal");
3434
3435 let err = run([
3436 OsString::from("run"),
3437 OsString::from("--journal"),
3438 OsString::from(journal_path.as_os_str()),
3439 OsString::from("--execute"),
3440 OsString::from("--dfx"),
3441 OsString::from("/bin/false"),
3442 OsString::from("--max-steps"),
3443 OsString::from("1"),
3444 OsString::from("--out"),
3445 OsString::from(out_path.as_os_str()),
3446 ])
3447 .expect_err("failing runner command should fail");
3448
3449 let run_summary: serde_json::Value =
3450 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3451 .expect("decode run summary");
3452 let updated: RestoreApplyJournal =
3453 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3454 .expect("decode updated journal");
3455
3456 fs::remove_dir_all(root).expect("remove temp root");
3457 assert!(matches!(
3458 err,
3459 RestoreCommandError::RestoreRunCommandFailed {
3460 sequence: 0,
3461 status,
3462 } if status == "1"
3463 ));
3464 assert_eq!(updated.failed_operations, 1);
3465 assert_eq!(updated.pending_operations, 0);
3466 assert_eq!(
3467 updated.operations[0].state,
3468 RestoreApplyOperationState::Failed
3469 );
3470 assert_eq!(run_summary["execute"], true);
3471 assert_eq!(run_summary["attention_required"], true);
3472 assert_eq!(run_summary["outcome"], "failed");
3473 assert_eq!(run_summary["stopped_reason"], "command-failed");
3474 assert_eq!(run_summary["next_action"], "inspect-failed-operation");
3475 assert_eq!(run_summary["executed_operation_count"], 1);
3476 assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
3477 assert_eq!(run_summary["executed_operations"][0]["status"], "1");
3478 assert_eq!(
3479 updated.operations[0].blocking_reasons,
3480 vec!["runner-command-exit-1".to_string()]
3481 );
3482 }
3483
3484 #[test]
3486 fn run_restore_run_require_no_attention_writes_summary_then_fails() {
3487 let root = temp_dir("canic-cli-restore-run-require-attention");
3488 fs::create_dir_all(&root).expect("create temp root");
3489 let journal_path = root.join("restore-apply-journal.json");
3490 let out_path = root.join("restore-run.json");
3491 let mut journal = ready_apply_journal();
3492 journal
3493 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3494 .expect("mark pending operation");
3495
3496 fs::write(
3497 &journal_path,
3498 serde_json::to_vec(&journal).expect("serialize journal"),
3499 )
3500 .expect("write journal");
3501
3502 let err = run([
3503 OsString::from("run"),
3504 OsString::from("--journal"),
3505 OsString::from(journal_path.as_os_str()),
3506 OsString::from("--dry-run"),
3507 OsString::from("--out"),
3508 OsString::from(out_path.as_os_str()),
3509 OsString::from("--require-no-attention"),
3510 ])
3511 .expect_err("attention run should fail requirement");
3512
3513 let run_summary: serde_json::Value =
3514 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3515 .expect("decode run summary");
3516
3517 fs::remove_dir_all(root).expect("remove temp root");
3518 assert_eq!(run_summary["attention_required"], true);
3519 assert_eq!(run_summary["outcome"], "pending");
3520 assert_eq!(run_summary["stopped_reason"], "pending");
3521 assert_eq!(run_summary["next_action"], "unclaim-pending");
3522 assert!(matches!(
3523 err,
3524 RestoreCommandError::RestoreApplyReportNeedsAttention {
3525 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3526 ..
3527 }
3528 ));
3529 }
3530
3531 #[test]
3533 fn run_restore_run_require_run_mode_writes_summary_then_fails() {
3534 let root = temp_dir("canic-cli-restore-run-require-run-mode");
3535 fs::create_dir_all(&root).expect("create temp root");
3536 let journal_path = root.join("restore-apply-journal.json");
3537 let out_path = root.join("restore-run.json");
3538 let journal = ready_apply_journal();
3539
3540 fs::write(
3541 &journal_path,
3542 serde_json::to_vec(&journal).expect("serialize journal"),
3543 )
3544 .expect("write journal");
3545
3546 let err = run([
3547 OsString::from("run"),
3548 OsString::from("--journal"),
3549 OsString::from(journal_path.as_os_str()),
3550 OsString::from("--dry-run"),
3551 OsString::from("--out"),
3552 OsString::from(out_path.as_os_str()),
3553 OsString::from("--require-run-mode"),
3554 OsString::from("execute"),
3555 ])
3556 .expect_err("run mode mismatch should fail requirement");
3557
3558 let run_summary: serde_json::Value =
3559 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3560 .expect("decode run summary");
3561
3562 fs::remove_dir_all(root).expect("remove temp root");
3563 assert_eq!(run_summary["run_mode"], "dry-run");
3564 assert!(matches!(
3565 err,
3566 RestoreCommandError::RestoreRunModeMismatch {
3567 expected,
3568 actual,
3569 ..
3570 } if expected == "execute" && actual == "dry-run"
3571 ));
3572 }
3573
3574 #[test]
3576 fn run_restore_run_require_executed_count_writes_summary_then_fails() {
3577 let root = temp_dir("canic-cli-restore-run-require-executed-count");
3578 fs::create_dir_all(&root).expect("create temp root");
3579 let journal_path = root.join("restore-apply-journal.json");
3580 let out_path = root.join("restore-run.json");
3581 let journal = ready_apply_journal();
3582
3583 fs::write(
3584 &journal_path,
3585 serde_json::to_vec(&journal).expect("serialize journal"),
3586 )
3587 .expect("write journal");
3588
3589 let err = run([
3590 OsString::from("run"),
3591 OsString::from("--journal"),
3592 OsString::from(journal_path.as_os_str()),
3593 OsString::from("--execute"),
3594 OsString::from("--dfx"),
3595 OsString::from("/bin/true"),
3596 OsString::from("--max-steps"),
3597 OsString::from("1"),
3598 OsString::from("--out"),
3599 OsString::from(out_path.as_os_str()),
3600 OsString::from("--require-executed-count"),
3601 OsString::from("2"),
3602 ])
3603 .expect_err("executed count mismatch should fail requirement");
3604
3605 let run_summary: serde_json::Value =
3606 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3607 .expect("decode run summary");
3608
3609 fs::remove_dir_all(root).expect("remove temp root");
3610 assert_eq!(run_summary["executed_operation_count"], 1);
3611 assert!(matches!(
3612 err,
3613 RestoreCommandError::RestoreRunExecutedCountMismatch {
3614 expected: 2,
3615 actual: 1,
3616 ..
3617 }
3618 ));
3619 }
3620
3621 #[test]
3623 fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
3624 let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
3625 fs::create_dir_all(&root).expect("create temp root");
3626 let journal_path = root.join("restore-apply-journal.json");
3627 let out_path = root.join("restore-run.json");
3628 let journal = ready_apply_journal();
3629
3630 fs::write(
3631 &journal_path,
3632 serde_json::to_vec(&journal).expect("serialize journal"),
3633 )
3634 .expect("write journal");
3635
3636 let err = run([
3637 OsString::from("run"),
3638 OsString::from("--journal"),
3639 OsString::from(journal_path.as_os_str()),
3640 OsString::from("--dry-run"),
3641 OsString::from("--out"),
3642 OsString::from(out_path.as_os_str()),
3643 OsString::from("--require-stopped-reason"),
3644 OsString::from("complete"),
3645 ])
3646 .expect_err("stopped reason mismatch should fail requirement");
3647
3648 let run_summary: serde_json::Value =
3649 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3650 .expect("decode run summary");
3651
3652 fs::remove_dir_all(root).expect("remove temp root");
3653 assert_eq!(run_summary["stopped_reason"], "preview");
3654 assert!(matches!(
3655 err,
3656 RestoreCommandError::RestoreRunStoppedReasonMismatch {
3657 expected,
3658 actual,
3659 ..
3660 } if expected == "complete" && actual == "preview"
3661 ));
3662 }
3663
3664 #[test]
3666 fn run_restore_run_require_next_action_writes_summary_then_fails() {
3667 let root = temp_dir("canic-cli-restore-run-require-next-action");
3668 fs::create_dir_all(&root).expect("create temp root");
3669 let journal_path = root.join("restore-apply-journal.json");
3670 let out_path = root.join("restore-run.json");
3671 let journal = ready_apply_journal();
3672
3673 fs::write(
3674 &journal_path,
3675 serde_json::to_vec(&journal).expect("serialize journal"),
3676 )
3677 .expect("write journal");
3678
3679 let err = run([
3680 OsString::from("run"),
3681 OsString::from("--journal"),
3682 OsString::from(journal_path.as_os_str()),
3683 OsString::from("--dry-run"),
3684 OsString::from("--out"),
3685 OsString::from(out_path.as_os_str()),
3686 OsString::from("--require-next-action"),
3687 OsString::from("done"),
3688 ])
3689 .expect_err("next action mismatch should fail requirement");
3690
3691 let run_summary: serde_json::Value =
3692 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3693 .expect("decode run summary");
3694
3695 fs::remove_dir_all(root).expect("remove temp root");
3696 assert_eq!(run_summary["next_action"], "rerun");
3697 assert!(matches!(
3698 err,
3699 RestoreCommandError::RestoreRunNextActionMismatch {
3700 expected,
3701 actual,
3702 ..
3703 } if expected == "done" && actual == "rerun"
3704 ));
3705 }
3706
3707 #[test]
3709 fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
3710 let root = temp_dir("canic-cli-restore-apply-report-attention");
3711 fs::create_dir_all(&root).expect("create temp root");
3712 let journal_path = root.join("restore-apply-journal.json");
3713 let out_path = root.join("restore-apply-report.json");
3714 let mut journal = ready_apply_journal();
3715 journal
3716 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3717 .expect("mark pending operation");
3718
3719 fs::write(
3720 &journal_path,
3721 serde_json::to_vec(&journal).expect("serialize journal"),
3722 )
3723 .expect("write journal");
3724
3725 let err = run([
3726 OsString::from("apply-report"),
3727 OsString::from("--journal"),
3728 OsString::from(journal_path.as_os_str()),
3729 OsString::from("--out"),
3730 OsString::from(out_path.as_os_str()),
3731 OsString::from("--require-no-attention"),
3732 ])
3733 .expect_err("attention report should fail requirement");
3734
3735 let report: RestoreApplyJournalReport =
3736 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3737 .expect("decode apply report");
3738
3739 fs::remove_dir_all(root).expect("remove temp root");
3740 assert!(report.attention_required);
3741 assert_eq!(report.pending_operations, 1);
3742 assert!(matches!(
3743 err,
3744 RestoreCommandError::RestoreApplyReportNeedsAttention {
3745 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3746 ..
3747 }
3748 ));
3749 }
3750
3751 #[test]
3753 fn run_restore_apply_status_require_complete_writes_status_then_fails() {
3754 let root = temp_dir("canic-cli-restore-apply-status-incomplete");
3755 fs::create_dir_all(&root).expect("create temp root");
3756 let journal_path = root.join("restore-apply-journal.json");
3757 let out_path = root.join("restore-apply-status.json");
3758 let journal = ready_apply_journal();
3759
3760 fs::write(
3761 &journal_path,
3762 serde_json::to_vec(&journal).expect("serialize journal"),
3763 )
3764 .expect("write journal");
3765
3766 let err = run([
3767 OsString::from("apply-status"),
3768 OsString::from("--journal"),
3769 OsString::from(journal_path.as_os_str()),
3770 OsString::from("--out"),
3771 OsString::from(out_path.as_os_str()),
3772 OsString::from("--require-complete"),
3773 ])
3774 .expect_err("incomplete journal should fail requirement");
3775
3776 assert!(out_path.exists());
3777 let status: RestoreApplyJournalStatus =
3778 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3779 .expect("decode apply status");
3780
3781 fs::remove_dir_all(root).expect("remove temp root");
3782 assert!(!status.complete);
3783 assert_eq!(status.completed_operations, 0);
3784 assert_eq!(status.operation_count, 8);
3785 assert!(matches!(
3786 err,
3787 RestoreCommandError::RestoreApplyIncomplete {
3788 completed_operations: 0,
3789 operation_count: 8,
3790 ..
3791 }
3792 ));
3793 }
3794
3795 #[test]
3797 fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
3798 let root = temp_dir("canic-cli-restore-apply-status-failed");
3799 fs::create_dir_all(&root).expect("create temp root");
3800 let journal_path = root.join("restore-apply-journal.json");
3801 let out_path = root.join("restore-apply-status.json");
3802 let mut journal = ready_apply_journal();
3803 journal
3804 .mark_operation_failed(0, "dfx-load-failed".to_string())
3805 .expect("mark failed operation");
3806
3807 fs::write(
3808 &journal_path,
3809 serde_json::to_vec(&journal).expect("serialize journal"),
3810 )
3811 .expect("write journal");
3812
3813 let err = run([
3814 OsString::from("apply-status"),
3815 OsString::from("--journal"),
3816 OsString::from(journal_path.as_os_str()),
3817 OsString::from("--out"),
3818 OsString::from(out_path.as_os_str()),
3819 OsString::from("--require-no-failed"),
3820 ])
3821 .expect_err("failed operation should fail requirement");
3822
3823 assert!(out_path.exists());
3824 let status: RestoreApplyJournalStatus =
3825 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3826 .expect("decode apply status");
3827
3828 fs::remove_dir_all(root).expect("remove temp root");
3829 assert_eq!(status.failed_operations, 1);
3830 assert!(matches!(
3831 err,
3832 RestoreCommandError::RestoreApplyFailed {
3833 failed_operations: 1,
3834 ..
3835 }
3836 ));
3837 }
3838
3839 #[test]
3841 fn run_restore_apply_status_require_complete_accepts_complete_journal() {
3842 let root = temp_dir("canic-cli-restore-apply-status-complete");
3843 fs::create_dir_all(&root).expect("create temp root");
3844 let journal_path = root.join("restore-apply-journal.json");
3845 let out_path = root.join("restore-apply-status.json");
3846 let mut journal = ready_apply_journal();
3847 for sequence in 0..journal.operation_count {
3848 journal
3849 .mark_operation_completed(sequence)
3850 .expect("complete operation");
3851 }
3852
3853 fs::write(
3854 &journal_path,
3855 serde_json::to_vec(&journal).expect("serialize journal"),
3856 )
3857 .expect("write journal");
3858
3859 run([
3860 OsString::from("apply-status"),
3861 OsString::from("--journal"),
3862 OsString::from(journal_path.as_os_str()),
3863 OsString::from("--out"),
3864 OsString::from(out_path.as_os_str()),
3865 OsString::from("--require-complete"),
3866 ])
3867 .expect("complete journal should pass requirement");
3868
3869 let status: RestoreApplyJournalStatus =
3870 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3871 .expect("decode apply status");
3872
3873 fs::remove_dir_all(root).expect("remove temp root");
3874 assert!(status.complete);
3875 assert_eq!(status.completed_operations, 8);
3876 assert_eq!(status.operation_count, 8);
3877 }
3878
3879 #[test]
3881 fn run_restore_apply_next_writes_next_ready_operation() {
3882 let root = temp_dir("canic-cli-restore-apply-next");
3883 fs::create_dir_all(&root).expect("create temp root");
3884 let journal_path = root.join("restore-apply-journal.json");
3885 let out_path = root.join("restore-apply-next.json");
3886 let mut journal = ready_apply_journal();
3887 journal
3888 .mark_operation_completed(0)
3889 .expect("mark first operation complete");
3890
3891 fs::write(
3892 &journal_path,
3893 serde_json::to_vec(&journal).expect("serialize journal"),
3894 )
3895 .expect("write journal");
3896
3897 run([
3898 OsString::from("apply-next"),
3899 OsString::from("--journal"),
3900 OsString::from(journal_path.as_os_str()),
3901 OsString::from("--out"),
3902 OsString::from(out_path.as_os_str()),
3903 ])
3904 .expect("write apply next");
3905
3906 let next: RestoreApplyNextOperation =
3907 serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
3908 .expect("decode next operation");
3909 let operation = next.operation.expect("operation should be available");
3910
3911 fs::remove_dir_all(root).expect("remove temp root");
3912 assert!(next.ready);
3913 assert!(next.operation_available);
3914 assert_eq!(operation.sequence, 1);
3915 assert_eq!(
3916 operation.operation,
3917 canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
3918 );
3919 }
3920
3921 #[test]
3923 fn run_restore_apply_command_writes_next_command_preview() {
3924 let root = temp_dir("canic-cli-restore-apply-command");
3925 fs::create_dir_all(&root).expect("create temp root");
3926 let journal_path = root.join("restore-apply-journal.json");
3927 let out_path = root.join("restore-apply-command.json");
3928 let journal = ready_apply_journal();
3929
3930 fs::write(
3931 &journal_path,
3932 serde_json::to_vec(&journal).expect("serialize journal"),
3933 )
3934 .expect("write journal");
3935
3936 run([
3937 OsString::from("apply-command"),
3938 OsString::from("--journal"),
3939 OsString::from(journal_path.as_os_str()),
3940 OsString::from("--dfx"),
3941 OsString::from("/tmp/dfx"),
3942 OsString::from("--network"),
3943 OsString::from("local"),
3944 OsString::from("--out"),
3945 OsString::from(out_path.as_os_str()),
3946 ])
3947 .expect("write command preview");
3948
3949 let preview: RestoreApplyCommandPreview =
3950 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
3951 .expect("decode command preview");
3952 let command = preview.command.expect("command should be available");
3953
3954 fs::remove_dir_all(root).expect("remove temp root");
3955 assert!(preview.ready);
3956 assert!(preview.command_available);
3957 assert_eq!(command.program, "/tmp/dfx");
3958 assert_eq!(
3959 command.args,
3960 vec![
3961 "canister".to_string(),
3962 "--network".to_string(),
3963 "local".to_string(),
3964 "snapshot".to_string(),
3965 "upload".to_string(),
3966 "--dir".to_string(),
3967 "artifacts/root".to_string(),
3968 ROOT.to_string(),
3969 ]
3970 );
3971 assert!(command.mutates);
3972 }
3973
3974 #[test]
3976 fn run_restore_apply_command_require_command_writes_preview_then_fails() {
3977 let root = temp_dir("canic-cli-restore-apply-command-require");
3978 fs::create_dir_all(&root).expect("create temp root");
3979 let journal_path = root.join("restore-apply-journal.json");
3980 let out_path = root.join("restore-apply-command.json");
3981 let mut journal = ready_apply_journal();
3982
3983 for sequence in 0..journal.operation_count {
3984 journal
3985 .mark_operation_completed(sequence)
3986 .expect("mark operation completed");
3987 }
3988
3989 fs::write(
3990 &journal_path,
3991 serde_json::to_vec(&journal).expect("serialize journal"),
3992 )
3993 .expect("write journal");
3994
3995 let err = run([
3996 OsString::from("apply-command"),
3997 OsString::from("--journal"),
3998 OsString::from(journal_path.as_os_str()),
3999 OsString::from("--out"),
4000 OsString::from(out_path.as_os_str()),
4001 OsString::from("--require-command"),
4002 ])
4003 .expect_err("missing command should fail");
4004
4005 let preview: RestoreApplyCommandPreview =
4006 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
4007 .expect("decode command preview");
4008
4009 fs::remove_dir_all(root).expect("remove temp root");
4010 assert!(preview.complete);
4011 assert!(!preview.operation_available);
4012 assert!(!preview.command_available);
4013 assert!(matches!(
4014 err,
4015 RestoreCommandError::RestoreApplyCommandUnavailable {
4016 operation_available: false,
4017 complete: true,
4018 ..
4019 }
4020 ));
4021 }
4022
4023 #[test]
4025 fn run_restore_apply_claim_marks_next_operation_pending() {
4026 let root = temp_dir("canic-cli-restore-apply-claim");
4027 fs::create_dir_all(&root).expect("create temp root");
4028 let journal_path = root.join("restore-apply-journal.json");
4029 let claimed_path = root.join("restore-apply-journal.claimed.json");
4030 let journal = ready_apply_journal();
4031
4032 fs::write(
4033 &journal_path,
4034 serde_json::to_vec(&journal).expect("serialize journal"),
4035 )
4036 .expect("write journal");
4037
4038 run([
4039 OsString::from("apply-claim"),
4040 OsString::from("--journal"),
4041 OsString::from(journal_path.as_os_str()),
4042 OsString::from("--sequence"),
4043 OsString::from("0"),
4044 OsString::from("--updated-at"),
4045 OsString::from("2026-05-04T12:00:00Z"),
4046 OsString::from("--out"),
4047 OsString::from(claimed_path.as_os_str()),
4048 ])
4049 .expect("claim operation");
4050
4051 let claimed: RestoreApplyJournal =
4052 serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
4053 .expect("decode claimed journal");
4054 let status = claimed.status();
4055 let next = claimed.next_operation();
4056
4057 fs::remove_dir_all(root).expect("remove temp root");
4058 assert_eq!(claimed.pending_operations, 1);
4059 assert_eq!(claimed.ready_operations, 7);
4060 assert_eq!(
4061 claimed.operations[0].state,
4062 RestoreApplyOperationState::Pending
4063 );
4064 assert_eq!(
4065 claimed.operations[0].state_updated_at.as_deref(),
4066 Some("2026-05-04T12:00:00Z")
4067 );
4068 assert_eq!(status.next_transition_sequence, Some(0));
4069 assert_eq!(
4070 status.next_transition_state,
4071 Some(RestoreApplyOperationState::Pending)
4072 );
4073 assert_eq!(
4074 status.next_transition_updated_at.as_deref(),
4075 Some("2026-05-04T12:00:00Z")
4076 );
4077 assert_eq!(
4078 next.operation.expect("next operation").state,
4079 RestoreApplyOperationState::Pending
4080 );
4081 }
4082
4083 #[test]
4085 fn run_restore_apply_claim_rejects_sequence_mismatch() {
4086 let root = temp_dir("canic-cli-restore-apply-claim-sequence");
4087 fs::create_dir_all(&root).expect("create temp root");
4088 let journal_path = root.join("restore-apply-journal.json");
4089 let claimed_path = root.join("restore-apply-journal.claimed.json");
4090 let journal = ready_apply_journal();
4091
4092 fs::write(
4093 &journal_path,
4094 serde_json::to_vec(&journal).expect("serialize journal"),
4095 )
4096 .expect("write journal");
4097
4098 let err = run([
4099 OsString::from("apply-claim"),
4100 OsString::from("--journal"),
4101 OsString::from(journal_path.as_os_str()),
4102 OsString::from("--sequence"),
4103 OsString::from("1"),
4104 OsString::from("--out"),
4105 OsString::from(claimed_path.as_os_str()),
4106 ])
4107 .expect_err("stale sequence should fail claim");
4108
4109 assert!(!claimed_path.exists());
4110 fs::remove_dir_all(root).expect("remove temp root");
4111 assert!(matches!(
4112 err,
4113 RestoreCommandError::RestoreApplyClaimSequenceMismatch {
4114 expected: 1,
4115 actual: Some(0),
4116 }
4117 ));
4118 }
4119
4120 #[test]
4122 fn run_restore_apply_unclaim_marks_pending_operation_ready() {
4123 let root = temp_dir("canic-cli-restore-apply-unclaim");
4124 fs::create_dir_all(&root).expect("create temp root");
4125 let journal_path = root.join("restore-apply-journal.json");
4126 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
4127 let mut journal = ready_apply_journal();
4128 journal
4129 .mark_next_operation_pending()
4130 .expect("claim operation");
4131
4132 fs::write(
4133 &journal_path,
4134 serde_json::to_vec(&journal).expect("serialize journal"),
4135 )
4136 .expect("write journal");
4137
4138 run([
4139 OsString::from("apply-unclaim"),
4140 OsString::from("--journal"),
4141 OsString::from(journal_path.as_os_str()),
4142 OsString::from("--sequence"),
4143 OsString::from("0"),
4144 OsString::from("--updated-at"),
4145 OsString::from("2026-05-04T12:01:00Z"),
4146 OsString::from("--out"),
4147 OsString::from(unclaimed_path.as_os_str()),
4148 ])
4149 .expect("unclaim operation");
4150
4151 let unclaimed: RestoreApplyJournal =
4152 serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
4153 .expect("decode unclaimed journal");
4154 let status = unclaimed.status();
4155
4156 fs::remove_dir_all(root).expect("remove temp root");
4157 assert_eq!(unclaimed.pending_operations, 0);
4158 assert_eq!(unclaimed.ready_operations, 8);
4159 assert_eq!(
4160 unclaimed.operations[0].state,
4161 RestoreApplyOperationState::Ready
4162 );
4163 assert_eq!(
4164 unclaimed.operations[0].state_updated_at.as_deref(),
4165 Some("2026-05-04T12:01:00Z")
4166 );
4167 assert_eq!(status.next_ready_sequence, Some(0));
4168 assert_eq!(
4169 status.next_transition_state,
4170 Some(RestoreApplyOperationState::Ready)
4171 );
4172 assert_eq!(
4173 status.next_transition_updated_at.as_deref(),
4174 Some("2026-05-04T12:01:00Z")
4175 );
4176 }
4177
4178 #[test]
4180 fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
4181 let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
4182 fs::create_dir_all(&root).expect("create temp root");
4183 let journal_path = root.join("restore-apply-journal.json");
4184 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
4185 let mut journal = ready_apply_journal();
4186 journal
4187 .mark_next_operation_pending()
4188 .expect("claim operation");
4189
4190 fs::write(
4191 &journal_path,
4192 serde_json::to_vec(&journal).expect("serialize journal"),
4193 )
4194 .expect("write journal");
4195
4196 let err = run([
4197 OsString::from("apply-unclaim"),
4198 OsString::from("--journal"),
4199 OsString::from(journal_path.as_os_str()),
4200 OsString::from("--sequence"),
4201 OsString::from("1"),
4202 OsString::from("--out"),
4203 OsString::from(unclaimed_path.as_os_str()),
4204 ])
4205 .expect_err("stale sequence should fail unclaim");
4206
4207 assert!(!unclaimed_path.exists());
4208 fs::remove_dir_all(root).expect("remove temp root");
4209 assert!(matches!(
4210 err,
4211 RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
4212 expected: 1,
4213 actual: Some(0),
4214 }
4215 ));
4216 }
4217
4218 #[test]
4220 fn run_restore_apply_mark_completes_operation() {
4221 let root = temp_dir("canic-cli-restore-apply-mark-complete");
4222 fs::create_dir_all(&root).expect("create temp root");
4223 let journal_path = root.join("restore-apply-journal.json");
4224 let updated_path = root.join("restore-apply-journal.updated.json");
4225 let journal = ready_apply_journal();
4226
4227 fs::write(
4228 &journal_path,
4229 serde_json::to_vec(&journal).expect("serialize journal"),
4230 )
4231 .expect("write journal");
4232
4233 run([
4234 OsString::from("apply-mark"),
4235 OsString::from("--journal"),
4236 OsString::from(journal_path.as_os_str()),
4237 OsString::from("--sequence"),
4238 OsString::from("0"),
4239 OsString::from("--state"),
4240 OsString::from("completed"),
4241 OsString::from("--updated-at"),
4242 OsString::from("2026-05-04T12:02:00Z"),
4243 OsString::from("--out"),
4244 OsString::from(updated_path.as_os_str()),
4245 ])
4246 .expect("mark operation completed");
4247
4248 let updated: RestoreApplyJournal =
4249 serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
4250 .expect("decode updated journal");
4251 let status = updated.status();
4252
4253 fs::remove_dir_all(root).expect("remove temp root");
4254 assert_eq!(updated.completed_operations, 1);
4255 assert_eq!(updated.ready_operations, 7);
4256 assert_eq!(
4257 updated.operations[0].state_updated_at.as_deref(),
4258 Some("2026-05-04T12:02:00Z")
4259 );
4260 assert_eq!(status.next_ready_sequence, Some(1));
4261 }
4262
4263 #[test]
4265 fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
4266 let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
4267 fs::create_dir_all(&root).expect("create temp root");
4268 let journal_path = root.join("restore-apply-journal.json");
4269 let updated_path = root.join("restore-apply-journal.updated.json");
4270 let journal = ready_apply_journal();
4271
4272 fs::write(
4273 &journal_path,
4274 serde_json::to_vec(&journal).expect("serialize journal"),
4275 )
4276 .expect("write journal");
4277
4278 let err = run([
4279 OsString::from("apply-mark"),
4280 OsString::from("--journal"),
4281 OsString::from(journal_path.as_os_str()),
4282 OsString::from("--sequence"),
4283 OsString::from("0"),
4284 OsString::from("--state"),
4285 OsString::from("completed"),
4286 OsString::from("--out"),
4287 OsString::from(updated_path.as_os_str()),
4288 OsString::from("--require-pending"),
4289 ])
4290 .expect_err("ready operation should fail pending requirement");
4291
4292 assert!(!updated_path.exists());
4293 fs::remove_dir_all(root).expect("remove temp root");
4294 assert!(matches!(
4295 err,
4296 RestoreCommandError::RestoreApplyMarkRequiresPending {
4297 sequence: 0,
4298 state: RestoreApplyOperationState::Ready,
4299 }
4300 ));
4301 }
4302
4303 #[test]
4305 fn run_restore_apply_mark_rejects_out_of_order_operation() {
4306 let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
4307 fs::create_dir_all(&root).expect("create temp root");
4308 let journal_path = root.join("restore-apply-journal.json");
4309 let updated_path = root.join("restore-apply-journal.updated.json");
4310 let journal = ready_apply_journal();
4311
4312 fs::write(
4313 &journal_path,
4314 serde_json::to_vec(&journal).expect("serialize journal"),
4315 )
4316 .expect("write journal");
4317
4318 let err = run([
4319 OsString::from("apply-mark"),
4320 OsString::from("--journal"),
4321 OsString::from(journal_path.as_os_str()),
4322 OsString::from("--sequence"),
4323 OsString::from("1"),
4324 OsString::from("--state"),
4325 OsString::from("completed"),
4326 OsString::from("--out"),
4327 OsString::from(updated_path.as_os_str()),
4328 ])
4329 .expect_err("out-of-order operation should fail");
4330
4331 assert!(!updated_path.exists());
4332 fs::remove_dir_all(root).expect("remove temp root");
4333 assert!(matches!(
4334 err,
4335 RestoreCommandError::RestoreApplyJournal(
4336 RestoreApplyJournalError::OutOfOrderOperationTransition {
4337 requested: 1,
4338 next: 0
4339 }
4340 )
4341 ));
4342 }
4343
4344 #[test]
4346 fn run_restore_apply_mark_failed_requires_reason() {
4347 let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
4348 fs::create_dir_all(&root).expect("create temp root");
4349 let journal_path = root.join("restore-apply-journal.json");
4350 let journal = ready_apply_journal();
4351
4352 fs::write(
4353 &journal_path,
4354 serde_json::to_vec(&journal).expect("serialize journal"),
4355 )
4356 .expect("write journal");
4357
4358 let err = run([
4359 OsString::from("apply-mark"),
4360 OsString::from("--journal"),
4361 OsString::from(journal_path.as_os_str()),
4362 OsString::from("--sequence"),
4363 OsString::from("0"),
4364 OsString::from("--state"),
4365 OsString::from("failed"),
4366 ])
4367 .expect_err("failed state should require reason");
4368
4369 fs::remove_dir_all(root).expect("remove temp root");
4370 assert!(matches!(
4371 err,
4372 RestoreCommandError::RestoreApplyJournal(
4373 RestoreApplyJournalError::FailureReasonRequired(0)
4374 )
4375 ));
4376 }
4377
4378 #[test]
4380 fn run_restore_apply_dry_run_rejects_mismatched_status() {
4381 let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
4382 fs::create_dir_all(&root).expect("create temp root");
4383 let plan_path = root.join("restore-plan.json");
4384 let status_path = root.join("restore-status.json");
4385 let out_path = root.join("restore-apply-dry-run.json");
4386 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4387 let mut status = RestoreStatus::from_plan(&plan);
4388 status.backup_id = "other-backup".to_string();
4389
4390 fs::write(
4391 &plan_path,
4392 serde_json::to_vec(&plan).expect("serialize plan"),
4393 )
4394 .expect("write plan");
4395 fs::write(
4396 &status_path,
4397 serde_json::to_vec(&status).expect("serialize status"),
4398 )
4399 .expect("write status");
4400
4401 let err = run([
4402 OsString::from("apply"),
4403 OsString::from("--plan"),
4404 OsString::from(plan_path.as_os_str()),
4405 OsString::from("--status"),
4406 OsString::from(status_path.as_os_str()),
4407 OsString::from("--dry-run"),
4408 OsString::from("--out"),
4409 OsString::from(out_path.as_os_str()),
4410 ])
4411 .expect_err("mismatched status should fail");
4412
4413 assert!(!out_path.exists());
4414 fs::remove_dir_all(root).expect("remove temp root");
4415 assert!(matches!(
4416 err,
4417 RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
4418 field: "backup_id",
4419 ..
4420 })
4421 ));
4422 }
4423
4424 fn ready_apply_journal() -> RestoreApplyJournal {
4426 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4427 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4428 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4429
4430 journal.ready = true;
4431 journal.blocked_reasons = Vec::new();
4432 for operation in &mut journal.operations {
4433 operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
4434 operation.blocking_reasons = Vec::new();
4435 }
4436 journal.blocked_operations = 0;
4437 journal.ready_operations = journal.operation_count;
4438 journal.validate().expect("journal should validate");
4439 journal
4440 }
4441
4442 fn valid_manifest() -> FleetBackupManifest {
4444 FleetBackupManifest {
4445 manifest_version: 1,
4446 backup_id: "backup-test".to_string(),
4447 created_at: "2026-05-03T00:00:00Z".to_string(),
4448 tool: ToolMetadata {
4449 name: "canic".to_string(),
4450 version: "0.30.1".to_string(),
4451 },
4452 source: SourceMetadata {
4453 environment: "local".to_string(),
4454 root_canister: ROOT.to_string(),
4455 },
4456 consistency: ConsistencySection {
4457 mode: ConsistencyMode::CrashConsistent,
4458 backup_units: vec![BackupUnit {
4459 unit_id: "fleet".to_string(),
4460 kind: BackupUnitKind::SubtreeRooted,
4461 roles: vec!["root".to_string(), "app".to_string()],
4462 consistency_reason: None,
4463 dependency_closure: Vec::new(),
4464 topology_validation: "subtree-closed".to_string(),
4465 quiescence_strategy: None,
4466 }],
4467 },
4468 fleet: FleetSection {
4469 topology_hash_algorithm: "sha256".to_string(),
4470 topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
4471 discovery_topology_hash: HASH.to_string(),
4472 pre_snapshot_topology_hash: HASH.to_string(),
4473 topology_hash: HASH.to_string(),
4474 members: vec![
4475 fleet_member("root", ROOT, None, IdentityMode::Fixed),
4476 fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
4477 ],
4478 },
4479 verification: VerificationPlan::default(),
4480 }
4481 }
4482
4483 fn restore_ready_manifest() -> FleetBackupManifest {
4485 let mut manifest = valid_manifest();
4486 for member in &mut manifest.fleet.members {
4487 member.source_snapshot.module_hash = Some(HASH.to_string());
4488 member.source_snapshot.wasm_hash = Some(HASH.to_string());
4489 member.source_snapshot.checksum = Some(HASH.to_string());
4490 }
4491 manifest
4492 }
4493
4494 fn fleet_member(
4496 role: &str,
4497 canister_id: &str,
4498 parent_canister_id: Option<&str>,
4499 identity_mode: IdentityMode,
4500 ) -> FleetMember {
4501 FleetMember {
4502 role: role.to_string(),
4503 canister_id: canister_id.to_string(),
4504 parent_canister_id: parent_canister_id.map(str::to_string),
4505 subnet_canister_id: Some(ROOT.to_string()),
4506 controller_hint: None,
4507 identity_mode,
4508 restore_group: 1,
4509 verification_class: "basic".to_string(),
4510 verification_checks: vec![VerificationCheck {
4511 kind: "status".to_string(),
4512 method: None,
4513 roles: vec![role.to_string()],
4514 }],
4515 source_snapshot: SourceSnapshot {
4516 snapshot_id: format!("{role}-snapshot"),
4517 module_hash: None,
4518 wasm_hash: None,
4519 code_version: Some("v0.30.1".to_string()),
4520 artifact_path: format!("artifacts/{role}"),
4521 checksum_algorithm: "sha256".to_string(),
4522 checksum: None,
4523 },
4524 }
4525 }
4526
4527 fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
4529 layout.write_manifest(manifest).expect("write manifest");
4530
4531 let artifacts = manifest
4532 .fleet
4533 .members
4534 .iter()
4535 .map(|member| {
4536 let bytes = format!("{} artifact", member.role);
4537 let artifact_path = root.join(&member.source_snapshot.artifact_path);
4538 if let Some(parent) = artifact_path.parent() {
4539 fs::create_dir_all(parent).expect("create artifact parent");
4540 }
4541 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4542 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4543
4544 ArtifactJournalEntry {
4545 canister_id: member.canister_id.clone(),
4546 snapshot_id: member.source_snapshot.snapshot_id.clone(),
4547 state: ArtifactState::Durable,
4548 temp_path: None,
4549 artifact_path: member.source_snapshot.artifact_path.clone(),
4550 checksum_algorithm: checksum.algorithm,
4551 checksum: Some(checksum.hash),
4552 updated_at: "2026-05-03T00:00:00Z".to_string(),
4553 }
4554 })
4555 .collect();
4556
4557 layout
4558 .write_journal(&DownloadJournal {
4559 journal_version: 1,
4560 backup_id: manifest.backup_id.clone(),
4561 discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
4562 pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
4563 operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
4564 artifacts,
4565 })
4566 .expect("write journal");
4567 }
4568
4569 fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
4571 for member in &mut manifest.fleet.members {
4572 let bytes = format!("{} apply artifact", member.role);
4573 let artifact_path = root.join(&member.source_snapshot.artifact_path);
4574 if let Some(parent) = artifact_path.parent() {
4575 fs::create_dir_all(parent).expect("create artifact parent");
4576 }
4577 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4578 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4579 member.source_snapshot.checksum = Some(checksum.hash);
4580 }
4581 }
4582
4583 fn temp_dir(prefix: &str) -> PathBuf {
4585 let nanos = SystemTime::now()
4586 .duration_since(UNIX_EPOCH)
4587 .expect("system time after epoch")
4588 .as_nanos();
4589 std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
4590 }
4591}