1use canic_backup::{
2 manifest::FleetBackupManifest,
3 persistence::{BackupLayout, PersistenceError},
4 restore::{
5 RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6 RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7 RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8 RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationKindCounts,
9 RestoreApplyOperationState, RestoreApplyPendingSummary, RestoreApplyProgressSummary,
10 RestoreApplyReportOperation, RestoreApplyReportOutcome, RestoreApplyRunnerCommand,
11 RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner, RestoreStatus,
12 },
13};
14use serde::Serialize;
15use std::{
16 ffi::OsString,
17 fs,
18 io::{self, Write},
19 path::PathBuf,
20 process::Command,
21};
22use thiserror::Error as ThisError;
23
24#[derive(Debug, ThisError)]
29pub enum RestoreCommandError {
30 #[error("{0}")]
31 Usage(&'static str),
32
33 #[error("missing required option {0}")]
34 MissingOption(&'static str),
35
36 #[error("use either --manifest or --backup-dir, not both")]
37 ConflictingManifestSources,
38
39 #[error("--require-verified requires --backup-dir")]
40 RequireVerifiedNeedsBackupDir,
41
42 #[error("restore apply currently requires --dry-run")]
43 ApplyRequiresDryRun,
44
45 #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
46 RestoreRunRequiresMode,
47
48 #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
49 RestoreRunConflictingModes,
50
51 #[error("restore run command failed for operation {sequence}: status={status}")]
52 RestoreRunCommandFailed { sequence: usize, status: String },
53
54 #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
55 RestoreRunModeMismatch {
56 backup_id: String,
57 expected: String,
58 actual: String,
59 },
60
61 #[error(
62 "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
63 )]
64 RestoreRunStoppedReasonMismatch {
65 backup_id: String,
66 expected: String,
67 actual: String,
68 },
69
70 #[error(
71 "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
72 )]
73 RestoreRunNextActionMismatch {
74 backup_id: String,
75 expected: String,
76 actual: String,
77 },
78
79 #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
80 RestoreRunExecutedCountMismatch {
81 backup_id: String,
82 expected: usize,
83 actual: usize,
84 },
85
86 #[error("restore run for backup {backup_id} wrote {actual} receipts, expected {expected}")]
87 RestoreRunReceiptCountMismatch {
88 backup_id: String,
89 expected: usize,
90 actual: usize,
91 },
92
93 #[error(
94 "restore run for backup {backup_id} wrote {actual} {receipt_kind} receipts, expected {expected}"
95 )]
96 RestoreRunReceiptKindCountMismatch {
97 backup_id: String,
98 receipt_kind: &'static str,
99 expected: usize,
100 actual: usize,
101 },
102
103 #[error(
104 "restore run for backup {backup_id} wrote {actual_receipts} receipts with {mismatched_receipts} updated_at mismatches, expected {expected}"
105 )]
106 RestoreRunReceiptUpdatedAtMismatch {
107 backup_id: String,
108 expected: String,
109 actual_receipts: usize,
110 mismatched_receipts: usize,
111 },
112
113 #[error(
114 "restore run for backup {backup_id} reported requested_state_updated_at={actual:?}, expected {expected}"
115 )]
116 RestoreRunStateUpdatedAtMismatch {
117 backup_id: String,
118 expected: String,
119 actual: Option<String>,
120 },
121
122 #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
123 RestoreNotReady {
124 backup_id: String,
125 reasons: Vec<String>,
126 },
127
128 #[error(
129 "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
130 )]
131 RestoreApplyPending {
132 backup_id: String,
133 pending_operations: usize,
134 next_transition_sequence: Option<usize>,
135 },
136
137 #[error(
138 "restore apply journal for backup {backup_id} has stale or untracked pending work before {cutoff_updated_at}: pending_sequence={pending_sequence:?}, pending_updated_at={pending_updated_at:?}"
139 )]
140 RestoreApplyPendingStale {
141 backup_id: String,
142 cutoff_updated_at: String,
143 pending_sequence: Option<usize>,
144 pending_updated_at: Option<String>,
145 },
146
147 #[error(
148 "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
149 )]
150 RestoreApplyIncomplete {
151 backup_id: String,
152 completed_operations: usize,
153 operation_count: usize,
154 },
155
156 #[error(
157 "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
158 )]
159 RestoreApplyFailed {
160 backup_id: String,
161 failed_operations: usize,
162 },
163
164 #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
165 RestoreApplyNotReady {
166 backup_id: String,
167 reasons: Vec<String>,
168 },
169
170 #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
171 RestoreApplyReportNeedsAttention {
172 backup_id: String,
173 outcome: canic_backup::restore::RestoreApplyReportOutcome,
174 },
175
176 #[error(
177 "restore apply progress for backup {backup_id} has unexpected {field}: expected={expected}, actual={actual}"
178 )]
179 RestoreApplyProgressMismatch {
180 backup_id: String,
181 field: &'static str,
182 expected: usize,
183 actual: usize,
184 },
185
186 #[error(
187 "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
188 )]
189 RestoreApplyCommandUnavailable {
190 backup_id: String,
191 operation_available: bool,
192 complete: bool,
193 blocked_reasons: Vec<String>,
194 },
195
196 #[error(
197 "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
198 )]
199 RestoreApplyMarkRequiresPending {
200 sequence: usize,
201 state: RestoreApplyOperationState,
202 },
203
204 #[error(
205 "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
206 )]
207 RestoreApplyClaimSequenceMismatch {
208 expected: usize,
209 actual: Option<usize>,
210 },
211
212 #[error(
213 "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
214 )]
215 RestoreApplyUnclaimSequenceMismatch {
216 expected: usize,
217 actual: Option<usize>,
218 },
219
220 #[error("unknown option {0}")]
221 UnknownOption(String),
222
223 #[error("option {0} requires a value")]
224 MissingValue(&'static str),
225
226 #[error("option --sequence requires a non-negative integer value")]
227 InvalidSequence,
228
229 #[error("unsupported apply-mark state {0}; use completed or failed")]
230 InvalidApplyMarkState(String),
231
232 #[error(transparent)]
233 Io(#[from] std::io::Error),
234
235 #[error(transparent)]
236 Json(#[from] serde_json::Error),
237
238 #[error(transparent)]
239 Persistence(#[from] PersistenceError),
240
241 #[error(transparent)]
242 RestorePlan(#[from] RestorePlanError),
243
244 #[error(transparent)]
245 RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
246
247 #[error(transparent)]
248 RestoreApplyJournal(#[from] RestoreApplyJournalError),
249}
250
251#[derive(Clone, Debug, Eq, PartialEq)]
256pub struct RestorePlanOptions {
257 pub manifest: Option<PathBuf>,
258 pub backup_dir: Option<PathBuf>,
259 pub mapping: Option<PathBuf>,
260 pub out: Option<PathBuf>,
261 pub require_verified: bool,
262 pub require_restore_ready: bool,
263}
264
265impl RestorePlanOptions {
266 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
268 where
269 I: IntoIterator<Item = OsString>,
270 {
271 let mut manifest = None;
272 let mut backup_dir = None;
273 let mut mapping = None;
274 let mut out = None;
275 let mut require_verified = false;
276 let mut require_restore_ready = false;
277
278 let mut args = args.into_iter();
279 while let Some(arg) = args.next() {
280 let arg = arg
281 .into_string()
282 .map_err(|_| RestoreCommandError::Usage(usage()))?;
283 match arg.as_str() {
284 "--manifest" => {
285 manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
286 }
287 "--backup-dir" => {
288 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
289 }
290 "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
291 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
292 "--require-verified" => require_verified = true,
293 "--require-restore-ready" => require_restore_ready = true,
294 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
295 _ => return Err(RestoreCommandError::UnknownOption(arg)),
296 }
297 }
298
299 if manifest.is_some() && backup_dir.is_some() {
300 return Err(RestoreCommandError::ConflictingManifestSources);
301 }
302
303 if manifest.is_none() && backup_dir.is_none() {
304 return Err(RestoreCommandError::MissingOption(
305 "--manifest or --backup-dir",
306 ));
307 }
308
309 if require_verified && backup_dir.is_none() {
310 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
311 }
312
313 Ok(Self {
314 manifest,
315 backup_dir,
316 mapping,
317 out,
318 require_verified,
319 require_restore_ready,
320 })
321 }
322}
323
324#[derive(Clone, Debug, Eq, PartialEq)]
329pub struct RestoreStatusOptions {
330 pub plan: PathBuf,
331 pub out: Option<PathBuf>,
332}
333
334impl RestoreStatusOptions {
335 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
337 where
338 I: IntoIterator<Item = OsString>,
339 {
340 let mut plan = None;
341 let mut out = None;
342
343 let mut args = args.into_iter();
344 while let Some(arg) = args.next() {
345 let arg = arg
346 .into_string()
347 .map_err(|_| RestoreCommandError::Usage(usage()))?;
348 match arg.as_str() {
349 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
350 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
351 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
352 _ => return Err(RestoreCommandError::UnknownOption(arg)),
353 }
354 }
355
356 Ok(Self {
357 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
358 out,
359 })
360 }
361}
362
363#[derive(Clone, Debug, Eq, PartialEq)]
368pub struct RestoreApplyOptions {
369 pub plan: PathBuf,
370 pub status: Option<PathBuf>,
371 pub backup_dir: Option<PathBuf>,
372 pub out: Option<PathBuf>,
373 pub journal_out: Option<PathBuf>,
374 pub dry_run: bool,
375}
376
377impl RestoreApplyOptions {
378 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
380 where
381 I: IntoIterator<Item = OsString>,
382 {
383 let mut plan = None;
384 let mut status = None;
385 let mut backup_dir = None;
386 let mut out = None;
387 let mut journal_out = None;
388 let mut dry_run = false;
389
390 let mut args = args.into_iter();
391 while let Some(arg) = args.next() {
392 let arg = arg
393 .into_string()
394 .map_err(|_| RestoreCommandError::Usage(usage()))?;
395 match arg.as_str() {
396 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
397 "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
398 "--backup-dir" => {
399 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
400 }
401 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
402 "--journal-out" => {
403 journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
404 }
405 "--dry-run" => dry_run = true,
406 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
407 _ => return Err(RestoreCommandError::UnknownOption(arg)),
408 }
409 }
410
411 if !dry_run {
412 return Err(RestoreCommandError::ApplyRequiresDryRun);
413 }
414
415 Ok(Self {
416 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
417 status,
418 backup_dir,
419 out,
420 journal_out,
421 dry_run,
422 })
423 }
424}
425
426#[derive(Clone, Debug, Eq, PartialEq)]
431#[expect(
432 clippy::struct_excessive_bools,
433 reason = "CLI status options mirror independent fail-closed guard flags"
434)]
435pub struct RestoreApplyStatusOptions {
436 pub journal: PathBuf,
437 pub require_ready: bool,
438 pub require_no_pending: bool,
439 pub require_no_failed: bool,
440 pub require_complete: bool,
441 pub require_remaining_count: Option<usize>,
442 pub require_attention_count: Option<usize>,
443 pub require_completion_basis_points: Option<usize>,
444 pub require_no_pending_before: Option<String>,
445 pub out: Option<PathBuf>,
446}
447
448impl RestoreApplyStatusOptions {
449 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
451 where
452 I: IntoIterator<Item = OsString>,
453 {
454 let mut journal = None;
455 let mut require_ready = false;
456 let mut require_no_pending = false;
457 let mut require_no_failed = false;
458 let mut require_complete = false;
459 let mut require_remaining_count = None;
460 let mut require_attention_count = None;
461 let mut require_completion_basis_points = None;
462 let mut require_no_pending_before = None;
463 let mut out = None;
464
465 let mut args = args.into_iter();
466 while let Some(arg) = args.next() {
467 let arg = arg
468 .into_string()
469 .map_err(|_| RestoreCommandError::Usage(usage()))?;
470 if parse_progress_requirement_option(
471 arg.as_str(),
472 &mut args,
473 &mut require_remaining_count,
474 &mut require_attention_count,
475 &mut require_completion_basis_points,
476 )? {
477 continue;
478 }
479 if parse_pending_requirement_option(
480 arg.as_str(),
481 &mut args,
482 &mut require_no_pending_before,
483 )? {
484 continue;
485 }
486 match arg.as_str() {
487 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
488 "--require-ready" => require_ready = true,
489 "--require-no-pending" => require_no_pending = true,
490 "--require-no-failed" => require_no_failed = true,
491 "--require-complete" => require_complete = true,
492 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
493 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
494 _ => return Err(RestoreCommandError::UnknownOption(arg)),
495 }
496 }
497
498 Ok(Self {
499 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
500 require_ready,
501 require_no_pending,
502 require_no_failed,
503 require_complete,
504 require_remaining_count,
505 require_attention_count,
506 require_completion_basis_points,
507 require_no_pending_before,
508 out,
509 })
510 }
511}
512
513#[derive(Clone, Debug, Eq, PartialEq)]
518pub struct RestoreApplyReportOptions {
519 pub journal: PathBuf,
520 pub require_no_attention: bool,
521 pub require_remaining_count: Option<usize>,
522 pub require_attention_count: Option<usize>,
523 pub require_completion_basis_points: Option<usize>,
524 pub require_no_pending_before: Option<String>,
525 pub out: Option<PathBuf>,
526}
527
528impl RestoreApplyReportOptions {
529 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
531 where
532 I: IntoIterator<Item = OsString>,
533 {
534 let mut journal = None;
535 let mut require_no_attention = false;
536 let mut require_remaining_count = None;
537 let mut require_attention_count = None;
538 let mut require_completion_basis_points = None;
539 let mut require_no_pending_before = None;
540 let mut out = None;
541
542 let mut args = args.into_iter();
543 while let Some(arg) = args.next() {
544 let arg = arg
545 .into_string()
546 .map_err(|_| RestoreCommandError::Usage(usage()))?;
547 if parse_progress_requirement_option(
548 arg.as_str(),
549 &mut args,
550 &mut require_remaining_count,
551 &mut require_attention_count,
552 &mut require_completion_basis_points,
553 )? {
554 continue;
555 }
556 if parse_pending_requirement_option(
557 arg.as_str(),
558 &mut args,
559 &mut require_no_pending_before,
560 )? {
561 continue;
562 }
563 match arg.as_str() {
564 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
565 "--require-no-attention" => require_no_attention = true,
566 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
567 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
568 _ => return Err(RestoreCommandError::UnknownOption(arg)),
569 }
570 }
571
572 Ok(Self {
573 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
574 require_no_attention,
575 require_remaining_count,
576 require_attention_count,
577 require_completion_basis_points,
578 require_no_pending_before,
579 out,
580 })
581 }
582}
583
584#[derive(Clone, Debug, Eq, PartialEq)]
589#[expect(
590 clippy::struct_excessive_bools,
591 reason = "CLI runner options mirror independent mode and fail-closed guard flags"
592)]
593pub struct RestoreRunOptions {
594 pub journal: PathBuf,
595 pub dfx: String,
596 pub network: Option<String>,
597 pub out: Option<PathBuf>,
598 pub dry_run: bool,
599 pub execute: bool,
600 pub unclaim_pending: bool,
601 pub max_steps: Option<usize>,
602 pub updated_at: Option<String>,
603 pub require_complete: bool,
604 pub require_no_attention: bool,
605 pub require_run_mode: Option<String>,
606 pub require_stopped_reason: Option<String>,
607 pub require_next_action: Option<String>,
608 pub require_executed_count: Option<usize>,
609 pub require_receipt_count: Option<usize>,
610 pub require_completed_receipt_count: Option<usize>,
611 pub require_failed_receipt_count: Option<usize>,
612 pub require_recovered_receipt_count: Option<usize>,
613 pub require_receipt_updated_at: Option<String>,
614 pub require_state_updated_at: Option<String>,
615 pub require_remaining_count: Option<usize>,
616 pub require_attention_count: Option<usize>,
617 pub require_completion_basis_points: Option<usize>,
618 pub require_no_pending_before: Option<String>,
619}
620
621impl RestoreRunOptions {
622 #[expect(
624 clippy::too_many_lines,
625 reason = "Restore runner options intentionally parse a broad flat CLI surface"
626 )]
627 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
628 where
629 I: IntoIterator<Item = OsString>,
630 {
631 let mut journal = None;
632 let mut dfx = "dfx".to_string();
633 let mut network = None;
634 let mut out = None;
635 let mut dry_run = false;
636 let mut execute = false;
637 let mut unclaim_pending = false;
638 let mut max_steps = None;
639 let mut updated_at = None;
640 let mut require_complete = false;
641 let mut require_no_attention = false;
642 let mut require_run_mode = None;
643 let mut require_stopped_reason = None;
644 let mut require_next_action = None;
645 let mut require_executed_count = None;
646 let mut require_receipt_count = None;
647 let mut require_completed_receipt_count = None;
648 let mut require_failed_receipt_count = None;
649 let mut require_recovered_receipt_count = None;
650 let mut require_receipt_updated_at = None;
651 let mut require_state_updated_at = None;
652 let mut require_remaining_count = None;
653 let mut require_attention_count = None;
654 let mut require_completion_basis_points = None;
655 let mut require_no_pending_before = None;
656
657 let mut args = args.into_iter();
658 while let Some(arg) = args.next() {
659 let arg = arg
660 .into_string()
661 .map_err(|_| RestoreCommandError::Usage(usage()))?;
662 if parse_progress_requirement_option(
663 arg.as_str(),
664 &mut args,
665 &mut require_remaining_count,
666 &mut require_attention_count,
667 &mut require_completion_basis_points,
668 )? {
669 continue;
670 }
671 if parse_pending_requirement_option(
672 arg.as_str(),
673 &mut args,
674 &mut require_no_pending_before,
675 )? {
676 continue;
677 }
678 if parse_run_count_requirement_option(
679 arg.as_str(),
680 &mut args,
681 &mut require_executed_count,
682 &mut require_receipt_count,
683 )? {
684 continue;
685 }
686 if parse_run_receipt_kind_requirement_option(
687 arg.as_str(),
688 &mut args,
689 &mut require_completed_receipt_count,
690 &mut require_failed_receipt_count,
691 &mut require_recovered_receipt_count,
692 )? {
693 continue;
694 }
695
696 match arg.as_str() {
697 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
698 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
699 "--network" => network = Some(next_value(&mut args, "--network")?),
700 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
701 "--dry-run" => dry_run = true,
702 "--execute" => execute = true,
703 "--unclaim-pending" => unclaim_pending = true,
704 "--max-steps" => {
705 max_steps = Some(parse_sequence(next_value(&mut args, "--max-steps")?)?);
706 }
707 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
708 "--require-complete" => require_complete = true,
709 "--require-no-attention" => require_no_attention = true,
710 "--require-run-mode" => {
711 require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
712 }
713 "--require-stopped-reason" => {
714 require_stopped_reason =
715 Some(next_value(&mut args, "--require-stopped-reason")?);
716 }
717 "--require-next-action" => {
718 require_next_action = Some(next_value(&mut args, "--require-next-action")?);
719 }
720 "--require-receipt-updated-at" => {
721 require_receipt_updated_at =
722 Some(next_value(&mut args, "--require-receipt-updated-at")?);
723 }
724 "--require-state-updated-at" => {
725 require_state_updated_at =
726 Some(next_value(&mut args, "--require-state-updated-at")?);
727 }
728 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
729 _ => return Err(RestoreCommandError::UnknownOption(arg)),
730 }
731 }
732
733 validate_restore_run_mode_selection(dry_run, execute, unclaim_pending)?;
734
735 Ok(Self {
736 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
737 dfx,
738 network,
739 out,
740 dry_run,
741 execute,
742 unclaim_pending,
743 max_steps,
744 updated_at,
745 require_complete,
746 require_no_attention,
747 require_run_mode,
748 require_stopped_reason,
749 require_next_action,
750 require_executed_count,
751 require_receipt_count,
752 require_completed_receipt_count,
753 require_failed_receipt_count,
754 require_recovered_receipt_count,
755 require_receipt_updated_at,
756 require_state_updated_at,
757 require_remaining_count,
758 require_attention_count,
759 require_completion_basis_points,
760 require_no_pending_before,
761 })
762 }
763}
764
765fn validate_restore_run_mode_selection(
767 dry_run: bool,
768 execute: bool,
769 unclaim_pending: bool,
770) -> Result<(), RestoreCommandError> {
771 let mode_count = [dry_run, execute, unclaim_pending]
772 .into_iter()
773 .filter(|enabled| *enabled)
774 .count();
775 if mode_count > 1 {
776 return Err(RestoreCommandError::RestoreRunConflictingModes);
777 }
778
779 if mode_count == 0 {
780 return Err(RestoreCommandError::RestoreRunRequiresMode);
781 }
782
783 Ok(())
784}
785
786struct RestoreRunResult {
791 response: RestoreRunResponse,
792 error: Option<RestoreCommandError>,
793}
794
795impl RestoreRunResult {
796 const fn ok(response: RestoreRunResponse) -> Self {
798 Self {
799 response,
800 error: None,
801 }
802 }
803}
804
805const RESTORE_RUN_MODE_DRY_RUN: &str = "dry-run";
806const RESTORE_RUN_MODE_EXECUTE: &str = "execute";
807const RESTORE_RUN_MODE_UNCLAIM_PENDING: &str = "unclaim-pending";
808
809const RESTORE_RUN_STOPPED_BLOCKED: &str = "blocked";
810const RESTORE_RUN_STOPPED_COMMAND_FAILED: &str = "command-failed";
811const RESTORE_RUN_STOPPED_COMPLETE: &str = "complete";
812const RESTORE_RUN_STOPPED_MAX_STEPS: &str = "max-steps-reached";
813const RESTORE_RUN_STOPPED_PENDING: &str = "pending";
814const RESTORE_RUN_STOPPED_PREVIEW: &str = "preview";
815const RESTORE_RUN_STOPPED_READY: &str = "ready";
816const RESTORE_RUN_STOPPED_RECOVERED_PENDING: &str = "recovered-pending";
817
818const RESTORE_RUN_ACTION_DONE: &str = "done";
819const RESTORE_RUN_ACTION_FIX_BLOCKED: &str = "fix-blocked-journal";
820const RESTORE_RUN_ACTION_INSPECT_FAILED: &str = "inspect-failed-operation";
821const RESTORE_RUN_ACTION_RERUN: &str = "rerun";
822const RESTORE_RUN_ACTION_UNCLAIM_PENDING: &str = "unclaim-pending";
823
824const RESTORE_RUN_EXECUTED_COMPLETED: &str = "completed";
825const RESTORE_RUN_EXECUTED_FAILED: &str = "failed";
826const RESTORE_RUN_RECEIPT_COMPLETED: &str = "command-completed";
827const RESTORE_RUN_RECEIPT_FAILED: &str = "command-failed";
828const RESTORE_RUN_RECEIPT_RECOVERED_PENDING: &str = "pending-recovered";
829const RESTORE_RUN_RECEIPT_STATE_READY: &str = "ready";
830const RESTORE_RUN_COMMAND_EXIT_PREFIX: &str = "runner-command-exit";
831const RESTORE_RUN_RESPONSE_VERSION: u16 = 1;
832
833#[derive(Clone, Debug, Serialize)]
838#[expect(
839 clippy::struct_excessive_bools,
840 reason = "Runner response exposes stable JSON status flags for operators and CI"
841)]
842pub struct RestoreRunResponse {
843 run_version: u16,
844 backup_id: String,
845 run_mode: &'static str,
846 dry_run: bool,
847 execute: bool,
848 unclaim_pending: bool,
849 stopped_reason: &'static str,
850 next_action: &'static str,
851 #[serde(skip_serializing_if = "Option::is_none")]
852 requested_state_updated_at: Option<String>,
853 #[serde(skip_serializing_if = "Option::is_none")]
854 max_steps_reached: Option<bool>,
855 #[serde(default, skip_serializing_if = "Vec::is_empty")]
856 executed_operations: Vec<RestoreRunExecutedOperation>,
857 #[serde(default, skip_serializing_if = "Vec::is_empty")]
858 operation_receipts: Vec<RestoreRunOperationReceipt>,
859 #[serde(skip_serializing_if = "Option::is_none")]
860 operation_receipt_count: Option<usize>,
861 operation_receipt_summary: RestoreRunReceiptSummary,
862 #[serde(skip_serializing_if = "Option::is_none")]
863 executed_operation_count: Option<usize>,
864 #[serde(skip_serializing_if = "Option::is_none")]
865 recovered_operation: Option<RestoreApplyJournalOperation>,
866 ready: bool,
867 complete: bool,
868 attention_required: bool,
869 outcome: RestoreApplyReportOutcome,
870 operation_count: usize,
871 operation_counts: RestoreApplyOperationKindCounts,
872 operation_counts_supplied: bool,
873 progress: RestoreApplyProgressSummary,
874 pending_summary: RestoreApplyPendingSummary,
875 pending_operations: usize,
876 ready_operations: usize,
877 blocked_operations: usize,
878 completed_operations: usize,
879 failed_operations: usize,
880 blocked_reasons: Vec<String>,
881 next_transition: Option<RestoreApplyReportOperation>,
882 #[serde(skip_serializing_if = "Option::is_none")]
883 operation_available: Option<bool>,
884 #[serde(skip_serializing_if = "Option::is_none")]
885 command_available: Option<bool>,
886 #[serde(skip_serializing_if = "Option::is_none")]
887 command: Option<RestoreApplyRunnerCommand>,
888}
889
890impl RestoreRunResponse {
891 fn from_report(
893 backup_id: String,
894 report: RestoreApplyJournalReport,
895 mode: RestoreRunResponseMode,
896 ) -> Self {
897 Self {
898 run_version: RESTORE_RUN_RESPONSE_VERSION,
899 backup_id,
900 run_mode: mode.run_mode,
901 dry_run: mode.dry_run,
902 execute: mode.execute,
903 unclaim_pending: mode.unclaim_pending,
904 stopped_reason: mode.stopped_reason,
905 next_action: mode.next_action,
906 requested_state_updated_at: None,
907 max_steps_reached: None,
908 executed_operations: Vec::new(),
909 operation_receipts: Vec::new(),
910 operation_receipt_count: Some(0),
911 operation_receipt_summary: RestoreRunReceiptSummary::default(),
912 executed_operation_count: None,
913 recovered_operation: None,
914 ready: report.ready,
915 complete: report.complete,
916 attention_required: report.attention_required,
917 outcome: report.outcome,
918 operation_count: report.operation_count,
919 operation_counts: report.operation_counts,
920 operation_counts_supplied: report.operation_counts_supplied,
921 progress: report.progress,
922 pending_summary: report.pending_summary,
923 pending_operations: report.pending_operations,
924 ready_operations: report.ready_operations,
925 blocked_operations: report.blocked_operations,
926 completed_operations: report.completed_operations,
927 failed_operations: report.failed_operations,
928 blocked_reasons: report.blocked_reasons,
929 next_transition: report.next_transition,
930 operation_available: None,
931 command_available: None,
932 command: None,
933 }
934 }
935
936 fn set_operation_receipts(&mut self, receipts: Vec<RestoreRunOperationReceipt>) {
938 self.operation_receipt_summary = RestoreRunReceiptSummary::from_receipts(&receipts);
939 self.operation_receipt_count = Some(receipts.len());
940 self.operation_receipts = receipts;
941 }
942
943 fn set_requested_state_updated_at(&mut self, updated_at: Option<&String>) {
945 self.requested_state_updated_at = updated_at.cloned();
946 }
947}
948
949#[derive(Clone, Debug, Default, Serialize)]
954struct RestoreRunReceiptSummary {
955 total_receipts: usize,
956 command_completed: usize,
957 command_failed: usize,
958 pending_recovered: usize,
959}
960
961impl RestoreRunReceiptSummary {
962 fn from_receipts(receipts: &[RestoreRunOperationReceipt]) -> Self {
964 let mut summary = Self {
965 total_receipts: receipts.len(),
966 ..Self::default()
967 };
968
969 for receipt in receipts {
970 match receipt.event {
971 RESTORE_RUN_RECEIPT_COMPLETED => summary.command_completed += 1,
972 RESTORE_RUN_RECEIPT_FAILED => summary.command_failed += 1,
973 RESTORE_RUN_RECEIPT_RECOVERED_PENDING => summary.pending_recovered += 1,
974 _ => {}
975 }
976 }
977
978 summary
979 }
980}
981
982#[derive(Clone, Debug, Serialize)]
987struct RestoreRunOperationReceipt {
988 event: &'static str,
989 sequence: usize,
990 operation: RestoreApplyOperationKind,
991 target_canister: String,
992 state: &'static str,
993 #[serde(skip_serializing_if = "Option::is_none")]
994 updated_at: Option<String>,
995 #[serde(skip_serializing_if = "Option::is_none")]
996 command: Option<RestoreApplyRunnerCommand>,
997 #[serde(skip_serializing_if = "Option::is_none")]
998 status: Option<String>,
999}
1000
1001impl RestoreRunOperationReceipt {
1002 fn completed(
1004 operation: RestoreApplyJournalOperation,
1005 command: RestoreApplyRunnerCommand,
1006 status: String,
1007 updated_at: Option<String>,
1008 ) -> Self {
1009 Self::from_operation(
1010 RESTORE_RUN_RECEIPT_COMPLETED,
1011 operation,
1012 RESTORE_RUN_EXECUTED_COMPLETED,
1013 updated_at,
1014 Some(command),
1015 Some(status),
1016 )
1017 }
1018
1019 fn failed(
1021 operation: RestoreApplyJournalOperation,
1022 command: RestoreApplyRunnerCommand,
1023 status: String,
1024 updated_at: Option<String>,
1025 ) -> Self {
1026 Self::from_operation(
1027 RESTORE_RUN_RECEIPT_FAILED,
1028 operation,
1029 RESTORE_RUN_EXECUTED_FAILED,
1030 updated_at,
1031 Some(command),
1032 Some(status),
1033 )
1034 }
1035
1036 fn recovered_pending(
1038 operation: RestoreApplyJournalOperation,
1039 updated_at: Option<String>,
1040 ) -> Self {
1041 Self::from_operation(
1042 RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
1043 operation,
1044 RESTORE_RUN_RECEIPT_STATE_READY,
1045 updated_at,
1046 None,
1047 None,
1048 )
1049 }
1050
1051 fn from_operation(
1053 event: &'static str,
1054 operation: RestoreApplyJournalOperation,
1055 state: &'static str,
1056 updated_at: Option<String>,
1057 command: Option<RestoreApplyRunnerCommand>,
1058 status: Option<String>,
1059 ) -> Self {
1060 Self {
1061 event,
1062 sequence: operation.sequence,
1063 operation: operation.operation,
1064 target_canister: operation.target_canister,
1065 state,
1066 updated_at,
1067 command,
1068 status,
1069 }
1070 }
1071}
1072
1073#[derive(Clone, Debug, Serialize)]
1078struct RestoreRunExecutedOperation {
1079 sequence: usize,
1080 operation: RestoreApplyOperationKind,
1081 target_canister: String,
1082 command: RestoreApplyRunnerCommand,
1083 status: String,
1084 state: &'static str,
1085}
1086
1087impl RestoreRunExecutedOperation {
1088 fn completed(
1090 operation: RestoreApplyJournalOperation,
1091 command: RestoreApplyRunnerCommand,
1092 status: String,
1093 ) -> Self {
1094 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_COMPLETED)
1095 }
1096
1097 fn failed(
1099 operation: RestoreApplyJournalOperation,
1100 command: RestoreApplyRunnerCommand,
1101 status: String,
1102 ) -> Self {
1103 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_FAILED)
1104 }
1105
1106 fn from_operation(
1108 operation: RestoreApplyJournalOperation,
1109 command: RestoreApplyRunnerCommand,
1110 status: String,
1111 state: &'static str,
1112 ) -> Self {
1113 Self {
1114 sequence: operation.sequence,
1115 operation: operation.operation,
1116 target_canister: operation.target_canister,
1117 command,
1118 status,
1119 state,
1120 }
1121 }
1122}
1123
1124struct RestoreRunResponseMode {
1129 run_mode: &'static str,
1130 dry_run: bool,
1131 execute: bool,
1132 unclaim_pending: bool,
1133 stopped_reason: &'static str,
1134 next_action: &'static str,
1135}
1136
1137impl RestoreRunResponseMode {
1138 const fn new(
1140 run_mode: &'static str,
1141 dry_run: bool,
1142 execute: bool,
1143 unclaim_pending: bool,
1144 stopped_reason: &'static str,
1145 next_action: &'static str,
1146 ) -> Self {
1147 Self {
1148 run_mode,
1149 dry_run,
1150 execute,
1151 unclaim_pending,
1152 stopped_reason,
1153 next_action,
1154 }
1155 }
1156
1157 const fn dry_run(stopped_reason: &'static str, next_action: &'static str) -> Self {
1159 Self::new(
1160 RESTORE_RUN_MODE_DRY_RUN,
1161 true,
1162 false,
1163 false,
1164 stopped_reason,
1165 next_action,
1166 )
1167 }
1168
1169 const fn execute(stopped_reason: &'static str, next_action: &'static str) -> Self {
1171 Self::new(
1172 RESTORE_RUN_MODE_EXECUTE,
1173 false,
1174 true,
1175 false,
1176 stopped_reason,
1177 next_action,
1178 )
1179 }
1180
1181 const fn unclaim_pending(next_action: &'static str) -> Self {
1183 Self::new(
1184 RESTORE_RUN_MODE_UNCLAIM_PENDING,
1185 false,
1186 false,
1187 true,
1188 RESTORE_RUN_STOPPED_RECOVERED_PENDING,
1189 next_action,
1190 )
1191 }
1192}
1193
1194#[derive(Clone, Debug, Eq, PartialEq)]
1199pub struct RestoreApplyNextOptions {
1200 pub journal: PathBuf,
1201 pub out: Option<PathBuf>,
1202}
1203
1204impl RestoreApplyNextOptions {
1205 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1207 where
1208 I: IntoIterator<Item = OsString>,
1209 {
1210 let mut journal = None;
1211 let mut out = None;
1212
1213 let mut args = args.into_iter();
1214 while let Some(arg) = args.next() {
1215 let arg = arg
1216 .into_string()
1217 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1218 match arg.as_str() {
1219 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1220 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1221 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1222 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1223 }
1224 }
1225
1226 Ok(Self {
1227 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1228 out,
1229 })
1230 }
1231}
1232
1233#[derive(Clone, Debug, Eq, PartialEq)]
1238pub struct RestoreApplyCommandOptions {
1239 pub journal: PathBuf,
1240 pub dfx: String,
1241 pub network: Option<String>,
1242 pub out: Option<PathBuf>,
1243 pub require_command: bool,
1244}
1245
1246impl RestoreApplyCommandOptions {
1247 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1249 where
1250 I: IntoIterator<Item = OsString>,
1251 {
1252 let mut journal = None;
1253 let mut dfx = "dfx".to_string();
1254 let mut network = None;
1255 let mut out = None;
1256 let mut require_command = false;
1257
1258 let mut args = args.into_iter();
1259 while let Some(arg) = args.next() {
1260 let arg = arg
1261 .into_string()
1262 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1263 match arg.as_str() {
1264 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1265 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
1266 "--network" => network = Some(next_value(&mut args, "--network")?),
1267 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1268 "--require-command" => require_command = true,
1269 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1270 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1271 }
1272 }
1273
1274 Ok(Self {
1275 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1276 dfx,
1277 network,
1278 out,
1279 require_command,
1280 })
1281 }
1282}
1283
1284#[derive(Clone, Debug, Eq, PartialEq)]
1289pub struct RestoreApplyClaimOptions {
1290 pub journal: PathBuf,
1291 pub sequence: Option<usize>,
1292 pub updated_at: Option<String>,
1293 pub out: Option<PathBuf>,
1294}
1295
1296impl RestoreApplyClaimOptions {
1297 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1299 where
1300 I: IntoIterator<Item = OsString>,
1301 {
1302 let mut journal = None;
1303 let mut sequence = None;
1304 let mut updated_at = None;
1305 let mut out = None;
1306
1307 let mut args = args.into_iter();
1308 while let Some(arg) = args.next() {
1309 let arg = arg
1310 .into_string()
1311 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1312 match arg.as_str() {
1313 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1314 "--sequence" => {
1315 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1316 }
1317 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1318 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1319 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1320 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1321 }
1322 }
1323
1324 Ok(Self {
1325 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1326 sequence,
1327 updated_at,
1328 out,
1329 })
1330 }
1331}
1332
1333#[derive(Clone, Debug, Eq, PartialEq)]
1338pub struct RestoreApplyUnclaimOptions {
1339 pub journal: PathBuf,
1340 pub sequence: Option<usize>,
1341 pub updated_at: Option<String>,
1342 pub out: Option<PathBuf>,
1343}
1344
1345impl RestoreApplyUnclaimOptions {
1346 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1348 where
1349 I: IntoIterator<Item = OsString>,
1350 {
1351 let mut journal = None;
1352 let mut sequence = None;
1353 let mut updated_at = None;
1354 let mut out = None;
1355
1356 let mut args = args.into_iter();
1357 while let Some(arg) = args.next() {
1358 let arg = arg
1359 .into_string()
1360 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1361 match arg.as_str() {
1362 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1363 "--sequence" => {
1364 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1365 }
1366 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1367 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1368 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1369 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1370 }
1371 }
1372
1373 Ok(Self {
1374 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1375 sequence,
1376 updated_at,
1377 out,
1378 })
1379 }
1380}
1381
1382#[derive(Clone, Debug, Eq, PartialEq)]
1387pub struct RestoreApplyMarkOptions {
1388 pub journal: PathBuf,
1389 pub sequence: usize,
1390 pub state: RestoreApplyMarkState,
1391 pub reason: Option<String>,
1392 pub updated_at: Option<String>,
1393 pub out: Option<PathBuf>,
1394 pub require_pending: bool,
1395}
1396
1397impl RestoreApplyMarkOptions {
1398 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1400 where
1401 I: IntoIterator<Item = OsString>,
1402 {
1403 let mut journal = None;
1404 let mut sequence = None;
1405 let mut state = None;
1406 let mut reason = None;
1407 let mut updated_at = None;
1408 let mut out = None;
1409 let mut require_pending = false;
1410
1411 let mut args = args.into_iter();
1412 while let Some(arg) = args.next() {
1413 let arg = arg
1414 .into_string()
1415 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1416 match arg.as_str() {
1417 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1418 "--sequence" => {
1419 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1420 }
1421 "--state" => {
1422 state = Some(RestoreApplyMarkState::parse(next_value(
1423 &mut args, "--state",
1424 )?)?);
1425 }
1426 "--reason" => reason = Some(next_value(&mut args, "--reason")?),
1427 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1428 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1429 "--require-pending" => require_pending = true,
1430 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1431 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1432 }
1433 }
1434
1435 Ok(Self {
1436 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1437 sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
1438 state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
1439 reason,
1440 updated_at,
1441 out,
1442 require_pending,
1443 })
1444 }
1445}
1446
1447#[derive(Clone, Debug, Eq, PartialEq)]
1452pub enum RestoreApplyMarkState {
1453 Completed,
1454 Failed,
1455}
1456
1457impl RestoreApplyMarkState {
1458 fn parse(value: String) -> Result<Self, RestoreCommandError> {
1460 match value.as_str() {
1461 "completed" => Ok(Self::Completed),
1462 "failed" => Ok(Self::Failed),
1463 _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
1464 }
1465 }
1466}
1467
1468pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
1470where
1471 I: IntoIterator<Item = OsString>,
1472{
1473 let mut args = args.into_iter();
1474 let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
1475 return Err(RestoreCommandError::Usage(usage()));
1476 };
1477
1478 match command.as_str() {
1479 "plan" => {
1480 let options = RestorePlanOptions::parse(args)?;
1481 let plan = plan_restore(&options)?;
1482 write_plan(&options, &plan)?;
1483 enforce_restore_plan_requirements(&options, &plan)?;
1484 Ok(())
1485 }
1486 "status" => {
1487 let options = RestoreStatusOptions::parse(args)?;
1488 let status = restore_status(&options)?;
1489 write_status(&options, &status)?;
1490 Ok(())
1491 }
1492 "apply" => {
1493 let options = RestoreApplyOptions::parse(args)?;
1494 let dry_run = restore_apply_dry_run(&options)?;
1495 write_apply_dry_run(&options, &dry_run)?;
1496 write_apply_journal_if_requested(&options, &dry_run)?;
1497 Ok(())
1498 }
1499 "apply-status" => {
1500 let options = RestoreApplyStatusOptions::parse(args)?;
1501 let status = restore_apply_status(&options)?;
1502 write_apply_status(&options, &status)?;
1503 enforce_apply_status_requirements(&options, &status)?;
1504 Ok(())
1505 }
1506 "apply-report" => {
1507 let options = RestoreApplyReportOptions::parse(args)?;
1508 let report = restore_apply_report(&options)?;
1509 write_apply_report(&options, &report)?;
1510 enforce_apply_report_requirements(&options, &report)?;
1511 Ok(())
1512 }
1513 "run" => {
1514 let options = RestoreRunOptions::parse(args)?;
1515 let run = if options.execute {
1516 restore_run_execute_result(&options)?
1517 } else if options.unclaim_pending {
1518 RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1519 } else {
1520 RestoreRunResult::ok(restore_run_dry_run(&options)?)
1521 };
1522 write_restore_run(&options, &run.response)?;
1523 if let Some(error) = run.error {
1524 return Err(error);
1525 }
1526 enforce_restore_run_requirements(&options, &run.response)?;
1527 Ok(())
1528 }
1529 "apply-next" => {
1530 let options = RestoreApplyNextOptions::parse(args)?;
1531 let next = restore_apply_next(&options)?;
1532 write_apply_next(&options, &next)?;
1533 Ok(())
1534 }
1535 "apply-command" => {
1536 let options = RestoreApplyCommandOptions::parse(args)?;
1537 let preview = restore_apply_command(&options)?;
1538 write_apply_command(&options, &preview)?;
1539 enforce_apply_command_requirements(&options, &preview)?;
1540 Ok(())
1541 }
1542 "apply-claim" => {
1543 let options = RestoreApplyClaimOptions::parse(args)?;
1544 let journal = restore_apply_claim(&options)?;
1545 write_apply_claim(&options, &journal)?;
1546 Ok(())
1547 }
1548 "apply-unclaim" => {
1549 let options = RestoreApplyUnclaimOptions::parse(args)?;
1550 let journal = restore_apply_unclaim(&options)?;
1551 write_apply_unclaim(&options, &journal)?;
1552 Ok(())
1553 }
1554 "apply-mark" => {
1555 let options = RestoreApplyMarkOptions::parse(args)?;
1556 let journal = restore_apply_mark(&options)?;
1557 write_apply_mark(&options, &journal)?;
1558 Ok(())
1559 }
1560 "help" | "--help" | "-h" => Err(RestoreCommandError::Usage(usage())),
1561 _ => Err(RestoreCommandError::UnknownOption(command)),
1562 }
1563}
1564
1565pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1567 verify_backup_layout_if_required(options)?;
1568
1569 let manifest = read_manifest_source(options)?;
1570 let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1571
1572 RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1573}
1574
1575pub fn restore_status(
1577 options: &RestoreStatusOptions,
1578) -> Result<RestoreStatus, RestoreCommandError> {
1579 let plan = read_plan(&options.plan)?;
1580 Ok(RestoreStatus::from_plan(&plan))
1581}
1582
1583pub fn restore_apply_dry_run(
1585 options: &RestoreApplyOptions,
1586) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1587 let plan = read_plan(&options.plan)?;
1588 let status = options.status.as_ref().map(read_status).transpose()?;
1589 if let Some(backup_dir) = &options.backup_dir {
1590 return RestoreApplyDryRun::try_from_plan_with_artifacts(
1591 &plan,
1592 status.as_ref(),
1593 backup_dir,
1594 )
1595 .map_err(RestoreCommandError::from);
1596 }
1597
1598 RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1599}
1600
1601pub fn restore_apply_status(
1603 options: &RestoreApplyStatusOptions,
1604) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1605 let journal = read_apply_journal(&options.journal)?;
1606 Ok(journal.status())
1607}
1608
1609pub fn restore_apply_report(
1611 options: &RestoreApplyReportOptions,
1612) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1613 let journal = read_apply_journal(&options.journal)?;
1614 Ok(journal.report())
1615}
1616
1617pub fn restore_run_dry_run(
1619 options: &RestoreRunOptions,
1620) -> Result<RestoreRunResponse, RestoreCommandError> {
1621 let journal = read_apply_journal(&options.journal)?;
1622 let report = journal.report();
1623 let preview = journal.next_command_preview_with_config(&restore_run_command_config(options));
1624 let stopped_reason = restore_run_stopped_reason(&report, false, false);
1625 let next_action = restore_run_next_action(&report, false);
1626
1627 let mut response = RestoreRunResponse::from_report(
1628 journal.backup_id,
1629 report,
1630 RestoreRunResponseMode::dry_run(stopped_reason, next_action),
1631 );
1632 response.set_requested_state_updated_at(options.updated_at.as_ref());
1633 response.operation_available = Some(preview.operation_available);
1634 response.command_available = Some(preview.command_available);
1635 response.command = preview.command;
1636 Ok(response)
1637}
1638
1639pub fn restore_run_unclaim_pending(
1641 options: &RestoreRunOptions,
1642) -> Result<RestoreRunResponse, RestoreCommandError> {
1643 let mut journal = read_apply_journal(&options.journal)?;
1644 let recovered_operation = journal
1645 .next_transition_operation()
1646 .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1647 .cloned()
1648 .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1649
1650 let recovered_updated_at = state_updated_at(options.updated_at.as_ref());
1651 journal.mark_next_operation_ready_at(Some(recovered_updated_at.clone()))?;
1652 write_apply_journal_file(&options.journal, &journal)?;
1653
1654 let report = journal.report();
1655 let next_action = restore_run_next_action(&report, true);
1656 let mut response = RestoreRunResponse::from_report(
1657 journal.backup_id,
1658 report,
1659 RestoreRunResponseMode::unclaim_pending(next_action),
1660 );
1661 response.set_requested_state_updated_at(options.updated_at.as_ref());
1662 response.set_operation_receipts(vec![RestoreRunOperationReceipt::recovered_pending(
1663 recovered_operation.clone(),
1664 Some(recovered_updated_at),
1665 )]);
1666 response.recovered_operation = Some(recovered_operation);
1667 Ok(response)
1668}
1669
1670pub fn restore_run_execute(
1672 options: &RestoreRunOptions,
1673) -> Result<RestoreRunResponse, RestoreCommandError> {
1674 let run = restore_run_execute_result(options)?;
1675 if let Some(error) = run.error {
1676 return Err(error);
1677 }
1678
1679 Ok(run.response)
1680}
1681
1682fn restore_run_execute_result(
1684 options: &RestoreRunOptions,
1685) -> Result<RestoreRunResult, RestoreCommandError> {
1686 let mut journal = read_apply_journal(&options.journal)?;
1687 let mut executed_operations = Vec::new();
1688 let mut operation_receipts = Vec::new();
1689 let config = restore_run_command_config(options);
1690
1691 loop {
1692 let report = journal.report();
1693 let max_steps_reached =
1694 restore_run_max_steps_reached(options, executed_operations.len(), &report);
1695 if report.complete || max_steps_reached {
1696 return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1697 &journal,
1698 executed_operations,
1699 operation_receipts,
1700 max_steps_reached,
1701 options.updated_at.as_ref(),
1702 )));
1703 }
1704
1705 enforce_restore_run_executable(&journal, &report)?;
1706 let preview = journal.next_command_preview_with_config(&config);
1707 enforce_restore_run_command_available(&preview)?;
1708
1709 let operation = preview
1710 .operation
1711 .clone()
1712 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1713 let command = preview
1714 .command
1715 .clone()
1716 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1717 let sequence = operation.sequence;
1718
1719 enforce_apply_claim_sequence(sequence, &journal)?;
1720 journal.mark_operation_pending_at(
1721 sequence,
1722 Some(state_updated_at(options.updated_at.as_ref())),
1723 )?;
1724 write_apply_journal_file(&options.journal, &journal)?;
1725
1726 let status = Command::new(&command.program)
1727 .args(&command.args)
1728 .status()?;
1729 let status_label = exit_status_label(status);
1730 if status.success() {
1731 let completed_updated_at = state_updated_at(options.updated_at.as_ref());
1732 journal.mark_operation_completed_at(sequence, Some(completed_updated_at.clone()))?;
1733 write_apply_journal_file(&options.journal, &journal)?;
1734 executed_operations.push(RestoreRunExecutedOperation::completed(
1735 operation.clone(),
1736 command.clone(),
1737 status_label.clone(),
1738 ));
1739 operation_receipts.push(RestoreRunOperationReceipt::completed(
1740 operation,
1741 command,
1742 status_label,
1743 Some(completed_updated_at),
1744 ));
1745 continue;
1746 }
1747
1748 let failed_updated_at = state_updated_at(options.updated_at.as_ref());
1749 journal.mark_operation_failed_at(
1750 sequence,
1751 format!("{RESTORE_RUN_COMMAND_EXIT_PREFIX}-{status_label}"),
1752 Some(failed_updated_at.clone()),
1753 )?;
1754 write_apply_journal_file(&options.journal, &journal)?;
1755 executed_operations.push(RestoreRunExecutedOperation::failed(
1756 operation.clone(),
1757 command.clone(),
1758 status_label.clone(),
1759 ));
1760 operation_receipts.push(RestoreRunOperationReceipt::failed(
1761 operation,
1762 command,
1763 status_label.clone(),
1764 Some(failed_updated_at),
1765 ));
1766 let response = restore_run_execute_summary(
1767 &journal,
1768 executed_operations,
1769 operation_receipts,
1770 false,
1771 options.updated_at.as_ref(),
1772 );
1773 return Ok(RestoreRunResult {
1774 response,
1775 error: Some(RestoreCommandError::RestoreRunCommandFailed {
1776 sequence,
1777 status: status_label,
1778 }),
1779 });
1780 }
1781}
1782
1783fn restore_run_command_config(options: &RestoreRunOptions) -> RestoreApplyCommandConfig {
1785 restore_command_config(&options.dfx, options.network.as_deref())
1786}
1787
1788fn restore_apply_command_config(options: &RestoreApplyCommandOptions) -> RestoreApplyCommandConfig {
1790 restore_command_config(&options.dfx, options.network.as_deref())
1791}
1792
1793fn restore_command_config(program: &str, network: Option<&str>) -> RestoreApplyCommandConfig {
1795 RestoreApplyCommandConfig {
1796 program: program.to_string(),
1797 network: network.map(str::to_string),
1798 }
1799}
1800
1801fn restore_run_max_steps_reached(
1803 options: &RestoreRunOptions,
1804 executed_operation_count: usize,
1805 report: &RestoreApplyJournalReport,
1806) -> bool {
1807 options.max_steps == Some(executed_operation_count) && !report.complete
1808}
1809
1810fn restore_run_execute_summary(
1812 journal: &RestoreApplyJournal,
1813 executed_operations: Vec<RestoreRunExecutedOperation>,
1814 operation_receipts: Vec<RestoreRunOperationReceipt>,
1815 max_steps_reached: bool,
1816 requested_state_updated_at: Option<&String>,
1817) -> RestoreRunResponse {
1818 let report = journal.report();
1819 let executed_operation_count = executed_operations.len();
1820 let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
1821 let next_action = restore_run_next_action(&report, false);
1822
1823 let mut response = RestoreRunResponse::from_report(
1824 journal.backup_id.clone(),
1825 report,
1826 RestoreRunResponseMode::execute(stopped_reason, next_action),
1827 );
1828 response.set_requested_state_updated_at(requested_state_updated_at);
1829 response.max_steps_reached = Some(max_steps_reached);
1830 response.executed_operation_count = Some(executed_operation_count);
1831 response.executed_operations = executed_operations;
1832 response.set_operation_receipts(operation_receipts);
1833 response
1834}
1835
1836const fn restore_run_stopped_reason(
1838 report: &RestoreApplyJournalReport,
1839 max_steps_reached: bool,
1840 executed: bool,
1841) -> &'static str {
1842 if report.complete {
1843 return RESTORE_RUN_STOPPED_COMPLETE;
1844 }
1845 if report.failed_operations > 0 {
1846 return RESTORE_RUN_STOPPED_COMMAND_FAILED;
1847 }
1848 if report.pending_operations > 0 {
1849 return RESTORE_RUN_STOPPED_PENDING;
1850 }
1851 if !report.ready || report.blocked_operations > 0 {
1852 return RESTORE_RUN_STOPPED_BLOCKED;
1853 }
1854 if max_steps_reached {
1855 return RESTORE_RUN_STOPPED_MAX_STEPS;
1856 }
1857 if executed {
1858 return RESTORE_RUN_STOPPED_READY;
1859 }
1860 RESTORE_RUN_STOPPED_PREVIEW
1861}
1862
1863const fn restore_run_next_action(
1865 report: &RestoreApplyJournalReport,
1866 recovered_pending: bool,
1867) -> &'static str {
1868 if report.complete {
1869 return RESTORE_RUN_ACTION_DONE;
1870 }
1871 if report.failed_operations > 0 {
1872 return RESTORE_RUN_ACTION_INSPECT_FAILED;
1873 }
1874 if report.pending_operations > 0 {
1875 return RESTORE_RUN_ACTION_UNCLAIM_PENDING;
1876 }
1877 if !report.ready || report.blocked_operations > 0 {
1878 return RESTORE_RUN_ACTION_FIX_BLOCKED;
1879 }
1880 if recovered_pending {
1881 return RESTORE_RUN_ACTION_RERUN;
1882 }
1883 RESTORE_RUN_ACTION_RERUN
1884}
1885
1886fn enforce_restore_run_executable(
1888 journal: &RestoreApplyJournal,
1889 report: &RestoreApplyJournalReport,
1890) -> Result<(), RestoreCommandError> {
1891 if report.pending_operations > 0 {
1892 return Err(RestoreCommandError::RestoreApplyPending {
1893 backup_id: report.backup_id.clone(),
1894 pending_operations: report.pending_operations,
1895 next_transition_sequence: report
1896 .next_transition
1897 .as_ref()
1898 .map(|operation| operation.sequence),
1899 });
1900 }
1901
1902 if report.failed_operations > 0 {
1903 return Err(RestoreCommandError::RestoreApplyFailed {
1904 backup_id: report.backup_id.clone(),
1905 failed_operations: report.failed_operations,
1906 });
1907 }
1908
1909 if report.ready {
1910 return Ok(());
1911 }
1912
1913 Err(RestoreCommandError::RestoreApplyNotReady {
1914 backup_id: journal.backup_id.clone(),
1915 reasons: report.blocked_reasons.clone(),
1916 })
1917}
1918
1919fn enforce_restore_run_command_available(
1921 preview: &RestoreApplyCommandPreview,
1922) -> Result<(), RestoreCommandError> {
1923 if preview.command_available {
1924 return Ok(());
1925 }
1926
1927 Err(restore_command_unavailable_error(preview))
1928}
1929
1930fn restore_command_unavailable_error(preview: &RestoreApplyCommandPreview) -> RestoreCommandError {
1932 RestoreCommandError::RestoreApplyCommandUnavailable {
1933 backup_id: preview.backup_id.clone(),
1934 operation_available: preview.operation_available,
1935 complete: preview.complete,
1936 blocked_reasons: preview.blocked_reasons.clone(),
1937 }
1938}
1939
1940fn exit_status_label(status: std::process::ExitStatus) -> String {
1942 status
1943 .code()
1944 .map_or_else(|| "signal".to_string(), |code| code.to_string())
1945}
1946
1947fn enforce_restore_run_requirements(
1949 options: &RestoreRunOptions,
1950 run: &RestoreRunResponse,
1951) -> Result<(), RestoreCommandError> {
1952 if options.require_complete && !run.complete {
1953 return Err(RestoreCommandError::RestoreApplyIncomplete {
1954 backup_id: run.backup_id.clone(),
1955 completed_operations: run.completed_operations,
1956 operation_count: run.operation_count,
1957 });
1958 }
1959
1960 if options.require_no_attention && run.attention_required {
1961 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1962 backup_id: run.backup_id.clone(),
1963 outcome: run.outcome.clone(),
1964 });
1965 }
1966
1967 if let Some(expected) = &options.require_run_mode
1968 && run.run_mode != expected
1969 {
1970 return Err(RestoreCommandError::RestoreRunModeMismatch {
1971 backup_id: run.backup_id.clone(),
1972 expected: expected.clone(),
1973 actual: run.run_mode.to_string(),
1974 });
1975 }
1976
1977 if let Some(expected) = &options.require_stopped_reason
1978 && run.stopped_reason != expected
1979 {
1980 return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
1981 backup_id: run.backup_id.clone(),
1982 expected: expected.clone(),
1983 actual: run.stopped_reason.to_string(),
1984 });
1985 }
1986
1987 if let Some(expected) = &options.require_next_action
1988 && run.next_action != expected
1989 {
1990 return Err(RestoreCommandError::RestoreRunNextActionMismatch {
1991 backup_id: run.backup_id.clone(),
1992 expected: expected.clone(),
1993 actual: run.next_action.to_string(),
1994 });
1995 }
1996
1997 if let Some(expected) = options.require_executed_count {
1998 let actual = run.executed_operation_count.unwrap_or(0);
1999 if actual != expected {
2000 return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
2001 backup_id: run.backup_id.clone(),
2002 expected,
2003 actual,
2004 });
2005 }
2006 }
2007
2008 enforce_restore_run_receipt_requirements(options, run)?;
2009
2010 enforce_progress_requirements(
2011 &run.backup_id,
2012 &run.progress,
2013 options.require_remaining_count,
2014 options.require_attention_count,
2015 options.require_completion_basis_points,
2016 )?;
2017 enforce_pending_before_requirement(
2018 &run.backup_id,
2019 &run.pending_summary,
2020 options.require_no_pending_before.as_deref(),
2021 )?;
2022
2023 Ok(())
2024}
2025
2026fn enforce_restore_run_receipt_requirements(
2028 options: &RestoreRunOptions,
2029 run: &RestoreRunResponse,
2030) -> Result<(), RestoreCommandError> {
2031 if let Some(expected) = options.require_receipt_count {
2032 let actual = run.operation_receipt_count.unwrap_or(0);
2033 if actual != expected {
2034 return Err(RestoreCommandError::RestoreRunReceiptCountMismatch {
2035 backup_id: run.backup_id.clone(),
2036 expected,
2037 actual,
2038 });
2039 }
2040 }
2041
2042 enforce_restore_run_receipt_kind_requirement(
2043 &run.backup_id,
2044 RESTORE_RUN_RECEIPT_COMPLETED,
2045 options.require_completed_receipt_count,
2046 run.operation_receipt_summary.command_completed,
2047 )?;
2048 enforce_restore_run_receipt_kind_requirement(
2049 &run.backup_id,
2050 RESTORE_RUN_RECEIPT_FAILED,
2051 options.require_failed_receipt_count,
2052 run.operation_receipt_summary.command_failed,
2053 )?;
2054 enforce_restore_run_receipt_kind_requirement(
2055 &run.backup_id,
2056 RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
2057 options.require_recovered_receipt_count,
2058 run.operation_receipt_summary.pending_recovered,
2059 )?;
2060 enforce_restore_run_receipt_updated_at_requirement(
2061 &run.backup_id,
2062 &run.operation_receipts,
2063 options.require_receipt_updated_at.as_deref(),
2064 )?;
2065 enforce_restore_run_state_updated_at_requirement(
2066 &run.backup_id,
2067 run.requested_state_updated_at.as_deref(),
2068 options.require_state_updated_at.as_deref(),
2069 )?;
2070
2071 Ok(())
2072}
2073
2074fn enforce_restore_run_state_updated_at_requirement(
2076 backup_id: &str,
2077 actual: Option<&str>,
2078 expected: Option<&str>,
2079) -> Result<(), RestoreCommandError> {
2080 if let Some(expected) = expected
2081 && actual != Some(expected)
2082 {
2083 return Err(RestoreCommandError::RestoreRunStateUpdatedAtMismatch {
2084 backup_id: backup_id.to_string(),
2085 expected: expected.to_string(),
2086 actual: actual.map(str::to_string),
2087 });
2088 }
2089
2090 Ok(())
2091}
2092
2093fn enforce_restore_run_receipt_updated_at_requirement(
2095 backup_id: &str,
2096 receipts: &[RestoreRunOperationReceipt],
2097 expected: Option<&str>,
2098) -> Result<(), RestoreCommandError> {
2099 let Some(expected) = expected else {
2100 return Ok(());
2101 };
2102
2103 let actual_receipts = receipts.len();
2104 let mismatched_receipts = receipts
2105 .iter()
2106 .filter(|receipt| receipt.updated_at.as_deref() != Some(expected))
2107 .count();
2108 if actual_receipts == 0 || mismatched_receipts > 0 {
2109 return Err(RestoreCommandError::RestoreRunReceiptUpdatedAtMismatch {
2110 backup_id: backup_id.to_string(),
2111 expected: expected.to_string(),
2112 actual_receipts,
2113 mismatched_receipts,
2114 });
2115 }
2116
2117 Ok(())
2118}
2119
2120fn enforce_restore_run_receipt_kind_requirement(
2122 backup_id: &str,
2123 receipt_kind: &'static str,
2124 expected: Option<usize>,
2125 actual: usize,
2126) -> Result<(), RestoreCommandError> {
2127 if let Some(expected) = expected
2128 && actual != expected
2129 {
2130 return Err(RestoreCommandError::RestoreRunReceiptKindCountMismatch {
2131 backup_id: backup_id.to_string(),
2132 receipt_kind,
2133 expected,
2134 actual,
2135 });
2136 }
2137
2138 Ok(())
2139}
2140
2141fn enforce_progress_requirements(
2143 backup_id: &str,
2144 progress: &RestoreApplyProgressSummary,
2145 require_remaining_count: Option<usize>,
2146 require_attention_count: Option<usize>,
2147 require_completion_basis_points: Option<usize>,
2148) -> Result<(), RestoreCommandError> {
2149 if let Some(expected) = require_remaining_count
2150 && progress.remaining_operations != expected
2151 {
2152 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2153 backup_id: backup_id.to_string(),
2154 field: "remaining_operations",
2155 expected,
2156 actual: progress.remaining_operations,
2157 });
2158 }
2159
2160 if let Some(expected) = require_attention_count
2161 && progress.attention_operations != expected
2162 {
2163 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2164 backup_id: backup_id.to_string(),
2165 field: "attention_operations",
2166 expected,
2167 actual: progress.attention_operations,
2168 });
2169 }
2170
2171 if let Some(expected) = require_completion_basis_points
2172 && progress.completion_basis_points != expected
2173 {
2174 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2175 backup_id: backup_id.to_string(),
2176 field: "completion_basis_points",
2177 expected,
2178 actual: progress.completion_basis_points,
2179 });
2180 }
2181
2182 Ok(())
2183}
2184
2185fn enforce_pending_before_requirement(
2187 backup_id: &str,
2188 pending: &RestoreApplyPendingSummary,
2189 require_no_pending_before: Option<&str>,
2190) -> Result<(), RestoreCommandError> {
2191 let Some(cutoff_updated_at) = require_no_pending_before else {
2192 return Ok(());
2193 };
2194
2195 if pending.pending_operations == 0 {
2196 return Ok(());
2197 }
2198
2199 if pending.pending_updated_at_known
2200 && pending
2201 .pending_updated_at
2202 .as_deref()
2203 .is_some_and(|updated_at| updated_at >= cutoff_updated_at)
2204 {
2205 return Ok(());
2206 }
2207
2208 Err(RestoreCommandError::RestoreApplyPendingStale {
2209 backup_id: backup_id.to_string(),
2210 cutoff_updated_at: cutoff_updated_at.to_string(),
2211 pending_sequence: pending.pending_sequence,
2212 pending_updated_at: pending.pending_updated_at.clone(),
2213 })
2214}
2215
2216fn enforce_apply_report_requirements(
2218 options: &RestoreApplyReportOptions,
2219 report: &RestoreApplyJournalReport,
2220) -> Result<(), RestoreCommandError> {
2221 if options.require_no_attention && report.attention_required {
2222 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
2223 backup_id: report.backup_id.clone(),
2224 outcome: report.outcome.clone(),
2225 });
2226 }
2227
2228 enforce_progress_requirements(
2229 &report.backup_id,
2230 &report.progress,
2231 options.require_remaining_count,
2232 options.require_attention_count,
2233 options.require_completion_basis_points,
2234 )?;
2235 enforce_pending_before_requirement(
2236 &report.backup_id,
2237 &report.pending_summary,
2238 options.require_no_pending_before.as_deref(),
2239 )
2240}
2241
2242fn enforce_apply_status_requirements(
2244 options: &RestoreApplyStatusOptions,
2245 status: &RestoreApplyJournalStatus,
2246) -> Result<(), RestoreCommandError> {
2247 if options.require_ready && !status.ready {
2248 return Err(RestoreCommandError::RestoreApplyNotReady {
2249 backup_id: status.backup_id.clone(),
2250 reasons: status.blocked_reasons.clone(),
2251 });
2252 }
2253
2254 if options.require_no_pending && status.pending_operations > 0 {
2255 return Err(RestoreCommandError::RestoreApplyPending {
2256 backup_id: status.backup_id.clone(),
2257 pending_operations: status.pending_operations,
2258 next_transition_sequence: status.next_transition_sequence,
2259 });
2260 }
2261
2262 if options.require_no_failed && status.failed_operations > 0 {
2263 return Err(RestoreCommandError::RestoreApplyFailed {
2264 backup_id: status.backup_id.clone(),
2265 failed_operations: status.failed_operations,
2266 });
2267 }
2268
2269 if options.require_complete && !status.complete {
2270 return Err(RestoreCommandError::RestoreApplyIncomplete {
2271 backup_id: status.backup_id.clone(),
2272 completed_operations: status.completed_operations,
2273 operation_count: status.operation_count,
2274 });
2275 }
2276
2277 enforce_progress_requirements(
2278 &status.backup_id,
2279 &status.progress,
2280 options.require_remaining_count,
2281 options.require_attention_count,
2282 options.require_completion_basis_points,
2283 )?;
2284 enforce_pending_before_requirement(
2285 &status.backup_id,
2286 &status.pending_summary,
2287 options.require_no_pending_before.as_deref(),
2288 )?;
2289
2290 Ok(())
2291}
2292
2293pub fn restore_apply_next(
2295 options: &RestoreApplyNextOptions,
2296) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
2297 let journal = read_apply_journal(&options.journal)?;
2298 Ok(journal.next_operation())
2299}
2300
2301pub fn restore_apply_command(
2303 options: &RestoreApplyCommandOptions,
2304) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
2305 let journal = read_apply_journal(&options.journal)?;
2306 Ok(journal.next_command_preview_with_config(&restore_apply_command_config(options)))
2307}
2308
2309fn enforce_apply_command_requirements(
2311 options: &RestoreApplyCommandOptions,
2312 preview: &RestoreApplyCommandPreview,
2313) -> Result<(), RestoreCommandError> {
2314 if !options.require_command || preview.command_available {
2315 return Ok(());
2316 }
2317
2318 Err(restore_command_unavailable_error(preview))
2319}
2320
2321pub fn restore_apply_claim(
2323 options: &RestoreApplyClaimOptions,
2324) -> Result<RestoreApplyJournal, RestoreCommandError> {
2325 let mut journal = read_apply_journal(&options.journal)?;
2326 let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
2327
2328 if let Some(sequence) = options.sequence {
2329 enforce_apply_claim_sequence(sequence, &journal)?;
2330 journal.mark_operation_pending_at(sequence, updated_at)?;
2331 return Ok(journal);
2332 }
2333
2334 journal.mark_next_operation_pending_at(updated_at)?;
2335 Ok(journal)
2336}
2337
2338fn enforce_apply_claim_sequence(
2340 expected: usize,
2341 journal: &RestoreApplyJournal,
2342) -> Result<(), RestoreCommandError> {
2343 let actual = journal
2344 .next_transition_operation()
2345 .map(|operation| operation.sequence);
2346
2347 if actual == Some(expected) {
2348 return Ok(());
2349 }
2350
2351 Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
2352}
2353
2354pub fn restore_apply_unclaim(
2356 options: &RestoreApplyUnclaimOptions,
2357) -> Result<RestoreApplyJournal, RestoreCommandError> {
2358 let mut journal = read_apply_journal(&options.journal)?;
2359 if let Some(sequence) = options.sequence {
2360 enforce_apply_unclaim_sequence(sequence, &journal)?;
2361 }
2362
2363 journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
2364 Ok(journal)
2365}
2366
2367fn enforce_apply_unclaim_sequence(
2369 expected: usize,
2370 journal: &RestoreApplyJournal,
2371) -> Result<(), RestoreCommandError> {
2372 let actual = journal
2373 .next_transition_operation()
2374 .map(|operation| operation.sequence);
2375
2376 if actual == Some(expected) {
2377 return Ok(());
2378 }
2379
2380 Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
2381}
2382
2383pub fn restore_apply_mark(
2385 options: &RestoreApplyMarkOptions,
2386) -> Result<RestoreApplyJournal, RestoreCommandError> {
2387 let mut journal = read_apply_journal(&options.journal)?;
2388 enforce_apply_mark_pending_requirement(options, &journal)?;
2389
2390 match options.state {
2391 RestoreApplyMarkState::Completed => {
2392 journal.mark_operation_completed_at(
2393 options.sequence,
2394 Some(state_updated_at(options.updated_at.as_ref())),
2395 )?;
2396 }
2397 RestoreApplyMarkState::Failed => {
2398 let reason =
2399 options
2400 .reason
2401 .clone()
2402 .ok_or(RestoreApplyJournalError::FailureReasonRequired(
2403 options.sequence,
2404 ))?;
2405 journal.mark_operation_failed_at(
2406 options.sequence,
2407 reason,
2408 Some(state_updated_at(options.updated_at.as_ref())),
2409 )?;
2410 }
2411 }
2412
2413 Ok(journal)
2414}
2415
2416fn enforce_apply_mark_pending_requirement(
2418 options: &RestoreApplyMarkOptions,
2419 journal: &RestoreApplyJournal,
2420) -> Result<(), RestoreCommandError> {
2421 if !options.require_pending {
2422 return Ok(());
2423 }
2424
2425 let state = journal
2426 .operations
2427 .iter()
2428 .find(|operation| operation.sequence == options.sequence)
2429 .map(|operation| operation.state.clone())
2430 .ok_or(RestoreApplyJournalError::OperationNotFound(
2431 options.sequence,
2432 ))?;
2433
2434 if state == RestoreApplyOperationState::Pending {
2435 return Ok(());
2436 }
2437
2438 Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
2439 sequence: options.sequence,
2440 state,
2441 })
2442}
2443
2444fn enforce_restore_plan_requirements(
2446 options: &RestorePlanOptions,
2447 plan: &RestorePlan,
2448) -> Result<(), RestoreCommandError> {
2449 if !options.require_restore_ready || plan.readiness_summary.ready {
2450 return Ok(());
2451 }
2452
2453 Err(RestoreCommandError::RestoreNotReady {
2454 backup_id: plan.backup_id.clone(),
2455 reasons: plan.readiness_summary.reasons.clone(),
2456 })
2457}
2458
2459fn verify_backup_layout_if_required(
2461 options: &RestorePlanOptions,
2462) -> Result<(), RestoreCommandError> {
2463 if !options.require_verified {
2464 return Ok(());
2465 }
2466
2467 let Some(dir) = &options.backup_dir else {
2468 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
2469 };
2470
2471 BackupLayout::new(dir.clone()).verify_integrity()?;
2472 Ok(())
2473}
2474
2475fn read_manifest_source(
2477 options: &RestorePlanOptions,
2478) -> Result<FleetBackupManifest, RestoreCommandError> {
2479 if let Some(path) = &options.manifest {
2480 return read_manifest(path);
2481 }
2482
2483 let Some(dir) = &options.backup_dir else {
2484 return Err(RestoreCommandError::MissingOption(
2485 "--manifest or --backup-dir",
2486 ));
2487 };
2488
2489 BackupLayout::new(dir.clone())
2490 .read_manifest()
2491 .map_err(RestoreCommandError::from)
2492}
2493
2494fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
2496 let data = fs::read_to_string(path)?;
2497 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2498}
2499
2500fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
2502 let data = fs::read_to_string(path)?;
2503 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2504}
2505
2506fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
2508 let data = fs::read_to_string(path)?;
2509 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2510}
2511
2512fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
2514 let data = fs::read_to_string(path)?;
2515 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2516}
2517
2518fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
2520 let data = fs::read_to_string(path)?;
2521 let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
2522 journal.validate()?;
2523 Ok(journal)
2524}
2525
2526fn parse_progress_requirement_option<I>(
2528 arg: &str,
2529 args: &mut I,
2530 require_remaining_count: &mut Option<usize>,
2531 require_attention_count: &mut Option<usize>,
2532 require_completion_basis_points: &mut Option<usize>,
2533) -> Result<bool, RestoreCommandError>
2534where
2535 I: Iterator<Item = OsString>,
2536{
2537 match arg {
2538 "--require-remaining-count" => {
2539 *require_remaining_count = Some(parse_sequence(next_value(
2540 args,
2541 "--require-remaining-count",
2542 )?)?);
2543 Ok(true)
2544 }
2545 "--require-attention-count" => {
2546 *require_attention_count = Some(parse_sequence(next_value(
2547 args,
2548 "--require-attention-count",
2549 )?)?);
2550 Ok(true)
2551 }
2552 "--require-completion-basis-points" => {
2553 *require_completion_basis_points = Some(parse_sequence(next_value(
2554 args,
2555 "--require-completion-basis-points",
2556 )?)?);
2557 Ok(true)
2558 }
2559 _ => Ok(false),
2560 }
2561}
2562
2563fn parse_pending_requirement_option<I>(
2565 arg: &str,
2566 args: &mut I,
2567 require_no_pending_before: &mut Option<String>,
2568) -> Result<bool, RestoreCommandError>
2569where
2570 I: Iterator<Item = OsString>,
2571{
2572 match arg {
2573 "--require-no-pending-before" => {
2574 *require_no_pending_before = Some(next_value(args, "--require-no-pending-before")?);
2575 Ok(true)
2576 }
2577 _ => Ok(false),
2578 }
2579}
2580
2581fn parse_run_count_requirement_option<I>(
2583 arg: &str,
2584 args: &mut I,
2585 require_executed_count: &mut Option<usize>,
2586 require_receipt_count: &mut Option<usize>,
2587) -> Result<bool, RestoreCommandError>
2588where
2589 I: Iterator<Item = OsString>,
2590{
2591 match arg {
2592 "--require-executed-count" => {
2593 *require_executed_count = Some(parse_sequence(next_value(
2594 args,
2595 "--require-executed-count",
2596 )?)?);
2597 Ok(true)
2598 }
2599 "--require-receipt-count" => {
2600 *require_receipt_count = Some(parse_sequence(next_value(
2601 args,
2602 "--require-receipt-count",
2603 )?)?);
2604 Ok(true)
2605 }
2606 _ => Ok(false),
2607 }
2608}
2609
2610fn parse_run_receipt_kind_requirement_option<I>(
2612 arg: &str,
2613 args: &mut I,
2614 require_completed_receipt_count: &mut Option<usize>,
2615 require_failed_receipt_count: &mut Option<usize>,
2616 require_recovered_receipt_count: &mut Option<usize>,
2617) -> Result<bool, RestoreCommandError>
2618where
2619 I: Iterator<Item = OsString>,
2620{
2621 match arg {
2622 "--require-completed-receipt-count" => {
2623 *require_completed_receipt_count = Some(parse_sequence(next_value(
2624 args,
2625 "--require-completed-receipt-count",
2626 )?)?);
2627 Ok(true)
2628 }
2629 "--require-failed-receipt-count" => {
2630 *require_failed_receipt_count = Some(parse_sequence(next_value(
2631 args,
2632 "--require-failed-receipt-count",
2633 )?)?);
2634 Ok(true)
2635 }
2636 "--require-recovered-receipt-count" => {
2637 *require_recovered_receipt_count = Some(parse_sequence(next_value(
2638 args,
2639 "--require-recovered-receipt-count",
2640 )?)?);
2641 Ok(true)
2642 }
2643 _ => Ok(false),
2644 }
2645}
2646
2647fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
2649 value
2650 .parse::<usize>()
2651 .map_err(|_| RestoreCommandError::InvalidSequence)
2652}
2653
2654fn state_updated_at(updated_at: Option<&String>) -> String {
2656 updated_at.cloned().unwrap_or_else(timestamp_placeholder)
2657}
2658
2659fn timestamp_placeholder() -> String {
2661 "unknown".to_string()
2662}
2663
2664fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
2666 if let Some(path) = &options.out {
2667 let data = serde_json::to_vec_pretty(plan)?;
2668 fs::write(path, data)?;
2669 return Ok(());
2670 }
2671
2672 let stdout = io::stdout();
2673 let mut handle = stdout.lock();
2674 serde_json::to_writer_pretty(&mut handle, plan)?;
2675 writeln!(handle)?;
2676 Ok(())
2677}
2678
2679fn write_status(
2681 options: &RestoreStatusOptions,
2682 status: &RestoreStatus,
2683) -> Result<(), RestoreCommandError> {
2684 if let Some(path) = &options.out {
2685 let data = serde_json::to_vec_pretty(status)?;
2686 fs::write(path, data)?;
2687 return Ok(());
2688 }
2689
2690 let stdout = io::stdout();
2691 let mut handle = stdout.lock();
2692 serde_json::to_writer_pretty(&mut handle, status)?;
2693 writeln!(handle)?;
2694 Ok(())
2695}
2696
2697fn write_apply_dry_run(
2699 options: &RestoreApplyOptions,
2700 dry_run: &RestoreApplyDryRun,
2701) -> Result<(), RestoreCommandError> {
2702 if let Some(path) = &options.out {
2703 let data = serde_json::to_vec_pretty(dry_run)?;
2704 fs::write(path, data)?;
2705 return Ok(());
2706 }
2707
2708 let stdout = io::stdout();
2709 let mut handle = stdout.lock();
2710 serde_json::to_writer_pretty(&mut handle, dry_run)?;
2711 writeln!(handle)?;
2712 Ok(())
2713}
2714
2715fn write_apply_journal_if_requested(
2717 options: &RestoreApplyOptions,
2718 dry_run: &RestoreApplyDryRun,
2719) -> Result<(), RestoreCommandError> {
2720 let Some(path) = &options.journal_out else {
2721 return Ok(());
2722 };
2723
2724 let journal = RestoreApplyJournal::from_dry_run(dry_run);
2725 let data = serde_json::to_vec_pretty(&journal)?;
2726 fs::write(path, data)?;
2727 Ok(())
2728}
2729
2730fn write_apply_status(
2732 options: &RestoreApplyStatusOptions,
2733 status: &RestoreApplyJournalStatus,
2734) -> Result<(), RestoreCommandError> {
2735 if let Some(path) = &options.out {
2736 let data = serde_json::to_vec_pretty(status)?;
2737 fs::write(path, data)?;
2738 return Ok(());
2739 }
2740
2741 let stdout = io::stdout();
2742 let mut handle = stdout.lock();
2743 serde_json::to_writer_pretty(&mut handle, status)?;
2744 writeln!(handle)?;
2745 Ok(())
2746}
2747
2748fn write_apply_report(
2750 options: &RestoreApplyReportOptions,
2751 report: &RestoreApplyJournalReport,
2752) -> Result<(), RestoreCommandError> {
2753 if let Some(path) = &options.out {
2754 let data = serde_json::to_vec_pretty(report)?;
2755 fs::write(path, data)?;
2756 return Ok(());
2757 }
2758
2759 let stdout = io::stdout();
2760 let mut handle = stdout.lock();
2761 serde_json::to_writer_pretty(&mut handle, report)?;
2762 writeln!(handle)?;
2763 Ok(())
2764}
2765
2766fn write_restore_run(
2768 options: &RestoreRunOptions,
2769 run: &RestoreRunResponse,
2770) -> Result<(), RestoreCommandError> {
2771 if let Some(path) = &options.out {
2772 let data = serde_json::to_vec_pretty(run)?;
2773 fs::write(path, data)?;
2774 return Ok(());
2775 }
2776
2777 let stdout = io::stdout();
2778 let mut handle = stdout.lock();
2779 serde_json::to_writer_pretty(&mut handle, run)?;
2780 writeln!(handle)?;
2781 Ok(())
2782}
2783
2784fn write_apply_journal_file(
2786 path: &PathBuf,
2787 journal: &RestoreApplyJournal,
2788) -> Result<(), RestoreCommandError> {
2789 let data = serde_json::to_vec_pretty(journal)?;
2790 fs::write(path, data)?;
2791 Ok(())
2792}
2793
2794fn write_apply_next(
2796 options: &RestoreApplyNextOptions,
2797 next: &RestoreApplyNextOperation,
2798) -> Result<(), RestoreCommandError> {
2799 if let Some(path) = &options.out {
2800 let data = serde_json::to_vec_pretty(next)?;
2801 fs::write(path, data)?;
2802 return Ok(());
2803 }
2804
2805 let stdout = io::stdout();
2806 let mut handle = stdout.lock();
2807 serde_json::to_writer_pretty(&mut handle, next)?;
2808 writeln!(handle)?;
2809 Ok(())
2810}
2811
2812fn write_apply_command(
2814 options: &RestoreApplyCommandOptions,
2815 preview: &RestoreApplyCommandPreview,
2816) -> Result<(), RestoreCommandError> {
2817 if let Some(path) = &options.out {
2818 let data = serde_json::to_vec_pretty(preview)?;
2819 fs::write(path, data)?;
2820 return Ok(());
2821 }
2822
2823 let stdout = io::stdout();
2824 let mut handle = stdout.lock();
2825 serde_json::to_writer_pretty(&mut handle, preview)?;
2826 writeln!(handle)?;
2827 Ok(())
2828}
2829
2830fn write_apply_claim(
2832 options: &RestoreApplyClaimOptions,
2833 journal: &RestoreApplyJournal,
2834) -> Result<(), RestoreCommandError> {
2835 if let Some(path) = &options.out {
2836 let data = serde_json::to_vec_pretty(journal)?;
2837 fs::write(path, data)?;
2838 return Ok(());
2839 }
2840
2841 let stdout = io::stdout();
2842 let mut handle = stdout.lock();
2843 serde_json::to_writer_pretty(&mut handle, journal)?;
2844 writeln!(handle)?;
2845 Ok(())
2846}
2847
2848fn write_apply_unclaim(
2850 options: &RestoreApplyUnclaimOptions,
2851 journal: &RestoreApplyJournal,
2852) -> Result<(), RestoreCommandError> {
2853 if let Some(path) = &options.out {
2854 let data = serde_json::to_vec_pretty(journal)?;
2855 fs::write(path, data)?;
2856 return Ok(());
2857 }
2858
2859 let stdout = io::stdout();
2860 let mut handle = stdout.lock();
2861 serde_json::to_writer_pretty(&mut handle, journal)?;
2862 writeln!(handle)?;
2863 Ok(())
2864}
2865
2866fn write_apply_mark(
2868 options: &RestoreApplyMarkOptions,
2869 journal: &RestoreApplyJournal,
2870) -> Result<(), RestoreCommandError> {
2871 if let Some(path) = &options.out {
2872 let data = serde_json::to_vec_pretty(journal)?;
2873 fs::write(path, data)?;
2874 return Ok(());
2875 }
2876
2877 let stdout = io::stdout();
2878 let mut handle = stdout.lock();
2879 serde_json::to_writer_pretty(&mut handle, journal)?;
2880 writeln!(handle)?;
2881 Ok(())
2882}
2883
2884fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
2886where
2887 I: Iterator<Item = OsString>,
2888{
2889 args.next()
2890 .and_then(|value| value.into_string().ok())
2891 .ok_or(RestoreCommandError::MissingValue(option))
2892}
2893
2894const fn usage() -> &'static str {
2896 "usage: canic restore plan (--manifest <file> | --backup-dir <dir>) [--mapping <file>] [--out <file>] [--require-verified] [--require-restore-ready]\n canic restore status --plan <file> [--out <file>]\n canic restore apply --plan <file> [--status <file>] [--backup-dir <dir>] --dry-run [--out <file>] [--journal-out <file>]\n canic restore apply-status --journal <file> [--out <file>] [--require-ready] [--require-no-pending] [--require-no-failed] [--require-complete] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n canic restore apply-report --journal <file> [--out <file>] [--require-no-attention] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n canic restore run --journal <file> (--dry-run | --execute | --unclaim-pending) [--dfx <path>] [--network <name>] [--max-steps <n>] [--updated-at <text>] [--out <file>] [--require-complete] [--require-no-attention] [--require-run-mode <text>] [--require-stopped-reason <text>] [--require-next-action <text>] [--require-executed-count <n>] [--require-receipt-count <n>] [--require-completed-receipt-count <n>] [--require-failed-receipt-count <n>] [--require-recovered-receipt-count <n>] [--require-receipt-updated-at <text>] [--require-state-updated-at <text>] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n canic restore apply-next --journal <file> [--out <file>]\n canic restore apply-command --journal <file> [--dfx <path>] [--network <name>] [--out <file>] [--require-command]\n canic restore apply-claim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-unclaim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-mark --journal <file> --sequence <n> --state completed|failed [--reason <text>] [--updated-at <text>] [--out <file>] [--require-pending]"
2897}
2898
2899#[cfg(test)]
2900mod tests {
2901 use super::*;
2902 use canic_backup::restore::RestoreApplyOperationState;
2903 use canic_backup::{
2904 artifacts::ArtifactChecksum,
2905 journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
2906 manifest::{
2907 BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
2908 FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
2909 VerificationCheck, VerificationPlan,
2910 },
2911 };
2912 use serde_json::json;
2913 use std::{
2914 path::Path,
2915 time::{SystemTime, UNIX_EPOCH},
2916 };
2917
2918 const ROOT: &str = "aaaaa-aa";
2919 const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2920 const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2921 const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2922
2923 struct RestoreCliFixture {
2928 root: PathBuf,
2929 journal_path: PathBuf,
2930 out_path: PathBuf,
2931 }
2932
2933 impl RestoreCliFixture {
2934 fn new(prefix: &str, out_file: &str) -> Self {
2936 let root = temp_dir(prefix);
2937 fs::create_dir_all(&root).expect("create temp root");
2938
2939 Self {
2940 journal_path: root.join("restore-apply-journal.json"),
2941 out_path: root.join(out_file),
2942 root,
2943 }
2944 }
2945
2946 fn write_journal(&self, journal: &RestoreApplyJournal) {
2948 fs::write(
2949 &self.journal_path,
2950 serde_json::to_vec(journal).expect("serialize journal"),
2951 )
2952 .expect("write journal");
2953 }
2954
2955 fn run_apply_status(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
2957 self.run_journal_command("apply-status", extra)
2958 }
2959
2960 fn run_apply_report(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
2962 self.run_journal_command("apply-report", extra)
2963 }
2964
2965 fn run_restore_run(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
2967 self.run_journal_command("run", extra)
2968 }
2969
2970 fn read_out<T>(&self, label: &str) -> T
2972 where
2973 T: serde::de::DeserializeOwned,
2974 {
2975 serde_json::from_slice(&fs::read(&self.out_path).expect(label)).expect(label)
2976 }
2977
2978 fn run_journal_command(
2980 &self,
2981 command: &str,
2982 extra: &[&str],
2983 ) -> Result<(), RestoreCommandError> {
2984 let mut args = vec![
2985 OsString::from(command),
2986 OsString::from("--journal"),
2987 OsString::from(self.journal_path.as_os_str()),
2988 OsString::from("--out"),
2989 OsString::from(self.out_path.as_os_str()),
2990 ];
2991 args.extend(extra.iter().map(OsString::from));
2992 run(args)
2993 }
2994 }
2995
2996 impl Drop for RestoreCliFixture {
2997 fn drop(&mut self) {
2999 let _ = fs::remove_dir_all(&self.root);
3000 }
3001 }
3002
3003 #[test]
3005 fn parses_restore_plan_options() {
3006 let options = RestorePlanOptions::parse([
3007 OsString::from("--manifest"),
3008 OsString::from("manifest.json"),
3009 OsString::from("--mapping"),
3010 OsString::from("mapping.json"),
3011 OsString::from("--out"),
3012 OsString::from("plan.json"),
3013 OsString::from("--require-restore-ready"),
3014 ])
3015 .expect("parse options");
3016
3017 assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
3018 assert_eq!(options.backup_dir, None);
3019 assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
3020 assert_eq!(options.out, Some(PathBuf::from("plan.json")));
3021 assert!(!options.require_verified);
3022 assert!(options.require_restore_ready);
3023 }
3024
3025 #[test]
3027 fn parses_verified_restore_plan_options() {
3028 let options = RestorePlanOptions::parse([
3029 OsString::from("--backup-dir"),
3030 OsString::from("backups/run"),
3031 OsString::from("--require-verified"),
3032 ])
3033 .expect("parse verified options");
3034
3035 assert_eq!(options.manifest, None);
3036 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
3037 assert_eq!(options.mapping, None);
3038 assert_eq!(options.out, None);
3039 assert!(options.require_verified);
3040 assert!(!options.require_restore_ready);
3041 }
3042
3043 #[test]
3045 fn parses_restore_status_options() {
3046 let options = RestoreStatusOptions::parse([
3047 OsString::from("--plan"),
3048 OsString::from("restore-plan.json"),
3049 OsString::from("--out"),
3050 OsString::from("restore-status.json"),
3051 ])
3052 .expect("parse status options");
3053
3054 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
3055 assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
3056 }
3057
3058 #[test]
3060 fn parses_restore_apply_dry_run_options() {
3061 let options = RestoreApplyOptions::parse([
3062 OsString::from("--plan"),
3063 OsString::from("restore-plan.json"),
3064 OsString::from("--status"),
3065 OsString::from("restore-status.json"),
3066 OsString::from("--backup-dir"),
3067 OsString::from("backups/run"),
3068 OsString::from("--dry-run"),
3069 OsString::from("--out"),
3070 OsString::from("restore-apply-dry-run.json"),
3071 OsString::from("--journal-out"),
3072 OsString::from("restore-apply-journal.json"),
3073 ])
3074 .expect("parse apply options");
3075
3076 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
3077 assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
3078 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
3079 assert_eq!(
3080 options.out,
3081 Some(PathBuf::from("restore-apply-dry-run.json"))
3082 );
3083 assert_eq!(
3084 options.journal_out,
3085 Some(PathBuf::from("restore-apply-journal.json"))
3086 );
3087 assert!(options.dry_run);
3088 }
3089
3090 #[test]
3092 fn parses_restore_apply_status_options() {
3093 let options = RestoreApplyStatusOptions::parse([
3094 OsString::from("--journal"),
3095 OsString::from("restore-apply-journal.json"),
3096 OsString::from("--out"),
3097 OsString::from("restore-apply-status.json"),
3098 OsString::from("--require-ready"),
3099 OsString::from("--require-no-pending"),
3100 OsString::from("--require-no-failed"),
3101 OsString::from("--require-complete"),
3102 OsString::from("--require-remaining-count"),
3103 OsString::from("7"),
3104 OsString::from("--require-attention-count"),
3105 OsString::from("0"),
3106 OsString::from("--require-completion-basis-points"),
3107 OsString::from("1250"),
3108 OsString::from("--require-no-pending-before"),
3109 OsString::from("2026-05-05T12:00:00Z"),
3110 ])
3111 .expect("parse apply-status options");
3112
3113 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3114 assert!(options.require_ready);
3115 assert!(options.require_no_pending);
3116 assert!(options.require_no_failed);
3117 assert!(options.require_complete);
3118 assert_eq!(options.require_remaining_count, Some(7));
3119 assert_eq!(options.require_attention_count, Some(0));
3120 assert_eq!(options.require_completion_basis_points, Some(1250));
3121 assert_eq!(
3122 options.require_no_pending_before.as_deref(),
3123 Some("2026-05-05T12:00:00Z")
3124 );
3125 assert_eq!(
3126 options.out,
3127 Some(PathBuf::from("restore-apply-status.json"))
3128 );
3129 }
3130
3131 #[test]
3133 fn parses_restore_apply_report_options() {
3134 let options = RestoreApplyReportOptions::parse([
3135 OsString::from("--journal"),
3136 OsString::from("restore-apply-journal.json"),
3137 OsString::from("--out"),
3138 OsString::from("restore-apply-report.json"),
3139 OsString::from("--require-no-attention"),
3140 OsString::from("--require-remaining-count"),
3141 OsString::from("8"),
3142 OsString::from("--require-attention-count"),
3143 OsString::from("0"),
3144 OsString::from("--require-completion-basis-points"),
3145 OsString::from("0"),
3146 OsString::from("--require-no-pending-before"),
3147 OsString::from("2026-05-05T12:00:00Z"),
3148 ])
3149 .expect("parse apply-report options");
3150
3151 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3152 assert!(options.require_no_attention);
3153 assert_eq!(options.require_remaining_count, Some(8));
3154 assert_eq!(options.require_attention_count, Some(0));
3155 assert_eq!(options.require_completion_basis_points, Some(0));
3156 assert_eq!(
3157 options.require_no_pending_before.as_deref(),
3158 Some("2026-05-05T12:00:00Z")
3159 );
3160 assert_eq!(
3161 options.out,
3162 Some(PathBuf::from("restore-apply-report.json"))
3163 );
3164 }
3165
3166 #[test]
3168 fn parses_restore_run_dry_run_options() {
3169 let options = RestoreRunOptions::parse([
3170 OsString::from("--journal"),
3171 OsString::from("restore-apply-journal.json"),
3172 OsString::from("--dry-run"),
3173 OsString::from("--dfx"),
3174 OsString::from("/tmp/dfx"),
3175 OsString::from("--network"),
3176 OsString::from("local"),
3177 OsString::from("--out"),
3178 OsString::from("restore-run-dry-run.json"),
3179 OsString::from("--max-steps"),
3180 OsString::from("1"),
3181 OsString::from("--updated-at"),
3182 OsString::from("2026-05-05T12:03:00Z"),
3183 OsString::from("--require-complete"),
3184 OsString::from("--require-no-attention"),
3185 OsString::from("--require-run-mode"),
3186 OsString::from("dry-run"),
3187 OsString::from("--require-stopped-reason"),
3188 OsString::from("preview"),
3189 OsString::from("--require-next-action"),
3190 OsString::from("rerun"),
3191 OsString::from("--require-executed-count"),
3192 OsString::from("0"),
3193 OsString::from("--require-receipt-count"),
3194 OsString::from("0"),
3195 OsString::from("--require-completed-receipt-count"),
3196 OsString::from("0"),
3197 OsString::from("--require-failed-receipt-count"),
3198 OsString::from("0"),
3199 OsString::from("--require-recovered-receipt-count"),
3200 OsString::from("0"),
3201 OsString::from("--require-receipt-updated-at"),
3202 OsString::from("2026-05-05T12:03:00Z"),
3203 OsString::from("--require-state-updated-at"),
3204 OsString::from("2026-05-05T12:03:00Z"),
3205 OsString::from("--require-remaining-count"),
3206 OsString::from("8"),
3207 OsString::from("--require-attention-count"),
3208 OsString::from("0"),
3209 OsString::from("--require-completion-basis-points"),
3210 OsString::from("0"),
3211 OsString::from("--require-no-pending-before"),
3212 OsString::from("2026-05-05T12:00:00Z"),
3213 ])
3214 .expect("parse restore run options");
3215
3216 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3217 assert_eq!(options.dfx, "/tmp/dfx");
3218 assert_eq!(options.network.as_deref(), Some("local"));
3219 assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
3220 assert!(options.dry_run);
3221 assert!(!options.execute);
3222 assert!(!options.unclaim_pending);
3223 assert_eq!(options.max_steps, Some(1));
3224 assert_eq!(options.updated_at.as_deref(), Some("2026-05-05T12:03:00Z"));
3225 assert!(options.require_complete);
3226 assert!(options.require_no_attention);
3227 assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
3228 assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
3229 assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
3230 assert_eq!(options.require_executed_count, Some(0));
3231 assert_eq!(options.require_receipt_count, Some(0));
3232 assert_eq!(options.require_completed_receipt_count, Some(0));
3233 assert_eq!(options.require_failed_receipt_count, Some(0));
3234 assert_eq!(options.require_recovered_receipt_count, Some(0));
3235 assert_eq!(
3236 options.require_receipt_updated_at.as_deref(),
3237 Some("2026-05-05T12:03:00Z")
3238 );
3239 assert_eq!(
3240 options.require_state_updated_at.as_deref(),
3241 Some("2026-05-05T12:03:00Z")
3242 );
3243 assert_eq!(options.require_remaining_count, Some(8));
3244 assert_eq!(options.require_attention_count, Some(0));
3245 assert_eq!(options.require_completion_basis_points, Some(0));
3246 assert_eq!(
3247 options.require_no_pending_before.as_deref(),
3248 Some("2026-05-05T12:00:00Z")
3249 );
3250 }
3251
3252 #[test]
3254 fn parses_restore_run_execute_options() {
3255 let options = RestoreRunOptions::parse([
3256 OsString::from("--journal"),
3257 OsString::from("restore-apply-journal.json"),
3258 OsString::from("--execute"),
3259 OsString::from("--dfx"),
3260 OsString::from("/bin/true"),
3261 OsString::from("--max-steps"),
3262 OsString::from("4"),
3263 ])
3264 .expect("parse restore run execute options");
3265
3266 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3267 assert_eq!(options.dfx, "/bin/true");
3268 assert_eq!(options.network, None);
3269 assert_eq!(options.out, None);
3270 assert!(!options.dry_run);
3271 assert!(options.execute);
3272 assert!(!options.unclaim_pending);
3273 assert_eq!(options.max_steps, Some(4));
3274 assert_eq!(options.updated_at, None);
3275 assert!(!options.require_complete);
3276 assert!(!options.require_no_attention);
3277 assert_eq!(options.require_run_mode, None);
3278 assert_eq!(options.require_stopped_reason, None);
3279 assert_eq!(options.require_next_action, None);
3280 assert_eq!(options.require_executed_count, None);
3281 assert_eq!(options.require_receipt_count, None);
3282 assert_eq!(options.require_completed_receipt_count, None);
3283 assert_eq!(options.require_failed_receipt_count, None);
3284 assert_eq!(options.require_recovered_receipt_count, None);
3285 assert_eq!(options.require_receipt_updated_at, None);
3286 assert_eq!(options.require_state_updated_at, None);
3287 }
3288
3289 #[test]
3291 fn parses_restore_run_unclaim_pending_options() {
3292 let options = RestoreRunOptions::parse([
3293 OsString::from("--journal"),
3294 OsString::from("restore-apply-journal.json"),
3295 OsString::from("--unclaim-pending"),
3296 OsString::from("--out"),
3297 OsString::from("restore-run.json"),
3298 ])
3299 .expect("parse restore run unclaim options");
3300
3301 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3302 assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
3303 assert!(!options.dry_run);
3304 assert!(!options.execute);
3305 assert!(options.unclaim_pending);
3306 }
3307
3308 #[test]
3310 fn parses_restore_apply_next_options() {
3311 let options = RestoreApplyNextOptions::parse([
3312 OsString::from("--journal"),
3313 OsString::from("restore-apply-journal.json"),
3314 OsString::from("--out"),
3315 OsString::from("restore-apply-next.json"),
3316 ])
3317 .expect("parse apply-next options");
3318
3319 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3320 assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
3321 }
3322
3323 #[test]
3325 fn parses_restore_apply_command_options() {
3326 let options = RestoreApplyCommandOptions::parse([
3327 OsString::from("--journal"),
3328 OsString::from("restore-apply-journal.json"),
3329 OsString::from("--dfx"),
3330 OsString::from("/tmp/dfx"),
3331 OsString::from("--network"),
3332 OsString::from("local"),
3333 OsString::from("--out"),
3334 OsString::from("restore-apply-command.json"),
3335 OsString::from("--require-command"),
3336 ])
3337 .expect("parse apply-command options");
3338
3339 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3340 assert_eq!(options.dfx, "/tmp/dfx");
3341 assert_eq!(options.network.as_deref(), Some("local"));
3342 assert!(options.require_command);
3343 assert_eq!(
3344 options.out,
3345 Some(PathBuf::from("restore-apply-command.json"))
3346 );
3347 }
3348
3349 #[test]
3351 fn parses_restore_apply_claim_options() {
3352 let options = RestoreApplyClaimOptions::parse([
3353 OsString::from("--journal"),
3354 OsString::from("restore-apply-journal.json"),
3355 OsString::from("--sequence"),
3356 OsString::from("0"),
3357 OsString::from("--updated-at"),
3358 OsString::from("2026-05-04T12:00:00Z"),
3359 OsString::from("--out"),
3360 OsString::from("restore-apply-journal.claimed.json"),
3361 ])
3362 .expect("parse apply-claim options");
3363
3364 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3365 assert_eq!(options.sequence, Some(0));
3366 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
3367 assert_eq!(
3368 options.out,
3369 Some(PathBuf::from("restore-apply-journal.claimed.json"))
3370 );
3371 }
3372
3373 #[test]
3375 fn parses_restore_apply_unclaim_options() {
3376 let options = RestoreApplyUnclaimOptions::parse([
3377 OsString::from("--journal"),
3378 OsString::from("restore-apply-journal.json"),
3379 OsString::from("--sequence"),
3380 OsString::from("0"),
3381 OsString::from("--updated-at"),
3382 OsString::from("2026-05-04T12:01:00Z"),
3383 OsString::from("--out"),
3384 OsString::from("restore-apply-journal.unclaimed.json"),
3385 ])
3386 .expect("parse apply-unclaim options");
3387
3388 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3389 assert_eq!(options.sequence, Some(0));
3390 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
3391 assert_eq!(
3392 options.out,
3393 Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
3394 );
3395 }
3396
3397 #[test]
3399 fn parses_restore_apply_mark_options() {
3400 let options = RestoreApplyMarkOptions::parse([
3401 OsString::from("--journal"),
3402 OsString::from("restore-apply-journal.json"),
3403 OsString::from("--sequence"),
3404 OsString::from("4"),
3405 OsString::from("--state"),
3406 OsString::from("failed"),
3407 OsString::from("--reason"),
3408 OsString::from("dfx-load-failed"),
3409 OsString::from("--updated-at"),
3410 OsString::from("2026-05-04T12:02:00Z"),
3411 OsString::from("--out"),
3412 OsString::from("restore-apply-journal.updated.json"),
3413 OsString::from("--require-pending"),
3414 ])
3415 .expect("parse apply-mark options");
3416
3417 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3418 assert_eq!(options.sequence, 4);
3419 assert_eq!(options.state, RestoreApplyMarkState::Failed);
3420 assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
3421 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
3422 assert!(options.require_pending);
3423 assert_eq!(
3424 options.out,
3425 Some(PathBuf::from("restore-apply-journal.updated.json"))
3426 );
3427 }
3428
3429 #[test]
3431 fn restore_apply_requires_dry_run() {
3432 let err = RestoreApplyOptions::parse([
3433 OsString::from("--plan"),
3434 OsString::from("restore-plan.json"),
3435 ])
3436 .expect_err("apply without dry-run should fail");
3437
3438 assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
3439 }
3440
3441 #[test]
3443 fn restore_run_requires_mode() {
3444 let err = RestoreRunOptions::parse([
3445 OsString::from("--journal"),
3446 OsString::from("restore-apply-journal.json"),
3447 ])
3448 .expect_err("restore run without dry-run should fail");
3449
3450 assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
3451 }
3452
3453 #[test]
3455 fn restore_run_rejects_conflicting_modes() {
3456 let err = RestoreRunOptions::parse([
3457 OsString::from("--journal"),
3458 OsString::from("restore-apply-journal.json"),
3459 OsString::from("--dry-run"),
3460 OsString::from("--execute"),
3461 OsString::from("--unclaim-pending"),
3462 ])
3463 .expect_err("restore run should reject conflicting modes");
3464
3465 assert!(matches!(
3466 err,
3467 RestoreCommandError::RestoreRunConflictingModes
3468 ));
3469 }
3470
3471 #[test]
3473 fn plan_restore_reads_manifest_from_backup_dir() {
3474 let root = temp_dir("canic-cli-restore-plan-layout");
3475 let layout = BackupLayout::new(root.clone());
3476 layout
3477 .write_manifest(&valid_manifest())
3478 .expect("write manifest");
3479
3480 let options = RestorePlanOptions {
3481 manifest: None,
3482 backup_dir: Some(root.clone()),
3483 mapping: None,
3484 out: None,
3485 require_verified: false,
3486 require_restore_ready: false,
3487 };
3488
3489 let plan = plan_restore(&options).expect("plan restore");
3490
3491 fs::remove_dir_all(root).expect("remove temp root");
3492 assert_eq!(plan.backup_id, "backup-test");
3493 assert_eq!(plan.member_count, 2);
3494 }
3495
3496 #[test]
3498 fn parse_rejects_conflicting_manifest_sources() {
3499 let err = RestorePlanOptions::parse([
3500 OsString::from("--manifest"),
3501 OsString::from("manifest.json"),
3502 OsString::from("--backup-dir"),
3503 OsString::from("backups/run"),
3504 ])
3505 .expect_err("conflicting sources should fail");
3506
3507 assert!(matches!(
3508 err,
3509 RestoreCommandError::ConflictingManifestSources
3510 ));
3511 }
3512
3513 #[test]
3515 fn parse_rejects_require_verified_with_manifest_source() {
3516 let err = RestorePlanOptions::parse([
3517 OsString::from("--manifest"),
3518 OsString::from("manifest.json"),
3519 OsString::from("--require-verified"),
3520 ])
3521 .expect_err("verification should require a backup layout");
3522
3523 assert!(matches!(
3524 err,
3525 RestoreCommandError::RequireVerifiedNeedsBackupDir
3526 ));
3527 }
3528
3529 #[test]
3531 fn plan_restore_requires_verified_backup_layout() {
3532 let root = temp_dir("canic-cli-restore-plan-verified");
3533 let layout = BackupLayout::new(root.clone());
3534 let manifest = valid_manifest();
3535 write_verified_layout(&root, &layout, &manifest);
3536
3537 let options = RestorePlanOptions {
3538 manifest: None,
3539 backup_dir: Some(root.clone()),
3540 mapping: None,
3541 out: None,
3542 require_verified: true,
3543 require_restore_ready: false,
3544 };
3545
3546 let plan = plan_restore(&options).expect("plan verified restore");
3547
3548 fs::remove_dir_all(root).expect("remove temp root");
3549 assert_eq!(plan.backup_id, "backup-test");
3550 assert_eq!(plan.member_count, 2);
3551 }
3552
3553 #[test]
3555 fn plan_restore_rejects_unverified_backup_layout() {
3556 let root = temp_dir("canic-cli-restore-plan-unverified");
3557 let layout = BackupLayout::new(root.clone());
3558 layout
3559 .write_manifest(&valid_manifest())
3560 .expect("write manifest");
3561
3562 let options = RestorePlanOptions {
3563 manifest: None,
3564 backup_dir: Some(root.clone()),
3565 mapping: None,
3566 out: None,
3567 require_verified: true,
3568 require_restore_ready: false,
3569 };
3570
3571 let err = plan_restore(&options).expect_err("missing journal should fail");
3572
3573 fs::remove_dir_all(root).expect("remove temp root");
3574 assert!(matches!(err, RestoreCommandError::Persistence(_)));
3575 }
3576
3577 #[test]
3579 fn plan_restore_reads_manifest_and_mapping() {
3580 let root = temp_dir("canic-cli-restore-plan");
3581 fs::create_dir_all(&root).expect("create temp root");
3582 let manifest_path = root.join("manifest.json");
3583 let mapping_path = root.join("mapping.json");
3584
3585 fs::write(
3586 &manifest_path,
3587 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
3588 )
3589 .expect("write manifest");
3590 fs::write(
3591 &mapping_path,
3592 json!({
3593 "members": [
3594 {"source_canister": ROOT, "target_canister": ROOT},
3595 {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
3596 ]
3597 })
3598 .to_string(),
3599 )
3600 .expect("write mapping");
3601
3602 let options = RestorePlanOptions {
3603 manifest: Some(manifest_path),
3604 backup_dir: None,
3605 mapping: Some(mapping_path),
3606 out: None,
3607 require_verified: false,
3608 require_restore_ready: false,
3609 };
3610
3611 let plan = plan_restore(&options).expect("plan restore");
3612
3613 fs::remove_dir_all(root).expect("remove temp root");
3614 let members = plan.ordered_members();
3615 assert_eq!(members.len(), 2);
3616 assert_eq!(members[0].source_canister, ROOT);
3617 assert_eq!(members[1].target_canister, MAPPED_CHILD);
3618 }
3619
3620 #[test]
3622 fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
3623 let root = temp_dir("canic-cli-restore-plan-require-ready");
3624 fs::create_dir_all(&root).expect("create temp root");
3625 let manifest_path = root.join("manifest.json");
3626 let out_path = root.join("plan.json");
3627
3628 fs::write(
3629 &manifest_path,
3630 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
3631 )
3632 .expect("write manifest");
3633
3634 let err = run([
3635 OsString::from("plan"),
3636 OsString::from("--manifest"),
3637 OsString::from(manifest_path.as_os_str()),
3638 OsString::from("--out"),
3639 OsString::from(out_path.as_os_str()),
3640 OsString::from("--require-restore-ready"),
3641 ])
3642 .expect_err("restore readiness should be enforced");
3643
3644 assert!(out_path.exists());
3645 let plan: RestorePlan =
3646 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
3647
3648 fs::remove_dir_all(root).expect("remove temp root");
3649 assert!(!plan.readiness_summary.ready);
3650 assert!(matches!(
3651 err,
3652 RestoreCommandError::RestoreNotReady {
3653 reasons,
3654 ..
3655 } if reasons == [
3656 "missing-module-hash",
3657 "missing-wasm-hash",
3658 "missing-snapshot-checksum"
3659 ]
3660 ));
3661 }
3662
3663 #[test]
3665 fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
3666 let root = temp_dir("canic-cli-restore-plan-ready");
3667 fs::create_dir_all(&root).expect("create temp root");
3668 let manifest_path = root.join("manifest.json");
3669 let out_path = root.join("plan.json");
3670
3671 fs::write(
3672 &manifest_path,
3673 serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
3674 )
3675 .expect("write manifest");
3676
3677 run([
3678 OsString::from("plan"),
3679 OsString::from("--manifest"),
3680 OsString::from(manifest_path.as_os_str()),
3681 OsString::from("--out"),
3682 OsString::from(out_path.as_os_str()),
3683 OsString::from("--require-restore-ready"),
3684 ])
3685 .expect("restore-ready plan should pass");
3686
3687 let plan: RestorePlan =
3688 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
3689
3690 fs::remove_dir_all(root).expect("remove temp root");
3691 assert!(plan.readiness_summary.ready);
3692 assert!(plan.readiness_summary.reasons.is_empty());
3693 }
3694
3695 #[test]
3697 fn run_restore_status_writes_planned_status() {
3698 let root = temp_dir("canic-cli-restore-status");
3699 fs::create_dir_all(&root).expect("create temp root");
3700 let plan_path = root.join("restore-plan.json");
3701 let out_path = root.join("restore-status.json");
3702 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3703
3704 fs::write(
3705 &plan_path,
3706 serde_json::to_vec(&plan).expect("serialize plan"),
3707 )
3708 .expect("write plan");
3709
3710 run([
3711 OsString::from("status"),
3712 OsString::from("--plan"),
3713 OsString::from(plan_path.as_os_str()),
3714 OsString::from("--out"),
3715 OsString::from(out_path.as_os_str()),
3716 ])
3717 .expect("write restore status");
3718
3719 let status: RestoreStatus =
3720 serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
3721 .expect("decode restore status");
3722 let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
3723
3724 fs::remove_dir_all(root).expect("remove temp root");
3725 assert_eq!(status.status_version, 1);
3726 assert_eq!(status.backup_id.as_str(), "backup-test");
3727 assert!(status.ready);
3728 assert!(status.readiness_reasons.is_empty());
3729 assert_eq!(status.member_count, 2);
3730 assert_eq!(status.phase_count, 1);
3731 assert_eq!(status.planned_snapshot_uploads, 2);
3732 assert_eq!(status.planned_snapshot_loads, 2);
3733 assert_eq!(status.planned_code_reinstalls, 2);
3734 assert_eq!(status.planned_verification_checks, 2);
3735 assert_eq!(status.planned_operations, 8);
3736 assert_eq!(status.phases[0].members[0].source_canister, ROOT);
3737 assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
3738 }
3739
3740 #[test]
3742 fn run_restore_apply_dry_run_writes_operations() {
3743 let root = temp_dir("canic-cli-restore-apply-dry-run");
3744 fs::create_dir_all(&root).expect("create temp root");
3745 let plan_path = root.join("restore-plan.json");
3746 let status_path = root.join("restore-status.json");
3747 let out_path = root.join("restore-apply-dry-run.json");
3748 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3749 let status = RestoreStatus::from_plan(&plan);
3750
3751 fs::write(
3752 &plan_path,
3753 serde_json::to_vec(&plan).expect("serialize plan"),
3754 )
3755 .expect("write plan");
3756 fs::write(
3757 &status_path,
3758 serde_json::to_vec(&status).expect("serialize status"),
3759 )
3760 .expect("write status");
3761
3762 run([
3763 OsString::from("apply"),
3764 OsString::from("--plan"),
3765 OsString::from(plan_path.as_os_str()),
3766 OsString::from("--status"),
3767 OsString::from(status_path.as_os_str()),
3768 OsString::from("--dry-run"),
3769 OsString::from("--out"),
3770 OsString::from(out_path.as_os_str()),
3771 ])
3772 .expect("write apply dry-run");
3773
3774 let dry_run: RestoreApplyDryRun =
3775 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3776 .expect("decode dry-run");
3777 let dry_run_json: serde_json::Value =
3778 serde_json::to_value(&dry_run).expect("encode dry-run");
3779
3780 fs::remove_dir_all(root).expect("remove temp root");
3781 assert_eq!(dry_run.dry_run_version, 1);
3782 assert_eq!(dry_run.backup_id.as_str(), "backup-test");
3783 assert!(dry_run.ready);
3784 assert!(dry_run.status_supplied);
3785 assert_eq!(dry_run.member_count, 2);
3786 assert_eq!(dry_run.phase_count, 1);
3787 assert_eq!(dry_run.planned_snapshot_uploads, 2);
3788 assert_eq!(dry_run.planned_operations, 8);
3789 assert_eq!(dry_run.rendered_operations, 8);
3790 assert_eq!(dry_run_json["operation_counts"]["snapshot_uploads"], 2);
3791 assert_eq!(dry_run_json["operation_counts"]["snapshot_loads"], 2);
3792 assert_eq!(dry_run_json["operation_counts"]["code_reinstalls"], 2);
3793 assert_eq!(dry_run_json["operation_counts"]["member_verifications"], 2);
3794 assert_eq!(dry_run_json["operation_counts"]["fleet_verifications"], 0);
3795 assert_eq!(
3796 dry_run_json["operation_counts"]["verification_operations"],
3797 2
3798 );
3799 assert_eq!(
3800 dry_run_json["phases"][0]["operations"][0]["operation"],
3801 "upload-snapshot"
3802 );
3803 assert_eq!(
3804 dry_run_json["phases"][0]["operations"][3]["operation"],
3805 "verify-member"
3806 );
3807 assert_eq!(
3808 dry_run_json["phases"][0]["operations"][3]["verification_kind"],
3809 "status"
3810 );
3811 assert_eq!(
3812 dry_run_json["phases"][0]["operations"][3]["verification_method"],
3813 serde_json::Value::Null
3814 );
3815 }
3816
3817 #[test]
3819 fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
3820 let root = temp_dir("canic-cli-restore-apply-artifacts");
3821 fs::create_dir_all(&root).expect("create temp root");
3822 let plan_path = root.join("restore-plan.json");
3823 let out_path = root.join("restore-apply-dry-run.json");
3824 let journal_path = root.join("restore-apply-journal.json");
3825 let status_path = root.join("restore-apply-status.json");
3826 let mut manifest = restore_ready_manifest();
3827 write_manifest_artifacts(&root, &mut manifest);
3828 let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
3829
3830 fs::write(
3831 &plan_path,
3832 serde_json::to_vec(&plan).expect("serialize plan"),
3833 )
3834 .expect("write plan");
3835
3836 run([
3837 OsString::from("apply"),
3838 OsString::from("--plan"),
3839 OsString::from(plan_path.as_os_str()),
3840 OsString::from("--backup-dir"),
3841 OsString::from(root.as_os_str()),
3842 OsString::from("--dry-run"),
3843 OsString::from("--out"),
3844 OsString::from(out_path.as_os_str()),
3845 OsString::from("--journal-out"),
3846 OsString::from(journal_path.as_os_str()),
3847 ])
3848 .expect("write apply dry-run");
3849 run([
3850 OsString::from("apply-status"),
3851 OsString::from("--journal"),
3852 OsString::from(journal_path.as_os_str()),
3853 OsString::from("--out"),
3854 OsString::from(status_path.as_os_str()),
3855 ])
3856 .expect("write apply status");
3857
3858 let dry_run: RestoreApplyDryRun =
3859 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3860 .expect("decode dry-run");
3861 let validation = dry_run
3862 .artifact_validation
3863 .expect("artifact validation should be present");
3864 let journal_json: serde_json::Value =
3865 serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
3866 .expect("decode journal");
3867 let status_json: serde_json::Value =
3868 serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
3869 .expect("decode apply status");
3870
3871 fs::remove_dir_all(root).expect("remove temp root");
3872 assert_eq!(validation.checked_members, 2);
3873 assert!(validation.artifacts_present);
3874 assert!(validation.checksums_verified);
3875 assert_eq!(validation.members_with_expected_checksums, 2);
3876 assert_eq!(journal_json["ready"], true);
3877 assert_eq!(journal_json["operation_count"], 8);
3878 assert_eq!(journal_json["operation_counts"]["snapshot_uploads"], 2);
3879 assert_eq!(journal_json["operation_counts"]["snapshot_loads"], 2);
3880 assert_eq!(journal_json["operation_counts"]["code_reinstalls"], 2);
3881 assert_eq!(journal_json["operation_counts"]["member_verifications"], 2);
3882 assert_eq!(journal_json["operation_counts"]["fleet_verifications"], 0);
3883 assert_eq!(
3884 journal_json["operation_counts"]["verification_operations"],
3885 2
3886 );
3887 assert_eq!(journal_json["ready_operations"], 8);
3888 assert_eq!(journal_json["blocked_operations"], 0);
3889 assert_eq!(journal_json["operations"][0]["state"], "ready");
3890 assert_eq!(status_json["ready"], true);
3891 assert_eq!(status_json["operation_count"], 8);
3892 assert_eq!(status_json["operation_counts"]["snapshot_uploads"], 2);
3893 assert_eq!(status_json["operation_counts"]["snapshot_loads"], 2);
3894 assert_eq!(status_json["operation_counts"]["code_reinstalls"], 2);
3895 assert_eq!(status_json["operation_counts"]["member_verifications"], 2);
3896 assert_eq!(status_json["operation_counts"]["fleet_verifications"], 0);
3897 assert_eq!(
3898 status_json["operation_counts"]["verification_operations"],
3899 2
3900 );
3901 assert_eq!(status_json["operation_counts_supplied"], true);
3902 assert_eq!(status_json["progress"]["operation_count"], 8);
3903 assert_eq!(status_json["progress"]["completed_operations"], 0);
3904 assert_eq!(status_json["progress"]["remaining_operations"], 8);
3905 assert_eq!(status_json["progress"]["transitionable_operations"], 8);
3906 assert_eq!(status_json["progress"]["attention_operations"], 0);
3907 assert_eq!(status_json["progress"]["completion_basis_points"], 0);
3908 assert_eq!(status_json["next_ready_sequence"], 0);
3909 assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
3910 }
3911
3912 #[test]
3914 fn run_restore_apply_status_rejects_invalid_journal() {
3915 let root = temp_dir("canic-cli-restore-apply-status-invalid");
3916 fs::create_dir_all(&root).expect("create temp root");
3917 let journal_path = root.join("restore-apply-journal.json");
3918 let out_path = root.join("restore-apply-status.json");
3919 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3920 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3921 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3922 journal.operation_count += 1;
3923
3924 fs::write(
3925 &journal_path,
3926 serde_json::to_vec(&journal).expect("serialize journal"),
3927 )
3928 .expect("write journal");
3929
3930 let err = run([
3931 OsString::from("apply-status"),
3932 OsString::from("--journal"),
3933 OsString::from(journal_path.as_os_str()),
3934 OsString::from("--out"),
3935 OsString::from(out_path.as_os_str()),
3936 ])
3937 .expect_err("invalid journal should fail");
3938
3939 assert!(!out_path.exists());
3940 fs::remove_dir_all(root).expect("remove temp root");
3941 assert!(matches!(
3942 err,
3943 RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
3944 field: "operation_count",
3945 ..
3946 })
3947 ));
3948 }
3949
3950 #[test]
3952 fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
3953 let fixture = RestoreCliFixture::new(
3954 "canic-cli-restore-apply-status-pending",
3955 "restore-apply-status.json",
3956 );
3957 let mut journal = ready_apply_journal();
3958 journal
3959 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
3960 .expect("claim operation");
3961 fixture.write_journal(&journal);
3962
3963 let err = fixture
3964 .run_apply_status(&["--require-no-pending"])
3965 .expect_err("pending operation should fail requirement");
3966
3967 assert!(fixture.out_path.exists());
3968 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
3969
3970 assert_eq!(status.pending_operations, 1);
3971 assert_eq!(status.next_transition_sequence, Some(0));
3972 assert_eq!(status.pending_summary.pending_operations, 1);
3973 assert_eq!(status.pending_summary.pending_sequence, Some(0));
3974 assert_eq!(
3975 status.pending_summary.pending_updated_at.as_deref(),
3976 Some("2026-05-04T12:00:00Z")
3977 );
3978 assert!(status.pending_summary.pending_updated_at_known);
3979 assert_eq!(
3980 status.next_transition_updated_at.as_deref(),
3981 Some("2026-05-04T12:00:00Z")
3982 );
3983 assert!(matches!(
3984 err,
3985 RestoreCommandError::RestoreApplyPending {
3986 pending_operations: 1,
3987 next_transition_sequence: Some(0),
3988 ..
3989 }
3990 ));
3991 }
3992
3993 #[test]
3995 fn run_restore_apply_status_require_no_pending_before_writes_status_then_fails() {
3996 let fixture = RestoreCliFixture::new(
3997 "canic-cli-restore-apply-status-stale-pending",
3998 "restore-apply-status.json",
3999 );
4000 let mut journal = ready_apply_journal();
4001 journal
4002 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4003 .expect("claim operation");
4004 fixture.write_journal(&journal);
4005
4006 let err = fixture
4007 .run_apply_status(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
4008 .expect_err("stale pending operation should fail requirement");
4009
4010 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4011
4012 assert_eq!(status.pending_summary.pending_sequence, Some(0));
4013 assert_eq!(
4014 status.pending_summary.pending_updated_at.as_deref(),
4015 Some("2026-05-04T12:00:00Z")
4016 );
4017 assert!(matches!(
4018 err,
4019 RestoreCommandError::RestoreApplyPendingStale {
4020 cutoff_updated_at,
4021 pending_sequence: Some(0),
4022 pending_updated_at,
4023 ..
4024 } if cutoff_updated_at == "2026-05-05T12:00:00Z"
4025 && pending_updated_at.as_deref() == Some("2026-05-04T12:00:00Z")
4026 ));
4027 }
4028
4029 #[test]
4031 fn run_restore_apply_status_require_progress_writes_status_then_fails() {
4032 let fixture = RestoreCliFixture::new(
4033 "canic-cli-restore-apply-status-progress",
4034 "restore-apply-status.json",
4035 );
4036 let journal = ready_apply_journal();
4037 fixture.write_journal(&journal);
4038
4039 let err = fixture
4040 .run_apply_status(&[
4041 "--require-remaining-count",
4042 "7",
4043 "--require-attention-count",
4044 "0",
4045 "--require-completion-basis-points",
4046 "0",
4047 ])
4048 .expect_err("remaining progress mismatch should fail requirement");
4049
4050 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4051
4052 assert_eq!(status.progress.remaining_operations, 8);
4053 assert_eq!(status.progress.attention_operations, 0);
4054 assert_eq!(status.progress.completion_basis_points, 0);
4055 assert!(matches!(
4056 err,
4057 RestoreCommandError::RestoreApplyProgressMismatch {
4058 field: "remaining_operations",
4059 expected: 7,
4060 actual: 8,
4061 ..
4062 }
4063 ));
4064 }
4065
4066 #[test]
4068 fn run_restore_apply_status_require_ready_writes_status_then_fails() {
4069 let root = temp_dir("canic-cli-restore-apply-status-ready");
4070 fs::create_dir_all(&root).expect("create temp root");
4071 let journal_path = root.join("restore-apply-journal.json");
4072 let out_path = root.join("restore-apply-status.json");
4073 let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
4074 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4075 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
4076
4077 fs::write(
4078 &journal_path,
4079 serde_json::to_vec(&journal).expect("serialize journal"),
4080 )
4081 .expect("write journal");
4082
4083 let err = run([
4084 OsString::from("apply-status"),
4085 OsString::from("--journal"),
4086 OsString::from(journal_path.as_os_str()),
4087 OsString::from("--out"),
4088 OsString::from(out_path.as_os_str()),
4089 OsString::from("--require-ready"),
4090 ])
4091 .expect_err("unready journal should fail requirement");
4092
4093 let status: RestoreApplyJournalStatus =
4094 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
4095 .expect("decode apply status");
4096
4097 fs::remove_dir_all(root).expect("remove temp root");
4098 assert!(!status.ready);
4099 assert_eq!(status.blocked_operations, status.operation_count);
4100 assert!(
4101 status
4102 .blocked_reasons
4103 .contains(&"missing-snapshot-checksum".to_string())
4104 );
4105 assert!(matches!(
4106 err,
4107 RestoreCommandError::RestoreApplyNotReady { reasons, .. }
4108 if reasons.contains(&"missing-snapshot-checksum".to_string())
4109 ));
4110 }
4111
4112 #[test]
4114 fn run_restore_apply_report_writes_attention_summary() {
4115 let root = temp_dir("canic-cli-restore-apply-report");
4116 fs::create_dir_all(&root).expect("create temp root");
4117 let journal_path = root.join("restore-apply-journal.json");
4118 let out_path = root.join("restore-apply-report.json");
4119 let mut journal = ready_apply_journal();
4120 journal
4121 .mark_operation_failed_at(
4122 0,
4123 "dfx-upload-failed".to_string(),
4124 Some("2026-05-05T12:00:00Z".to_string()),
4125 )
4126 .expect("mark failed operation");
4127 journal
4128 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4129 .expect("mark pending operation");
4130
4131 fs::write(
4132 &journal_path,
4133 serde_json::to_vec(&journal).expect("serialize journal"),
4134 )
4135 .expect("write journal");
4136
4137 run([
4138 OsString::from("apply-report"),
4139 OsString::from("--journal"),
4140 OsString::from(journal_path.as_os_str()),
4141 OsString::from("--out"),
4142 OsString::from(out_path.as_os_str()),
4143 ])
4144 .expect("write apply report");
4145
4146 let report: RestoreApplyJournalReport =
4147 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
4148 .expect("decode apply report");
4149 let report_json: serde_json::Value =
4150 serde_json::to_value(&report).expect("encode apply report");
4151
4152 fs::remove_dir_all(root).expect("remove temp root");
4153 assert_eq!(report.backup_id, "backup-test");
4154 assert!(report.attention_required);
4155 assert_eq!(report.failed_operations, 1);
4156 assert_eq!(report.pending_operations, 1);
4157 assert_eq!(report.operation_counts.snapshot_uploads, 2);
4158 assert_eq!(report.operation_counts.snapshot_loads, 2);
4159 assert_eq!(report.operation_counts.code_reinstalls, 2);
4160 assert_eq!(report.operation_counts.member_verifications, 2);
4161 assert_eq!(report.operation_counts.fleet_verifications, 0);
4162 assert_eq!(report.operation_counts.verification_operations, 2);
4163 assert!(report.operation_counts_supplied);
4164 assert_eq!(report.progress.operation_count, 8);
4165 assert_eq!(report.progress.completed_operations, 0);
4166 assert_eq!(report.progress.remaining_operations, 8);
4167 assert_eq!(report.progress.transitionable_operations, 7);
4168 assert_eq!(report.progress.attention_operations, 2);
4169 assert_eq!(report.progress.completion_basis_points, 0);
4170 assert_eq!(report.pending_summary.pending_operations, 1);
4171 assert_eq!(report.pending_summary.pending_sequence, Some(1));
4172 assert_eq!(
4173 report.pending_summary.pending_updated_at.as_deref(),
4174 Some("2026-05-05T12:01:00Z")
4175 );
4176 assert!(report.pending_summary.pending_updated_at_known);
4177 assert_eq!(report.failed.len(), 1);
4178 assert_eq!(report.pending.len(), 1);
4179 assert_eq!(report.failed[0].sequence, 0);
4180 assert_eq!(report.pending[0].sequence, 1);
4181 assert_eq!(
4182 report.next_transition.as_ref().map(|op| op.sequence),
4183 Some(1)
4184 );
4185 assert_eq!(report_json["outcome"], "failed");
4186 assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
4187 }
4188
4189 #[test]
4191 fn run_restore_apply_report_require_progress_writes_report_then_fails() {
4192 let fixture = RestoreCliFixture::new(
4193 "canic-cli-restore-apply-report-progress",
4194 "restore-apply-report.json",
4195 );
4196 let journal = ready_apply_journal();
4197 fixture.write_journal(&journal);
4198
4199 let err = fixture
4200 .run_apply_report(&[
4201 "--require-remaining-count",
4202 "8",
4203 "--require-attention-count",
4204 "1",
4205 "--require-completion-basis-points",
4206 "0",
4207 ])
4208 .expect_err("attention progress mismatch should fail requirement");
4209
4210 let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4211
4212 assert_eq!(report.progress.remaining_operations, 8);
4213 assert_eq!(report.progress.attention_operations, 0);
4214 assert_eq!(report.progress.completion_basis_points, 0);
4215 assert!(matches!(
4216 err,
4217 RestoreCommandError::RestoreApplyProgressMismatch {
4218 field: "attention_operations",
4219 expected: 1,
4220 actual: 0,
4221 ..
4222 }
4223 ));
4224 }
4225
4226 #[test]
4228 fn run_restore_apply_report_require_no_pending_before_writes_report_then_fails() {
4229 let fixture = RestoreCliFixture::new(
4230 "canic-cli-restore-apply-report-stale-pending",
4231 "restore-apply-report.json",
4232 );
4233 let mut journal = ready_apply_journal();
4234 journal
4235 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4236 .expect("mark pending operation");
4237 fixture.write_journal(&journal);
4238
4239 let err = fixture
4240 .run_apply_report(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
4241 .expect_err("stale pending report should fail requirement");
4242
4243 let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4244
4245 assert_eq!(report.pending_summary.pending_sequence, Some(0));
4246 assert!(matches!(
4247 err,
4248 RestoreCommandError::RestoreApplyPendingStale {
4249 pending_sequence: Some(0),
4250 ..
4251 }
4252 ));
4253 }
4254
4255 #[test]
4257 fn run_restore_run_dry_run_writes_native_runner_preview() {
4258 let root = temp_dir("canic-cli-restore-run-dry-run");
4259 fs::create_dir_all(&root).expect("create temp root");
4260 let journal_path = root.join("restore-apply-journal.json");
4261 let out_path = root.join("restore-run-dry-run.json");
4262 let journal = ready_apply_journal();
4263
4264 fs::write(
4265 &journal_path,
4266 serde_json::to_vec(&journal).expect("serialize journal"),
4267 )
4268 .expect("write journal");
4269
4270 run([
4271 OsString::from("run"),
4272 OsString::from("--journal"),
4273 OsString::from(journal_path.as_os_str()),
4274 OsString::from("--dry-run"),
4275 OsString::from("--dfx"),
4276 OsString::from("/tmp/dfx"),
4277 OsString::from("--network"),
4278 OsString::from("local"),
4279 OsString::from("--updated-at"),
4280 OsString::from("2026-05-05T12:00:00Z"),
4281 OsString::from("--out"),
4282 OsString::from(out_path.as_os_str()),
4283 OsString::from("--require-state-updated-at"),
4284 OsString::from("2026-05-05T12:00:00Z"),
4285 ])
4286 .expect("write restore run dry-run");
4287
4288 let dry_run: serde_json::Value =
4289 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4290 .expect("decode dry-run");
4291
4292 fs::remove_dir_all(root).expect("remove temp root");
4293 assert_eq!(dry_run["run_version"], 1);
4294 assert_eq!(dry_run["backup_id"], "backup-test");
4295 assert_eq!(dry_run["run_mode"], "dry-run");
4296 assert_eq!(dry_run["dry_run"], true);
4297 assert_eq!(
4298 dry_run["requested_state_updated_at"],
4299 "2026-05-05T12:00:00Z"
4300 );
4301 assert_eq!(dry_run["ready"], true);
4302 assert_eq!(dry_run["complete"], false);
4303 assert_eq!(dry_run["attention_required"], false);
4304 assert_eq!(dry_run["operation_counts"]["snapshot_uploads"], 2);
4305 assert_eq!(dry_run["operation_counts"]["snapshot_loads"], 2);
4306 assert_eq!(dry_run["operation_counts"]["code_reinstalls"], 2);
4307 assert_eq!(dry_run["operation_counts"]["member_verifications"], 2);
4308 assert_eq!(dry_run["operation_counts"]["fleet_verifications"], 0);
4309 assert_eq!(dry_run["operation_counts"]["verification_operations"], 2);
4310 assert_eq!(dry_run["operation_counts_supplied"], true);
4311 assert_eq!(dry_run["progress"]["operation_count"], 8);
4312 assert_eq!(dry_run["progress"]["completed_operations"], 0);
4313 assert_eq!(dry_run["progress"]["remaining_operations"], 8);
4314 assert_eq!(dry_run["progress"]["transitionable_operations"], 8);
4315 assert_eq!(dry_run["progress"]["attention_operations"], 0);
4316 assert_eq!(dry_run["progress"]["completion_basis_points"], 0);
4317 assert_eq!(dry_run["pending_summary"]["pending_operations"], 0);
4318 assert_eq!(
4319 dry_run["pending_summary"]["pending_operation_available"],
4320 false
4321 );
4322 assert_eq!(dry_run["operation_receipt_count"], 0);
4323 assert_eq!(dry_run["operation_receipt_summary"]["total_receipts"], 0);
4324 assert_eq!(dry_run["operation_receipt_summary"]["command_completed"], 0);
4325 assert_eq!(dry_run["operation_receipt_summary"]["command_failed"], 0);
4326 assert_eq!(dry_run["operation_receipt_summary"]["pending_recovered"], 0);
4327 assert_eq!(dry_run["stopped_reason"], "preview");
4328 assert_eq!(dry_run["next_action"], "rerun");
4329 assert_eq!(dry_run["operation_available"], true);
4330 assert_eq!(dry_run["command_available"], true);
4331 assert_eq!(dry_run["next_transition"]["sequence"], 0);
4332 assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
4333 assert_eq!(
4334 dry_run["command"]["args"],
4335 json!([
4336 "canister",
4337 "--network",
4338 "local",
4339 "snapshot",
4340 "upload",
4341 "--dir",
4342 "artifacts/root",
4343 ROOT
4344 ])
4345 );
4346 assert_eq!(dry_run["command"]["mutates"], true);
4347 }
4348
4349 #[test]
4351 fn run_restore_run_unclaim_pending_marks_operation_ready() {
4352 let root = temp_dir("canic-cli-restore-run-unclaim-pending");
4353 fs::create_dir_all(&root).expect("create temp root");
4354 let journal_path = root.join("restore-apply-journal.json");
4355 let out_path = root.join("restore-run.json");
4356 let mut journal = ready_apply_journal();
4357 journal
4358 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4359 .expect("mark pending operation");
4360
4361 fs::write(
4362 &journal_path,
4363 serde_json::to_vec(&journal).expect("serialize journal"),
4364 )
4365 .expect("write journal");
4366
4367 run([
4368 OsString::from("run"),
4369 OsString::from("--journal"),
4370 OsString::from(journal_path.as_os_str()),
4371 OsString::from("--unclaim-pending"),
4372 OsString::from("--updated-at"),
4373 OsString::from("2026-05-05T12:02:00Z"),
4374 OsString::from("--out"),
4375 OsString::from(out_path.as_os_str()),
4376 ])
4377 .expect("unclaim pending operation");
4378
4379 let run_summary: serde_json::Value =
4380 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4381 .expect("decode run summary");
4382 let updated: RestoreApplyJournal =
4383 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4384 .expect("decode updated journal");
4385
4386 fs::remove_dir_all(root).expect("remove temp root");
4387 assert_eq!(run_summary["run_mode"], "unclaim-pending");
4388 assert_eq!(run_summary["unclaim_pending"], true);
4389 assert_eq!(run_summary["stopped_reason"], "recovered-pending");
4390 assert_eq!(run_summary["next_action"], "rerun");
4391 assert_eq!(
4392 run_summary["requested_state_updated_at"],
4393 "2026-05-05T12:02:00Z"
4394 );
4395 assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
4396 assert_eq!(run_summary["recovered_operation"]["state"], "pending");
4397 assert_eq!(run_summary["operation_receipt_count"], 1);
4398 assert_eq!(
4399 run_summary["operation_receipt_summary"]["total_receipts"],
4400 1
4401 );
4402 assert_eq!(
4403 run_summary["operation_receipt_summary"]["command_completed"],
4404 0
4405 );
4406 assert_eq!(
4407 run_summary["operation_receipt_summary"]["command_failed"],
4408 0
4409 );
4410 assert_eq!(
4411 run_summary["operation_receipt_summary"]["pending_recovered"],
4412 1
4413 );
4414 assert_eq!(
4415 run_summary["operation_receipts"][0]["event"],
4416 "pending-recovered"
4417 );
4418 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
4419 assert_eq!(run_summary["operation_receipts"][0]["state"], "ready");
4420 assert_eq!(
4421 run_summary["operation_receipts"][0]["updated_at"],
4422 "2026-05-05T12:02:00Z"
4423 );
4424 assert_eq!(run_summary["pending_operations"], 0);
4425 assert_eq!(run_summary["ready_operations"], 8);
4426 assert_eq!(run_summary["attention_required"], false);
4427 assert_eq!(updated.pending_operations, 0);
4428 assert_eq!(updated.ready_operations, 8);
4429 assert_eq!(
4430 updated.operations[0].state,
4431 RestoreApplyOperationState::Ready
4432 );
4433 assert_eq!(
4434 updated.operations[0].state_updated_at.as_deref(),
4435 Some("2026-05-05T12:02:00Z")
4436 );
4437 }
4438
4439 #[test]
4441 fn run_restore_run_execute_marks_completed_operation() {
4442 let root = temp_dir("canic-cli-restore-run-execute");
4443 fs::create_dir_all(&root).expect("create temp root");
4444 let journal_path = root.join("restore-apply-journal.json");
4445 let out_path = root.join("restore-run.json");
4446 let journal = ready_apply_journal();
4447
4448 fs::write(
4449 &journal_path,
4450 serde_json::to_vec(&journal).expect("serialize journal"),
4451 )
4452 .expect("write journal");
4453
4454 run([
4455 OsString::from("run"),
4456 OsString::from("--journal"),
4457 OsString::from(journal_path.as_os_str()),
4458 OsString::from("--execute"),
4459 OsString::from("--dfx"),
4460 OsString::from("/bin/true"),
4461 OsString::from("--max-steps"),
4462 OsString::from("1"),
4463 OsString::from("--updated-at"),
4464 OsString::from("2026-05-05T12:03:00Z"),
4465 OsString::from("--out"),
4466 OsString::from(out_path.as_os_str()),
4467 OsString::from("--require-receipt-updated-at"),
4468 OsString::from("2026-05-05T12:03:00Z"),
4469 ])
4470 .expect("execute one restore run step");
4471
4472 let run_summary: serde_json::Value =
4473 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4474 .expect("decode run summary");
4475 let updated: RestoreApplyJournal =
4476 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4477 .expect("decode updated journal");
4478
4479 fs::remove_dir_all(root).expect("remove temp root");
4480 assert_eq!(run_summary["run_mode"], "execute");
4481 assert_eq!(run_summary["execute"], true);
4482 assert_eq!(run_summary["dry_run"], false);
4483 assert_eq!(run_summary["max_steps_reached"], true);
4484 assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
4485 assert_eq!(run_summary["next_action"], "rerun");
4486 assert_eq!(
4487 run_summary["requested_state_updated_at"],
4488 "2026-05-05T12:03:00Z"
4489 );
4490 assert_eq!(run_summary["executed_operation_count"], 1);
4491 assert_eq!(run_summary["operation_receipt_count"], 1);
4492 assert_eq!(
4493 run_summary["operation_receipt_summary"]["total_receipts"],
4494 1
4495 );
4496 assert_eq!(
4497 run_summary["operation_receipt_summary"]["command_completed"],
4498 1
4499 );
4500 assert_eq!(
4501 run_summary["operation_receipt_summary"]["command_failed"],
4502 0
4503 );
4504 assert_eq!(
4505 run_summary["operation_receipt_summary"]["pending_recovered"],
4506 0
4507 );
4508 assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
4509 assert_eq!(
4510 run_summary["executed_operations"][0]["command"]["program"],
4511 "/bin/true"
4512 );
4513 assert_eq!(
4514 run_summary["operation_receipts"][0]["event"],
4515 "command-completed"
4516 );
4517 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
4518 assert_eq!(run_summary["operation_receipts"][0]["state"], "completed");
4519 assert_eq!(
4520 run_summary["operation_receipts"][0]["command"]["program"],
4521 "/bin/true"
4522 );
4523 assert_eq!(run_summary["operation_receipts"][0]["status"], "0");
4524 assert_eq!(
4525 run_summary["operation_receipts"][0]["updated_at"],
4526 "2026-05-05T12:03:00Z"
4527 );
4528 assert_eq!(updated.completed_operations, 1);
4529 assert_eq!(updated.pending_operations, 0);
4530 assert_eq!(updated.failed_operations, 0);
4531 assert_eq!(
4532 updated.operations[0].state,
4533 RestoreApplyOperationState::Completed
4534 );
4535 assert_eq!(
4536 updated.operations[0].state_updated_at.as_deref(),
4537 Some("2026-05-05T12:03:00Z")
4538 );
4539 }
4540
4541 #[test]
4543 fn run_restore_run_require_complete_writes_summary_then_fails() {
4544 let root = temp_dir("canic-cli-restore-run-require-complete");
4545 fs::create_dir_all(&root).expect("create temp root");
4546 let journal_path = root.join("restore-apply-journal.json");
4547 let out_path = root.join("restore-run.json");
4548 let journal = ready_apply_journal();
4549
4550 fs::write(
4551 &journal_path,
4552 serde_json::to_vec(&journal).expect("serialize journal"),
4553 )
4554 .expect("write journal");
4555
4556 let err = run([
4557 OsString::from("run"),
4558 OsString::from("--journal"),
4559 OsString::from(journal_path.as_os_str()),
4560 OsString::from("--execute"),
4561 OsString::from("--dfx"),
4562 OsString::from("/bin/true"),
4563 OsString::from("--max-steps"),
4564 OsString::from("1"),
4565 OsString::from("--out"),
4566 OsString::from(out_path.as_os_str()),
4567 OsString::from("--require-complete"),
4568 ])
4569 .expect_err("incomplete run should fail requirement");
4570
4571 let run_summary: serde_json::Value =
4572 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4573 .expect("decode run summary");
4574
4575 fs::remove_dir_all(root).expect("remove temp root");
4576 assert_eq!(run_summary["executed_operation_count"], 1);
4577 assert_eq!(run_summary["complete"], false);
4578 assert!(matches!(
4579 err,
4580 RestoreCommandError::RestoreApplyIncomplete {
4581 completed_operations: 1,
4582 operation_count: 8,
4583 ..
4584 }
4585 ));
4586 }
4587
4588 #[test]
4590 fn run_restore_run_execute_marks_failed_operation() {
4591 let root = temp_dir("canic-cli-restore-run-execute-failed");
4592 fs::create_dir_all(&root).expect("create temp root");
4593 let journal_path = root.join("restore-apply-journal.json");
4594 let out_path = root.join("restore-run.json");
4595 let journal = ready_apply_journal();
4596
4597 fs::write(
4598 &journal_path,
4599 serde_json::to_vec(&journal).expect("serialize journal"),
4600 )
4601 .expect("write journal");
4602
4603 let err = run([
4604 OsString::from("run"),
4605 OsString::from("--journal"),
4606 OsString::from(journal_path.as_os_str()),
4607 OsString::from("--execute"),
4608 OsString::from("--dfx"),
4609 OsString::from("/bin/false"),
4610 OsString::from("--max-steps"),
4611 OsString::from("1"),
4612 OsString::from("--updated-at"),
4613 OsString::from("2026-05-05T12:04:00Z"),
4614 OsString::from("--out"),
4615 OsString::from(out_path.as_os_str()),
4616 ])
4617 .expect_err("failing runner command should fail");
4618
4619 let run_summary: serde_json::Value =
4620 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4621 .expect("decode run summary");
4622 let updated: RestoreApplyJournal =
4623 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4624 .expect("decode updated journal");
4625
4626 fs::remove_dir_all(root).expect("remove temp root");
4627 assert!(matches!(
4628 err,
4629 RestoreCommandError::RestoreRunCommandFailed {
4630 sequence: 0,
4631 status,
4632 } if status == "1"
4633 ));
4634 assert_eq!(updated.failed_operations, 1);
4635 assert_eq!(updated.pending_operations, 0);
4636 assert_eq!(
4637 updated.operations[0].state,
4638 RestoreApplyOperationState::Failed
4639 );
4640 assert_eq!(run_summary["execute"], true);
4641 assert_eq!(run_summary["attention_required"], true);
4642 assert_eq!(run_summary["outcome"], "failed");
4643 assert_eq!(run_summary["stopped_reason"], "command-failed");
4644 assert_eq!(run_summary["next_action"], "inspect-failed-operation");
4645 assert_eq!(
4646 run_summary["requested_state_updated_at"],
4647 "2026-05-05T12:04:00Z"
4648 );
4649 assert_eq!(run_summary["executed_operation_count"], 1);
4650 assert_eq!(run_summary["operation_receipt_count"], 1);
4651 assert_eq!(
4652 run_summary["operation_receipt_summary"]["total_receipts"],
4653 1
4654 );
4655 assert_eq!(
4656 run_summary["operation_receipt_summary"]["command_completed"],
4657 0
4658 );
4659 assert_eq!(
4660 run_summary["operation_receipt_summary"]["command_failed"],
4661 1
4662 );
4663 assert_eq!(
4664 run_summary["operation_receipt_summary"]["pending_recovered"],
4665 0
4666 );
4667 assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
4668 assert_eq!(run_summary["executed_operations"][0]["status"], "1");
4669 assert_eq!(
4670 run_summary["operation_receipts"][0]["event"],
4671 "command-failed"
4672 );
4673 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
4674 assert_eq!(run_summary["operation_receipts"][0]["state"], "failed");
4675 assert_eq!(
4676 run_summary["operation_receipts"][0]["command"]["program"],
4677 "/bin/false"
4678 );
4679 assert_eq!(run_summary["operation_receipts"][0]["status"], "1");
4680 assert_eq!(
4681 run_summary["operation_receipts"][0]["updated_at"],
4682 "2026-05-05T12:04:00Z"
4683 );
4684 assert_eq!(
4685 updated.operations[0].state_updated_at.as_deref(),
4686 Some("2026-05-05T12:04:00Z")
4687 );
4688 assert_eq!(
4689 updated.operations[0].blocking_reasons,
4690 vec!["runner-command-exit-1".to_string()]
4691 );
4692 }
4693
4694 #[test]
4696 fn run_restore_run_require_no_attention_writes_summary_then_fails() {
4697 let fixture = RestoreCliFixture::new(
4698 "canic-cli-restore-run-require-attention",
4699 "restore-run.json",
4700 );
4701 let mut journal = ready_apply_journal();
4702 journal
4703 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4704 .expect("mark pending operation");
4705 fixture.write_journal(&journal);
4706
4707 let err = fixture
4708 .run_restore_run(&["--dry-run", "--require-no-attention"])
4709 .expect_err("attention run should fail requirement");
4710
4711 let run_summary: serde_json::Value = fixture.read_out("read run summary");
4712
4713 assert_eq!(run_summary["attention_required"], true);
4714 assert_eq!(run_summary["outcome"], "pending");
4715 assert_eq!(run_summary["stopped_reason"], "pending");
4716 assert_eq!(run_summary["next_action"], "unclaim-pending");
4717 assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
4718 assert_eq!(
4719 run_summary["pending_summary"]["pending_updated_at"],
4720 "2026-05-05T12:01:00Z"
4721 );
4722 assert!(matches!(
4723 err,
4724 RestoreCommandError::RestoreApplyReportNeedsAttention {
4725 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
4726 ..
4727 }
4728 ));
4729 }
4730
4731 #[test]
4733 fn run_restore_run_require_no_pending_before_writes_summary_then_fails() {
4734 let fixture = RestoreCliFixture::new(
4735 "canic-cli-restore-run-require-stale-pending",
4736 "restore-run.json",
4737 );
4738 let mut journal = ready_apply_journal();
4739 journal
4740 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4741 .expect("mark pending operation");
4742 fixture.write_journal(&journal);
4743
4744 let err = fixture
4745 .run_restore_run(&[
4746 "--dry-run",
4747 "--require-no-pending-before",
4748 "2026-05-05T12:00:00Z",
4749 ])
4750 .expect_err("stale pending run should fail requirement");
4751
4752 let run_summary: serde_json::Value = fixture.read_out("read run summary");
4753
4754 assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
4755 assert!(matches!(
4756 err,
4757 RestoreCommandError::RestoreApplyPendingStale {
4758 pending_sequence: Some(0),
4759 ..
4760 }
4761 ));
4762 }
4763
4764 #[test]
4766 fn run_restore_run_require_run_mode_writes_summary_then_fails() {
4767 let fixture =
4768 RestoreCliFixture::new("canic-cli-restore-run-require-run-mode", "restore-run.json");
4769 let journal = ready_apply_journal();
4770 fixture.write_journal(&journal);
4771
4772 let err = fixture
4773 .run_restore_run(&["--dry-run", "--require-run-mode", "execute"])
4774 .expect_err("run mode mismatch should fail requirement");
4775
4776 let run_summary: serde_json::Value = fixture.read_out("read run summary");
4777
4778 assert_eq!(run_summary["run_mode"], "dry-run");
4779 assert!(matches!(
4780 err,
4781 RestoreCommandError::RestoreRunModeMismatch {
4782 expected,
4783 actual,
4784 ..
4785 } if expected == "execute" && actual == "dry-run"
4786 ));
4787 }
4788
4789 #[test]
4791 fn run_restore_run_require_executed_count_writes_summary_then_fails() {
4792 let root = temp_dir("canic-cli-restore-run-require-executed-count");
4793 fs::create_dir_all(&root).expect("create temp root");
4794 let journal_path = root.join("restore-apply-journal.json");
4795 let out_path = root.join("restore-run.json");
4796 let journal = ready_apply_journal();
4797
4798 fs::write(
4799 &journal_path,
4800 serde_json::to_vec(&journal).expect("serialize journal"),
4801 )
4802 .expect("write journal");
4803
4804 let err = run([
4805 OsString::from("run"),
4806 OsString::from("--journal"),
4807 OsString::from(journal_path.as_os_str()),
4808 OsString::from("--execute"),
4809 OsString::from("--dfx"),
4810 OsString::from("/bin/true"),
4811 OsString::from("--max-steps"),
4812 OsString::from("1"),
4813 OsString::from("--out"),
4814 OsString::from(out_path.as_os_str()),
4815 OsString::from("--require-executed-count"),
4816 OsString::from("2"),
4817 ])
4818 .expect_err("executed count mismatch should fail requirement");
4819
4820 let run_summary: serde_json::Value =
4821 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4822 .expect("decode run summary");
4823
4824 fs::remove_dir_all(root).expect("remove temp root");
4825 assert_eq!(run_summary["executed_operation_count"], 1);
4826 assert!(matches!(
4827 err,
4828 RestoreCommandError::RestoreRunExecutedCountMismatch {
4829 expected: 2,
4830 actual: 1,
4831 ..
4832 }
4833 ));
4834 }
4835
4836 #[test]
4838 fn run_restore_run_require_receipt_count_writes_summary_then_fails() {
4839 let fixture = RestoreCliFixture::new(
4840 "canic-cli-restore-run-require-receipt-count",
4841 "restore-run.json",
4842 );
4843 let journal = ready_apply_journal();
4844 fixture.write_journal(&journal);
4845
4846 let err = fixture
4847 .run_restore_run(&[
4848 "--execute",
4849 "--dfx",
4850 "/bin/true",
4851 "--max-steps",
4852 "1",
4853 "--require-receipt-count",
4854 "2",
4855 ])
4856 .expect_err("receipt count mismatch should fail requirement");
4857
4858 let run_summary: serde_json::Value = fixture.read_out("read run summary");
4859
4860 assert_eq!(run_summary["operation_receipt_count"], 1);
4861 assert_eq!(
4862 run_summary["operation_receipt_summary"]["total_receipts"],
4863 1
4864 );
4865 assert!(matches!(
4866 err,
4867 RestoreCommandError::RestoreRunReceiptCountMismatch {
4868 expected: 2,
4869 actual: 1,
4870 ..
4871 }
4872 ));
4873 }
4874
4875 #[test]
4877 fn run_restore_run_require_receipt_kind_count_writes_summary_then_fails() {
4878 let fixture = RestoreCliFixture::new(
4879 "canic-cli-restore-run-require-receipt-kind-count",
4880 "restore-run.json",
4881 );
4882 let journal = ready_apply_journal();
4883 fixture.write_journal(&journal);
4884
4885 let err = fixture
4886 .run_restore_run(&[
4887 "--execute",
4888 "--dfx",
4889 "/bin/true",
4890 "--max-steps",
4891 "1",
4892 "--require-failed-receipt-count",
4893 "1",
4894 ])
4895 .expect_err("receipt kind count mismatch should fail requirement");
4896
4897 let run_summary: serde_json::Value = fixture.read_out("read run summary");
4898
4899 assert_eq!(
4900 run_summary["operation_receipt_summary"]["command_failed"],
4901 0
4902 );
4903 assert_eq!(
4904 run_summary["operation_receipt_summary"]["command_completed"],
4905 1
4906 );
4907 assert!(matches!(
4908 err,
4909 RestoreCommandError::RestoreRunReceiptKindCountMismatch {
4910 receipt_kind: "command-failed",
4911 expected: 1,
4912 actual: 0,
4913 ..
4914 }
4915 ));
4916 }
4917
4918 #[test]
4920 fn run_restore_run_require_receipt_updated_at_writes_summary_then_fails() {
4921 let fixture = RestoreCliFixture::new(
4922 "canic-cli-restore-run-require-receipt-updated-at",
4923 "restore-run.json",
4924 );
4925 let journal = ready_apply_journal();
4926 fixture.write_journal(&journal);
4927
4928 let err = fixture
4929 .run_restore_run(&[
4930 "--execute",
4931 "--dfx",
4932 "/bin/true",
4933 "--max-steps",
4934 "1",
4935 "--updated-at",
4936 "2026-05-05T12:03:00Z",
4937 "--require-receipt-updated-at",
4938 "2026-05-05T12:04:00Z",
4939 ])
4940 .expect_err("receipt updated-at mismatch should fail requirement");
4941
4942 let run_summary: serde_json::Value = fixture.read_out("read run summary");
4943
4944 assert_eq!(
4945 run_summary["operation_receipts"][0]["updated_at"],
4946 "2026-05-05T12:03:00Z"
4947 );
4948 assert!(matches!(
4949 err,
4950 RestoreCommandError::RestoreRunReceiptUpdatedAtMismatch {
4951 expected,
4952 actual_receipts: 1,
4953 mismatched_receipts: 1,
4954 ..
4955 } if expected == "2026-05-05T12:04:00Z"
4956 ));
4957 }
4958
4959 #[test]
4961 fn run_restore_run_require_state_updated_at_writes_summary_then_fails() {
4962 let fixture = RestoreCliFixture::new(
4963 "canic-cli-restore-run-require-state-updated-at",
4964 "restore-run.json",
4965 );
4966 let journal = ready_apply_journal();
4967 fixture.write_journal(&journal);
4968
4969 let err = fixture
4970 .run_restore_run(&[
4971 "--dry-run",
4972 "--updated-at",
4973 "2026-05-05T12:03:00Z",
4974 "--require-state-updated-at",
4975 "2026-05-05T12:04:00Z",
4976 ])
4977 .expect_err("state updated-at mismatch should fail requirement");
4978
4979 let run_summary: serde_json::Value = fixture.read_out("read run summary");
4980
4981 assert_eq!(
4982 run_summary["requested_state_updated_at"],
4983 "2026-05-05T12:03:00Z"
4984 );
4985 assert_eq!(run_summary["operation_receipt_count"], 0);
4986 assert!(matches!(
4987 err,
4988 RestoreCommandError::RestoreRunStateUpdatedAtMismatch {
4989 expected,
4990 actual: Some(actual),
4991 ..
4992 } if expected == "2026-05-05T12:04:00Z"
4993 && actual == "2026-05-05T12:03:00Z"
4994 ));
4995 }
4996
4997 #[test]
4999 fn run_restore_run_require_progress_writes_summary_then_fails() {
5000 let root = temp_dir("canic-cli-restore-run-require-progress");
5001 fs::create_dir_all(&root).expect("create temp root");
5002 let journal_path = root.join("restore-apply-journal.json");
5003 let out_path = root.join("restore-run.json");
5004 let journal = ready_apply_journal();
5005
5006 fs::write(
5007 &journal_path,
5008 serde_json::to_vec(&journal).expect("serialize journal"),
5009 )
5010 .expect("write journal");
5011
5012 let err = run([
5013 OsString::from("run"),
5014 OsString::from("--journal"),
5015 OsString::from(journal_path.as_os_str()),
5016 OsString::from("--execute"),
5017 OsString::from("--dfx"),
5018 OsString::from("/bin/true"),
5019 OsString::from("--max-steps"),
5020 OsString::from("1"),
5021 OsString::from("--out"),
5022 OsString::from(out_path.as_os_str()),
5023 OsString::from("--require-remaining-count"),
5024 OsString::from("7"),
5025 OsString::from("--require-attention-count"),
5026 OsString::from("0"),
5027 OsString::from("--require-completion-basis-points"),
5028 OsString::from("0"),
5029 ])
5030 .expect_err("completion progress mismatch should fail requirement");
5031
5032 let run_summary: serde_json::Value =
5033 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5034 .expect("decode run summary");
5035
5036 fs::remove_dir_all(root).expect("remove temp root");
5037 assert_eq!(run_summary["progress"]["remaining_operations"], 7);
5038 assert_eq!(run_summary["progress"]["attention_operations"], 0);
5039 assert_eq!(run_summary["progress"]["completion_basis_points"], 1250);
5040 assert!(matches!(
5041 err,
5042 RestoreCommandError::RestoreApplyProgressMismatch {
5043 field: "completion_basis_points",
5044 expected: 0,
5045 actual: 1250,
5046 ..
5047 }
5048 ));
5049 }
5050
5051 #[test]
5053 fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
5054 let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
5055 fs::create_dir_all(&root).expect("create temp root");
5056 let journal_path = root.join("restore-apply-journal.json");
5057 let out_path = root.join("restore-run.json");
5058 let journal = ready_apply_journal();
5059
5060 fs::write(
5061 &journal_path,
5062 serde_json::to_vec(&journal).expect("serialize journal"),
5063 )
5064 .expect("write journal");
5065
5066 let err = run([
5067 OsString::from("run"),
5068 OsString::from("--journal"),
5069 OsString::from(journal_path.as_os_str()),
5070 OsString::from("--dry-run"),
5071 OsString::from("--out"),
5072 OsString::from(out_path.as_os_str()),
5073 OsString::from("--require-stopped-reason"),
5074 OsString::from("complete"),
5075 ])
5076 .expect_err("stopped reason mismatch should fail requirement");
5077
5078 let run_summary: serde_json::Value =
5079 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5080 .expect("decode run summary");
5081
5082 fs::remove_dir_all(root).expect("remove temp root");
5083 assert_eq!(run_summary["stopped_reason"], "preview");
5084 assert!(matches!(
5085 err,
5086 RestoreCommandError::RestoreRunStoppedReasonMismatch {
5087 expected,
5088 actual,
5089 ..
5090 } if expected == "complete" && actual == "preview"
5091 ));
5092 }
5093
5094 #[test]
5096 fn run_restore_run_require_next_action_writes_summary_then_fails() {
5097 let root = temp_dir("canic-cli-restore-run-require-next-action");
5098 fs::create_dir_all(&root).expect("create temp root");
5099 let journal_path = root.join("restore-apply-journal.json");
5100 let out_path = root.join("restore-run.json");
5101 let journal = ready_apply_journal();
5102
5103 fs::write(
5104 &journal_path,
5105 serde_json::to_vec(&journal).expect("serialize journal"),
5106 )
5107 .expect("write journal");
5108
5109 let err = run([
5110 OsString::from("run"),
5111 OsString::from("--journal"),
5112 OsString::from(journal_path.as_os_str()),
5113 OsString::from("--dry-run"),
5114 OsString::from("--out"),
5115 OsString::from(out_path.as_os_str()),
5116 OsString::from("--require-next-action"),
5117 OsString::from("done"),
5118 ])
5119 .expect_err("next action mismatch should fail requirement");
5120
5121 let run_summary: serde_json::Value =
5122 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5123 .expect("decode run summary");
5124
5125 fs::remove_dir_all(root).expect("remove temp root");
5126 assert_eq!(run_summary["next_action"], "rerun");
5127 assert!(matches!(
5128 err,
5129 RestoreCommandError::RestoreRunNextActionMismatch {
5130 expected,
5131 actual,
5132 ..
5133 } if expected == "done" && actual == "rerun"
5134 ));
5135 }
5136
5137 #[test]
5139 fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
5140 let root = temp_dir("canic-cli-restore-apply-report-attention");
5141 fs::create_dir_all(&root).expect("create temp root");
5142 let journal_path = root.join("restore-apply-journal.json");
5143 let out_path = root.join("restore-apply-report.json");
5144 let mut journal = ready_apply_journal();
5145 journal
5146 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
5147 .expect("mark pending operation");
5148
5149 fs::write(
5150 &journal_path,
5151 serde_json::to_vec(&journal).expect("serialize journal"),
5152 )
5153 .expect("write journal");
5154
5155 let err = run([
5156 OsString::from("apply-report"),
5157 OsString::from("--journal"),
5158 OsString::from(journal_path.as_os_str()),
5159 OsString::from("--out"),
5160 OsString::from(out_path.as_os_str()),
5161 OsString::from("--require-no-attention"),
5162 ])
5163 .expect_err("attention report should fail requirement");
5164
5165 let report: RestoreApplyJournalReport =
5166 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
5167 .expect("decode apply report");
5168
5169 fs::remove_dir_all(root).expect("remove temp root");
5170 assert!(report.attention_required);
5171 assert_eq!(report.pending_operations, 1);
5172 assert!(matches!(
5173 err,
5174 RestoreCommandError::RestoreApplyReportNeedsAttention {
5175 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
5176 ..
5177 }
5178 ));
5179 }
5180
5181 #[test]
5183 fn run_restore_apply_status_require_complete_writes_status_then_fails() {
5184 let root = temp_dir("canic-cli-restore-apply-status-incomplete");
5185 fs::create_dir_all(&root).expect("create temp root");
5186 let journal_path = root.join("restore-apply-journal.json");
5187 let out_path = root.join("restore-apply-status.json");
5188 let journal = ready_apply_journal();
5189
5190 fs::write(
5191 &journal_path,
5192 serde_json::to_vec(&journal).expect("serialize journal"),
5193 )
5194 .expect("write journal");
5195
5196 let err = run([
5197 OsString::from("apply-status"),
5198 OsString::from("--journal"),
5199 OsString::from(journal_path.as_os_str()),
5200 OsString::from("--out"),
5201 OsString::from(out_path.as_os_str()),
5202 OsString::from("--require-complete"),
5203 ])
5204 .expect_err("incomplete journal should fail requirement");
5205
5206 assert!(out_path.exists());
5207 let status: RestoreApplyJournalStatus =
5208 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
5209 .expect("decode apply status");
5210
5211 fs::remove_dir_all(root).expect("remove temp root");
5212 assert!(!status.complete);
5213 assert_eq!(status.completed_operations, 0);
5214 assert_eq!(status.operation_count, 8);
5215 assert!(matches!(
5216 err,
5217 RestoreCommandError::RestoreApplyIncomplete {
5218 completed_operations: 0,
5219 operation_count: 8,
5220 ..
5221 }
5222 ));
5223 }
5224
5225 #[test]
5227 fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
5228 let root = temp_dir("canic-cli-restore-apply-status-failed");
5229 fs::create_dir_all(&root).expect("create temp root");
5230 let journal_path = root.join("restore-apply-journal.json");
5231 let out_path = root.join("restore-apply-status.json");
5232 let mut journal = ready_apply_journal();
5233 journal
5234 .mark_operation_failed(0, "dfx-load-failed".to_string())
5235 .expect("mark failed operation");
5236
5237 fs::write(
5238 &journal_path,
5239 serde_json::to_vec(&journal).expect("serialize journal"),
5240 )
5241 .expect("write journal");
5242
5243 let err = run([
5244 OsString::from("apply-status"),
5245 OsString::from("--journal"),
5246 OsString::from(journal_path.as_os_str()),
5247 OsString::from("--out"),
5248 OsString::from(out_path.as_os_str()),
5249 OsString::from("--require-no-failed"),
5250 ])
5251 .expect_err("failed operation should fail requirement");
5252
5253 assert!(out_path.exists());
5254 let status: RestoreApplyJournalStatus =
5255 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
5256 .expect("decode apply status");
5257
5258 fs::remove_dir_all(root).expect("remove temp root");
5259 assert_eq!(status.failed_operations, 1);
5260 assert!(matches!(
5261 err,
5262 RestoreCommandError::RestoreApplyFailed {
5263 failed_operations: 1,
5264 ..
5265 }
5266 ));
5267 }
5268
5269 #[test]
5271 fn run_restore_apply_status_require_complete_accepts_complete_journal() {
5272 let root = temp_dir("canic-cli-restore-apply-status-complete");
5273 fs::create_dir_all(&root).expect("create temp root");
5274 let journal_path = root.join("restore-apply-journal.json");
5275 let out_path = root.join("restore-apply-status.json");
5276 let mut journal = ready_apply_journal();
5277 for sequence in 0..journal.operation_count {
5278 journal
5279 .mark_operation_completed(sequence)
5280 .expect("complete operation");
5281 }
5282
5283 fs::write(
5284 &journal_path,
5285 serde_json::to_vec(&journal).expect("serialize journal"),
5286 )
5287 .expect("write journal");
5288
5289 run([
5290 OsString::from("apply-status"),
5291 OsString::from("--journal"),
5292 OsString::from(journal_path.as_os_str()),
5293 OsString::from("--out"),
5294 OsString::from(out_path.as_os_str()),
5295 OsString::from("--require-complete"),
5296 ])
5297 .expect("complete journal should pass requirement");
5298
5299 let status: RestoreApplyJournalStatus =
5300 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
5301 .expect("decode apply status");
5302
5303 fs::remove_dir_all(root).expect("remove temp root");
5304 assert!(status.complete);
5305 assert_eq!(status.completed_operations, 8);
5306 assert_eq!(status.operation_count, 8);
5307 }
5308
5309 #[test]
5311 fn run_restore_apply_next_writes_next_ready_operation() {
5312 let root = temp_dir("canic-cli-restore-apply-next");
5313 fs::create_dir_all(&root).expect("create temp root");
5314 let journal_path = root.join("restore-apply-journal.json");
5315 let out_path = root.join("restore-apply-next.json");
5316 let mut journal = ready_apply_journal();
5317 journal
5318 .mark_operation_completed(0)
5319 .expect("mark first operation complete");
5320
5321 fs::write(
5322 &journal_path,
5323 serde_json::to_vec(&journal).expect("serialize journal"),
5324 )
5325 .expect("write journal");
5326
5327 run([
5328 OsString::from("apply-next"),
5329 OsString::from("--journal"),
5330 OsString::from(journal_path.as_os_str()),
5331 OsString::from("--out"),
5332 OsString::from(out_path.as_os_str()),
5333 ])
5334 .expect("write apply next");
5335
5336 let next: RestoreApplyNextOperation =
5337 serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
5338 .expect("decode next operation");
5339 let operation = next.operation.expect("operation should be available");
5340
5341 fs::remove_dir_all(root).expect("remove temp root");
5342 assert!(next.ready);
5343 assert!(next.operation_available);
5344 assert_eq!(operation.sequence, 1);
5345 assert_eq!(
5346 operation.operation,
5347 canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
5348 );
5349 }
5350
5351 #[test]
5353 fn run_restore_apply_command_writes_next_command_preview() {
5354 let root = temp_dir("canic-cli-restore-apply-command");
5355 fs::create_dir_all(&root).expect("create temp root");
5356 let journal_path = root.join("restore-apply-journal.json");
5357 let out_path = root.join("restore-apply-command.json");
5358 let journal = ready_apply_journal();
5359
5360 fs::write(
5361 &journal_path,
5362 serde_json::to_vec(&journal).expect("serialize journal"),
5363 )
5364 .expect("write journal");
5365
5366 run([
5367 OsString::from("apply-command"),
5368 OsString::from("--journal"),
5369 OsString::from(journal_path.as_os_str()),
5370 OsString::from("--dfx"),
5371 OsString::from("/tmp/dfx"),
5372 OsString::from("--network"),
5373 OsString::from("local"),
5374 OsString::from("--out"),
5375 OsString::from(out_path.as_os_str()),
5376 ])
5377 .expect("write command preview");
5378
5379 let preview: RestoreApplyCommandPreview =
5380 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
5381 .expect("decode command preview");
5382 let command = preview.command.expect("command should be available");
5383
5384 fs::remove_dir_all(root).expect("remove temp root");
5385 assert!(preview.ready);
5386 assert!(preview.command_available);
5387 assert_eq!(command.program, "/tmp/dfx");
5388 assert_eq!(
5389 command.args,
5390 vec![
5391 "canister".to_string(),
5392 "--network".to_string(),
5393 "local".to_string(),
5394 "snapshot".to_string(),
5395 "upload".to_string(),
5396 "--dir".to_string(),
5397 "artifacts/root".to_string(),
5398 ROOT.to_string(),
5399 ]
5400 );
5401 assert!(command.mutates);
5402 }
5403
5404 #[test]
5406 fn run_restore_apply_command_require_command_writes_preview_then_fails() {
5407 let root = temp_dir("canic-cli-restore-apply-command-require");
5408 fs::create_dir_all(&root).expect("create temp root");
5409 let journal_path = root.join("restore-apply-journal.json");
5410 let out_path = root.join("restore-apply-command.json");
5411 let mut journal = ready_apply_journal();
5412
5413 for sequence in 0..journal.operation_count {
5414 journal
5415 .mark_operation_completed(sequence)
5416 .expect("mark operation completed");
5417 }
5418
5419 fs::write(
5420 &journal_path,
5421 serde_json::to_vec(&journal).expect("serialize journal"),
5422 )
5423 .expect("write journal");
5424
5425 let err = run([
5426 OsString::from("apply-command"),
5427 OsString::from("--journal"),
5428 OsString::from(journal_path.as_os_str()),
5429 OsString::from("--out"),
5430 OsString::from(out_path.as_os_str()),
5431 OsString::from("--require-command"),
5432 ])
5433 .expect_err("missing command should fail");
5434
5435 let preview: RestoreApplyCommandPreview =
5436 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
5437 .expect("decode command preview");
5438
5439 fs::remove_dir_all(root).expect("remove temp root");
5440 assert!(preview.complete);
5441 assert!(!preview.operation_available);
5442 assert!(!preview.command_available);
5443 assert!(matches!(
5444 err,
5445 RestoreCommandError::RestoreApplyCommandUnavailable {
5446 operation_available: false,
5447 complete: true,
5448 ..
5449 }
5450 ));
5451 }
5452
5453 #[test]
5455 fn run_restore_apply_claim_marks_next_operation_pending() {
5456 let root = temp_dir("canic-cli-restore-apply-claim");
5457 fs::create_dir_all(&root).expect("create temp root");
5458 let journal_path = root.join("restore-apply-journal.json");
5459 let claimed_path = root.join("restore-apply-journal.claimed.json");
5460 let journal = ready_apply_journal();
5461
5462 fs::write(
5463 &journal_path,
5464 serde_json::to_vec(&journal).expect("serialize journal"),
5465 )
5466 .expect("write journal");
5467
5468 run([
5469 OsString::from("apply-claim"),
5470 OsString::from("--journal"),
5471 OsString::from(journal_path.as_os_str()),
5472 OsString::from("--sequence"),
5473 OsString::from("0"),
5474 OsString::from("--updated-at"),
5475 OsString::from("2026-05-04T12:00:00Z"),
5476 OsString::from("--out"),
5477 OsString::from(claimed_path.as_os_str()),
5478 ])
5479 .expect("claim operation");
5480
5481 let claimed: RestoreApplyJournal =
5482 serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
5483 .expect("decode claimed journal");
5484 let status = claimed.status();
5485 let next = claimed.next_operation();
5486
5487 fs::remove_dir_all(root).expect("remove temp root");
5488 assert_eq!(claimed.pending_operations, 1);
5489 assert_eq!(claimed.ready_operations, 7);
5490 assert_eq!(
5491 claimed.operations[0].state,
5492 RestoreApplyOperationState::Pending
5493 );
5494 assert_eq!(
5495 claimed.operations[0].state_updated_at.as_deref(),
5496 Some("2026-05-04T12:00:00Z")
5497 );
5498 assert_eq!(status.next_transition_sequence, Some(0));
5499 assert_eq!(
5500 status.next_transition_state,
5501 Some(RestoreApplyOperationState::Pending)
5502 );
5503 assert_eq!(
5504 status.next_transition_updated_at.as_deref(),
5505 Some("2026-05-04T12:00:00Z")
5506 );
5507 assert_eq!(
5508 next.operation.expect("next operation").state,
5509 RestoreApplyOperationState::Pending
5510 );
5511 }
5512
5513 #[test]
5515 fn run_restore_apply_claim_rejects_sequence_mismatch() {
5516 let root = temp_dir("canic-cli-restore-apply-claim-sequence");
5517 fs::create_dir_all(&root).expect("create temp root");
5518 let journal_path = root.join("restore-apply-journal.json");
5519 let claimed_path = root.join("restore-apply-journal.claimed.json");
5520 let journal = ready_apply_journal();
5521
5522 fs::write(
5523 &journal_path,
5524 serde_json::to_vec(&journal).expect("serialize journal"),
5525 )
5526 .expect("write journal");
5527
5528 let err = run([
5529 OsString::from("apply-claim"),
5530 OsString::from("--journal"),
5531 OsString::from(journal_path.as_os_str()),
5532 OsString::from("--sequence"),
5533 OsString::from("1"),
5534 OsString::from("--out"),
5535 OsString::from(claimed_path.as_os_str()),
5536 ])
5537 .expect_err("stale sequence should fail claim");
5538
5539 assert!(!claimed_path.exists());
5540 fs::remove_dir_all(root).expect("remove temp root");
5541 assert!(matches!(
5542 err,
5543 RestoreCommandError::RestoreApplyClaimSequenceMismatch {
5544 expected: 1,
5545 actual: Some(0),
5546 }
5547 ));
5548 }
5549
5550 #[test]
5552 fn run_restore_apply_unclaim_marks_pending_operation_ready() {
5553 let root = temp_dir("canic-cli-restore-apply-unclaim");
5554 fs::create_dir_all(&root).expect("create temp root");
5555 let journal_path = root.join("restore-apply-journal.json");
5556 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
5557 let mut journal = ready_apply_journal();
5558 journal
5559 .mark_next_operation_pending()
5560 .expect("claim operation");
5561
5562 fs::write(
5563 &journal_path,
5564 serde_json::to_vec(&journal).expect("serialize journal"),
5565 )
5566 .expect("write journal");
5567
5568 run([
5569 OsString::from("apply-unclaim"),
5570 OsString::from("--journal"),
5571 OsString::from(journal_path.as_os_str()),
5572 OsString::from("--sequence"),
5573 OsString::from("0"),
5574 OsString::from("--updated-at"),
5575 OsString::from("2026-05-04T12:01:00Z"),
5576 OsString::from("--out"),
5577 OsString::from(unclaimed_path.as_os_str()),
5578 ])
5579 .expect("unclaim operation");
5580
5581 let unclaimed: RestoreApplyJournal =
5582 serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
5583 .expect("decode unclaimed journal");
5584 let status = unclaimed.status();
5585
5586 fs::remove_dir_all(root).expect("remove temp root");
5587 assert_eq!(unclaimed.pending_operations, 0);
5588 assert_eq!(unclaimed.ready_operations, 8);
5589 assert_eq!(
5590 unclaimed.operations[0].state,
5591 RestoreApplyOperationState::Ready
5592 );
5593 assert_eq!(
5594 unclaimed.operations[0].state_updated_at.as_deref(),
5595 Some("2026-05-04T12:01:00Z")
5596 );
5597 assert_eq!(status.next_ready_sequence, Some(0));
5598 assert_eq!(
5599 status.next_transition_state,
5600 Some(RestoreApplyOperationState::Ready)
5601 );
5602 assert_eq!(
5603 status.next_transition_updated_at.as_deref(),
5604 Some("2026-05-04T12:01:00Z")
5605 );
5606 }
5607
5608 #[test]
5610 fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
5611 let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
5612 fs::create_dir_all(&root).expect("create temp root");
5613 let journal_path = root.join("restore-apply-journal.json");
5614 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
5615 let mut journal = ready_apply_journal();
5616 journal
5617 .mark_next_operation_pending()
5618 .expect("claim operation");
5619
5620 fs::write(
5621 &journal_path,
5622 serde_json::to_vec(&journal).expect("serialize journal"),
5623 )
5624 .expect("write journal");
5625
5626 let err = run([
5627 OsString::from("apply-unclaim"),
5628 OsString::from("--journal"),
5629 OsString::from(journal_path.as_os_str()),
5630 OsString::from("--sequence"),
5631 OsString::from("1"),
5632 OsString::from("--out"),
5633 OsString::from(unclaimed_path.as_os_str()),
5634 ])
5635 .expect_err("stale sequence should fail unclaim");
5636
5637 assert!(!unclaimed_path.exists());
5638 fs::remove_dir_all(root).expect("remove temp root");
5639 assert!(matches!(
5640 err,
5641 RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
5642 expected: 1,
5643 actual: Some(0),
5644 }
5645 ));
5646 }
5647
5648 #[test]
5650 fn run_restore_apply_mark_completes_operation() {
5651 let root = temp_dir("canic-cli-restore-apply-mark-complete");
5652 fs::create_dir_all(&root).expect("create temp root");
5653 let journal_path = root.join("restore-apply-journal.json");
5654 let updated_path = root.join("restore-apply-journal.updated.json");
5655 let journal = ready_apply_journal();
5656
5657 fs::write(
5658 &journal_path,
5659 serde_json::to_vec(&journal).expect("serialize journal"),
5660 )
5661 .expect("write journal");
5662
5663 run([
5664 OsString::from("apply-mark"),
5665 OsString::from("--journal"),
5666 OsString::from(journal_path.as_os_str()),
5667 OsString::from("--sequence"),
5668 OsString::from("0"),
5669 OsString::from("--state"),
5670 OsString::from("completed"),
5671 OsString::from("--updated-at"),
5672 OsString::from("2026-05-04T12:02:00Z"),
5673 OsString::from("--out"),
5674 OsString::from(updated_path.as_os_str()),
5675 ])
5676 .expect("mark operation completed");
5677
5678 let updated: RestoreApplyJournal =
5679 serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
5680 .expect("decode updated journal");
5681 let status = updated.status();
5682
5683 fs::remove_dir_all(root).expect("remove temp root");
5684 assert_eq!(updated.completed_operations, 1);
5685 assert_eq!(updated.ready_operations, 7);
5686 assert_eq!(
5687 updated.operations[0].state_updated_at.as_deref(),
5688 Some("2026-05-04T12:02:00Z")
5689 );
5690 assert_eq!(status.next_ready_sequence, Some(1));
5691 }
5692
5693 #[test]
5695 fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
5696 let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
5697 fs::create_dir_all(&root).expect("create temp root");
5698 let journal_path = root.join("restore-apply-journal.json");
5699 let updated_path = root.join("restore-apply-journal.updated.json");
5700 let journal = ready_apply_journal();
5701
5702 fs::write(
5703 &journal_path,
5704 serde_json::to_vec(&journal).expect("serialize journal"),
5705 )
5706 .expect("write journal");
5707
5708 let err = run([
5709 OsString::from("apply-mark"),
5710 OsString::from("--journal"),
5711 OsString::from(journal_path.as_os_str()),
5712 OsString::from("--sequence"),
5713 OsString::from("0"),
5714 OsString::from("--state"),
5715 OsString::from("completed"),
5716 OsString::from("--out"),
5717 OsString::from(updated_path.as_os_str()),
5718 OsString::from("--require-pending"),
5719 ])
5720 .expect_err("ready operation should fail pending requirement");
5721
5722 assert!(!updated_path.exists());
5723 fs::remove_dir_all(root).expect("remove temp root");
5724 assert!(matches!(
5725 err,
5726 RestoreCommandError::RestoreApplyMarkRequiresPending {
5727 sequence: 0,
5728 state: RestoreApplyOperationState::Ready,
5729 }
5730 ));
5731 }
5732
5733 #[test]
5735 fn run_restore_apply_mark_rejects_out_of_order_operation() {
5736 let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
5737 fs::create_dir_all(&root).expect("create temp root");
5738 let journal_path = root.join("restore-apply-journal.json");
5739 let updated_path = root.join("restore-apply-journal.updated.json");
5740 let journal = ready_apply_journal();
5741
5742 fs::write(
5743 &journal_path,
5744 serde_json::to_vec(&journal).expect("serialize journal"),
5745 )
5746 .expect("write journal");
5747
5748 let err = run([
5749 OsString::from("apply-mark"),
5750 OsString::from("--journal"),
5751 OsString::from(journal_path.as_os_str()),
5752 OsString::from("--sequence"),
5753 OsString::from("1"),
5754 OsString::from("--state"),
5755 OsString::from("completed"),
5756 OsString::from("--out"),
5757 OsString::from(updated_path.as_os_str()),
5758 ])
5759 .expect_err("out-of-order operation should fail");
5760
5761 assert!(!updated_path.exists());
5762 fs::remove_dir_all(root).expect("remove temp root");
5763 assert!(matches!(
5764 err,
5765 RestoreCommandError::RestoreApplyJournal(
5766 RestoreApplyJournalError::OutOfOrderOperationTransition {
5767 requested: 1,
5768 next: 0
5769 }
5770 )
5771 ));
5772 }
5773
5774 #[test]
5776 fn run_restore_apply_mark_failed_requires_reason() {
5777 let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
5778 fs::create_dir_all(&root).expect("create temp root");
5779 let journal_path = root.join("restore-apply-journal.json");
5780 let journal = ready_apply_journal();
5781
5782 fs::write(
5783 &journal_path,
5784 serde_json::to_vec(&journal).expect("serialize journal"),
5785 )
5786 .expect("write journal");
5787
5788 let err = run([
5789 OsString::from("apply-mark"),
5790 OsString::from("--journal"),
5791 OsString::from(journal_path.as_os_str()),
5792 OsString::from("--sequence"),
5793 OsString::from("0"),
5794 OsString::from("--state"),
5795 OsString::from("failed"),
5796 ])
5797 .expect_err("failed state should require reason");
5798
5799 fs::remove_dir_all(root).expect("remove temp root");
5800 assert!(matches!(
5801 err,
5802 RestoreCommandError::RestoreApplyJournal(
5803 RestoreApplyJournalError::FailureReasonRequired(0)
5804 )
5805 ));
5806 }
5807
5808 #[test]
5810 fn run_restore_apply_dry_run_rejects_mismatched_status() {
5811 let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
5812 fs::create_dir_all(&root).expect("create temp root");
5813 let plan_path = root.join("restore-plan.json");
5814 let status_path = root.join("restore-status.json");
5815 let out_path = root.join("restore-apply-dry-run.json");
5816 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
5817 let mut status = RestoreStatus::from_plan(&plan);
5818 status.backup_id = "other-backup".to_string();
5819
5820 fs::write(
5821 &plan_path,
5822 serde_json::to_vec(&plan).expect("serialize plan"),
5823 )
5824 .expect("write plan");
5825 fs::write(
5826 &status_path,
5827 serde_json::to_vec(&status).expect("serialize status"),
5828 )
5829 .expect("write status");
5830
5831 let err = run([
5832 OsString::from("apply"),
5833 OsString::from("--plan"),
5834 OsString::from(plan_path.as_os_str()),
5835 OsString::from("--status"),
5836 OsString::from(status_path.as_os_str()),
5837 OsString::from("--dry-run"),
5838 OsString::from("--out"),
5839 OsString::from(out_path.as_os_str()),
5840 ])
5841 .expect_err("mismatched status should fail");
5842
5843 assert!(!out_path.exists());
5844 fs::remove_dir_all(root).expect("remove temp root");
5845 assert!(matches!(
5846 err,
5847 RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
5848 field: "backup_id",
5849 ..
5850 })
5851 ));
5852 }
5853
5854 fn ready_apply_journal() -> RestoreApplyJournal {
5856 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
5857 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
5858 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
5859
5860 journal.ready = true;
5861 journal.blocked_reasons = Vec::new();
5862 for operation in &mut journal.operations {
5863 operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
5864 operation.blocking_reasons = Vec::new();
5865 }
5866 journal.blocked_operations = 0;
5867 journal.ready_operations = journal.operation_count;
5868 journal.validate().expect("journal should validate");
5869 journal
5870 }
5871
5872 fn valid_manifest() -> FleetBackupManifest {
5874 FleetBackupManifest {
5875 manifest_version: 1,
5876 backup_id: "backup-test".to_string(),
5877 created_at: "2026-05-03T00:00:00Z".to_string(),
5878 tool: ToolMetadata {
5879 name: "canic".to_string(),
5880 version: "0.30.1".to_string(),
5881 },
5882 source: SourceMetadata {
5883 environment: "local".to_string(),
5884 root_canister: ROOT.to_string(),
5885 },
5886 consistency: ConsistencySection {
5887 mode: ConsistencyMode::CrashConsistent,
5888 backup_units: vec![BackupUnit {
5889 unit_id: "fleet".to_string(),
5890 kind: BackupUnitKind::SubtreeRooted,
5891 roles: vec!["root".to_string(), "app".to_string()],
5892 consistency_reason: None,
5893 dependency_closure: Vec::new(),
5894 topology_validation: "subtree-closed".to_string(),
5895 quiescence_strategy: None,
5896 }],
5897 },
5898 fleet: FleetSection {
5899 topology_hash_algorithm: "sha256".to_string(),
5900 topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
5901 discovery_topology_hash: HASH.to_string(),
5902 pre_snapshot_topology_hash: HASH.to_string(),
5903 topology_hash: HASH.to_string(),
5904 members: vec![
5905 fleet_member("root", ROOT, None, IdentityMode::Fixed),
5906 fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
5907 ],
5908 },
5909 verification: VerificationPlan::default(),
5910 }
5911 }
5912
5913 fn restore_ready_manifest() -> FleetBackupManifest {
5915 let mut manifest = valid_manifest();
5916 for member in &mut manifest.fleet.members {
5917 member.source_snapshot.module_hash = Some(HASH.to_string());
5918 member.source_snapshot.wasm_hash = Some(HASH.to_string());
5919 member.source_snapshot.checksum = Some(HASH.to_string());
5920 }
5921 manifest
5922 }
5923
5924 fn fleet_member(
5926 role: &str,
5927 canister_id: &str,
5928 parent_canister_id: Option<&str>,
5929 identity_mode: IdentityMode,
5930 ) -> FleetMember {
5931 FleetMember {
5932 role: role.to_string(),
5933 canister_id: canister_id.to_string(),
5934 parent_canister_id: parent_canister_id.map(str::to_string),
5935 subnet_canister_id: Some(ROOT.to_string()),
5936 controller_hint: None,
5937 identity_mode,
5938 restore_group: 1,
5939 verification_class: "basic".to_string(),
5940 verification_checks: vec![VerificationCheck {
5941 kind: "status".to_string(),
5942 method: None,
5943 roles: vec![role.to_string()],
5944 }],
5945 source_snapshot: SourceSnapshot {
5946 snapshot_id: format!("{role}-snapshot"),
5947 module_hash: None,
5948 wasm_hash: None,
5949 code_version: Some("v0.30.1".to_string()),
5950 artifact_path: format!("artifacts/{role}"),
5951 checksum_algorithm: "sha256".to_string(),
5952 checksum: None,
5953 },
5954 }
5955 }
5956
5957 fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
5959 layout.write_manifest(manifest).expect("write manifest");
5960
5961 let artifacts = manifest
5962 .fleet
5963 .members
5964 .iter()
5965 .map(|member| {
5966 let bytes = format!("{} artifact", member.role);
5967 let artifact_path = root.join(&member.source_snapshot.artifact_path);
5968 if let Some(parent) = artifact_path.parent() {
5969 fs::create_dir_all(parent).expect("create artifact parent");
5970 }
5971 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
5972 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
5973
5974 ArtifactJournalEntry {
5975 canister_id: member.canister_id.clone(),
5976 snapshot_id: member.source_snapshot.snapshot_id.clone(),
5977 state: ArtifactState::Durable,
5978 temp_path: None,
5979 artifact_path: member.source_snapshot.artifact_path.clone(),
5980 checksum_algorithm: checksum.algorithm,
5981 checksum: Some(checksum.hash),
5982 updated_at: "2026-05-03T00:00:00Z".to_string(),
5983 }
5984 })
5985 .collect();
5986
5987 layout
5988 .write_journal(&DownloadJournal {
5989 journal_version: 1,
5990 backup_id: manifest.backup_id.clone(),
5991 discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
5992 pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
5993 operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
5994 artifacts,
5995 })
5996 .expect("write journal");
5997 }
5998
5999 fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
6001 for member in &mut manifest.fleet.members {
6002 let bytes = format!("{} apply artifact", member.role);
6003 let artifact_path = root.join(&member.source_snapshot.artifact_path);
6004 if let Some(parent) = artifact_path.parent() {
6005 fs::create_dir_all(parent).expect("create artifact parent");
6006 }
6007 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
6008 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
6009 member.source_snapshot.checksum = Some(checksum.hash);
6010 }
6011 }
6012
6013 fn temp_dir(prefix: &str) -> PathBuf {
6015 let nanos = SystemTime::now()
6016 .duration_since(UNIX_EPOCH)
6017 .expect("system time after epoch")
6018 .as_nanos();
6019 std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
6020 }
6021}