1use canic_backup::{
2 manifest::FleetBackupManifest,
3 persistence::{BackupLayout, PersistenceError},
4 restore::{
5 RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6 RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7 RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8 RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationKindCounts,
9 RestoreApplyOperationState, RestoreApplyPendingSummary, RestoreApplyProgressSummary,
10 RestoreApplyReportOperation, RestoreApplyReportOutcome, RestoreApplyRunnerCommand,
11 RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner, RestoreStatus,
12 },
13};
14use serde::Serialize;
15use std::{
16 ffi::OsString,
17 fs,
18 io::{self, Write},
19 path::PathBuf,
20 process::Command,
21};
22use thiserror::Error as ThisError;
23
24#[derive(Debug, ThisError)]
29pub enum RestoreCommandError {
30 #[error("{0}")]
31 Usage(&'static str),
32
33 #[error("missing required option {0}")]
34 MissingOption(&'static str),
35
36 #[error("use either --manifest or --backup-dir, not both")]
37 ConflictingManifestSources,
38
39 #[error("--require-verified requires --backup-dir")]
40 RequireVerifiedNeedsBackupDir,
41
42 #[error("restore apply currently requires --dry-run")]
43 ApplyRequiresDryRun,
44
45 #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
46 RestoreRunRequiresMode,
47
48 #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
49 RestoreRunConflictingModes,
50
51 #[error("restore run command failed for operation {sequence}: status={status}")]
52 RestoreRunCommandFailed { sequence: usize, status: String },
53
54 #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
55 RestoreRunModeMismatch {
56 backup_id: String,
57 expected: String,
58 actual: String,
59 },
60
61 #[error(
62 "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
63 )]
64 RestoreRunStoppedReasonMismatch {
65 backup_id: String,
66 expected: String,
67 actual: String,
68 },
69
70 #[error(
71 "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
72 )]
73 RestoreRunNextActionMismatch {
74 backup_id: String,
75 expected: String,
76 actual: String,
77 },
78
79 #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
80 RestoreRunExecutedCountMismatch {
81 backup_id: String,
82 expected: usize,
83 actual: usize,
84 },
85
86 #[error("restore run for backup {backup_id} wrote {actual} receipts, expected {expected}")]
87 RestoreRunReceiptCountMismatch {
88 backup_id: String,
89 expected: usize,
90 actual: usize,
91 },
92
93 #[error(
94 "restore run for backup {backup_id} wrote {actual} {receipt_kind} receipts, expected {expected}"
95 )]
96 RestoreRunReceiptKindCountMismatch {
97 backup_id: String,
98 receipt_kind: &'static str,
99 expected: usize,
100 actual: usize,
101 },
102
103 #[error(
104 "restore run for backup {backup_id} wrote {actual_receipts} receipts with {mismatched_receipts} updated_at mismatches, expected {expected}"
105 )]
106 RestoreRunReceiptUpdatedAtMismatch {
107 backup_id: String,
108 expected: String,
109 actual_receipts: usize,
110 mismatched_receipts: usize,
111 },
112
113 #[error(
114 "restore run for backup {backup_id} has {actual} remaining ready operations, expected {expected}"
115 )]
116 RestoreRunBatchRemainingReadyCountMismatch {
117 backup_id: String,
118 expected: usize,
119 actual: usize,
120 },
121
122 #[error(
123 "restore run for backup {backup_id} started with {actual} ready operations, expected {expected}"
124 )]
125 RestoreRunBatchInitialReadyCountMismatch {
126 backup_id: String,
127 expected: usize,
128 actual: usize,
129 },
130
131 #[error(
132 "restore run for backup {backup_id} executed {actual} batch operations, expected {expected}"
133 )]
134 RestoreRunBatchExecutedCountMismatch {
135 backup_id: String,
136 expected: usize,
137 actual: usize,
138 },
139
140 #[error("restore run for backup {backup_id} has ready delta {actual}, expected {expected}")]
141 RestoreRunBatchReadyDeltaMismatch {
142 backup_id: String,
143 expected: isize,
144 actual: isize,
145 },
146
147 #[error("restore run for backup {backup_id} has remaining delta {actual}, expected {expected}")]
148 RestoreRunBatchRemainingDeltaMismatch {
149 backup_id: String,
150 expected: isize,
151 actual: isize,
152 },
153
154 #[error(
155 "restore run for backup {backup_id} stopped_by_max_steps={actual}, expected {expected}"
156 )]
157 RestoreRunBatchStoppedByMaxStepsMismatch {
158 backup_id: String,
159 expected: bool,
160 actual: bool,
161 },
162
163 #[error(
164 "restore run for backup {backup_id} reported requested_state_updated_at={actual:?}, expected {expected}"
165 )]
166 RestoreRunStateUpdatedAtMismatch {
167 backup_id: String,
168 expected: String,
169 actual: Option<String>,
170 },
171
172 #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
173 RestoreNotReady {
174 backup_id: String,
175 reasons: Vec<String>,
176 },
177
178 #[error("restore manifest {backup_id} is not design-v1 ready")]
179 DesignConformanceNotReady { backup_id: String },
180
181 #[error(
182 "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
183 )]
184 RestoreApplyPending {
185 backup_id: String,
186 pending_operations: usize,
187 next_transition_sequence: Option<usize>,
188 },
189
190 #[error(
191 "restore apply journal for backup {backup_id} has stale or untracked pending work before {cutoff_updated_at}: pending_sequence={pending_sequence:?}, pending_updated_at={pending_updated_at:?}"
192 )]
193 RestoreApplyPendingStale {
194 backup_id: String,
195 cutoff_updated_at: String,
196 pending_sequence: Option<usize>,
197 pending_updated_at: Option<String>,
198 },
199
200 #[error(
201 "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
202 )]
203 RestoreApplyIncomplete {
204 backup_id: String,
205 completed_operations: usize,
206 operation_count: usize,
207 },
208
209 #[error(
210 "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
211 )]
212 RestoreApplyFailed {
213 backup_id: String,
214 failed_operations: usize,
215 },
216
217 #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
218 RestoreApplyNotReady {
219 backup_id: String,
220 reasons: Vec<String>,
221 },
222
223 #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
224 RestoreApplyReportNeedsAttention {
225 backup_id: String,
226 outcome: canic_backup::restore::RestoreApplyReportOutcome,
227 },
228
229 #[error(
230 "restore apply progress for backup {backup_id} has unexpected {field}: expected={expected}, actual={actual}"
231 )]
232 RestoreApplyProgressMismatch {
233 backup_id: String,
234 field: &'static str,
235 expected: usize,
236 actual: usize,
237 },
238
239 #[error(
240 "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
241 )]
242 RestoreApplyCommandUnavailable {
243 backup_id: String,
244 operation_available: bool,
245 complete: bool,
246 blocked_reasons: Vec<String>,
247 },
248
249 #[error(
250 "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
251 )]
252 RestoreApplyMarkRequiresPending {
253 sequence: usize,
254 state: RestoreApplyOperationState,
255 },
256
257 #[error(
258 "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
259 )]
260 RestoreApplyClaimSequenceMismatch {
261 expected: usize,
262 actual: Option<usize>,
263 },
264
265 #[error(
266 "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
267 )]
268 RestoreApplyUnclaimSequenceMismatch {
269 expected: usize,
270 actual: Option<usize>,
271 },
272
273 #[error("unknown option {0}")]
274 UnknownOption(String),
275
276 #[error("option {0} requires a value")]
277 MissingValue(&'static str),
278
279 #[error("option --sequence requires a non-negative integer value")]
280 InvalidSequence,
281
282 #[error("option {option} requires a positive integer value")]
283 InvalidPositiveInteger { option: &'static str },
284
285 #[error("option {option} requires an integer value")]
286 InvalidInteger { option: &'static str },
287
288 #[error("option {option} requires true or false, got {value}")]
289 InvalidBoolean { option: &'static str, value: String },
290
291 #[error("unsupported apply-mark state {0}; use completed or failed")]
292 InvalidApplyMarkState(String),
293
294 #[error(transparent)]
295 Io(#[from] std::io::Error),
296
297 #[error(transparent)]
298 Json(#[from] serde_json::Error),
299
300 #[error(transparent)]
301 Persistence(#[from] PersistenceError),
302
303 #[error(transparent)]
304 RestorePlan(#[from] RestorePlanError),
305
306 #[error(transparent)]
307 RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
308
309 #[error(transparent)]
310 RestoreApplyJournal(#[from] RestoreApplyJournalError),
311}
312
313#[derive(Clone, Debug, Eq, PartialEq)]
318pub struct RestorePlanOptions {
319 pub manifest: Option<PathBuf>,
320 pub backup_dir: Option<PathBuf>,
321 pub mapping: Option<PathBuf>,
322 pub out: Option<PathBuf>,
323 pub require_verified: bool,
324 pub require_design_v1: bool,
325 pub require_restore_ready: bool,
326}
327
328impl RestorePlanOptions {
329 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
331 where
332 I: IntoIterator<Item = OsString>,
333 {
334 let mut manifest = None;
335 let mut backup_dir = None;
336 let mut mapping = None;
337 let mut out = None;
338 let mut require_verified = false;
339 let mut require_design_v1 = false;
340 let mut require_restore_ready = false;
341
342 let mut args = args.into_iter();
343 while let Some(arg) = args.next() {
344 let arg = arg
345 .into_string()
346 .map_err(|_| RestoreCommandError::Usage(usage()))?;
347 match arg.as_str() {
348 "--manifest" => {
349 manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
350 }
351 "--backup-dir" => {
352 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
353 }
354 "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
355 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
356 "--require-verified" => require_verified = true,
357 "--require-design-v1" => require_design_v1 = true,
358 "--require-restore-ready" => require_restore_ready = true,
359 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
360 _ => return Err(RestoreCommandError::UnknownOption(arg)),
361 }
362 }
363
364 if manifest.is_some() && backup_dir.is_some() {
365 return Err(RestoreCommandError::ConflictingManifestSources);
366 }
367
368 if manifest.is_none() && backup_dir.is_none() {
369 return Err(RestoreCommandError::MissingOption(
370 "--manifest or --backup-dir",
371 ));
372 }
373
374 if require_verified && backup_dir.is_none() {
375 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
376 }
377
378 Ok(Self {
379 manifest,
380 backup_dir,
381 mapping,
382 out,
383 require_verified,
384 require_design_v1,
385 require_restore_ready,
386 })
387 }
388}
389
390#[derive(Clone, Debug, Eq, PartialEq)]
395pub struct RestoreStatusOptions {
396 pub plan: PathBuf,
397 pub out: Option<PathBuf>,
398}
399
400impl RestoreStatusOptions {
401 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
403 where
404 I: IntoIterator<Item = OsString>,
405 {
406 let mut plan = None;
407 let mut out = None;
408
409 let mut args = args.into_iter();
410 while let Some(arg) = args.next() {
411 let arg = arg
412 .into_string()
413 .map_err(|_| RestoreCommandError::Usage(usage()))?;
414 match arg.as_str() {
415 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
416 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
417 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
418 _ => return Err(RestoreCommandError::UnknownOption(arg)),
419 }
420 }
421
422 Ok(Self {
423 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
424 out,
425 })
426 }
427}
428
429#[derive(Clone, Debug, Eq, PartialEq)]
434pub struct RestoreApplyOptions {
435 pub plan: PathBuf,
436 pub status: Option<PathBuf>,
437 pub backup_dir: Option<PathBuf>,
438 pub out: Option<PathBuf>,
439 pub journal_out: Option<PathBuf>,
440 pub dry_run: bool,
441}
442
443impl RestoreApplyOptions {
444 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
446 where
447 I: IntoIterator<Item = OsString>,
448 {
449 let mut plan = None;
450 let mut status = None;
451 let mut backup_dir = None;
452 let mut out = None;
453 let mut journal_out = None;
454 let mut dry_run = false;
455
456 let mut args = args.into_iter();
457 while let Some(arg) = args.next() {
458 let arg = arg
459 .into_string()
460 .map_err(|_| RestoreCommandError::Usage(usage()))?;
461 match arg.as_str() {
462 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
463 "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
464 "--backup-dir" => {
465 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
466 }
467 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
468 "--journal-out" => {
469 journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
470 }
471 "--dry-run" => dry_run = true,
472 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
473 _ => return Err(RestoreCommandError::UnknownOption(arg)),
474 }
475 }
476
477 if !dry_run {
478 return Err(RestoreCommandError::ApplyRequiresDryRun);
479 }
480
481 Ok(Self {
482 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
483 status,
484 backup_dir,
485 out,
486 journal_out,
487 dry_run,
488 })
489 }
490}
491
492#[derive(Clone, Debug, Eq, PartialEq)]
497#[expect(
498 clippy::struct_excessive_bools,
499 reason = "CLI status options mirror independent fail-closed guard flags"
500)]
501pub struct RestoreApplyStatusOptions {
502 pub journal: PathBuf,
503 pub require_ready: bool,
504 pub require_no_pending: bool,
505 pub require_no_failed: bool,
506 pub require_complete: bool,
507 pub require_remaining_count: Option<usize>,
508 pub require_attention_count: Option<usize>,
509 pub require_completion_basis_points: Option<usize>,
510 pub require_no_pending_before: Option<String>,
511 pub out: Option<PathBuf>,
512}
513
514impl RestoreApplyStatusOptions {
515 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
517 where
518 I: IntoIterator<Item = OsString>,
519 {
520 let mut journal = None;
521 let mut require_ready = false;
522 let mut require_no_pending = false;
523 let mut require_no_failed = false;
524 let mut require_complete = false;
525 let mut require_remaining_count = None;
526 let mut require_attention_count = None;
527 let mut require_completion_basis_points = None;
528 let mut require_no_pending_before = None;
529 let mut out = None;
530
531 let mut args = args.into_iter();
532 while let Some(arg) = args.next() {
533 let arg = arg
534 .into_string()
535 .map_err(|_| RestoreCommandError::Usage(usage()))?;
536 if parse_progress_requirement_option(
537 arg.as_str(),
538 &mut args,
539 &mut require_remaining_count,
540 &mut require_attention_count,
541 &mut require_completion_basis_points,
542 )? {
543 continue;
544 }
545 if parse_pending_requirement_option(
546 arg.as_str(),
547 &mut args,
548 &mut require_no_pending_before,
549 )? {
550 continue;
551 }
552 match arg.as_str() {
553 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
554 "--require-ready" => require_ready = true,
555 "--require-no-pending" => require_no_pending = true,
556 "--require-no-failed" => require_no_failed = true,
557 "--require-complete" => require_complete = true,
558 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
559 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
560 _ => return Err(RestoreCommandError::UnknownOption(arg)),
561 }
562 }
563
564 Ok(Self {
565 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
566 require_ready,
567 require_no_pending,
568 require_no_failed,
569 require_complete,
570 require_remaining_count,
571 require_attention_count,
572 require_completion_basis_points,
573 require_no_pending_before,
574 out,
575 })
576 }
577}
578
579#[derive(Clone, Debug, Eq, PartialEq)]
584pub struct RestoreApplyReportOptions {
585 pub journal: PathBuf,
586 pub require_no_attention: bool,
587 pub require_remaining_count: Option<usize>,
588 pub require_attention_count: Option<usize>,
589 pub require_completion_basis_points: Option<usize>,
590 pub require_no_pending_before: Option<String>,
591 pub out: Option<PathBuf>,
592}
593
594impl RestoreApplyReportOptions {
595 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
597 where
598 I: IntoIterator<Item = OsString>,
599 {
600 let mut journal = None;
601 let mut require_no_attention = false;
602 let mut require_remaining_count = None;
603 let mut require_attention_count = None;
604 let mut require_completion_basis_points = None;
605 let mut require_no_pending_before = None;
606 let mut out = None;
607
608 let mut args = args.into_iter();
609 while let Some(arg) = args.next() {
610 let arg = arg
611 .into_string()
612 .map_err(|_| RestoreCommandError::Usage(usage()))?;
613 if parse_progress_requirement_option(
614 arg.as_str(),
615 &mut args,
616 &mut require_remaining_count,
617 &mut require_attention_count,
618 &mut require_completion_basis_points,
619 )? {
620 continue;
621 }
622 if parse_pending_requirement_option(
623 arg.as_str(),
624 &mut args,
625 &mut require_no_pending_before,
626 )? {
627 continue;
628 }
629 match arg.as_str() {
630 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
631 "--require-no-attention" => require_no_attention = true,
632 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
633 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
634 _ => return Err(RestoreCommandError::UnknownOption(arg)),
635 }
636 }
637
638 Ok(Self {
639 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
640 require_no_attention,
641 require_remaining_count,
642 require_attention_count,
643 require_completion_basis_points,
644 require_no_pending_before,
645 out,
646 })
647 }
648}
649
650#[derive(Clone, Debug, Eq, PartialEq)]
655#[expect(
656 clippy::struct_excessive_bools,
657 reason = "CLI runner options mirror independent mode and fail-closed guard flags"
658)]
659pub struct RestoreRunOptions {
660 pub journal: PathBuf,
661 pub dfx: String,
662 pub network: Option<String>,
663 pub out: Option<PathBuf>,
664 pub dry_run: bool,
665 pub execute: bool,
666 pub unclaim_pending: bool,
667 pub max_steps: Option<usize>,
668 pub updated_at: Option<String>,
669 pub require_complete: bool,
670 pub require_no_attention: bool,
671 pub require_run_mode: Option<String>,
672 pub require_stopped_reason: Option<String>,
673 pub require_next_action: Option<String>,
674 pub require_executed_count: Option<usize>,
675 pub require_receipt_count: Option<usize>,
676 pub require_completed_receipt_count: Option<usize>,
677 pub require_failed_receipt_count: Option<usize>,
678 pub require_recovered_receipt_count: Option<usize>,
679 pub require_receipt_updated_at: Option<String>,
680 pub require_state_updated_at: Option<String>,
681 pub require_batch_initial_ready_count: Option<usize>,
682 pub require_batch_executed_count: Option<usize>,
683 pub require_batch_remaining_ready_count: Option<usize>,
684 pub require_batch_ready_delta: Option<isize>,
685 pub require_batch_remaining_delta: Option<isize>,
686 pub require_batch_stopped_by_max_steps: Option<bool>,
687 pub require_remaining_count: Option<usize>,
688 pub require_attention_count: Option<usize>,
689 pub require_completion_basis_points: Option<usize>,
690 pub require_no_pending_before: Option<String>,
691}
692
693impl RestoreRunOptions {
694 #[expect(
696 clippy::too_many_lines,
697 reason = "Restore runner options intentionally parse a broad flat CLI surface"
698 )]
699 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
700 where
701 I: IntoIterator<Item = OsString>,
702 {
703 let mut journal = None;
704 let mut dfx = "dfx".to_string();
705 let mut network = None;
706 let mut out = None;
707 let mut dry_run = false;
708 let mut execute = false;
709 let mut unclaim_pending = false;
710 let mut max_steps = None;
711 let mut updated_at = None;
712 let mut require_complete = false;
713 let mut require_no_attention = false;
714 let mut require_run_mode = None;
715 let mut require_stopped_reason = None;
716 let mut require_next_action = None;
717 let mut require_executed_count = None;
718 let mut require_receipt_count = None;
719 let mut require_completed_receipt_count = None;
720 let mut require_failed_receipt_count = None;
721 let mut require_recovered_receipt_count = None;
722 let mut require_receipt_updated_at = None;
723 let mut require_state_updated_at = None;
724 let mut require_batch_initial_ready_count = None;
725 let mut require_batch_executed_count = None;
726 let mut require_batch_remaining_ready_count = None;
727 let mut require_batch_ready_delta = None;
728 let mut require_batch_remaining_delta = None;
729 let mut require_batch_stopped_by_max_steps = None;
730 let mut require_remaining_count = None;
731 let mut require_attention_count = None;
732 let mut require_completion_basis_points = None;
733 let mut require_no_pending_before = None;
734
735 let mut args = args.into_iter();
736 while let Some(arg) = args.next() {
737 let arg = arg
738 .into_string()
739 .map_err(|_| RestoreCommandError::Usage(usage()))?;
740 if parse_progress_requirement_option(
741 arg.as_str(),
742 &mut args,
743 &mut require_remaining_count,
744 &mut require_attention_count,
745 &mut require_completion_basis_points,
746 )? {
747 continue;
748 }
749 if parse_pending_requirement_option(
750 arg.as_str(),
751 &mut args,
752 &mut require_no_pending_before,
753 )? {
754 continue;
755 }
756 if parse_run_count_requirement_option(
757 arg.as_str(),
758 &mut args,
759 &mut require_executed_count,
760 &mut require_receipt_count,
761 )? {
762 continue;
763 }
764 if parse_run_receipt_kind_requirement_option(
765 arg.as_str(),
766 &mut args,
767 &mut require_completed_receipt_count,
768 &mut require_failed_receipt_count,
769 &mut require_recovered_receipt_count,
770 )? {
771 continue;
772 }
773
774 match arg.as_str() {
775 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
776 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
777 "--network" => network = Some(next_value(&mut args, "--network")?),
778 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
779 "--dry-run" => dry_run = true,
780 "--execute" => execute = true,
781 "--unclaim-pending" => unclaim_pending = true,
782 "--max-steps" => {
783 max_steps = Some(parse_positive_integer(
784 "--max-steps",
785 next_value(&mut args, "--max-steps")?,
786 )?);
787 }
788 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
789 "--require-complete" => require_complete = true,
790 "--require-no-attention" => require_no_attention = true,
791 "--require-run-mode" => {
792 require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
793 }
794 "--require-stopped-reason" => {
795 require_stopped_reason =
796 Some(next_value(&mut args, "--require-stopped-reason")?);
797 }
798 "--require-next-action" => {
799 require_next_action = Some(next_value(&mut args, "--require-next-action")?);
800 }
801 "--require-receipt-updated-at" => {
802 require_receipt_updated_at =
803 Some(next_value(&mut args, "--require-receipt-updated-at")?);
804 }
805 "--require-state-updated-at" => {
806 require_state_updated_at =
807 Some(next_value(&mut args, "--require-state-updated-at")?);
808 }
809 "--require-batch-initial-ready-count" => {
810 require_batch_initial_ready_count = Some(parse_sequence(next_value(
811 &mut args,
812 "--require-batch-initial-ready-count",
813 )?)?);
814 }
815 "--require-batch-executed-count" => {
816 require_batch_executed_count = Some(parse_sequence(next_value(
817 &mut args,
818 "--require-batch-executed-count",
819 )?)?);
820 }
821 "--require-batch-remaining-ready-count" => {
822 require_batch_remaining_ready_count = Some(parse_sequence(next_value(
823 &mut args,
824 "--require-batch-remaining-ready-count",
825 )?)?);
826 }
827 "--require-batch-ready-delta" => {
828 require_batch_ready_delta = Some(parse_integer(
829 "--require-batch-ready-delta",
830 next_value(&mut args, "--require-batch-ready-delta")?,
831 )?);
832 }
833 "--require-batch-remaining-delta" => {
834 require_batch_remaining_delta = Some(parse_integer(
835 "--require-batch-remaining-delta",
836 next_value(&mut args, "--require-batch-remaining-delta")?,
837 )?);
838 }
839 "--require-batch-stopped-by-max-steps" => {
840 require_batch_stopped_by_max_steps = Some(parse_bool(
841 "--require-batch-stopped-by-max-steps",
842 next_value(&mut args, "--require-batch-stopped-by-max-steps")?,
843 )?);
844 }
845 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
846 _ => return Err(RestoreCommandError::UnknownOption(arg)),
847 }
848 }
849
850 validate_restore_run_mode_selection(dry_run, execute, unclaim_pending)?;
851
852 Ok(Self {
853 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
854 dfx,
855 network,
856 out,
857 dry_run,
858 execute,
859 unclaim_pending,
860 max_steps,
861 updated_at,
862 require_complete,
863 require_no_attention,
864 require_run_mode,
865 require_stopped_reason,
866 require_next_action,
867 require_executed_count,
868 require_receipt_count,
869 require_completed_receipt_count,
870 require_failed_receipt_count,
871 require_recovered_receipt_count,
872 require_receipt_updated_at,
873 require_state_updated_at,
874 require_batch_initial_ready_count,
875 require_batch_executed_count,
876 require_batch_remaining_ready_count,
877 require_batch_ready_delta,
878 require_batch_remaining_delta,
879 require_batch_stopped_by_max_steps,
880 require_remaining_count,
881 require_attention_count,
882 require_completion_basis_points,
883 require_no_pending_before,
884 })
885 }
886}
887
888fn validate_restore_run_mode_selection(
890 dry_run: bool,
891 execute: bool,
892 unclaim_pending: bool,
893) -> Result<(), RestoreCommandError> {
894 let mode_count = [dry_run, execute, unclaim_pending]
895 .into_iter()
896 .filter(|enabled| *enabled)
897 .count();
898 if mode_count > 1 {
899 return Err(RestoreCommandError::RestoreRunConflictingModes);
900 }
901
902 if mode_count == 0 {
903 return Err(RestoreCommandError::RestoreRunRequiresMode);
904 }
905
906 Ok(())
907}
908
909struct RestoreRunResult {
914 response: RestoreRunResponse,
915 error: Option<RestoreCommandError>,
916}
917
918impl RestoreRunResult {
919 const fn ok(response: RestoreRunResponse) -> Self {
921 Self {
922 response,
923 error: None,
924 }
925 }
926}
927
928const RESTORE_RUN_MODE_DRY_RUN: &str = "dry-run";
929const RESTORE_RUN_MODE_EXECUTE: &str = "execute";
930const RESTORE_RUN_MODE_UNCLAIM_PENDING: &str = "unclaim-pending";
931
932const RESTORE_RUN_STOPPED_BLOCKED: &str = "blocked";
933const RESTORE_RUN_STOPPED_COMMAND_FAILED: &str = "command-failed";
934const RESTORE_RUN_STOPPED_COMPLETE: &str = "complete";
935const RESTORE_RUN_STOPPED_MAX_STEPS: &str = "max-steps-reached";
936const RESTORE_RUN_STOPPED_PENDING: &str = "pending";
937const RESTORE_RUN_STOPPED_PREVIEW: &str = "preview";
938const RESTORE_RUN_STOPPED_READY: &str = "ready";
939const RESTORE_RUN_STOPPED_RECOVERED_PENDING: &str = "recovered-pending";
940
941const RESTORE_RUN_ACTION_DONE: &str = "done";
942const RESTORE_RUN_ACTION_FIX_BLOCKED: &str = "fix-blocked-journal";
943const RESTORE_RUN_ACTION_INSPECT_FAILED: &str = "inspect-failed-operation";
944const RESTORE_RUN_ACTION_RERUN: &str = "rerun";
945const RESTORE_RUN_ACTION_UNCLAIM_PENDING: &str = "unclaim-pending";
946
947const RESTORE_RUN_EXECUTED_COMPLETED: &str = "completed";
948const RESTORE_RUN_EXECUTED_FAILED: &str = "failed";
949const RESTORE_RUN_RECEIPT_COMPLETED: &str = "command-completed";
950const RESTORE_RUN_RECEIPT_FAILED: &str = "command-failed";
951const RESTORE_RUN_RECEIPT_RECOVERED_PENDING: &str = "pending-recovered";
952const RESTORE_RUN_RECEIPT_STATE_READY: &str = "ready";
953const RESTORE_RUN_COMMAND_EXIT_PREFIX: &str = "runner-command-exit";
954const RESTORE_RUN_RESPONSE_VERSION: u16 = 1;
955
956#[derive(Clone, Debug, Serialize)]
961#[expect(
962 clippy::struct_excessive_bools,
963 reason = "Runner response exposes stable JSON status flags for operators and CI"
964)]
965pub struct RestoreRunResponse {
966 run_version: u16,
967 backup_id: String,
968 run_mode: &'static str,
969 dry_run: bool,
970 execute: bool,
971 unclaim_pending: bool,
972 stopped_reason: &'static str,
973 next_action: &'static str,
974 #[serde(skip_serializing_if = "Option::is_none")]
975 requested_state_updated_at: Option<String>,
976 #[serde(skip_serializing_if = "Option::is_none")]
977 max_steps_reached: Option<bool>,
978 #[serde(default, skip_serializing_if = "Vec::is_empty")]
979 executed_operations: Vec<RestoreRunExecutedOperation>,
980 #[serde(default, skip_serializing_if = "Vec::is_empty")]
981 operation_receipts: Vec<RestoreRunOperationReceipt>,
982 #[serde(skip_serializing_if = "Option::is_none")]
983 operation_receipt_count: Option<usize>,
984 operation_receipt_summary: RestoreRunReceiptSummary,
985 #[serde(skip_serializing_if = "Option::is_none")]
986 executed_operation_count: Option<usize>,
987 #[serde(skip_serializing_if = "Option::is_none")]
988 recovered_operation: Option<RestoreApplyJournalOperation>,
989 batch_summary: RestoreRunBatchSummary,
990 ready: bool,
991 complete: bool,
992 attention_required: bool,
993 outcome: RestoreApplyReportOutcome,
994 operation_count: usize,
995 operation_counts: RestoreApplyOperationKindCounts,
996 operation_counts_supplied: bool,
997 progress: RestoreApplyProgressSummary,
998 pending_summary: RestoreApplyPendingSummary,
999 pending_operations: usize,
1000 ready_operations: usize,
1001 blocked_operations: usize,
1002 completed_operations: usize,
1003 failed_operations: usize,
1004 blocked_reasons: Vec<String>,
1005 next_transition: Option<RestoreApplyReportOperation>,
1006 #[serde(skip_serializing_if = "Option::is_none")]
1007 operation_available: Option<bool>,
1008 #[serde(skip_serializing_if = "Option::is_none")]
1009 command_available: Option<bool>,
1010 #[serde(skip_serializing_if = "Option::is_none")]
1011 command: Option<RestoreApplyRunnerCommand>,
1012}
1013
1014impl RestoreRunResponse {
1015 fn from_report(
1017 backup_id: String,
1018 report: RestoreApplyJournalReport,
1019 mode: RestoreRunResponseMode,
1020 ) -> Self {
1021 Self {
1022 run_version: RESTORE_RUN_RESPONSE_VERSION,
1023 backup_id,
1024 run_mode: mode.run_mode,
1025 dry_run: mode.dry_run,
1026 execute: mode.execute,
1027 unclaim_pending: mode.unclaim_pending,
1028 stopped_reason: mode.stopped_reason,
1029 next_action: mode.next_action,
1030 requested_state_updated_at: None,
1031 max_steps_reached: None,
1032 executed_operations: Vec::new(),
1033 operation_receipts: Vec::new(),
1034 operation_receipt_count: Some(0),
1035 operation_receipt_summary: RestoreRunReceiptSummary::default(),
1036 executed_operation_count: None,
1037 recovered_operation: None,
1038 batch_summary: RestoreRunBatchSummary::from_counts(
1039 RestoreRunBatchStart::new(
1040 None,
1041 report.ready_operations,
1042 report.progress.remaining_operations,
1043 ),
1044 0,
1045 report.ready_operations,
1046 report.progress.remaining_operations,
1047 false,
1048 report.complete,
1049 ),
1050 ready: report.ready,
1051 complete: report.complete,
1052 attention_required: report.attention_required,
1053 outcome: report.outcome,
1054 operation_count: report.operation_count,
1055 operation_counts: report.operation_counts,
1056 operation_counts_supplied: report.operation_counts_supplied,
1057 progress: report.progress,
1058 pending_summary: report.pending_summary,
1059 pending_operations: report.pending_operations,
1060 ready_operations: report.ready_operations,
1061 blocked_operations: report.blocked_operations,
1062 completed_operations: report.completed_operations,
1063 failed_operations: report.failed_operations,
1064 blocked_reasons: report.blocked_reasons,
1065 next_transition: report.next_transition,
1066 operation_available: None,
1067 command_available: None,
1068 command: None,
1069 }
1070 }
1071
1072 fn set_operation_receipts(&mut self, receipts: Vec<RestoreRunOperationReceipt>) {
1074 self.operation_receipt_summary = RestoreRunReceiptSummary::from_receipts(&receipts);
1075 self.operation_receipt_count = Some(receipts.len());
1076 self.operation_receipts = receipts;
1077 }
1078
1079 fn set_requested_state_updated_at(&mut self, updated_at: Option<&String>) {
1081 self.requested_state_updated_at = updated_at.cloned();
1082 }
1083
1084 const fn set_batch_summary(
1086 &mut self,
1087 batch_start: RestoreRunBatchStart,
1088 executed_operations: usize,
1089 stopped_by_max_steps: bool,
1090 ) {
1091 self.batch_summary = RestoreRunBatchSummary::from_counts(
1092 batch_start,
1093 executed_operations,
1094 self.ready_operations,
1095 self.progress.remaining_operations,
1096 stopped_by_max_steps,
1097 self.complete,
1098 );
1099 }
1100}
1101
1102#[derive(Clone, Copy, Debug)]
1107struct RestoreRunBatchStart {
1108 requested_max_steps: Option<usize>,
1109 initial_ready_operations: usize,
1110 initial_remaining_operations: usize,
1111}
1112
1113impl RestoreRunBatchStart {
1114 const fn new(
1116 requested_max_steps: Option<usize>,
1117 initial_ready_operations: usize,
1118 initial_remaining_operations: usize,
1119 ) -> Self {
1120 Self {
1121 requested_max_steps,
1122 initial_ready_operations,
1123 initial_remaining_operations,
1124 }
1125 }
1126}
1127
1128#[derive(Clone, Debug, Serialize)]
1133struct RestoreRunBatchSummary {
1134 requested_max_steps: Option<usize>,
1135 initial_ready_operations: usize,
1136 initial_remaining_operations: usize,
1137 executed_operations: usize,
1138 remaining_ready_operations: usize,
1139 remaining_operations: usize,
1140 ready_operations_delta: isize,
1141 remaining_operations_delta: isize,
1142 stopped_by_max_steps: bool,
1143 complete: bool,
1144}
1145
1146impl RestoreRunBatchSummary {
1147 const fn from_counts(
1149 batch_start: RestoreRunBatchStart,
1150 executed_operations: usize,
1151 remaining_ready_operations: usize,
1152 remaining_operations: usize,
1153 stopped_by_max_steps: bool,
1154 complete: bool,
1155 ) -> Self {
1156 Self {
1157 requested_max_steps: batch_start.requested_max_steps,
1158 initial_ready_operations: batch_start.initial_ready_operations,
1159 initial_remaining_operations: batch_start.initial_remaining_operations,
1160 executed_operations,
1161 remaining_ready_operations,
1162 remaining_operations,
1163 ready_operations_delta: remaining_ready_operations.cast_signed()
1164 - batch_start.initial_ready_operations.cast_signed(),
1165 remaining_operations_delta: remaining_operations.cast_signed()
1166 - batch_start.initial_remaining_operations.cast_signed(),
1167 stopped_by_max_steps,
1168 complete,
1169 }
1170 }
1171}
1172
1173#[derive(Clone, Debug, Default, Serialize)]
1178struct RestoreRunReceiptSummary {
1179 total_receipts: usize,
1180 command_completed: usize,
1181 command_failed: usize,
1182 pending_recovered: usize,
1183}
1184
1185impl RestoreRunReceiptSummary {
1186 fn from_receipts(receipts: &[RestoreRunOperationReceipt]) -> Self {
1188 let mut summary = Self {
1189 total_receipts: receipts.len(),
1190 ..Self::default()
1191 };
1192
1193 for receipt in receipts {
1194 match receipt.event {
1195 RESTORE_RUN_RECEIPT_COMPLETED => summary.command_completed += 1,
1196 RESTORE_RUN_RECEIPT_FAILED => summary.command_failed += 1,
1197 RESTORE_RUN_RECEIPT_RECOVERED_PENDING => summary.pending_recovered += 1,
1198 _ => {}
1199 }
1200 }
1201
1202 summary
1203 }
1204}
1205
1206#[derive(Clone, Debug, Serialize)]
1211struct RestoreRunOperationReceipt {
1212 event: &'static str,
1213 sequence: usize,
1214 operation: RestoreApplyOperationKind,
1215 target_canister: String,
1216 state: &'static str,
1217 #[serde(skip_serializing_if = "Option::is_none")]
1218 updated_at: Option<String>,
1219 #[serde(skip_serializing_if = "Option::is_none")]
1220 command: Option<RestoreApplyRunnerCommand>,
1221 #[serde(skip_serializing_if = "Option::is_none")]
1222 status: Option<String>,
1223}
1224
1225impl RestoreRunOperationReceipt {
1226 fn completed(
1228 operation: RestoreApplyJournalOperation,
1229 command: RestoreApplyRunnerCommand,
1230 status: String,
1231 updated_at: Option<String>,
1232 ) -> Self {
1233 Self::from_operation(
1234 RESTORE_RUN_RECEIPT_COMPLETED,
1235 operation,
1236 RESTORE_RUN_EXECUTED_COMPLETED,
1237 updated_at,
1238 Some(command),
1239 Some(status),
1240 )
1241 }
1242
1243 fn failed(
1245 operation: RestoreApplyJournalOperation,
1246 command: RestoreApplyRunnerCommand,
1247 status: String,
1248 updated_at: Option<String>,
1249 ) -> Self {
1250 Self::from_operation(
1251 RESTORE_RUN_RECEIPT_FAILED,
1252 operation,
1253 RESTORE_RUN_EXECUTED_FAILED,
1254 updated_at,
1255 Some(command),
1256 Some(status),
1257 )
1258 }
1259
1260 fn recovered_pending(
1262 operation: RestoreApplyJournalOperation,
1263 updated_at: Option<String>,
1264 ) -> Self {
1265 Self::from_operation(
1266 RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
1267 operation,
1268 RESTORE_RUN_RECEIPT_STATE_READY,
1269 updated_at,
1270 None,
1271 None,
1272 )
1273 }
1274
1275 fn from_operation(
1277 event: &'static str,
1278 operation: RestoreApplyJournalOperation,
1279 state: &'static str,
1280 updated_at: Option<String>,
1281 command: Option<RestoreApplyRunnerCommand>,
1282 status: Option<String>,
1283 ) -> Self {
1284 Self {
1285 event,
1286 sequence: operation.sequence,
1287 operation: operation.operation,
1288 target_canister: operation.target_canister,
1289 state,
1290 updated_at,
1291 command,
1292 status,
1293 }
1294 }
1295}
1296
1297#[derive(Clone, Debug, Serialize)]
1302struct RestoreRunExecutedOperation {
1303 sequence: usize,
1304 operation: RestoreApplyOperationKind,
1305 target_canister: String,
1306 command: RestoreApplyRunnerCommand,
1307 status: String,
1308 state: &'static str,
1309}
1310
1311impl RestoreRunExecutedOperation {
1312 fn completed(
1314 operation: RestoreApplyJournalOperation,
1315 command: RestoreApplyRunnerCommand,
1316 status: String,
1317 ) -> Self {
1318 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_COMPLETED)
1319 }
1320
1321 fn failed(
1323 operation: RestoreApplyJournalOperation,
1324 command: RestoreApplyRunnerCommand,
1325 status: String,
1326 ) -> Self {
1327 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_FAILED)
1328 }
1329
1330 fn from_operation(
1332 operation: RestoreApplyJournalOperation,
1333 command: RestoreApplyRunnerCommand,
1334 status: String,
1335 state: &'static str,
1336 ) -> Self {
1337 Self {
1338 sequence: operation.sequence,
1339 operation: operation.operation,
1340 target_canister: operation.target_canister,
1341 command,
1342 status,
1343 state,
1344 }
1345 }
1346}
1347
1348struct RestoreRunResponseMode {
1353 run_mode: &'static str,
1354 dry_run: bool,
1355 execute: bool,
1356 unclaim_pending: bool,
1357 stopped_reason: &'static str,
1358 next_action: &'static str,
1359}
1360
1361impl RestoreRunResponseMode {
1362 const fn new(
1364 run_mode: &'static str,
1365 dry_run: bool,
1366 execute: bool,
1367 unclaim_pending: bool,
1368 stopped_reason: &'static str,
1369 next_action: &'static str,
1370 ) -> Self {
1371 Self {
1372 run_mode,
1373 dry_run,
1374 execute,
1375 unclaim_pending,
1376 stopped_reason,
1377 next_action,
1378 }
1379 }
1380
1381 const fn dry_run(stopped_reason: &'static str, next_action: &'static str) -> Self {
1383 Self::new(
1384 RESTORE_RUN_MODE_DRY_RUN,
1385 true,
1386 false,
1387 false,
1388 stopped_reason,
1389 next_action,
1390 )
1391 }
1392
1393 const fn execute(stopped_reason: &'static str, next_action: &'static str) -> Self {
1395 Self::new(
1396 RESTORE_RUN_MODE_EXECUTE,
1397 false,
1398 true,
1399 false,
1400 stopped_reason,
1401 next_action,
1402 )
1403 }
1404
1405 const fn unclaim_pending(next_action: &'static str) -> Self {
1407 Self::new(
1408 RESTORE_RUN_MODE_UNCLAIM_PENDING,
1409 false,
1410 false,
1411 true,
1412 RESTORE_RUN_STOPPED_RECOVERED_PENDING,
1413 next_action,
1414 )
1415 }
1416}
1417
1418#[derive(Clone, Debug, Eq, PartialEq)]
1423pub struct RestoreApplyNextOptions {
1424 pub journal: PathBuf,
1425 pub out: Option<PathBuf>,
1426}
1427
1428impl RestoreApplyNextOptions {
1429 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1431 where
1432 I: IntoIterator<Item = OsString>,
1433 {
1434 let mut journal = None;
1435 let mut out = None;
1436
1437 let mut args = args.into_iter();
1438 while let Some(arg) = args.next() {
1439 let arg = arg
1440 .into_string()
1441 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1442 match arg.as_str() {
1443 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1444 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1445 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1446 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1447 }
1448 }
1449
1450 Ok(Self {
1451 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1452 out,
1453 })
1454 }
1455}
1456
1457#[derive(Clone, Debug, Eq, PartialEq)]
1462pub struct RestoreApplyCommandOptions {
1463 pub journal: PathBuf,
1464 pub dfx: String,
1465 pub network: Option<String>,
1466 pub out: Option<PathBuf>,
1467 pub require_command: bool,
1468}
1469
1470impl RestoreApplyCommandOptions {
1471 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1473 where
1474 I: IntoIterator<Item = OsString>,
1475 {
1476 let mut journal = None;
1477 let mut dfx = "dfx".to_string();
1478 let mut network = None;
1479 let mut out = None;
1480 let mut require_command = false;
1481
1482 let mut args = args.into_iter();
1483 while let Some(arg) = args.next() {
1484 let arg = arg
1485 .into_string()
1486 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1487 match arg.as_str() {
1488 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1489 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
1490 "--network" => network = Some(next_value(&mut args, "--network")?),
1491 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1492 "--require-command" => require_command = true,
1493 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1494 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1495 }
1496 }
1497
1498 Ok(Self {
1499 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1500 dfx,
1501 network,
1502 out,
1503 require_command,
1504 })
1505 }
1506}
1507
1508#[derive(Clone, Debug, Eq, PartialEq)]
1513pub struct RestoreApplyClaimOptions {
1514 pub journal: PathBuf,
1515 pub sequence: Option<usize>,
1516 pub updated_at: Option<String>,
1517 pub out: Option<PathBuf>,
1518}
1519
1520impl RestoreApplyClaimOptions {
1521 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1523 where
1524 I: IntoIterator<Item = OsString>,
1525 {
1526 let mut journal = None;
1527 let mut sequence = None;
1528 let mut updated_at = None;
1529 let mut out = None;
1530
1531 let mut args = args.into_iter();
1532 while let Some(arg) = args.next() {
1533 let arg = arg
1534 .into_string()
1535 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1536 match arg.as_str() {
1537 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1538 "--sequence" => {
1539 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1540 }
1541 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1542 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1543 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1544 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1545 }
1546 }
1547
1548 Ok(Self {
1549 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1550 sequence,
1551 updated_at,
1552 out,
1553 })
1554 }
1555}
1556
1557#[derive(Clone, Debug, Eq, PartialEq)]
1562pub struct RestoreApplyUnclaimOptions {
1563 pub journal: PathBuf,
1564 pub sequence: Option<usize>,
1565 pub updated_at: Option<String>,
1566 pub out: Option<PathBuf>,
1567}
1568
1569impl RestoreApplyUnclaimOptions {
1570 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1572 where
1573 I: IntoIterator<Item = OsString>,
1574 {
1575 let mut journal = None;
1576 let mut sequence = None;
1577 let mut updated_at = None;
1578 let mut out = None;
1579
1580 let mut args = args.into_iter();
1581 while let Some(arg) = args.next() {
1582 let arg = arg
1583 .into_string()
1584 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1585 match arg.as_str() {
1586 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1587 "--sequence" => {
1588 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1589 }
1590 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1591 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1592 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1593 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1594 }
1595 }
1596
1597 Ok(Self {
1598 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1599 sequence,
1600 updated_at,
1601 out,
1602 })
1603 }
1604}
1605
1606#[derive(Clone, Debug, Eq, PartialEq)]
1611pub struct RestoreApplyMarkOptions {
1612 pub journal: PathBuf,
1613 pub sequence: usize,
1614 pub state: RestoreApplyMarkState,
1615 pub reason: Option<String>,
1616 pub updated_at: Option<String>,
1617 pub out: Option<PathBuf>,
1618 pub require_pending: bool,
1619}
1620
1621impl RestoreApplyMarkOptions {
1622 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1624 where
1625 I: IntoIterator<Item = OsString>,
1626 {
1627 let mut journal = None;
1628 let mut sequence = None;
1629 let mut state = None;
1630 let mut reason = None;
1631 let mut updated_at = None;
1632 let mut out = None;
1633 let mut require_pending = false;
1634
1635 let mut args = args.into_iter();
1636 while let Some(arg) = args.next() {
1637 let arg = arg
1638 .into_string()
1639 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1640 match arg.as_str() {
1641 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1642 "--sequence" => {
1643 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1644 }
1645 "--state" => {
1646 state = Some(RestoreApplyMarkState::parse(next_value(
1647 &mut args, "--state",
1648 )?)?);
1649 }
1650 "--reason" => reason = Some(next_value(&mut args, "--reason")?),
1651 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1652 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1653 "--require-pending" => require_pending = true,
1654 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1655 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1656 }
1657 }
1658
1659 Ok(Self {
1660 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1661 sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
1662 state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
1663 reason,
1664 updated_at,
1665 out,
1666 require_pending,
1667 })
1668 }
1669}
1670
1671#[derive(Clone, Debug, Eq, PartialEq)]
1676pub enum RestoreApplyMarkState {
1677 Completed,
1678 Failed,
1679}
1680
1681impl RestoreApplyMarkState {
1682 fn parse(value: String) -> Result<Self, RestoreCommandError> {
1684 match value.as_str() {
1685 "completed" => Ok(Self::Completed),
1686 "failed" => Ok(Self::Failed),
1687 _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
1688 }
1689 }
1690}
1691
1692pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
1694where
1695 I: IntoIterator<Item = OsString>,
1696{
1697 let mut args = args.into_iter();
1698 let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
1699 return Err(RestoreCommandError::Usage(usage()));
1700 };
1701
1702 match command.as_str() {
1703 "plan" => {
1704 let options = RestorePlanOptions::parse(args)?;
1705 let plan = plan_restore(&options)?;
1706 write_plan(&options, &plan)?;
1707 enforce_restore_plan_requirements(&options, &plan)?;
1708 Ok(())
1709 }
1710 "status" => {
1711 let options = RestoreStatusOptions::parse(args)?;
1712 let status = restore_status(&options)?;
1713 write_status(&options, &status)?;
1714 Ok(())
1715 }
1716 "apply" => {
1717 let options = RestoreApplyOptions::parse(args)?;
1718 let dry_run = restore_apply_dry_run(&options)?;
1719 write_apply_dry_run(&options, &dry_run)?;
1720 write_apply_journal_if_requested(&options, &dry_run)?;
1721 Ok(())
1722 }
1723 "apply-status" => {
1724 let options = RestoreApplyStatusOptions::parse(args)?;
1725 let status = restore_apply_status(&options)?;
1726 write_apply_status(&options, &status)?;
1727 enforce_apply_status_requirements(&options, &status)?;
1728 Ok(())
1729 }
1730 "apply-report" => {
1731 let options = RestoreApplyReportOptions::parse(args)?;
1732 let report = restore_apply_report(&options)?;
1733 write_apply_report(&options, &report)?;
1734 enforce_apply_report_requirements(&options, &report)?;
1735 Ok(())
1736 }
1737 "run" => {
1738 let options = RestoreRunOptions::parse(args)?;
1739 let run = if options.execute {
1740 restore_run_execute_result(&options)?
1741 } else if options.unclaim_pending {
1742 RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1743 } else {
1744 RestoreRunResult::ok(restore_run_dry_run(&options)?)
1745 };
1746 write_restore_run(&options, &run.response)?;
1747 if let Some(error) = run.error {
1748 return Err(error);
1749 }
1750 enforce_restore_run_requirements(&options, &run.response)?;
1751 Ok(())
1752 }
1753 "apply-next" => {
1754 let options = RestoreApplyNextOptions::parse(args)?;
1755 let next = restore_apply_next(&options)?;
1756 write_apply_next(&options, &next)?;
1757 Ok(())
1758 }
1759 "apply-command" => {
1760 let options = RestoreApplyCommandOptions::parse(args)?;
1761 let preview = restore_apply_command(&options)?;
1762 write_apply_command(&options, &preview)?;
1763 enforce_apply_command_requirements(&options, &preview)?;
1764 Ok(())
1765 }
1766 "apply-claim" => {
1767 let options = RestoreApplyClaimOptions::parse(args)?;
1768 let journal = restore_apply_claim(&options)?;
1769 write_apply_claim(&options, &journal)?;
1770 Ok(())
1771 }
1772 "apply-unclaim" => {
1773 let options = RestoreApplyUnclaimOptions::parse(args)?;
1774 let journal = restore_apply_unclaim(&options)?;
1775 write_apply_unclaim(&options, &journal)?;
1776 Ok(())
1777 }
1778 "apply-mark" => {
1779 let options = RestoreApplyMarkOptions::parse(args)?;
1780 let journal = restore_apply_mark(&options)?;
1781 write_apply_mark(&options, &journal)?;
1782 Ok(())
1783 }
1784 "help" | "--help" | "-h" => {
1785 println!("{}", usage());
1786 Ok(())
1787 }
1788 _ => Err(RestoreCommandError::UnknownOption(command)),
1789 }
1790}
1791
1792pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1794 verify_backup_layout_if_required(options)?;
1795
1796 let manifest = read_manifest_source(options)?;
1797 let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1798
1799 RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1800}
1801
1802pub fn restore_status(
1804 options: &RestoreStatusOptions,
1805) -> Result<RestoreStatus, RestoreCommandError> {
1806 let plan = read_plan(&options.plan)?;
1807 Ok(RestoreStatus::from_plan(&plan))
1808}
1809
1810pub fn restore_apply_dry_run(
1812 options: &RestoreApplyOptions,
1813) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1814 let plan = read_plan(&options.plan)?;
1815 let status = options.status.as_ref().map(read_status).transpose()?;
1816 if let Some(backup_dir) = &options.backup_dir {
1817 return RestoreApplyDryRun::try_from_plan_with_artifacts(
1818 &plan,
1819 status.as_ref(),
1820 backup_dir,
1821 )
1822 .map_err(RestoreCommandError::from);
1823 }
1824
1825 RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1826}
1827
1828pub fn restore_apply_status(
1830 options: &RestoreApplyStatusOptions,
1831) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1832 let journal = read_apply_journal(&options.journal)?;
1833 Ok(journal.status())
1834}
1835
1836pub fn restore_apply_report(
1838 options: &RestoreApplyReportOptions,
1839) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1840 let journal = read_apply_journal(&options.journal)?;
1841 Ok(journal.report())
1842}
1843
1844pub fn restore_run_dry_run(
1846 options: &RestoreRunOptions,
1847) -> Result<RestoreRunResponse, RestoreCommandError> {
1848 let journal = read_apply_journal(&options.journal)?;
1849 let report = journal.report();
1850 let initial_ready_operations = report.ready_operations;
1851 let initial_remaining_operations = report.progress.remaining_operations;
1852 let preview = journal.next_command_preview_with_config(&restore_run_command_config(options));
1853 let stopped_reason = restore_run_stopped_reason(&report, false, false);
1854 let next_action = restore_run_next_action(&report, false);
1855
1856 let mut response = RestoreRunResponse::from_report(
1857 journal.backup_id,
1858 report,
1859 RestoreRunResponseMode::dry_run(stopped_reason, next_action),
1860 );
1861 response.set_requested_state_updated_at(options.updated_at.as_ref());
1862 response.set_batch_summary(
1863 RestoreRunBatchStart::new(
1864 options.max_steps,
1865 initial_ready_operations,
1866 initial_remaining_operations,
1867 ),
1868 0,
1869 false,
1870 );
1871 response.operation_available = Some(preview.operation_available);
1872 response.command_available = Some(preview.command_available);
1873 response.command = preview.command;
1874 Ok(response)
1875}
1876
1877pub fn restore_run_unclaim_pending(
1879 options: &RestoreRunOptions,
1880) -> Result<RestoreRunResponse, RestoreCommandError> {
1881 let mut journal = read_apply_journal(&options.journal)?;
1882 let initial_report = journal.report();
1883 let initial_ready_operations = initial_report.ready_operations;
1884 let initial_remaining_operations = initial_report.progress.remaining_operations;
1885 let recovered_operation = journal
1886 .next_transition_operation()
1887 .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1888 .cloned()
1889 .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1890
1891 let recovered_updated_at = state_updated_at(options.updated_at.as_ref());
1892 journal.mark_next_operation_ready_at(Some(recovered_updated_at.clone()))?;
1893 write_apply_journal_file(&options.journal, &journal)?;
1894
1895 let report = journal.report();
1896 let next_action = restore_run_next_action(&report, true);
1897 let mut response = RestoreRunResponse::from_report(
1898 journal.backup_id,
1899 report,
1900 RestoreRunResponseMode::unclaim_pending(next_action),
1901 );
1902 response.set_requested_state_updated_at(options.updated_at.as_ref());
1903 response.set_batch_summary(
1904 RestoreRunBatchStart::new(
1905 options.max_steps,
1906 initial_ready_operations,
1907 initial_remaining_operations,
1908 ),
1909 0,
1910 false,
1911 );
1912 response.set_operation_receipts(vec![RestoreRunOperationReceipt::recovered_pending(
1913 recovered_operation.clone(),
1914 Some(recovered_updated_at),
1915 )]);
1916 response.recovered_operation = Some(recovered_operation);
1917 Ok(response)
1918}
1919
1920pub fn restore_run_execute(
1922 options: &RestoreRunOptions,
1923) -> Result<RestoreRunResponse, RestoreCommandError> {
1924 let run = restore_run_execute_result(options)?;
1925 if let Some(error) = run.error {
1926 return Err(error);
1927 }
1928
1929 Ok(run.response)
1930}
1931
1932fn restore_run_execute_result(
1934 options: &RestoreRunOptions,
1935) -> Result<RestoreRunResult, RestoreCommandError> {
1936 let mut journal = read_apply_journal(&options.journal)?;
1937 let initial_report = journal.report();
1938 let batch_start = RestoreRunBatchStart::new(
1939 options.max_steps,
1940 initial_report.ready_operations,
1941 initial_report.progress.remaining_operations,
1942 );
1943 let mut executed_operations = Vec::new();
1944 let mut operation_receipts = Vec::new();
1945 let config = restore_run_command_config(options);
1946
1947 loop {
1948 let report = journal.report();
1949 let max_steps_reached =
1950 restore_run_max_steps_reached(options, executed_operations.len(), &report);
1951 if report.complete || max_steps_reached {
1952 return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1953 &journal,
1954 executed_operations,
1955 operation_receipts,
1956 max_steps_reached,
1957 options.updated_at.as_ref(),
1958 batch_start,
1959 )));
1960 }
1961
1962 enforce_restore_run_executable(&journal, &report)?;
1963 let preview = journal.next_command_preview_with_config(&config);
1964 enforce_restore_run_command_available(&preview)?;
1965
1966 let operation = preview
1967 .operation
1968 .clone()
1969 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1970 let command = preview
1971 .command
1972 .clone()
1973 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1974 let sequence = operation.sequence;
1975
1976 enforce_apply_claim_sequence(sequence, &journal)?;
1977 journal.mark_operation_pending_at(
1978 sequence,
1979 Some(state_updated_at(options.updated_at.as_ref())),
1980 )?;
1981 write_apply_journal_file(&options.journal, &journal)?;
1982
1983 let status = Command::new(&command.program)
1984 .args(&command.args)
1985 .status()?;
1986 let status_label = exit_status_label(status);
1987 if status.success() {
1988 let completed_updated_at = state_updated_at(options.updated_at.as_ref());
1989 journal.mark_operation_completed_at(sequence, Some(completed_updated_at.clone()))?;
1990 write_apply_journal_file(&options.journal, &journal)?;
1991 executed_operations.push(RestoreRunExecutedOperation::completed(
1992 operation.clone(),
1993 command.clone(),
1994 status_label.clone(),
1995 ));
1996 operation_receipts.push(RestoreRunOperationReceipt::completed(
1997 operation,
1998 command,
1999 status_label,
2000 Some(completed_updated_at),
2001 ));
2002 continue;
2003 }
2004
2005 let failed_updated_at = state_updated_at(options.updated_at.as_ref());
2006 journal.mark_operation_failed_at(
2007 sequence,
2008 format!("{RESTORE_RUN_COMMAND_EXIT_PREFIX}-{status_label}"),
2009 Some(failed_updated_at.clone()),
2010 )?;
2011 write_apply_journal_file(&options.journal, &journal)?;
2012 executed_operations.push(RestoreRunExecutedOperation::failed(
2013 operation.clone(),
2014 command.clone(),
2015 status_label.clone(),
2016 ));
2017 operation_receipts.push(RestoreRunOperationReceipt::failed(
2018 operation,
2019 command,
2020 status_label.clone(),
2021 Some(failed_updated_at),
2022 ));
2023 let response = restore_run_execute_summary(
2024 &journal,
2025 executed_operations,
2026 operation_receipts,
2027 false,
2028 options.updated_at.as_ref(),
2029 batch_start,
2030 );
2031 return Ok(RestoreRunResult {
2032 response,
2033 error: Some(RestoreCommandError::RestoreRunCommandFailed {
2034 sequence,
2035 status: status_label,
2036 }),
2037 });
2038 }
2039}
2040
2041fn restore_run_command_config(options: &RestoreRunOptions) -> RestoreApplyCommandConfig {
2043 restore_command_config(&options.dfx, options.network.as_deref())
2044}
2045
2046fn restore_apply_command_config(options: &RestoreApplyCommandOptions) -> RestoreApplyCommandConfig {
2048 restore_command_config(&options.dfx, options.network.as_deref())
2049}
2050
2051fn restore_command_config(program: &str, network: Option<&str>) -> RestoreApplyCommandConfig {
2053 RestoreApplyCommandConfig {
2054 program: program.to_string(),
2055 network: network.map(str::to_string),
2056 }
2057}
2058
2059fn restore_run_max_steps_reached(
2061 options: &RestoreRunOptions,
2062 executed_operation_count: usize,
2063 report: &RestoreApplyJournalReport,
2064) -> bool {
2065 options.max_steps == Some(executed_operation_count) && !report.complete
2066}
2067
2068fn restore_run_execute_summary(
2070 journal: &RestoreApplyJournal,
2071 executed_operations: Vec<RestoreRunExecutedOperation>,
2072 operation_receipts: Vec<RestoreRunOperationReceipt>,
2073 max_steps_reached: bool,
2074 requested_state_updated_at: Option<&String>,
2075 batch_start: RestoreRunBatchStart,
2076) -> RestoreRunResponse {
2077 let report = journal.report();
2078 let executed_operation_count = executed_operations.len();
2079 let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
2080 let next_action = restore_run_next_action(&report, false);
2081
2082 let mut response = RestoreRunResponse::from_report(
2083 journal.backup_id.clone(),
2084 report,
2085 RestoreRunResponseMode::execute(stopped_reason, next_action),
2086 );
2087 response.set_requested_state_updated_at(requested_state_updated_at);
2088 response.set_batch_summary(batch_start, executed_operation_count, max_steps_reached);
2089 response.max_steps_reached = Some(max_steps_reached);
2090 response.executed_operation_count = Some(executed_operation_count);
2091 response.executed_operations = executed_operations;
2092 response.set_operation_receipts(operation_receipts);
2093 response
2094}
2095
2096const fn restore_run_stopped_reason(
2098 report: &RestoreApplyJournalReport,
2099 max_steps_reached: bool,
2100 executed: bool,
2101) -> &'static str {
2102 if report.complete {
2103 return RESTORE_RUN_STOPPED_COMPLETE;
2104 }
2105 if report.failed_operations > 0 {
2106 return RESTORE_RUN_STOPPED_COMMAND_FAILED;
2107 }
2108 if report.pending_operations > 0 {
2109 return RESTORE_RUN_STOPPED_PENDING;
2110 }
2111 if !report.ready || report.blocked_operations > 0 {
2112 return RESTORE_RUN_STOPPED_BLOCKED;
2113 }
2114 if max_steps_reached {
2115 return RESTORE_RUN_STOPPED_MAX_STEPS;
2116 }
2117 if executed {
2118 return RESTORE_RUN_STOPPED_READY;
2119 }
2120 RESTORE_RUN_STOPPED_PREVIEW
2121}
2122
2123const fn restore_run_next_action(
2125 report: &RestoreApplyJournalReport,
2126 recovered_pending: bool,
2127) -> &'static str {
2128 if report.complete {
2129 return RESTORE_RUN_ACTION_DONE;
2130 }
2131 if report.failed_operations > 0 {
2132 return RESTORE_RUN_ACTION_INSPECT_FAILED;
2133 }
2134 if report.pending_operations > 0 {
2135 return RESTORE_RUN_ACTION_UNCLAIM_PENDING;
2136 }
2137 if !report.ready || report.blocked_operations > 0 {
2138 return RESTORE_RUN_ACTION_FIX_BLOCKED;
2139 }
2140 if recovered_pending {
2141 return RESTORE_RUN_ACTION_RERUN;
2142 }
2143 RESTORE_RUN_ACTION_RERUN
2144}
2145
2146fn enforce_restore_run_executable(
2148 journal: &RestoreApplyJournal,
2149 report: &RestoreApplyJournalReport,
2150) -> Result<(), RestoreCommandError> {
2151 if report.pending_operations > 0 {
2152 return Err(RestoreCommandError::RestoreApplyPending {
2153 backup_id: report.backup_id.clone(),
2154 pending_operations: report.pending_operations,
2155 next_transition_sequence: report
2156 .next_transition
2157 .as_ref()
2158 .map(|operation| operation.sequence),
2159 });
2160 }
2161
2162 if report.failed_operations > 0 {
2163 return Err(RestoreCommandError::RestoreApplyFailed {
2164 backup_id: report.backup_id.clone(),
2165 failed_operations: report.failed_operations,
2166 });
2167 }
2168
2169 if report.ready {
2170 return Ok(());
2171 }
2172
2173 Err(RestoreCommandError::RestoreApplyNotReady {
2174 backup_id: journal.backup_id.clone(),
2175 reasons: report.blocked_reasons.clone(),
2176 })
2177}
2178
2179fn enforce_restore_run_command_available(
2181 preview: &RestoreApplyCommandPreview,
2182) -> Result<(), RestoreCommandError> {
2183 if preview.command_available {
2184 return Ok(());
2185 }
2186
2187 Err(restore_command_unavailable_error(preview))
2188}
2189
2190fn restore_command_unavailable_error(preview: &RestoreApplyCommandPreview) -> RestoreCommandError {
2192 RestoreCommandError::RestoreApplyCommandUnavailable {
2193 backup_id: preview.backup_id.clone(),
2194 operation_available: preview.operation_available,
2195 complete: preview.complete,
2196 blocked_reasons: preview.blocked_reasons.clone(),
2197 }
2198}
2199
2200fn exit_status_label(status: std::process::ExitStatus) -> String {
2202 status
2203 .code()
2204 .map_or_else(|| "signal".to_string(), |code| code.to_string())
2205}
2206
2207fn enforce_restore_run_requirements(
2209 options: &RestoreRunOptions,
2210 run: &RestoreRunResponse,
2211) -> Result<(), RestoreCommandError> {
2212 if options.require_complete && !run.complete {
2213 return Err(RestoreCommandError::RestoreApplyIncomplete {
2214 backup_id: run.backup_id.clone(),
2215 completed_operations: run.completed_operations,
2216 operation_count: run.operation_count,
2217 });
2218 }
2219
2220 if options.require_no_attention && run.attention_required {
2221 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
2222 backup_id: run.backup_id.clone(),
2223 outcome: run.outcome.clone(),
2224 });
2225 }
2226
2227 if let Some(expected) = &options.require_run_mode
2228 && run.run_mode != expected
2229 {
2230 return Err(RestoreCommandError::RestoreRunModeMismatch {
2231 backup_id: run.backup_id.clone(),
2232 expected: expected.clone(),
2233 actual: run.run_mode.to_string(),
2234 });
2235 }
2236
2237 if let Some(expected) = &options.require_stopped_reason
2238 && run.stopped_reason != expected
2239 {
2240 return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
2241 backup_id: run.backup_id.clone(),
2242 expected: expected.clone(),
2243 actual: run.stopped_reason.to_string(),
2244 });
2245 }
2246
2247 if let Some(expected) = &options.require_next_action
2248 && run.next_action != expected
2249 {
2250 return Err(RestoreCommandError::RestoreRunNextActionMismatch {
2251 backup_id: run.backup_id.clone(),
2252 expected: expected.clone(),
2253 actual: run.next_action.to_string(),
2254 });
2255 }
2256
2257 if let Some(expected) = options.require_executed_count {
2258 let actual = run.executed_operation_count.unwrap_or(0);
2259 if actual != expected {
2260 return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
2261 backup_id: run.backup_id.clone(),
2262 expected,
2263 actual,
2264 });
2265 }
2266 }
2267
2268 enforce_restore_run_receipt_requirements(options, run)?;
2269 enforce_restore_run_batch_requirements(options, run)?;
2270
2271 enforce_progress_requirements(
2272 &run.backup_id,
2273 &run.progress,
2274 options.require_remaining_count,
2275 options.require_attention_count,
2276 options.require_completion_basis_points,
2277 )?;
2278 enforce_pending_before_requirement(
2279 &run.backup_id,
2280 &run.pending_summary,
2281 options.require_no_pending_before.as_deref(),
2282 )?;
2283
2284 Ok(())
2285}
2286
2287fn enforce_restore_run_batch_requirements(
2289 options: &RestoreRunOptions,
2290 run: &RestoreRunResponse,
2291) -> Result<(), RestoreCommandError> {
2292 if let Some(expected) = options.require_batch_initial_ready_count {
2293 let actual = run.batch_summary.initial_ready_operations;
2294 if actual != expected {
2295 return Err(
2296 RestoreCommandError::RestoreRunBatchInitialReadyCountMismatch {
2297 backup_id: run.backup_id.clone(),
2298 expected,
2299 actual,
2300 },
2301 );
2302 }
2303 }
2304
2305 if let Some(expected) = options.require_batch_executed_count {
2306 let actual = run.batch_summary.executed_operations;
2307 if actual != expected {
2308 return Err(RestoreCommandError::RestoreRunBatchExecutedCountMismatch {
2309 backup_id: run.backup_id.clone(),
2310 expected,
2311 actual,
2312 });
2313 }
2314 }
2315
2316 if let Some(expected) = options.require_batch_remaining_ready_count {
2317 let actual = run.batch_summary.remaining_ready_operations;
2318 if actual != expected {
2319 return Err(
2320 RestoreCommandError::RestoreRunBatchRemainingReadyCountMismatch {
2321 backup_id: run.backup_id.clone(),
2322 expected,
2323 actual,
2324 },
2325 );
2326 }
2327 }
2328
2329 if let Some(expected) = options.require_batch_ready_delta {
2330 let actual = run.batch_summary.ready_operations_delta;
2331 if actual != expected {
2332 return Err(RestoreCommandError::RestoreRunBatchReadyDeltaMismatch {
2333 backup_id: run.backup_id.clone(),
2334 expected,
2335 actual,
2336 });
2337 }
2338 }
2339
2340 if let Some(expected) = options.require_batch_remaining_delta {
2341 let actual = run.batch_summary.remaining_operations_delta;
2342 if actual != expected {
2343 return Err(RestoreCommandError::RestoreRunBatchRemainingDeltaMismatch {
2344 backup_id: run.backup_id.clone(),
2345 expected,
2346 actual,
2347 });
2348 }
2349 }
2350
2351 if let Some(expected) = options.require_batch_stopped_by_max_steps {
2352 let actual = run.batch_summary.stopped_by_max_steps;
2353 if actual != expected {
2354 return Err(
2355 RestoreCommandError::RestoreRunBatchStoppedByMaxStepsMismatch {
2356 backup_id: run.backup_id.clone(),
2357 expected,
2358 actual,
2359 },
2360 );
2361 }
2362 }
2363
2364 Ok(())
2365}
2366
2367fn enforce_restore_run_receipt_requirements(
2369 options: &RestoreRunOptions,
2370 run: &RestoreRunResponse,
2371) -> Result<(), RestoreCommandError> {
2372 if let Some(expected) = options.require_receipt_count {
2373 let actual = run.operation_receipt_count.unwrap_or(0);
2374 if actual != expected {
2375 return Err(RestoreCommandError::RestoreRunReceiptCountMismatch {
2376 backup_id: run.backup_id.clone(),
2377 expected,
2378 actual,
2379 });
2380 }
2381 }
2382
2383 enforce_restore_run_receipt_kind_requirement(
2384 &run.backup_id,
2385 RESTORE_RUN_RECEIPT_COMPLETED,
2386 options.require_completed_receipt_count,
2387 run.operation_receipt_summary.command_completed,
2388 )?;
2389 enforce_restore_run_receipt_kind_requirement(
2390 &run.backup_id,
2391 RESTORE_RUN_RECEIPT_FAILED,
2392 options.require_failed_receipt_count,
2393 run.operation_receipt_summary.command_failed,
2394 )?;
2395 enforce_restore_run_receipt_kind_requirement(
2396 &run.backup_id,
2397 RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
2398 options.require_recovered_receipt_count,
2399 run.operation_receipt_summary.pending_recovered,
2400 )?;
2401 enforce_restore_run_receipt_updated_at_requirement(
2402 &run.backup_id,
2403 &run.operation_receipts,
2404 options.require_receipt_updated_at.as_deref(),
2405 )?;
2406 enforce_restore_run_state_updated_at_requirement(
2407 &run.backup_id,
2408 run.requested_state_updated_at.as_deref(),
2409 options.require_state_updated_at.as_deref(),
2410 )?;
2411
2412 Ok(())
2413}
2414
2415fn enforce_restore_run_state_updated_at_requirement(
2417 backup_id: &str,
2418 actual: Option<&str>,
2419 expected: Option<&str>,
2420) -> Result<(), RestoreCommandError> {
2421 if let Some(expected) = expected
2422 && actual != Some(expected)
2423 {
2424 return Err(RestoreCommandError::RestoreRunStateUpdatedAtMismatch {
2425 backup_id: backup_id.to_string(),
2426 expected: expected.to_string(),
2427 actual: actual.map(str::to_string),
2428 });
2429 }
2430
2431 Ok(())
2432}
2433
2434fn enforce_restore_run_receipt_updated_at_requirement(
2436 backup_id: &str,
2437 receipts: &[RestoreRunOperationReceipt],
2438 expected: Option<&str>,
2439) -> Result<(), RestoreCommandError> {
2440 let Some(expected) = expected else {
2441 return Ok(());
2442 };
2443
2444 let actual_receipts = receipts.len();
2445 let mismatched_receipts = receipts
2446 .iter()
2447 .filter(|receipt| receipt.updated_at.as_deref() != Some(expected))
2448 .count();
2449 if actual_receipts == 0 || mismatched_receipts > 0 {
2450 return Err(RestoreCommandError::RestoreRunReceiptUpdatedAtMismatch {
2451 backup_id: backup_id.to_string(),
2452 expected: expected.to_string(),
2453 actual_receipts,
2454 mismatched_receipts,
2455 });
2456 }
2457
2458 Ok(())
2459}
2460
2461fn enforce_restore_run_receipt_kind_requirement(
2463 backup_id: &str,
2464 receipt_kind: &'static str,
2465 expected: Option<usize>,
2466 actual: usize,
2467) -> Result<(), RestoreCommandError> {
2468 if let Some(expected) = expected
2469 && actual != expected
2470 {
2471 return Err(RestoreCommandError::RestoreRunReceiptKindCountMismatch {
2472 backup_id: backup_id.to_string(),
2473 receipt_kind,
2474 expected,
2475 actual,
2476 });
2477 }
2478
2479 Ok(())
2480}
2481
2482fn enforce_progress_requirements(
2484 backup_id: &str,
2485 progress: &RestoreApplyProgressSummary,
2486 require_remaining_count: Option<usize>,
2487 require_attention_count: Option<usize>,
2488 require_completion_basis_points: Option<usize>,
2489) -> Result<(), RestoreCommandError> {
2490 if let Some(expected) = require_remaining_count
2491 && progress.remaining_operations != expected
2492 {
2493 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2494 backup_id: backup_id.to_string(),
2495 field: "remaining_operations",
2496 expected,
2497 actual: progress.remaining_operations,
2498 });
2499 }
2500
2501 if let Some(expected) = require_attention_count
2502 && progress.attention_operations != expected
2503 {
2504 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2505 backup_id: backup_id.to_string(),
2506 field: "attention_operations",
2507 expected,
2508 actual: progress.attention_operations,
2509 });
2510 }
2511
2512 if let Some(expected) = require_completion_basis_points
2513 && progress.completion_basis_points != expected
2514 {
2515 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2516 backup_id: backup_id.to_string(),
2517 field: "completion_basis_points",
2518 expected,
2519 actual: progress.completion_basis_points,
2520 });
2521 }
2522
2523 Ok(())
2524}
2525
2526fn enforce_pending_before_requirement(
2528 backup_id: &str,
2529 pending: &RestoreApplyPendingSummary,
2530 require_no_pending_before: Option<&str>,
2531) -> Result<(), RestoreCommandError> {
2532 let Some(cutoff_updated_at) = require_no_pending_before else {
2533 return Ok(());
2534 };
2535
2536 if pending.pending_operations == 0 {
2537 return Ok(());
2538 }
2539
2540 if pending.pending_updated_at_known
2541 && pending
2542 .pending_updated_at
2543 .as_deref()
2544 .is_some_and(|updated_at| updated_at >= cutoff_updated_at)
2545 {
2546 return Ok(());
2547 }
2548
2549 Err(RestoreCommandError::RestoreApplyPendingStale {
2550 backup_id: backup_id.to_string(),
2551 cutoff_updated_at: cutoff_updated_at.to_string(),
2552 pending_sequence: pending.pending_sequence,
2553 pending_updated_at: pending.pending_updated_at.clone(),
2554 })
2555}
2556
2557fn enforce_apply_report_requirements(
2559 options: &RestoreApplyReportOptions,
2560 report: &RestoreApplyJournalReport,
2561) -> Result<(), RestoreCommandError> {
2562 if options.require_no_attention && report.attention_required {
2563 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
2564 backup_id: report.backup_id.clone(),
2565 outcome: report.outcome.clone(),
2566 });
2567 }
2568
2569 enforce_progress_requirements(
2570 &report.backup_id,
2571 &report.progress,
2572 options.require_remaining_count,
2573 options.require_attention_count,
2574 options.require_completion_basis_points,
2575 )?;
2576 enforce_pending_before_requirement(
2577 &report.backup_id,
2578 &report.pending_summary,
2579 options.require_no_pending_before.as_deref(),
2580 )
2581}
2582
2583fn enforce_apply_status_requirements(
2585 options: &RestoreApplyStatusOptions,
2586 status: &RestoreApplyJournalStatus,
2587) -> Result<(), RestoreCommandError> {
2588 if options.require_ready && !status.ready {
2589 return Err(RestoreCommandError::RestoreApplyNotReady {
2590 backup_id: status.backup_id.clone(),
2591 reasons: status.blocked_reasons.clone(),
2592 });
2593 }
2594
2595 if options.require_no_pending && status.pending_operations > 0 {
2596 return Err(RestoreCommandError::RestoreApplyPending {
2597 backup_id: status.backup_id.clone(),
2598 pending_operations: status.pending_operations,
2599 next_transition_sequence: status.next_transition_sequence,
2600 });
2601 }
2602
2603 if options.require_no_failed && status.failed_operations > 0 {
2604 return Err(RestoreCommandError::RestoreApplyFailed {
2605 backup_id: status.backup_id.clone(),
2606 failed_operations: status.failed_operations,
2607 });
2608 }
2609
2610 if options.require_complete && !status.complete {
2611 return Err(RestoreCommandError::RestoreApplyIncomplete {
2612 backup_id: status.backup_id.clone(),
2613 completed_operations: status.completed_operations,
2614 operation_count: status.operation_count,
2615 });
2616 }
2617
2618 enforce_progress_requirements(
2619 &status.backup_id,
2620 &status.progress,
2621 options.require_remaining_count,
2622 options.require_attention_count,
2623 options.require_completion_basis_points,
2624 )?;
2625 enforce_pending_before_requirement(
2626 &status.backup_id,
2627 &status.pending_summary,
2628 options.require_no_pending_before.as_deref(),
2629 )?;
2630
2631 Ok(())
2632}
2633
2634pub fn restore_apply_next(
2636 options: &RestoreApplyNextOptions,
2637) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
2638 let journal = read_apply_journal(&options.journal)?;
2639 Ok(journal.next_operation())
2640}
2641
2642pub fn restore_apply_command(
2644 options: &RestoreApplyCommandOptions,
2645) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
2646 let journal = read_apply_journal(&options.journal)?;
2647 Ok(journal.next_command_preview_with_config(&restore_apply_command_config(options)))
2648}
2649
2650fn enforce_apply_command_requirements(
2652 options: &RestoreApplyCommandOptions,
2653 preview: &RestoreApplyCommandPreview,
2654) -> Result<(), RestoreCommandError> {
2655 if !options.require_command || preview.command_available {
2656 return Ok(());
2657 }
2658
2659 Err(restore_command_unavailable_error(preview))
2660}
2661
2662pub fn restore_apply_claim(
2664 options: &RestoreApplyClaimOptions,
2665) -> Result<RestoreApplyJournal, RestoreCommandError> {
2666 let mut journal = read_apply_journal(&options.journal)?;
2667 let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
2668
2669 if let Some(sequence) = options.sequence {
2670 enforce_apply_claim_sequence(sequence, &journal)?;
2671 journal.mark_operation_pending_at(sequence, updated_at)?;
2672 return Ok(journal);
2673 }
2674
2675 journal.mark_next_operation_pending_at(updated_at)?;
2676 Ok(journal)
2677}
2678
2679fn enforce_apply_claim_sequence(
2681 expected: usize,
2682 journal: &RestoreApplyJournal,
2683) -> Result<(), RestoreCommandError> {
2684 let actual = journal
2685 .next_transition_operation()
2686 .map(|operation| operation.sequence);
2687
2688 if actual == Some(expected) {
2689 return Ok(());
2690 }
2691
2692 Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
2693}
2694
2695pub fn restore_apply_unclaim(
2697 options: &RestoreApplyUnclaimOptions,
2698) -> Result<RestoreApplyJournal, RestoreCommandError> {
2699 let mut journal = read_apply_journal(&options.journal)?;
2700 if let Some(sequence) = options.sequence {
2701 enforce_apply_unclaim_sequence(sequence, &journal)?;
2702 }
2703
2704 journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
2705 Ok(journal)
2706}
2707
2708fn enforce_apply_unclaim_sequence(
2710 expected: usize,
2711 journal: &RestoreApplyJournal,
2712) -> Result<(), RestoreCommandError> {
2713 let actual = journal
2714 .next_transition_operation()
2715 .map(|operation| operation.sequence);
2716
2717 if actual == Some(expected) {
2718 return Ok(());
2719 }
2720
2721 Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
2722}
2723
2724pub fn restore_apply_mark(
2726 options: &RestoreApplyMarkOptions,
2727) -> Result<RestoreApplyJournal, RestoreCommandError> {
2728 let mut journal = read_apply_journal(&options.journal)?;
2729 enforce_apply_mark_pending_requirement(options, &journal)?;
2730
2731 match options.state {
2732 RestoreApplyMarkState::Completed => {
2733 journal.mark_operation_completed_at(
2734 options.sequence,
2735 Some(state_updated_at(options.updated_at.as_ref())),
2736 )?;
2737 }
2738 RestoreApplyMarkState::Failed => {
2739 let reason =
2740 options
2741 .reason
2742 .clone()
2743 .ok_or(RestoreApplyJournalError::FailureReasonRequired(
2744 options.sequence,
2745 ))?;
2746 journal.mark_operation_failed_at(
2747 options.sequence,
2748 reason,
2749 Some(state_updated_at(options.updated_at.as_ref())),
2750 )?;
2751 }
2752 }
2753
2754 Ok(journal)
2755}
2756
2757fn enforce_apply_mark_pending_requirement(
2759 options: &RestoreApplyMarkOptions,
2760 journal: &RestoreApplyJournal,
2761) -> Result<(), RestoreCommandError> {
2762 if !options.require_pending {
2763 return Ok(());
2764 }
2765
2766 let state = journal
2767 .operations
2768 .iter()
2769 .find(|operation| operation.sequence == options.sequence)
2770 .map(|operation| operation.state.clone())
2771 .ok_or(RestoreApplyJournalError::OperationNotFound(
2772 options.sequence,
2773 ))?;
2774
2775 if state == RestoreApplyOperationState::Pending {
2776 return Ok(());
2777 }
2778
2779 Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
2780 sequence: options.sequence,
2781 state,
2782 })
2783}
2784
2785fn enforce_restore_plan_requirements(
2787 options: &RestorePlanOptions,
2788 plan: &RestorePlan,
2789) -> Result<(), RestoreCommandError> {
2790 if options.require_design_v1 {
2791 let manifest = read_manifest_source(options)?;
2792 if !manifest.design_conformance_report().design_v1_ready {
2793 return Err(RestoreCommandError::DesignConformanceNotReady {
2794 backup_id: plan.backup_id.clone(),
2795 });
2796 }
2797 }
2798
2799 if !options.require_restore_ready || plan.readiness_summary.ready {
2800 return Ok(());
2801 }
2802
2803 Err(RestoreCommandError::RestoreNotReady {
2804 backup_id: plan.backup_id.clone(),
2805 reasons: plan.readiness_summary.reasons.clone(),
2806 })
2807}
2808
2809fn verify_backup_layout_if_required(
2811 options: &RestorePlanOptions,
2812) -> Result<(), RestoreCommandError> {
2813 if !options.require_verified {
2814 return Ok(());
2815 }
2816
2817 let Some(dir) = &options.backup_dir else {
2818 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
2819 };
2820
2821 BackupLayout::new(dir.clone()).verify_integrity()?;
2822 Ok(())
2823}
2824
2825fn read_manifest_source(
2827 options: &RestorePlanOptions,
2828) -> Result<FleetBackupManifest, RestoreCommandError> {
2829 if let Some(path) = &options.manifest {
2830 return read_manifest(path);
2831 }
2832
2833 let Some(dir) = &options.backup_dir else {
2834 return Err(RestoreCommandError::MissingOption(
2835 "--manifest or --backup-dir",
2836 ));
2837 };
2838
2839 BackupLayout::new(dir.clone())
2840 .read_manifest()
2841 .map_err(RestoreCommandError::from)
2842}
2843
2844fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
2846 let data = fs::read_to_string(path)?;
2847 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2848}
2849
2850fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
2852 let data = fs::read_to_string(path)?;
2853 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2854}
2855
2856fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
2858 let data = fs::read_to_string(path)?;
2859 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2860}
2861
2862fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
2864 let data = fs::read_to_string(path)?;
2865 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2866}
2867
2868fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
2870 let data = fs::read_to_string(path)?;
2871 let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
2872 journal.validate()?;
2873 Ok(journal)
2874}
2875
2876fn parse_progress_requirement_option<I>(
2878 arg: &str,
2879 args: &mut I,
2880 require_remaining_count: &mut Option<usize>,
2881 require_attention_count: &mut Option<usize>,
2882 require_completion_basis_points: &mut Option<usize>,
2883) -> Result<bool, RestoreCommandError>
2884where
2885 I: Iterator<Item = OsString>,
2886{
2887 match arg {
2888 "--require-remaining-count" => {
2889 *require_remaining_count = Some(parse_sequence(next_value(
2890 args,
2891 "--require-remaining-count",
2892 )?)?);
2893 Ok(true)
2894 }
2895 "--require-attention-count" => {
2896 *require_attention_count = Some(parse_sequence(next_value(
2897 args,
2898 "--require-attention-count",
2899 )?)?);
2900 Ok(true)
2901 }
2902 "--require-completion-basis-points" => {
2903 *require_completion_basis_points = Some(parse_sequence(next_value(
2904 args,
2905 "--require-completion-basis-points",
2906 )?)?);
2907 Ok(true)
2908 }
2909 _ => Ok(false),
2910 }
2911}
2912
2913fn parse_pending_requirement_option<I>(
2915 arg: &str,
2916 args: &mut I,
2917 require_no_pending_before: &mut Option<String>,
2918) -> Result<bool, RestoreCommandError>
2919where
2920 I: Iterator<Item = OsString>,
2921{
2922 match arg {
2923 "--require-no-pending-before" => {
2924 *require_no_pending_before = Some(next_value(args, "--require-no-pending-before")?);
2925 Ok(true)
2926 }
2927 _ => Ok(false),
2928 }
2929}
2930
2931fn parse_run_count_requirement_option<I>(
2933 arg: &str,
2934 args: &mut I,
2935 require_executed_count: &mut Option<usize>,
2936 require_receipt_count: &mut Option<usize>,
2937) -> Result<bool, RestoreCommandError>
2938where
2939 I: Iterator<Item = OsString>,
2940{
2941 match arg {
2942 "--require-executed-count" => {
2943 *require_executed_count = Some(parse_sequence(next_value(
2944 args,
2945 "--require-executed-count",
2946 )?)?);
2947 Ok(true)
2948 }
2949 "--require-receipt-count" => {
2950 *require_receipt_count = Some(parse_sequence(next_value(
2951 args,
2952 "--require-receipt-count",
2953 )?)?);
2954 Ok(true)
2955 }
2956 _ => Ok(false),
2957 }
2958}
2959
2960fn parse_run_receipt_kind_requirement_option<I>(
2962 arg: &str,
2963 args: &mut I,
2964 require_completed_receipt_count: &mut Option<usize>,
2965 require_failed_receipt_count: &mut Option<usize>,
2966 require_recovered_receipt_count: &mut Option<usize>,
2967) -> Result<bool, RestoreCommandError>
2968where
2969 I: Iterator<Item = OsString>,
2970{
2971 match arg {
2972 "--require-completed-receipt-count" => {
2973 *require_completed_receipt_count = Some(parse_sequence(next_value(
2974 args,
2975 "--require-completed-receipt-count",
2976 )?)?);
2977 Ok(true)
2978 }
2979 "--require-failed-receipt-count" => {
2980 *require_failed_receipt_count = Some(parse_sequence(next_value(
2981 args,
2982 "--require-failed-receipt-count",
2983 )?)?);
2984 Ok(true)
2985 }
2986 "--require-recovered-receipt-count" => {
2987 *require_recovered_receipt_count = Some(parse_sequence(next_value(
2988 args,
2989 "--require-recovered-receipt-count",
2990 )?)?);
2991 Ok(true)
2992 }
2993 _ => Ok(false),
2994 }
2995}
2996
2997fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
2999 value
3000 .parse::<usize>()
3001 .map_err(|_| RestoreCommandError::InvalidSequence)
3002}
3003
3004fn parse_integer(option: &'static str, value: String) -> Result<isize, RestoreCommandError> {
3006 value
3007 .parse::<isize>()
3008 .map_err(|_| RestoreCommandError::InvalidInteger { option })
3009}
3010
3011fn parse_positive_integer(
3013 option: &'static str,
3014 value: String,
3015) -> Result<usize, RestoreCommandError> {
3016 let parsed = parse_sequence(value)?;
3017 if parsed == 0 {
3018 return Err(RestoreCommandError::InvalidPositiveInteger { option });
3019 }
3020
3021 Ok(parsed)
3022}
3023
3024fn parse_bool(option: &'static str, value: String) -> Result<bool, RestoreCommandError> {
3026 match value.as_str() {
3027 "true" => Ok(true),
3028 "false" => Ok(false),
3029 _ => Err(RestoreCommandError::InvalidBoolean { option, value }),
3030 }
3031}
3032
3033fn state_updated_at(updated_at: Option<&String>) -> String {
3035 updated_at.cloned().unwrap_or_else(timestamp_placeholder)
3036}
3037
3038fn timestamp_placeholder() -> String {
3040 "unknown".to_string()
3041}
3042
3043fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
3045 if let Some(path) = &options.out {
3046 let data = serde_json::to_vec_pretty(plan)?;
3047 fs::write(path, data)?;
3048 return Ok(());
3049 }
3050
3051 let stdout = io::stdout();
3052 let mut handle = stdout.lock();
3053 serde_json::to_writer_pretty(&mut handle, plan)?;
3054 writeln!(handle)?;
3055 Ok(())
3056}
3057
3058fn write_status(
3060 options: &RestoreStatusOptions,
3061 status: &RestoreStatus,
3062) -> Result<(), RestoreCommandError> {
3063 if let Some(path) = &options.out {
3064 let data = serde_json::to_vec_pretty(status)?;
3065 fs::write(path, data)?;
3066 return Ok(());
3067 }
3068
3069 let stdout = io::stdout();
3070 let mut handle = stdout.lock();
3071 serde_json::to_writer_pretty(&mut handle, status)?;
3072 writeln!(handle)?;
3073 Ok(())
3074}
3075
3076fn write_apply_dry_run(
3078 options: &RestoreApplyOptions,
3079 dry_run: &RestoreApplyDryRun,
3080) -> Result<(), RestoreCommandError> {
3081 if let Some(path) = &options.out {
3082 let data = serde_json::to_vec_pretty(dry_run)?;
3083 fs::write(path, data)?;
3084 return Ok(());
3085 }
3086
3087 let stdout = io::stdout();
3088 let mut handle = stdout.lock();
3089 serde_json::to_writer_pretty(&mut handle, dry_run)?;
3090 writeln!(handle)?;
3091 Ok(())
3092}
3093
3094fn write_apply_journal_if_requested(
3096 options: &RestoreApplyOptions,
3097 dry_run: &RestoreApplyDryRun,
3098) -> Result<(), RestoreCommandError> {
3099 let Some(path) = &options.journal_out else {
3100 return Ok(());
3101 };
3102
3103 let journal = RestoreApplyJournal::from_dry_run(dry_run);
3104 let data = serde_json::to_vec_pretty(&journal)?;
3105 fs::write(path, data)?;
3106 Ok(())
3107}
3108
3109fn write_apply_status(
3111 options: &RestoreApplyStatusOptions,
3112 status: &RestoreApplyJournalStatus,
3113) -> Result<(), RestoreCommandError> {
3114 if let Some(path) = &options.out {
3115 let data = serde_json::to_vec_pretty(status)?;
3116 fs::write(path, data)?;
3117 return Ok(());
3118 }
3119
3120 let stdout = io::stdout();
3121 let mut handle = stdout.lock();
3122 serde_json::to_writer_pretty(&mut handle, status)?;
3123 writeln!(handle)?;
3124 Ok(())
3125}
3126
3127fn write_apply_report(
3129 options: &RestoreApplyReportOptions,
3130 report: &RestoreApplyJournalReport,
3131) -> Result<(), RestoreCommandError> {
3132 if let Some(path) = &options.out {
3133 let data = serde_json::to_vec_pretty(report)?;
3134 fs::write(path, data)?;
3135 return Ok(());
3136 }
3137
3138 let stdout = io::stdout();
3139 let mut handle = stdout.lock();
3140 serde_json::to_writer_pretty(&mut handle, report)?;
3141 writeln!(handle)?;
3142 Ok(())
3143}
3144
3145fn write_restore_run(
3147 options: &RestoreRunOptions,
3148 run: &RestoreRunResponse,
3149) -> Result<(), RestoreCommandError> {
3150 if let Some(path) = &options.out {
3151 let data = serde_json::to_vec_pretty(run)?;
3152 fs::write(path, data)?;
3153 return Ok(());
3154 }
3155
3156 let stdout = io::stdout();
3157 let mut handle = stdout.lock();
3158 serde_json::to_writer_pretty(&mut handle, run)?;
3159 writeln!(handle)?;
3160 Ok(())
3161}
3162
3163fn write_apply_journal_file(
3165 path: &PathBuf,
3166 journal: &RestoreApplyJournal,
3167) -> Result<(), RestoreCommandError> {
3168 let data = serde_json::to_vec_pretty(journal)?;
3169 fs::write(path, data)?;
3170 Ok(())
3171}
3172
3173fn write_apply_next(
3175 options: &RestoreApplyNextOptions,
3176 next: &RestoreApplyNextOperation,
3177) -> Result<(), RestoreCommandError> {
3178 if let Some(path) = &options.out {
3179 let data = serde_json::to_vec_pretty(next)?;
3180 fs::write(path, data)?;
3181 return Ok(());
3182 }
3183
3184 let stdout = io::stdout();
3185 let mut handle = stdout.lock();
3186 serde_json::to_writer_pretty(&mut handle, next)?;
3187 writeln!(handle)?;
3188 Ok(())
3189}
3190
3191fn write_apply_command(
3193 options: &RestoreApplyCommandOptions,
3194 preview: &RestoreApplyCommandPreview,
3195) -> Result<(), RestoreCommandError> {
3196 if let Some(path) = &options.out {
3197 let data = serde_json::to_vec_pretty(preview)?;
3198 fs::write(path, data)?;
3199 return Ok(());
3200 }
3201
3202 let stdout = io::stdout();
3203 let mut handle = stdout.lock();
3204 serde_json::to_writer_pretty(&mut handle, preview)?;
3205 writeln!(handle)?;
3206 Ok(())
3207}
3208
3209fn write_apply_claim(
3211 options: &RestoreApplyClaimOptions,
3212 journal: &RestoreApplyJournal,
3213) -> Result<(), RestoreCommandError> {
3214 if let Some(path) = &options.out {
3215 let data = serde_json::to_vec_pretty(journal)?;
3216 fs::write(path, data)?;
3217 return Ok(());
3218 }
3219
3220 let stdout = io::stdout();
3221 let mut handle = stdout.lock();
3222 serde_json::to_writer_pretty(&mut handle, journal)?;
3223 writeln!(handle)?;
3224 Ok(())
3225}
3226
3227fn write_apply_unclaim(
3229 options: &RestoreApplyUnclaimOptions,
3230 journal: &RestoreApplyJournal,
3231) -> Result<(), RestoreCommandError> {
3232 if let Some(path) = &options.out {
3233 let data = serde_json::to_vec_pretty(journal)?;
3234 fs::write(path, data)?;
3235 return Ok(());
3236 }
3237
3238 let stdout = io::stdout();
3239 let mut handle = stdout.lock();
3240 serde_json::to_writer_pretty(&mut handle, journal)?;
3241 writeln!(handle)?;
3242 Ok(())
3243}
3244
3245fn write_apply_mark(
3247 options: &RestoreApplyMarkOptions,
3248 journal: &RestoreApplyJournal,
3249) -> Result<(), RestoreCommandError> {
3250 if let Some(path) = &options.out {
3251 let data = serde_json::to_vec_pretty(journal)?;
3252 fs::write(path, data)?;
3253 return Ok(());
3254 }
3255
3256 let stdout = io::stdout();
3257 let mut handle = stdout.lock();
3258 serde_json::to_writer_pretty(&mut handle, journal)?;
3259 writeln!(handle)?;
3260 Ok(())
3261}
3262
3263fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
3265where
3266 I: Iterator<Item = OsString>,
3267{
3268 args.next()
3269 .and_then(|value| value.into_string().ok())
3270 .ok_or(RestoreCommandError::MissingValue(option))
3271}
3272
3273const fn usage() -> &'static str {
3275 "usage: canic restore <command> [<args>]\n\ncommands:\n plan Build a no-mutation restore plan.\n status Build initial restore status from a plan.\n apply Render restore operations and optionally write an apply journal.\n apply-status Summarize apply journal state for scripts.\n apply-report Write an operator-focused apply journal report.\n run Preview, execute, or recover the native restore runner.\n apply-next Show the next transitionable journal operation.\n apply-command Preview the next generated dfx command.\n apply-claim Mark the next operation pending.\n apply-unclaim Move a pending operation back to ready.\n apply-mark Mark a pending operation completed or failed."
3276}
3277
3278#[cfg(test)]
3279mod tests {
3280 use super::*;
3281 use canic_backup::restore::RestoreApplyOperationState;
3282 use canic_backup::{
3283 artifacts::ArtifactChecksum,
3284 journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
3285 manifest::{
3286 BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
3287 FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
3288 VerificationCheck, VerificationPlan,
3289 },
3290 };
3291 use serde_json::json;
3292 use std::{
3293 path::Path,
3294 time::{SystemTime, UNIX_EPOCH},
3295 };
3296
3297 const ROOT: &str = "aaaaa-aa";
3298 const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
3299 const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
3300 const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
3301
3302 struct RestoreCliFixture {
3307 root: PathBuf,
3308 journal_path: PathBuf,
3309 out_path: PathBuf,
3310 }
3311
3312 impl RestoreCliFixture {
3313 fn new(prefix: &str, out_file: &str) -> Self {
3315 let root = temp_dir(prefix);
3316 fs::create_dir_all(&root).expect("create temp root");
3317
3318 Self {
3319 journal_path: root.join("restore-apply-journal.json"),
3320 out_path: root.join(out_file),
3321 root,
3322 }
3323 }
3324
3325 fn write_journal(&self, journal: &RestoreApplyJournal) {
3327 fs::write(
3328 &self.journal_path,
3329 serde_json::to_vec(journal).expect("serialize journal"),
3330 )
3331 .expect("write journal");
3332 }
3333
3334 fn run_apply_status(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
3336 self.run_journal_command("apply-status", extra)
3337 }
3338
3339 fn run_apply_report(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
3341 self.run_journal_command("apply-report", extra)
3342 }
3343
3344 fn run_restore_run(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
3346 self.run_journal_command("run", extra)
3347 }
3348
3349 fn read_out<T>(&self, label: &str) -> T
3351 where
3352 T: serde::de::DeserializeOwned,
3353 {
3354 serde_json::from_slice(&fs::read(&self.out_path).expect(label)).expect(label)
3355 }
3356
3357 fn run_journal_command(
3359 &self,
3360 command: &str,
3361 extra: &[&str],
3362 ) -> Result<(), RestoreCommandError> {
3363 let mut args = vec![
3364 OsString::from(command),
3365 OsString::from("--journal"),
3366 OsString::from(self.journal_path.as_os_str()),
3367 OsString::from("--out"),
3368 OsString::from(self.out_path.as_os_str()),
3369 ];
3370 args.extend(extra.iter().map(OsString::from));
3371 run(args)
3372 }
3373 }
3374
3375 impl Drop for RestoreCliFixture {
3376 fn drop(&mut self) {
3378 let _ = fs::remove_dir_all(&self.root);
3379 }
3380 }
3381
3382 fn assert_batch_summary(summary: &serde_json::Value, expected: serde_json::Value) {
3384 assert_eq!(summary, &expected);
3385 }
3386
3387 fn assert_completed_execute_batch_summary(run_summary: &serde_json::Value) {
3389 assert_batch_summary(
3390 &run_summary["batch_summary"],
3391 json!({
3392 "requested_max_steps": 1,
3393 "initial_ready_operations": 8,
3394 "initial_remaining_operations": 8,
3395 "executed_operations": 1,
3396 "remaining_ready_operations": 7,
3397 "remaining_operations": 7,
3398 "ready_operations_delta": -1,
3399 "remaining_operations_delta": -1,
3400 "stopped_by_max_steps": true,
3401 "complete": false,
3402 }),
3403 );
3404 }
3405
3406 #[test]
3408 fn parses_restore_plan_options() {
3409 let options = RestorePlanOptions::parse([
3410 OsString::from("--manifest"),
3411 OsString::from("manifest.json"),
3412 OsString::from("--mapping"),
3413 OsString::from("mapping.json"),
3414 OsString::from("--out"),
3415 OsString::from("plan.json"),
3416 OsString::from("--require-design-v1"),
3417 OsString::from("--require-restore-ready"),
3418 ])
3419 .expect("parse options");
3420
3421 assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
3422 assert_eq!(options.backup_dir, None);
3423 assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
3424 assert_eq!(options.out, Some(PathBuf::from("plan.json")));
3425 assert!(!options.require_verified);
3426 assert!(options.require_design_v1);
3427 assert!(options.require_restore_ready);
3428 }
3429
3430 #[test]
3432 fn restore_usage_lists_commands_without_runner_flag_dump() {
3433 let text = usage();
3434
3435 assert!(text.contains("usage: canic restore <command> [<args>]"));
3436 assert!(text.contains("plan"));
3437 assert!(text.contains("apply-status"));
3438 assert!(text.contains("run"));
3439 assert!(!text.contains("--require-batch-ready-delta"));
3440 assert!(!text.contains("--require-no-pending-before"));
3441 }
3442
3443 #[test]
3445 fn parses_verified_restore_plan_options() {
3446 let options = RestorePlanOptions::parse([
3447 OsString::from("--backup-dir"),
3448 OsString::from("backups/run"),
3449 OsString::from("--require-verified"),
3450 ])
3451 .expect("parse verified options");
3452
3453 assert_eq!(options.manifest, None);
3454 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
3455 assert_eq!(options.mapping, None);
3456 assert_eq!(options.out, None);
3457 assert!(options.require_verified);
3458 assert!(!options.require_design_v1);
3459 assert!(!options.require_restore_ready);
3460 }
3461
3462 #[test]
3464 fn parses_restore_status_options() {
3465 let options = RestoreStatusOptions::parse([
3466 OsString::from("--plan"),
3467 OsString::from("restore-plan.json"),
3468 OsString::from("--out"),
3469 OsString::from("restore-status.json"),
3470 ])
3471 .expect("parse status options");
3472
3473 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
3474 assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
3475 }
3476
3477 #[test]
3479 fn parses_restore_apply_dry_run_options() {
3480 let options = RestoreApplyOptions::parse([
3481 OsString::from("--plan"),
3482 OsString::from("restore-plan.json"),
3483 OsString::from("--status"),
3484 OsString::from("restore-status.json"),
3485 OsString::from("--backup-dir"),
3486 OsString::from("backups/run"),
3487 OsString::from("--dry-run"),
3488 OsString::from("--out"),
3489 OsString::from("restore-apply-dry-run.json"),
3490 OsString::from("--journal-out"),
3491 OsString::from("restore-apply-journal.json"),
3492 ])
3493 .expect("parse apply options");
3494
3495 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
3496 assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
3497 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
3498 assert_eq!(
3499 options.out,
3500 Some(PathBuf::from("restore-apply-dry-run.json"))
3501 );
3502 assert_eq!(
3503 options.journal_out,
3504 Some(PathBuf::from("restore-apply-journal.json"))
3505 );
3506 assert!(options.dry_run);
3507 }
3508
3509 #[test]
3511 fn parses_restore_apply_status_options() {
3512 let options = RestoreApplyStatusOptions::parse([
3513 OsString::from("--journal"),
3514 OsString::from("restore-apply-journal.json"),
3515 OsString::from("--out"),
3516 OsString::from("restore-apply-status.json"),
3517 OsString::from("--require-ready"),
3518 OsString::from("--require-no-pending"),
3519 OsString::from("--require-no-failed"),
3520 OsString::from("--require-complete"),
3521 OsString::from("--require-remaining-count"),
3522 OsString::from("7"),
3523 OsString::from("--require-attention-count"),
3524 OsString::from("0"),
3525 OsString::from("--require-completion-basis-points"),
3526 OsString::from("1250"),
3527 OsString::from("--require-no-pending-before"),
3528 OsString::from("2026-05-05T12:00:00Z"),
3529 ])
3530 .expect("parse apply-status options");
3531
3532 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3533 assert!(options.require_ready);
3534 assert!(options.require_no_pending);
3535 assert!(options.require_no_failed);
3536 assert!(options.require_complete);
3537 assert_eq!(options.require_remaining_count, Some(7));
3538 assert_eq!(options.require_attention_count, Some(0));
3539 assert_eq!(options.require_completion_basis_points, Some(1250));
3540 assert_eq!(
3541 options.require_no_pending_before.as_deref(),
3542 Some("2026-05-05T12:00:00Z")
3543 );
3544 assert_eq!(
3545 options.out,
3546 Some(PathBuf::from("restore-apply-status.json"))
3547 );
3548 }
3549
3550 #[test]
3552 fn parses_restore_apply_report_options() {
3553 let options = RestoreApplyReportOptions::parse([
3554 OsString::from("--journal"),
3555 OsString::from("restore-apply-journal.json"),
3556 OsString::from("--out"),
3557 OsString::from("restore-apply-report.json"),
3558 OsString::from("--require-no-attention"),
3559 OsString::from("--require-remaining-count"),
3560 OsString::from("8"),
3561 OsString::from("--require-attention-count"),
3562 OsString::from("0"),
3563 OsString::from("--require-completion-basis-points"),
3564 OsString::from("0"),
3565 OsString::from("--require-no-pending-before"),
3566 OsString::from("2026-05-05T12:00:00Z"),
3567 ])
3568 .expect("parse apply-report options");
3569
3570 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3571 assert!(options.require_no_attention);
3572 assert_eq!(options.require_remaining_count, Some(8));
3573 assert_eq!(options.require_attention_count, Some(0));
3574 assert_eq!(options.require_completion_basis_points, Some(0));
3575 assert_eq!(
3576 options.require_no_pending_before.as_deref(),
3577 Some("2026-05-05T12:00:00Z")
3578 );
3579 assert_eq!(
3580 options.out,
3581 Some(PathBuf::from("restore-apply-report.json"))
3582 );
3583 }
3584
3585 #[test]
3587 fn parses_restore_run_dry_run_options() {
3588 let options = RestoreRunOptions::parse([
3589 OsString::from("--journal"),
3590 OsString::from("restore-apply-journal.json"),
3591 OsString::from("--dry-run"),
3592 OsString::from("--dfx"),
3593 OsString::from("/tmp/dfx"),
3594 OsString::from("--network"),
3595 OsString::from("local"),
3596 OsString::from("--out"),
3597 OsString::from("restore-run-dry-run.json"),
3598 OsString::from("--max-steps"),
3599 OsString::from("1"),
3600 OsString::from("--updated-at"),
3601 OsString::from("2026-05-05T12:03:00Z"),
3602 OsString::from("--require-complete"),
3603 OsString::from("--require-no-attention"),
3604 OsString::from("--require-run-mode"),
3605 OsString::from("dry-run"),
3606 OsString::from("--require-stopped-reason"),
3607 OsString::from("preview"),
3608 OsString::from("--require-next-action"),
3609 OsString::from("rerun"),
3610 OsString::from("--require-executed-count"),
3611 OsString::from("0"),
3612 OsString::from("--require-receipt-count"),
3613 OsString::from("0"),
3614 OsString::from("--require-completed-receipt-count"),
3615 OsString::from("0"),
3616 OsString::from("--require-failed-receipt-count"),
3617 OsString::from("0"),
3618 OsString::from("--require-recovered-receipt-count"),
3619 OsString::from("0"),
3620 OsString::from("--require-receipt-updated-at"),
3621 OsString::from("2026-05-05T12:03:00Z"),
3622 OsString::from("--require-state-updated-at"),
3623 OsString::from("2026-05-05T12:03:00Z"),
3624 OsString::from("--require-batch-initial-ready-count"),
3625 OsString::from("8"),
3626 OsString::from("--require-batch-executed-count"),
3627 OsString::from("0"),
3628 OsString::from("--require-batch-remaining-ready-count"),
3629 OsString::from("8"),
3630 OsString::from("--require-batch-ready-delta"),
3631 OsString::from("0"),
3632 OsString::from("--require-batch-remaining-delta"),
3633 OsString::from("0"),
3634 OsString::from("--require-batch-stopped-by-max-steps"),
3635 OsString::from("false"),
3636 OsString::from("--require-remaining-count"),
3637 OsString::from("8"),
3638 OsString::from("--require-attention-count"),
3639 OsString::from("0"),
3640 OsString::from("--require-completion-basis-points"),
3641 OsString::from("0"),
3642 OsString::from("--require-no-pending-before"),
3643 OsString::from("2026-05-05T12:00:00Z"),
3644 ])
3645 .expect("parse restore run options");
3646
3647 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3648 assert_eq!(options.dfx, "/tmp/dfx");
3649 assert_eq!(options.network.as_deref(), Some("local"));
3650 assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
3651 assert!(options.dry_run);
3652 assert!(!options.execute);
3653 assert!(!options.unclaim_pending);
3654 assert_eq!(options.max_steps, Some(1));
3655 assert_eq!(options.updated_at.as_deref(), Some("2026-05-05T12:03:00Z"));
3656 assert!(options.require_complete);
3657 assert!(options.require_no_attention);
3658 assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
3659 assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
3660 assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
3661 assert_eq!(options.require_executed_count, Some(0));
3662 assert_eq!(options.require_receipt_count, Some(0));
3663 assert_eq!(options.require_completed_receipt_count, Some(0));
3664 assert_eq!(options.require_failed_receipt_count, Some(0));
3665 assert_eq!(options.require_recovered_receipt_count, Some(0));
3666 assert_eq!(
3667 options.require_receipt_updated_at.as_deref(),
3668 Some("2026-05-05T12:03:00Z")
3669 );
3670 assert_eq!(
3671 options.require_state_updated_at.as_deref(),
3672 Some("2026-05-05T12:03:00Z")
3673 );
3674 assert_eq!(options.require_batch_initial_ready_count, Some(8));
3675 assert_eq!(options.require_batch_executed_count, Some(0));
3676 assert_eq!(options.require_batch_remaining_ready_count, Some(8));
3677 assert_eq!(options.require_batch_ready_delta, Some(0));
3678 assert_eq!(options.require_batch_remaining_delta, Some(0));
3679 assert_eq!(options.require_batch_stopped_by_max_steps, Some(false));
3680 assert_eq!(options.require_remaining_count, Some(8));
3681 assert_eq!(options.require_attention_count, Some(0));
3682 assert_eq!(options.require_completion_basis_points, Some(0));
3683 assert_eq!(
3684 options.require_no_pending_before.as_deref(),
3685 Some("2026-05-05T12:00:00Z")
3686 );
3687 }
3688
3689 #[test]
3691 fn parses_restore_run_execute_options() {
3692 let options = RestoreRunOptions::parse([
3693 OsString::from("--journal"),
3694 OsString::from("restore-apply-journal.json"),
3695 OsString::from("--execute"),
3696 OsString::from("--dfx"),
3697 OsString::from("/bin/true"),
3698 OsString::from("--max-steps"),
3699 OsString::from("4"),
3700 ])
3701 .expect("parse restore run execute options");
3702
3703 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3704 assert_eq!(options.dfx, "/bin/true");
3705 assert_eq!(options.network, None);
3706 assert_eq!(options.out, None);
3707 assert!(!options.dry_run);
3708 assert!(options.execute);
3709 assert!(!options.unclaim_pending);
3710 assert_eq!(options.max_steps, Some(4));
3711 assert_eq!(options.updated_at, None);
3712 assert!(!options.require_complete);
3713 assert!(!options.require_no_attention);
3714 assert_eq!(options.require_run_mode, None);
3715 assert_eq!(options.require_stopped_reason, None);
3716 assert_eq!(options.require_next_action, None);
3717 assert_eq!(options.require_executed_count, None);
3718 assert_eq!(options.require_receipt_count, None);
3719 assert_eq!(options.require_completed_receipt_count, None);
3720 assert_eq!(options.require_failed_receipt_count, None);
3721 assert_eq!(options.require_recovered_receipt_count, None);
3722 assert_eq!(options.require_receipt_updated_at, None);
3723 assert_eq!(options.require_state_updated_at, None);
3724 assert_eq!(options.require_batch_initial_ready_count, None);
3725 assert_eq!(options.require_batch_executed_count, None);
3726 assert_eq!(options.require_batch_remaining_ready_count, None);
3727 assert_eq!(options.require_batch_ready_delta, None);
3728 assert_eq!(options.require_batch_remaining_delta, None);
3729 assert_eq!(options.require_batch_stopped_by_max_steps, None);
3730 }
3731
3732 #[test]
3734 fn parses_restore_run_unclaim_pending_options() {
3735 let options = RestoreRunOptions::parse([
3736 OsString::from("--journal"),
3737 OsString::from("restore-apply-journal.json"),
3738 OsString::from("--unclaim-pending"),
3739 OsString::from("--out"),
3740 OsString::from("restore-run.json"),
3741 ])
3742 .expect("parse restore run unclaim options");
3743
3744 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3745 assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
3746 assert!(!options.dry_run);
3747 assert!(!options.execute);
3748 assert!(options.unclaim_pending);
3749 }
3750
3751 #[test]
3753 fn parses_restore_apply_next_options() {
3754 let options = RestoreApplyNextOptions::parse([
3755 OsString::from("--journal"),
3756 OsString::from("restore-apply-journal.json"),
3757 OsString::from("--out"),
3758 OsString::from("restore-apply-next.json"),
3759 ])
3760 .expect("parse apply-next options");
3761
3762 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3763 assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
3764 }
3765
3766 #[test]
3768 fn parses_restore_apply_command_options() {
3769 let options = RestoreApplyCommandOptions::parse([
3770 OsString::from("--journal"),
3771 OsString::from("restore-apply-journal.json"),
3772 OsString::from("--dfx"),
3773 OsString::from("/tmp/dfx"),
3774 OsString::from("--network"),
3775 OsString::from("local"),
3776 OsString::from("--out"),
3777 OsString::from("restore-apply-command.json"),
3778 OsString::from("--require-command"),
3779 ])
3780 .expect("parse apply-command options");
3781
3782 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3783 assert_eq!(options.dfx, "/tmp/dfx");
3784 assert_eq!(options.network.as_deref(), Some("local"));
3785 assert!(options.require_command);
3786 assert_eq!(
3787 options.out,
3788 Some(PathBuf::from("restore-apply-command.json"))
3789 );
3790 }
3791
3792 #[test]
3794 fn parses_restore_apply_claim_options() {
3795 let options = RestoreApplyClaimOptions::parse([
3796 OsString::from("--journal"),
3797 OsString::from("restore-apply-journal.json"),
3798 OsString::from("--sequence"),
3799 OsString::from("0"),
3800 OsString::from("--updated-at"),
3801 OsString::from("2026-05-04T12:00:00Z"),
3802 OsString::from("--out"),
3803 OsString::from("restore-apply-journal.claimed.json"),
3804 ])
3805 .expect("parse apply-claim options");
3806
3807 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3808 assert_eq!(options.sequence, Some(0));
3809 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
3810 assert_eq!(
3811 options.out,
3812 Some(PathBuf::from("restore-apply-journal.claimed.json"))
3813 );
3814 }
3815
3816 #[test]
3818 fn parses_restore_apply_unclaim_options() {
3819 let options = RestoreApplyUnclaimOptions::parse([
3820 OsString::from("--journal"),
3821 OsString::from("restore-apply-journal.json"),
3822 OsString::from("--sequence"),
3823 OsString::from("0"),
3824 OsString::from("--updated-at"),
3825 OsString::from("2026-05-04T12:01:00Z"),
3826 OsString::from("--out"),
3827 OsString::from("restore-apply-journal.unclaimed.json"),
3828 ])
3829 .expect("parse apply-unclaim options");
3830
3831 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3832 assert_eq!(options.sequence, Some(0));
3833 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
3834 assert_eq!(
3835 options.out,
3836 Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
3837 );
3838 }
3839
3840 #[test]
3842 fn parses_restore_apply_mark_options() {
3843 let options = RestoreApplyMarkOptions::parse([
3844 OsString::from("--journal"),
3845 OsString::from("restore-apply-journal.json"),
3846 OsString::from("--sequence"),
3847 OsString::from("4"),
3848 OsString::from("--state"),
3849 OsString::from("failed"),
3850 OsString::from("--reason"),
3851 OsString::from("dfx-load-failed"),
3852 OsString::from("--updated-at"),
3853 OsString::from("2026-05-04T12:02:00Z"),
3854 OsString::from("--out"),
3855 OsString::from("restore-apply-journal.updated.json"),
3856 OsString::from("--require-pending"),
3857 ])
3858 .expect("parse apply-mark options");
3859
3860 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3861 assert_eq!(options.sequence, 4);
3862 assert_eq!(options.state, RestoreApplyMarkState::Failed);
3863 assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
3864 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
3865 assert!(options.require_pending);
3866 assert_eq!(
3867 options.out,
3868 Some(PathBuf::from("restore-apply-journal.updated.json"))
3869 );
3870 }
3871
3872 #[test]
3874 fn restore_apply_requires_dry_run() {
3875 let err = RestoreApplyOptions::parse([
3876 OsString::from("--plan"),
3877 OsString::from("restore-plan.json"),
3878 ])
3879 .expect_err("apply without dry-run should fail");
3880
3881 assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
3882 }
3883
3884 #[test]
3886 fn restore_run_requires_mode() {
3887 let err = RestoreRunOptions::parse([
3888 OsString::from("--journal"),
3889 OsString::from("restore-apply-journal.json"),
3890 ])
3891 .expect_err("restore run without dry-run should fail");
3892
3893 assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
3894 }
3895
3896 #[test]
3898 fn restore_run_rejects_conflicting_modes() {
3899 let err = RestoreRunOptions::parse([
3900 OsString::from("--journal"),
3901 OsString::from("restore-apply-journal.json"),
3902 OsString::from("--dry-run"),
3903 OsString::from("--execute"),
3904 OsString::from("--unclaim-pending"),
3905 ])
3906 .expect_err("restore run should reject conflicting modes");
3907
3908 assert!(matches!(
3909 err,
3910 RestoreCommandError::RestoreRunConflictingModes
3911 ));
3912 }
3913
3914 #[test]
3916 fn restore_run_rejects_zero_max_steps() {
3917 let err = RestoreRunOptions::parse([
3918 OsString::from("--journal"),
3919 OsString::from("restore-apply-journal.json"),
3920 OsString::from("--execute"),
3921 OsString::from("--max-steps"),
3922 OsString::from("0"),
3923 ])
3924 .expect_err("restore run should reject zero max steps");
3925
3926 assert!(matches!(
3927 err,
3928 RestoreCommandError::InvalidPositiveInteger {
3929 option: "--max-steps"
3930 }
3931 ));
3932 }
3933
3934 #[test]
3936 fn restore_run_rejects_invalid_batch_bool() {
3937 let err = RestoreRunOptions::parse([
3938 OsString::from("--journal"),
3939 OsString::from("restore-apply-journal.json"),
3940 OsString::from("--dry-run"),
3941 OsString::from("--require-batch-stopped-by-max-steps"),
3942 OsString::from("maybe"),
3943 ])
3944 .expect_err("restore run should reject invalid boolean gates");
3945
3946 assert!(matches!(
3947 err,
3948 RestoreCommandError::InvalidBoolean {
3949 option: "--require-batch-stopped-by-max-steps",
3950 value,
3951 } if value == "maybe"
3952 ));
3953 }
3954
3955 #[test]
3957 fn restore_run_rejects_invalid_batch_delta() {
3958 let err = RestoreRunOptions::parse([
3959 OsString::from("--journal"),
3960 OsString::from("restore-apply-journal.json"),
3961 OsString::from("--dry-run"),
3962 OsString::from("--require-batch-ready-delta"),
3963 OsString::from("not-an-int"),
3964 ])
3965 .expect_err("restore run should reject invalid signed integer gates");
3966
3967 assert!(matches!(
3968 err,
3969 RestoreCommandError::InvalidInteger {
3970 option: "--require-batch-ready-delta"
3971 }
3972 ));
3973 }
3974
3975 #[test]
3977 fn plan_restore_reads_manifest_from_backup_dir() {
3978 let root = temp_dir("canic-cli-restore-plan-layout");
3979 let layout = BackupLayout::new(root.clone());
3980 layout
3981 .write_manifest(&valid_manifest())
3982 .expect("write manifest");
3983
3984 let options = RestorePlanOptions {
3985 manifest: None,
3986 backup_dir: Some(root.clone()),
3987 mapping: None,
3988 out: None,
3989 require_verified: false,
3990 require_design_v1: false,
3991 require_restore_ready: false,
3992 };
3993
3994 let plan = plan_restore(&options).expect("plan restore");
3995
3996 fs::remove_dir_all(root).expect("remove temp root");
3997 assert_eq!(plan.backup_id, "backup-test");
3998 assert_eq!(plan.member_count, 2);
3999 }
4000
4001 #[test]
4003 fn parse_rejects_conflicting_manifest_sources() {
4004 let err = RestorePlanOptions::parse([
4005 OsString::from("--manifest"),
4006 OsString::from("manifest.json"),
4007 OsString::from("--backup-dir"),
4008 OsString::from("backups/run"),
4009 ])
4010 .expect_err("conflicting sources should fail");
4011
4012 assert!(matches!(
4013 err,
4014 RestoreCommandError::ConflictingManifestSources
4015 ));
4016 }
4017
4018 #[test]
4020 fn parse_rejects_require_verified_with_manifest_source() {
4021 let err = RestorePlanOptions::parse([
4022 OsString::from("--manifest"),
4023 OsString::from("manifest.json"),
4024 OsString::from("--require-verified"),
4025 ])
4026 .expect_err("verification should require a backup layout");
4027
4028 assert!(matches!(
4029 err,
4030 RestoreCommandError::RequireVerifiedNeedsBackupDir
4031 ));
4032 }
4033
4034 #[test]
4036 fn plan_restore_requires_verified_backup_layout() {
4037 let root = temp_dir("canic-cli-restore-plan-verified");
4038 let layout = BackupLayout::new(root.clone());
4039 let manifest = valid_manifest();
4040 write_verified_layout(&root, &layout, &manifest);
4041
4042 let options = RestorePlanOptions {
4043 manifest: None,
4044 backup_dir: Some(root.clone()),
4045 mapping: None,
4046 out: None,
4047 require_verified: true,
4048 require_design_v1: false,
4049 require_restore_ready: false,
4050 };
4051
4052 let plan = plan_restore(&options).expect("plan verified restore");
4053
4054 fs::remove_dir_all(root).expect("remove temp root");
4055 assert_eq!(plan.backup_id, "backup-test");
4056 assert_eq!(plan.member_count, 2);
4057 }
4058
4059 #[test]
4061 fn plan_restore_rejects_unverified_backup_layout() {
4062 let root = temp_dir("canic-cli-restore-plan-unverified");
4063 let layout = BackupLayout::new(root.clone());
4064 layout
4065 .write_manifest(&valid_manifest())
4066 .expect("write manifest");
4067
4068 let options = RestorePlanOptions {
4069 manifest: None,
4070 backup_dir: Some(root.clone()),
4071 mapping: None,
4072 out: None,
4073 require_verified: true,
4074 require_design_v1: false,
4075 require_restore_ready: false,
4076 };
4077
4078 let err = plan_restore(&options).expect_err("missing journal should fail");
4079
4080 fs::remove_dir_all(root).expect("remove temp root");
4081 assert!(matches!(err, RestoreCommandError::Persistence(_)));
4082 }
4083
4084 #[test]
4086 fn plan_restore_reads_manifest_and_mapping() {
4087 let root = temp_dir("canic-cli-restore-plan");
4088 fs::create_dir_all(&root).expect("create temp root");
4089 let manifest_path = root.join("manifest.json");
4090 let mapping_path = root.join("mapping.json");
4091
4092 fs::write(
4093 &manifest_path,
4094 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
4095 )
4096 .expect("write manifest");
4097 fs::write(
4098 &mapping_path,
4099 json!({
4100 "members": [
4101 {"source_canister": ROOT, "target_canister": ROOT},
4102 {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
4103 ]
4104 })
4105 .to_string(),
4106 )
4107 .expect("write mapping");
4108
4109 let options = RestorePlanOptions {
4110 manifest: Some(manifest_path),
4111 backup_dir: None,
4112 mapping: Some(mapping_path),
4113 out: None,
4114 require_verified: false,
4115 require_design_v1: false,
4116 require_restore_ready: false,
4117 };
4118
4119 let plan = plan_restore(&options).expect("plan restore");
4120
4121 fs::remove_dir_all(root).expect("remove temp root");
4122 let members = plan.ordered_members();
4123 assert_eq!(members.len(), 2);
4124 assert_eq!(members[0].source_canister, ROOT);
4125 assert_eq!(members[1].target_canister, MAPPED_CHILD);
4126 }
4127
4128 #[test]
4130 fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
4131 let root = temp_dir("canic-cli-restore-plan-require-ready");
4132 fs::create_dir_all(&root).expect("create temp root");
4133 let manifest_path = root.join("manifest.json");
4134 let out_path = root.join("plan.json");
4135
4136 fs::write(
4137 &manifest_path,
4138 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
4139 )
4140 .expect("write manifest");
4141
4142 let err = run([
4143 OsString::from("plan"),
4144 OsString::from("--manifest"),
4145 OsString::from(manifest_path.as_os_str()),
4146 OsString::from("--out"),
4147 OsString::from(out_path.as_os_str()),
4148 OsString::from("--require-restore-ready"),
4149 ])
4150 .expect_err("restore readiness should be enforced");
4151
4152 assert!(out_path.exists());
4153 let plan: RestorePlan =
4154 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
4155
4156 fs::remove_dir_all(root).expect("remove temp root");
4157 assert!(!plan.readiness_summary.ready);
4158 assert!(matches!(
4159 err,
4160 RestoreCommandError::RestoreNotReady {
4161 reasons,
4162 ..
4163 } if reasons == [
4164 "missing-module-hash",
4165 "missing-wasm-hash",
4166 "missing-snapshot-checksum"
4167 ]
4168 ));
4169 }
4170
4171 #[test]
4173 fn run_restore_plan_require_design_v1_writes_plan_then_fails() {
4174 let root = temp_dir("canic-cli-restore-plan-require-design-v1");
4175 fs::create_dir_all(&root).expect("create temp root");
4176 let manifest_path = root.join("manifest.json");
4177 let out_path = root.join("plan.json");
4178
4179 fs::write(
4180 &manifest_path,
4181 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
4182 )
4183 .expect("write manifest");
4184
4185 let err = run([
4186 OsString::from("plan"),
4187 OsString::from("--manifest"),
4188 OsString::from(manifest_path.as_os_str()),
4189 OsString::from("--out"),
4190 OsString::from(out_path.as_os_str()),
4191 OsString::from("--require-design-v1"),
4192 ])
4193 .expect_err("design-v1 readiness should be enforced");
4194
4195 assert!(out_path.exists());
4196 let plan: RestorePlan =
4197 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
4198
4199 fs::remove_dir_all(root).expect("remove temp root");
4200 assert_eq!(plan.backup_id, "backup-test");
4201 assert!(matches!(
4202 err,
4203 RestoreCommandError::DesignConformanceNotReady { .. }
4204 ));
4205 }
4206
4207 #[test]
4209 fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
4210 let root = temp_dir("canic-cli-restore-plan-ready");
4211 fs::create_dir_all(&root).expect("create temp root");
4212 let manifest_path = root.join("manifest.json");
4213 let out_path = root.join("plan.json");
4214
4215 fs::write(
4216 &manifest_path,
4217 serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
4218 )
4219 .expect("write manifest");
4220
4221 run([
4222 OsString::from("plan"),
4223 OsString::from("--manifest"),
4224 OsString::from(manifest_path.as_os_str()),
4225 OsString::from("--out"),
4226 OsString::from(out_path.as_os_str()),
4227 OsString::from("--require-restore-ready"),
4228 ])
4229 .expect("restore-ready plan should pass");
4230
4231 let plan: RestorePlan =
4232 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
4233
4234 fs::remove_dir_all(root).expect("remove temp root");
4235 assert!(plan.readiness_summary.ready);
4236 assert!(plan.readiness_summary.reasons.is_empty());
4237 }
4238
4239 #[test]
4241 fn run_restore_plan_require_design_v1_accepts_ready_manifest() {
4242 let root = temp_dir("canic-cli-restore-plan-design-v1-ready");
4243 fs::create_dir_all(&root).expect("create temp root");
4244 let manifest_path = root.join("manifest.json");
4245 let out_path = root.join("plan.json");
4246
4247 fs::write(
4248 &manifest_path,
4249 serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
4250 )
4251 .expect("write manifest");
4252
4253 run([
4254 OsString::from("plan"),
4255 OsString::from("--manifest"),
4256 OsString::from(manifest_path.as_os_str()),
4257 OsString::from("--out"),
4258 OsString::from(out_path.as_os_str()),
4259 OsString::from("--require-design-v1"),
4260 ])
4261 .expect("design-v1 ready plan should pass");
4262
4263 let plan: RestorePlan =
4264 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
4265
4266 fs::remove_dir_all(root).expect("remove temp root");
4267 assert_eq!(plan.backup_id, "backup-test");
4268 assert!(plan.readiness_summary.ready);
4269 }
4270
4271 #[test]
4273 fn run_restore_status_writes_planned_status() {
4274 let root = temp_dir("canic-cli-restore-status");
4275 fs::create_dir_all(&root).expect("create temp root");
4276 let plan_path = root.join("restore-plan.json");
4277 let out_path = root.join("restore-status.json");
4278 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4279
4280 fs::write(
4281 &plan_path,
4282 serde_json::to_vec(&plan).expect("serialize plan"),
4283 )
4284 .expect("write plan");
4285
4286 run([
4287 OsString::from("status"),
4288 OsString::from("--plan"),
4289 OsString::from(plan_path.as_os_str()),
4290 OsString::from("--out"),
4291 OsString::from(out_path.as_os_str()),
4292 ])
4293 .expect("write restore status");
4294
4295 let status: RestoreStatus =
4296 serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
4297 .expect("decode restore status");
4298 let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
4299
4300 fs::remove_dir_all(root).expect("remove temp root");
4301 assert_eq!(status.status_version, 1);
4302 assert_eq!(status.backup_id.as_str(), "backup-test");
4303 assert!(status.ready);
4304 assert!(status.readiness_reasons.is_empty());
4305 assert_eq!(status.member_count, 2);
4306 assert_eq!(status.phase_count, 1);
4307 assert_eq!(status.planned_snapshot_uploads, 2);
4308 assert_eq!(status.planned_snapshot_loads, 2);
4309 assert_eq!(status.planned_code_reinstalls, 2);
4310 assert_eq!(status.planned_verification_checks, 2);
4311 assert_eq!(status.planned_operations, 8);
4312 assert_eq!(status.phases[0].members[0].source_canister, ROOT);
4313 assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
4314 }
4315
4316 #[test]
4318 fn run_restore_apply_dry_run_writes_operations() {
4319 let root = temp_dir("canic-cli-restore-apply-dry-run");
4320 fs::create_dir_all(&root).expect("create temp root");
4321 let plan_path = root.join("restore-plan.json");
4322 let status_path = root.join("restore-status.json");
4323 let out_path = root.join("restore-apply-dry-run.json");
4324 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4325 let status = RestoreStatus::from_plan(&plan);
4326
4327 fs::write(
4328 &plan_path,
4329 serde_json::to_vec(&plan).expect("serialize plan"),
4330 )
4331 .expect("write plan");
4332 fs::write(
4333 &status_path,
4334 serde_json::to_vec(&status).expect("serialize status"),
4335 )
4336 .expect("write status");
4337
4338 run([
4339 OsString::from("apply"),
4340 OsString::from("--plan"),
4341 OsString::from(plan_path.as_os_str()),
4342 OsString::from("--status"),
4343 OsString::from(status_path.as_os_str()),
4344 OsString::from("--dry-run"),
4345 OsString::from("--out"),
4346 OsString::from(out_path.as_os_str()),
4347 ])
4348 .expect("write apply dry-run");
4349
4350 let dry_run: RestoreApplyDryRun =
4351 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4352 .expect("decode dry-run");
4353 let dry_run_json: serde_json::Value =
4354 serde_json::to_value(&dry_run).expect("encode dry-run");
4355
4356 fs::remove_dir_all(root).expect("remove temp root");
4357 assert_eq!(dry_run.dry_run_version, 1);
4358 assert_eq!(dry_run.backup_id.as_str(), "backup-test");
4359 assert!(dry_run.ready);
4360 assert!(dry_run.status_supplied);
4361 assert_eq!(dry_run.member_count, 2);
4362 assert_eq!(dry_run.phase_count, 1);
4363 assert_eq!(dry_run.planned_snapshot_uploads, 2);
4364 assert_eq!(dry_run.planned_operations, 8);
4365 assert_eq!(dry_run.rendered_operations, 8);
4366 assert_eq!(dry_run_json["operation_counts"]["snapshot_uploads"], 2);
4367 assert_eq!(dry_run_json["operation_counts"]["snapshot_loads"], 2);
4368 assert_eq!(dry_run_json["operation_counts"]["code_reinstalls"], 2);
4369 assert_eq!(dry_run_json["operation_counts"]["member_verifications"], 2);
4370 assert_eq!(dry_run_json["operation_counts"]["fleet_verifications"], 0);
4371 assert_eq!(
4372 dry_run_json["operation_counts"]["verification_operations"],
4373 2
4374 );
4375 assert_eq!(
4376 dry_run_json["phases"][0]["operations"][0]["operation"],
4377 "upload-snapshot"
4378 );
4379 assert_eq!(
4380 dry_run_json["phases"][0]["operations"][3]["operation"],
4381 "verify-member"
4382 );
4383 assert_eq!(
4384 dry_run_json["phases"][0]["operations"][3]["verification_kind"],
4385 "status"
4386 );
4387 assert_eq!(
4388 dry_run_json["phases"][0]["operations"][3]["verification_method"],
4389 serde_json::Value::Null
4390 );
4391 }
4392
4393 #[test]
4395 fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
4396 let root = temp_dir("canic-cli-restore-apply-artifacts");
4397 fs::create_dir_all(&root).expect("create temp root");
4398 let plan_path = root.join("restore-plan.json");
4399 let out_path = root.join("restore-apply-dry-run.json");
4400 let journal_path = root.join("restore-apply-journal.json");
4401 let status_path = root.join("restore-apply-status.json");
4402 let mut manifest = restore_ready_manifest();
4403 write_manifest_artifacts(&root, &mut manifest);
4404 let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
4405
4406 fs::write(
4407 &plan_path,
4408 serde_json::to_vec(&plan).expect("serialize plan"),
4409 )
4410 .expect("write plan");
4411
4412 run([
4413 OsString::from("apply"),
4414 OsString::from("--plan"),
4415 OsString::from(plan_path.as_os_str()),
4416 OsString::from("--backup-dir"),
4417 OsString::from(root.as_os_str()),
4418 OsString::from("--dry-run"),
4419 OsString::from("--out"),
4420 OsString::from(out_path.as_os_str()),
4421 OsString::from("--journal-out"),
4422 OsString::from(journal_path.as_os_str()),
4423 ])
4424 .expect("write apply dry-run");
4425 run([
4426 OsString::from("apply-status"),
4427 OsString::from("--journal"),
4428 OsString::from(journal_path.as_os_str()),
4429 OsString::from("--out"),
4430 OsString::from(status_path.as_os_str()),
4431 ])
4432 .expect("write apply status");
4433
4434 let dry_run: RestoreApplyDryRun =
4435 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4436 .expect("decode dry-run");
4437 let validation = dry_run
4438 .artifact_validation
4439 .expect("artifact validation should be present");
4440 let journal_json: serde_json::Value =
4441 serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
4442 .expect("decode journal");
4443 let status_json: serde_json::Value =
4444 serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
4445 .expect("decode apply status");
4446
4447 fs::remove_dir_all(root).expect("remove temp root");
4448 assert_eq!(validation.checked_members, 2);
4449 assert!(validation.artifacts_present);
4450 assert!(validation.checksums_verified);
4451 assert_eq!(validation.members_with_expected_checksums, 2);
4452 assert_eq!(journal_json["ready"], true);
4453 assert_eq!(journal_json["operation_count"], 8);
4454 assert_eq!(journal_json["operation_counts"]["snapshot_uploads"], 2);
4455 assert_eq!(journal_json["operation_counts"]["snapshot_loads"], 2);
4456 assert_eq!(journal_json["operation_counts"]["code_reinstalls"], 2);
4457 assert_eq!(journal_json["operation_counts"]["member_verifications"], 2);
4458 assert_eq!(journal_json["operation_counts"]["fleet_verifications"], 0);
4459 assert_eq!(
4460 journal_json["operation_counts"]["verification_operations"],
4461 2
4462 );
4463 assert_eq!(journal_json["ready_operations"], 8);
4464 assert_eq!(journal_json["blocked_operations"], 0);
4465 assert_eq!(journal_json["operations"][0]["state"], "ready");
4466 assert_eq!(status_json["ready"], true);
4467 assert_eq!(status_json["operation_count"], 8);
4468 assert_eq!(status_json["operation_counts"]["snapshot_uploads"], 2);
4469 assert_eq!(status_json["operation_counts"]["snapshot_loads"], 2);
4470 assert_eq!(status_json["operation_counts"]["code_reinstalls"], 2);
4471 assert_eq!(status_json["operation_counts"]["member_verifications"], 2);
4472 assert_eq!(status_json["operation_counts"]["fleet_verifications"], 0);
4473 assert_eq!(
4474 status_json["operation_counts"]["verification_operations"],
4475 2
4476 );
4477 assert_eq!(status_json["operation_counts_supplied"], true);
4478 assert_eq!(status_json["progress"]["operation_count"], 8);
4479 assert_eq!(status_json["progress"]["completed_operations"], 0);
4480 assert_eq!(status_json["progress"]["remaining_operations"], 8);
4481 assert_eq!(status_json["progress"]["transitionable_operations"], 8);
4482 assert_eq!(status_json["progress"]["attention_operations"], 0);
4483 assert_eq!(status_json["progress"]["completion_basis_points"], 0);
4484 assert_eq!(status_json["next_ready_sequence"], 0);
4485 assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
4486 }
4487
4488 #[test]
4490 fn run_restore_apply_status_rejects_invalid_journal() {
4491 let root = temp_dir("canic-cli-restore-apply-status-invalid");
4492 fs::create_dir_all(&root).expect("create temp root");
4493 let journal_path = root.join("restore-apply-journal.json");
4494 let out_path = root.join("restore-apply-status.json");
4495 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4496 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4497 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4498 journal.operation_count += 1;
4499
4500 fs::write(
4501 &journal_path,
4502 serde_json::to_vec(&journal).expect("serialize journal"),
4503 )
4504 .expect("write journal");
4505
4506 let err = run([
4507 OsString::from("apply-status"),
4508 OsString::from("--journal"),
4509 OsString::from(journal_path.as_os_str()),
4510 OsString::from("--out"),
4511 OsString::from(out_path.as_os_str()),
4512 ])
4513 .expect_err("invalid journal should fail");
4514
4515 assert!(!out_path.exists());
4516 fs::remove_dir_all(root).expect("remove temp root");
4517 assert!(matches!(
4518 err,
4519 RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
4520 field: "operation_count",
4521 ..
4522 })
4523 ));
4524 }
4525
4526 #[test]
4528 fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
4529 let fixture = RestoreCliFixture::new(
4530 "canic-cli-restore-apply-status-pending",
4531 "restore-apply-status.json",
4532 );
4533 let mut journal = ready_apply_journal();
4534 journal
4535 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4536 .expect("claim operation");
4537 fixture.write_journal(&journal);
4538
4539 let err = fixture
4540 .run_apply_status(&["--require-no-pending"])
4541 .expect_err("pending operation should fail requirement");
4542
4543 assert!(fixture.out_path.exists());
4544 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4545
4546 assert_eq!(status.pending_operations, 1);
4547 assert_eq!(status.next_transition_sequence, Some(0));
4548 assert_eq!(status.pending_summary.pending_operations, 1);
4549 assert_eq!(status.pending_summary.pending_sequence, Some(0));
4550 assert_eq!(
4551 status.pending_summary.pending_updated_at.as_deref(),
4552 Some("2026-05-04T12:00:00Z")
4553 );
4554 assert!(status.pending_summary.pending_updated_at_known);
4555 assert_eq!(
4556 status.next_transition_updated_at.as_deref(),
4557 Some("2026-05-04T12:00:00Z")
4558 );
4559 assert!(matches!(
4560 err,
4561 RestoreCommandError::RestoreApplyPending {
4562 pending_operations: 1,
4563 next_transition_sequence: Some(0),
4564 ..
4565 }
4566 ));
4567 }
4568
4569 #[test]
4571 fn run_restore_apply_status_require_no_pending_before_writes_status_then_fails() {
4572 let fixture = RestoreCliFixture::new(
4573 "canic-cli-restore-apply-status-stale-pending",
4574 "restore-apply-status.json",
4575 );
4576 let mut journal = ready_apply_journal();
4577 journal
4578 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4579 .expect("claim operation");
4580 fixture.write_journal(&journal);
4581
4582 let err = fixture
4583 .run_apply_status(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
4584 .expect_err("stale pending operation should fail requirement");
4585
4586 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4587
4588 assert_eq!(status.pending_summary.pending_sequence, Some(0));
4589 assert_eq!(
4590 status.pending_summary.pending_updated_at.as_deref(),
4591 Some("2026-05-04T12:00:00Z")
4592 );
4593 assert!(matches!(
4594 err,
4595 RestoreCommandError::RestoreApplyPendingStale {
4596 cutoff_updated_at,
4597 pending_sequence: Some(0),
4598 pending_updated_at,
4599 ..
4600 } if cutoff_updated_at == "2026-05-05T12:00:00Z"
4601 && pending_updated_at.as_deref() == Some("2026-05-04T12:00:00Z")
4602 ));
4603 }
4604
4605 #[test]
4607 fn run_restore_apply_status_require_progress_writes_status_then_fails() {
4608 let fixture = RestoreCliFixture::new(
4609 "canic-cli-restore-apply-status-progress",
4610 "restore-apply-status.json",
4611 );
4612 let journal = ready_apply_journal();
4613 fixture.write_journal(&journal);
4614
4615 let err = fixture
4616 .run_apply_status(&[
4617 "--require-remaining-count",
4618 "7",
4619 "--require-attention-count",
4620 "0",
4621 "--require-completion-basis-points",
4622 "0",
4623 ])
4624 .expect_err("remaining progress mismatch should fail requirement");
4625
4626 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4627
4628 assert_eq!(status.progress.remaining_operations, 8);
4629 assert_eq!(status.progress.attention_operations, 0);
4630 assert_eq!(status.progress.completion_basis_points, 0);
4631 assert!(matches!(
4632 err,
4633 RestoreCommandError::RestoreApplyProgressMismatch {
4634 field: "remaining_operations",
4635 expected: 7,
4636 actual: 8,
4637 ..
4638 }
4639 ));
4640 }
4641
4642 #[test]
4644 fn run_restore_apply_status_require_ready_writes_status_then_fails() {
4645 let root = temp_dir("canic-cli-restore-apply-status-ready");
4646 fs::create_dir_all(&root).expect("create temp root");
4647 let journal_path = root.join("restore-apply-journal.json");
4648 let out_path = root.join("restore-apply-status.json");
4649 let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
4650 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4651 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
4652
4653 fs::write(
4654 &journal_path,
4655 serde_json::to_vec(&journal).expect("serialize journal"),
4656 )
4657 .expect("write journal");
4658
4659 let err = run([
4660 OsString::from("apply-status"),
4661 OsString::from("--journal"),
4662 OsString::from(journal_path.as_os_str()),
4663 OsString::from("--out"),
4664 OsString::from(out_path.as_os_str()),
4665 OsString::from("--require-ready"),
4666 ])
4667 .expect_err("unready journal should fail requirement");
4668
4669 let status: RestoreApplyJournalStatus =
4670 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
4671 .expect("decode apply status");
4672
4673 fs::remove_dir_all(root).expect("remove temp root");
4674 assert!(!status.ready);
4675 assert_eq!(status.blocked_operations, status.operation_count);
4676 assert!(
4677 status
4678 .blocked_reasons
4679 .contains(&"missing-snapshot-checksum".to_string())
4680 );
4681 assert!(matches!(
4682 err,
4683 RestoreCommandError::RestoreApplyNotReady { reasons, .. }
4684 if reasons.contains(&"missing-snapshot-checksum".to_string())
4685 ));
4686 }
4687
4688 #[test]
4690 fn run_restore_apply_report_writes_attention_summary() {
4691 let root = temp_dir("canic-cli-restore-apply-report");
4692 fs::create_dir_all(&root).expect("create temp root");
4693 let journal_path = root.join("restore-apply-journal.json");
4694 let out_path = root.join("restore-apply-report.json");
4695 let mut journal = ready_apply_journal();
4696 journal
4697 .mark_operation_failed_at(
4698 0,
4699 "dfx-upload-failed".to_string(),
4700 Some("2026-05-05T12:00:00Z".to_string()),
4701 )
4702 .expect("mark failed operation");
4703 journal
4704 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4705 .expect("mark pending operation");
4706
4707 fs::write(
4708 &journal_path,
4709 serde_json::to_vec(&journal).expect("serialize journal"),
4710 )
4711 .expect("write journal");
4712
4713 run([
4714 OsString::from("apply-report"),
4715 OsString::from("--journal"),
4716 OsString::from(journal_path.as_os_str()),
4717 OsString::from("--out"),
4718 OsString::from(out_path.as_os_str()),
4719 ])
4720 .expect("write apply report");
4721
4722 let report: RestoreApplyJournalReport =
4723 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
4724 .expect("decode apply report");
4725 let report_json: serde_json::Value =
4726 serde_json::to_value(&report).expect("encode apply report");
4727
4728 fs::remove_dir_all(root).expect("remove temp root");
4729 assert_eq!(report.backup_id, "backup-test");
4730 assert!(report.attention_required);
4731 assert_eq!(report.failed_operations, 1);
4732 assert_eq!(report.pending_operations, 1);
4733 assert_eq!(report.operation_counts.snapshot_uploads, 2);
4734 assert_eq!(report.operation_counts.snapshot_loads, 2);
4735 assert_eq!(report.operation_counts.code_reinstalls, 2);
4736 assert_eq!(report.operation_counts.member_verifications, 2);
4737 assert_eq!(report.operation_counts.fleet_verifications, 0);
4738 assert_eq!(report.operation_counts.verification_operations, 2);
4739 assert!(report.operation_counts_supplied);
4740 assert_eq!(report.progress.operation_count, 8);
4741 assert_eq!(report.progress.completed_operations, 0);
4742 assert_eq!(report.progress.remaining_operations, 8);
4743 assert_eq!(report.progress.transitionable_operations, 7);
4744 assert_eq!(report.progress.attention_operations, 2);
4745 assert_eq!(report.progress.completion_basis_points, 0);
4746 assert_eq!(report.pending_summary.pending_operations, 1);
4747 assert_eq!(report.pending_summary.pending_sequence, Some(1));
4748 assert_eq!(
4749 report.pending_summary.pending_updated_at.as_deref(),
4750 Some("2026-05-05T12:01:00Z")
4751 );
4752 assert!(report.pending_summary.pending_updated_at_known);
4753 assert_eq!(report.failed.len(), 1);
4754 assert_eq!(report.pending.len(), 1);
4755 assert_eq!(report.failed[0].sequence, 0);
4756 assert_eq!(report.pending[0].sequence, 1);
4757 assert_eq!(
4758 report.next_transition.as_ref().map(|op| op.sequence),
4759 Some(1)
4760 );
4761 assert_eq!(report_json["outcome"], "failed");
4762 assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
4763 }
4764
4765 #[test]
4767 fn run_restore_apply_report_require_progress_writes_report_then_fails() {
4768 let fixture = RestoreCliFixture::new(
4769 "canic-cli-restore-apply-report-progress",
4770 "restore-apply-report.json",
4771 );
4772 let journal = ready_apply_journal();
4773 fixture.write_journal(&journal);
4774
4775 let err = fixture
4776 .run_apply_report(&[
4777 "--require-remaining-count",
4778 "8",
4779 "--require-attention-count",
4780 "1",
4781 "--require-completion-basis-points",
4782 "0",
4783 ])
4784 .expect_err("attention progress mismatch should fail requirement");
4785
4786 let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4787
4788 assert_eq!(report.progress.remaining_operations, 8);
4789 assert_eq!(report.progress.attention_operations, 0);
4790 assert_eq!(report.progress.completion_basis_points, 0);
4791 assert!(matches!(
4792 err,
4793 RestoreCommandError::RestoreApplyProgressMismatch {
4794 field: "attention_operations",
4795 expected: 1,
4796 actual: 0,
4797 ..
4798 }
4799 ));
4800 }
4801
4802 #[test]
4804 fn run_restore_apply_report_require_no_pending_before_writes_report_then_fails() {
4805 let fixture = RestoreCliFixture::new(
4806 "canic-cli-restore-apply-report-stale-pending",
4807 "restore-apply-report.json",
4808 );
4809 let mut journal = ready_apply_journal();
4810 journal
4811 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4812 .expect("mark pending operation");
4813 fixture.write_journal(&journal);
4814
4815 let err = fixture
4816 .run_apply_report(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
4817 .expect_err("stale pending report should fail requirement");
4818
4819 let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4820
4821 assert_eq!(report.pending_summary.pending_sequence, Some(0));
4822 assert!(matches!(
4823 err,
4824 RestoreCommandError::RestoreApplyPendingStale {
4825 pending_sequence: Some(0),
4826 ..
4827 }
4828 ));
4829 }
4830
4831 #[test]
4833 fn run_restore_run_dry_run_writes_native_runner_preview() {
4834 let root = temp_dir("canic-cli-restore-run-dry-run");
4835 fs::create_dir_all(&root).expect("create temp root");
4836 let journal_path = root.join("restore-apply-journal.json");
4837 let out_path = root.join("restore-run-dry-run.json");
4838 let journal = ready_apply_journal();
4839
4840 fs::write(
4841 &journal_path,
4842 serde_json::to_vec(&journal).expect("serialize journal"),
4843 )
4844 .expect("write journal");
4845
4846 run([
4847 OsString::from("run"),
4848 OsString::from("--journal"),
4849 OsString::from(journal_path.as_os_str()),
4850 OsString::from("--dry-run"),
4851 OsString::from("--dfx"),
4852 OsString::from("/tmp/dfx"),
4853 OsString::from("--network"),
4854 OsString::from("local"),
4855 OsString::from("--updated-at"),
4856 OsString::from("2026-05-05T12:00:00Z"),
4857 OsString::from("--out"),
4858 OsString::from(out_path.as_os_str()),
4859 OsString::from("--require-state-updated-at"),
4860 OsString::from("2026-05-05T12:00:00Z"),
4861 ])
4862 .expect("write restore run dry-run");
4863
4864 let dry_run: serde_json::Value =
4865 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4866 .expect("decode dry-run");
4867
4868 fs::remove_dir_all(root).expect("remove temp root");
4869 assert_eq!(dry_run["run_version"], 1);
4870 assert_eq!(dry_run["backup_id"], "backup-test");
4871 assert_eq!(dry_run["run_mode"], "dry-run");
4872 assert_eq!(dry_run["dry_run"], true);
4873 assert_eq!(
4874 dry_run["requested_state_updated_at"],
4875 "2026-05-05T12:00:00Z"
4876 );
4877 assert_eq!(dry_run["ready"], true);
4878 assert_eq!(dry_run["complete"], false);
4879 assert_eq!(dry_run["attention_required"], false);
4880 assert_eq!(dry_run["operation_counts"]["snapshot_uploads"], 2);
4881 assert_eq!(dry_run["operation_counts"]["snapshot_loads"], 2);
4882 assert_eq!(dry_run["operation_counts"]["code_reinstalls"], 2);
4883 assert_eq!(dry_run["operation_counts"]["member_verifications"], 2);
4884 assert_eq!(dry_run["operation_counts"]["fleet_verifications"], 0);
4885 assert_eq!(dry_run["operation_counts"]["verification_operations"], 2);
4886 assert_eq!(dry_run["operation_counts_supplied"], true);
4887 assert_eq!(dry_run["progress"]["operation_count"], 8);
4888 assert_eq!(dry_run["progress"]["completed_operations"], 0);
4889 assert_eq!(dry_run["progress"]["remaining_operations"], 8);
4890 assert_eq!(dry_run["progress"]["transitionable_operations"], 8);
4891 assert_eq!(dry_run["progress"]["attention_operations"], 0);
4892 assert_eq!(dry_run["progress"]["completion_basis_points"], 0);
4893 assert_eq!(dry_run["pending_summary"]["pending_operations"], 0);
4894 assert_eq!(
4895 dry_run["pending_summary"]["pending_operation_available"],
4896 false
4897 );
4898 assert_eq!(dry_run["operation_receipt_count"], 0);
4899 assert_eq!(dry_run["operation_receipt_summary"]["total_receipts"], 0);
4900 assert_eq!(dry_run["operation_receipt_summary"]["command_completed"], 0);
4901 assert_eq!(dry_run["operation_receipt_summary"]["command_failed"], 0);
4902 assert_eq!(dry_run["operation_receipt_summary"]["pending_recovered"], 0);
4903 assert_batch_summary(
4904 &dry_run["batch_summary"],
4905 json!({
4906 "requested_max_steps": null,
4907 "initial_ready_operations": 8,
4908 "initial_remaining_operations": 8,
4909 "executed_operations": 0,
4910 "remaining_ready_operations": 8,
4911 "remaining_operations": 8,
4912 "ready_operations_delta": 0,
4913 "remaining_operations_delta": 0,
4914 "stopped_by_max_steps": false,
4915 "complete": false,
4916 }),
4917 );
4918 assert_eq!(dry_run["stopped_reason"], "preview");
4919 assert_eq!(dry_run["next_action"], "rerun");
4920 assert_eq!(dry_run["operation_available"], true);
4921 assert_eq!(dry_run["command_available"], true);
4922 assert_eq!(dry_run["next_transition"]["sequence"], 0);
4923 assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
4924 assert_eq!(
4925 dry_run["command"]["args"],
4926 json!([
4927 "canister",
4928 "--network",
4929 "local",
4930 "snapshot",
4931 "upload",
4932 "--dir",
4933 "artifacts/root",
4934 ROOT
4935 ])
4936 );
4937 assert_eq!(dry_run["command"]["mutates"], true);
4938 }
4939
4940 #[test]
4942 fn run_restore_run_unclaim_pending_marks_operation_ready() {
4943 let root = temp_dir("canic-cli-restore-run-unclaim-pending");
4944 fs::create_dir_all(&root).expect("create temp root");
4945 let journal_path = root.join("restore-apply-journal.json");
4946 let out_path = root.join("restore-run.json");
4947 let mut journal = ready_apply_journal();
4948 journal
4949 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4950 .expect("mark pending operation");
4951
4952 fs::write(
4953 &journal_path,
4954 serde_json::to_vec(&journal).expect("serialize journal"),
4955 )
4956 .expect("write journal");
4957
4958 run([
4959 OsString::from("run"),
4960 OsString::from("--journal"),
4961 OsString::from(journal_path.as_os_str()),
4962 OsString::from("--unclaim-pending"),
4963 OsString::from("--updated-at"),
4964 OsString::from("2026-05-05T12:02:00Z"),
4965 OsString::from("--out"),
4966 OsString::from(out_path.as_os_str()),
4967 ])
4968 .expect("unclaim pending operation");
4969
4970 let run_summary: serde_json::Value =
4971 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4972 .expect("decode run summary");
4973 let updated: RestoreApplyJournal =
4974 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4975 .expect("decode updated journal");
4976
4977 fs::remove_dir_all(root).expect("remove temp root");
4978 assert_eq!(run_summary["run_mode"], "unclaim-pending");
4979 assert_eq!(run_summary["unclaim_pending"], true);
4980 assert_eq!(run_summary["stopped_reason"], "recovered-pending");
4981 assert_eq!(run_summary["next_action"], "rerun");
4982 assert_eq!(
4983 run_summary["requested_state_updated_at"],
4984 "2026-05-05T12:02:00Z"
4985 );
4986 assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
4987 assert_eq!(run_summary["recovered_operation"]["state"], "pending");
4988 assert_eq!(run_summary["operation_receipt_count"], 1);
4989 assert_eq!(
4990 run_summary["operation_receipt_summary"]["total_receipts"],
4991 1
4992 );
4993 assert_batch_summary(
4994 &run_summary["batch_summary"],
4995 json!({
4996 "requested_max_steps": null,
4997 "initial_ready_operations": 7,
4998 "initial_remaining_operations": 8,
4999 "executed_operations": 0,
5000 "remaining_ready_operations": 8,
5001 "remaining_operations": 8,
5002 "ready_operations_delta": 1,
5003 "remaining_operations_delta": 0,
5004 "stopped_by_max_steps": false,
5005 "complete": false,
5006 }),
5007 );
5008 assert_eq!(
5009 run_summary["operation_receipt_summary"]["command_completed"],
5010 0
5011 );
5012 assert_eq!(
5013 run_summary["operation_receipt_summary"]["command_failed"],
5014 0
5015 );
5016 assert_eq!(
5017 run_summary["operation_receipt_summary"]["pending_recovered"],
5018 1
5019 );
5020 assert_eq!(
5021 run_summary["operation_receipts"][0]["event"],
5022 "pending-recovered"
5023 );
5024 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
5025 assert_eq!(run_summary["operation_receipts"][0]["state"], "ready");
5026 assert_eq!(
5027 run_summary["operation_receipts"][0]["updated_at"],
5028 "2026-05-05T12:02:00Z"
5029 );
5030 assert_eq!(run_summary["pending_operations"], 0);
5031 assert_eq!(run_summary["ready_operations"], 8);
5032 assert_eq!(run_summary["attention_required"], false);
5033 assert_eq!(updated.pending_operations, 0);
5034 assert_eq!(updated.ready_operations, 8);
5035 assert_eq!(
5036 updated.operations[0].state,
5037 RestoreApplyOperationState::Ready
5038 );
5039 assert_eq!(
5040 updated.operations[0].state_updated_at.as_deref(),
5041 Some("2026-05-05T12:02:00Z")
5042 );
5043 }
5044
5045 #[test]
5047 fn run_restore_run_execute_marks_completed_operation() {
5048 let root = temp_dir("canic-cli-restore-run-execute");
5049 fs::create_dir_all(&root).expect("create temp root");
5050 let journal_path = root.join("restore-apply-journal.json");
5051 let out_path = root.join("restore-run.json");
5052 let journal = ready_apply_journal();
5053
5054 fs::write(
5055 &journal_path,
5056 serde_json::to_vec(&journal).expect("serialize journal"),
5057 )
5058 .expect("write journal");
5059
5060 run([
5061 OsString::from("run"),
5062 OsString::from("--journal"),
5063 OsString::from(journal_path.as_os_str()),
5064 OsString::from("--execute"),
5065 OsString::from("--dfx"),
5066 OsString::from("/bin/true"),
5067 OsString::from("--max-steps"),
5068 OsString::from("1"),
5069 OsString::from("--updated-at"),
5070 OsString::from("2026-05-05T12:03:00Z"),
5071 OsString::from("--out"),
5072 OsString::from(out_path.as_os_str()),
5073 OsString::from("--require-receipt-updated-at"),
5074 OsString::from("2026-05-05T12:03:00Z"),
5075 ])
5076 .expect("execute one restore run step");
5077
5078 let run_summary: serde_json::Value =
5079 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5080 .expect("decode run summary");
5081 let updated: RestoreApplyJournal =
5082 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
5083 .expect("decode updated journal");
5084
5085 fs::remove_dir_all(root).expect("remove temp root");
5086 assert_eq!(run_summary["run_mode"], "execute");
5087 assert_eq!(run_summary["execute"], true);
5088 assert_eq!(run_summary["dry_run"], false);
5089 assert_eq!(run_summary["max_steps_reached"], true);
5090 assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
5091 assert_eq!(run_summary["next_action"], "rerun");
5092 assert_eq!(
5093 run_summary["requested_state_updated_at"],
5094 "2026-05-05T12:03:00Z"
5095 );
5096 assert_eq!(run_summary["executed_operation_count"], 1);
5097 assert_completed_execute_batch_summary(&run_summary);
5098 assert_eq!(run_summary["operation_receipt_count"], 1);
5099 assert_eq!(
5100 run_summary["operation_receipt_summary"]["total_receipts"],
5101 1
5102 );
5103 assert_eq!(
5104 run_summary["operation_receipt_summary"]["command_completed"],
5105 1
5106 );
5107 assert_eq!(
5108 run_summary["operation_receipt_summary"]["command_failed"],
5109 0
5110 );
5111 assert_eq!(
5112 run_summary["operation_receipt_summary"]["pending_recovered"],
5113 0
5114 );
5115 assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
5116 assert_eq!(
5117 run_summary["executed_operations"][0]["command"]["program"],
5118 "/bin/true"
5119 );
5120 assert_eq!(
5121 run_summary["operation_receipts"][0]["event"],
5122 "command-completed"
5123 );
5124 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
5125 assert_eq!(run_summary["operation_receipts"][0]["state"], "completed");
5126 assert_eq!(
5127 run_summary["operation_receipts"][0]["command"]["program"],
5128 "/bin/true"
5129 );
5130 assert_eq!(run_summary["operation_receipts"][0]["status"], "0");
5131 assert_eq!(
5132 run_summary["operation_receipts"][0]["updated_at"],
5133 "2026-05-05T12:03:00Z"
5134 );
5135 assert_eq!(updated.completed_operations, 1);
5136 assert_eq!(updated.pending_operations, 0);
5137 assert_eq!(updated.failed_operations, 0);
5138 assert_eq!(
5139 updated.operations[0].state,
5140 RestoreApplyOperationState::Completed
5141 );
5142 assert_eq!(
5143 updated.operations[0].state_updated_at.as_deref(),
5144 Some("2026-05-05T12:03:00Z")
5145 );
5146 }
5147
5148 #[test]
5150 fn run_restore_run_require_complete_writes_summary_then_fails() {
5151 let root = temp_dir("canic-cli-restore-run-require-complete");
5152 fs::create_dir_all(&root).expect("create temp root");
5153 let journal_path = root.join("restore-apply-journal.json");
5154 let out_path = root.join("restore-run.json");
5155 let journal = ready_apply_journal();
5156
5157 fs::write(
5158 &journal_path,
5159 serde_json::to_vec(&journal).expect("serialize journal"),
5160 )
5161 .expect("write journal");
5162
5163 let err = run([
5164 OsString::from("run"),
5165 OsString::from("--journal"),
5166 OsString::from(journal_path.as_os_str()),
5167 OsString::from("--execute"),
5168 OsString::from("--dfx"),
5169 OsString::from("/bin/true"),
5170 OsString::from("--max-steps"),
5171 OsString::from("1"),
5172 OsString::from("--out"),
5173 OsString::from(out_path.as_os_str()),
5174 OsString::from("--require-complete"),
5175 ])
5176 .expect_err("incomplete run should fail requirement");
5177
5178 let run_summary: serde_json::Value =
5179 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5180 .expect("decode run summary");
5181
5182 fs::remove_dir_all(root).expect("remove temp root");
5183 assert_eq!(run_summary["executed_operation_count"], 1);
5184 assert_eq!(run_summary["complete"], false);
5185 assert!(matches!(
5186 err,
5187 RestoreCommandError::RestoreApplyIncomplete {
5188 completed_operations: 1,
5189 operation_count: 8,
5190 ..
5191 }
5192 ));
5193 }
5194
5195 #[test]
5197 fn run_restore_run_execute_marks_failed_operation() {
5198 let root = temp_dir("canic-cli-restore-run-execute-failed");
5199 fs::create_dir_all(&root).expect("create temp root");
5200 let journal_path = root.join("restore-apply-journal.json");
5201 let out_path = root.join("restore-run.json");
5202 let journal = ready_apply_journal();
5203
5204 fs::write(
5205 &journal_path,
5206 serde_json::to_vec(&journal).expect("serialize journal"),
5207 )
5208 .expect("write journal");
5209
5210 let err = run([
5211 OsString::from("run"),
5212 OsString::from("--journal"),
5213 OsString::from(journal_path.as_os_str()),
5214 OsString::from("--execute"),
5215 OsString::from("--dfx"),
5216 OsString::from("/bin/false"),
5217 OsString::from("--max-steps"),
5218 OsString::from("1"),
5219 OsString::from("--updated-at"),
5220 OsString::from("2026-05-05T12:04:00Z"),
5221 OsString::from("--out"),
5222 OsString::from(out_path.as_os_str()),
5223 ])
5224 .expect_err("failing runner command should fail");
5225
5226 let run_summary: serde_json::Value =
5227 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5228 .expect("decode run summary");
5229 let updated: RestoreApplyJournal =
5230 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
5231 .expect("decode updated journal");
5232
5233 fs::remove_dir_all(root).expect("remove temp root");
5234 assert!(matches!(
5235 err,
5236 RestoreCommandError::RestoreRunCommandFailed {
5237 sequence: 0,
5238 status,
5239 } if status == "1"
5240 ));
5241 assert_eq!(updated.failed_operations, 1);
5242 assert_eq!(updated.pending_operations, 0);
5243 assert_eq!(
5244 updated.operations[0].state,
5245 RestoreApplyOperationState::Failed
5246 );
5247 assert_eq!(run_summary["execute"], true);
5248 assert_eq!(run_summary["attention_required"], true);
5249 assert_eq!(run_summary["outcome"], "failed");
5250 assert_eq!(run_summary["stopped_reason"], "command-failed");
5251 assert_eq!(run_summary["next_action"], "inspect-failed-operation");
5252 assert_eq!(
5253 run_summary["requested_state_updated_at"],
5254 "2026-05-05T12:04:00Z"
5255 );
5256 assert_eq!(run_summary["executed_operation_count"], 1);
5257 assert_eq!(run_summary["operation_receipt_count"], 1);
5258 assert_eq!(
5259 run_summary["operation_receipt_summary"]["total_receipts"],
5260 1
5261 );
5262 assert_eq!(
5263 run_summary["operation_receipt_summary"]["command_completed"],
5264 0
5265 );
5266 assert_eq!(
5267 run_summary["operation_receipt_summary"]["command_failed"],
5268 1
5269 );
5270 assert_eq!(
5271 run_summary["operation_receipt_summary"]["pending_recovered"],
5272 0
5273 );
5274 assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
5275 assert_eq!(run_summary["executed_operations"][0]["status"], "1");
5276 assert_eq!(
5277 run_summary["operation_receipts"][0]["event"],
5278 "command-failed"
5279 );
5280 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
5281 assert_eq!(run_summary["operation_receipts"][0]["state"], "failed");
5282 assert_eq!(
5283 run_summary["operation_receipts"][0]["command"]["program"],
5284 "/bin/false"
5285 );
5286 assert_eq!(run_summary["operation_receipts"][0]["status"], "1");
5287 assert_eq!(
5288 run_summary["operation_receipts"][0]["updated_at"],
5289 "2026-05-05T12:04:00Z"
5290 );
5291 assert_eq!(
5292 updated.operations[0].state_updated_at.as_deref(),
5293 Some("2026-05-05T12:04:00Z")
5294 );
5295 assert_eq!(
5296 updated.operations[0].blocking_reasons,
5297 vec!["runner-command-exit-1".to_string()]
5298 );
5299 }
5300
5301 #[test]
5303 fn run_restore_run_require_no_attention_writes_summary_then_fails() {
5304 let fixture = RestoreCliFixture::new(
5305 "canic-cli-restore-run-require-attention",
5306 "restore-run.json",
5307 );
5308 let mut journal = ready_apply_journal();
5309 journal
5310 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
5311 .expect("mark pending operation");
5312 fixture.write_journal(&journal);
5313
5314 let err = fixture
5315 .run_restore_run(&["--dry-run", "--require-no-attention"])
5316 .expect_err("attention run should fail requirement");
5317
5318 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5319
5320 assert_eq!(run_summary["attention_required"], true);
5321 assert_eq!(run_summary["outcome"], "pending");
5322 assert_eq!(run_summary["stopped_reason"], "pending");
5323 assert_eq!(run_summary["next_action"], "unclaim-pending");
5324 assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
5325 assert_eq!(
5326 run_summary["pending_summary"]["pending_updated_at"],
5327 "2026-05-05T12:01:00Z"
5328 );
5329 assert!(matches!(
5330 err,
5331 RestoreCommandError::RestoreApplyReportNeedsAttention {
5332 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
5333 ..
5334 }
5335 ));
5336 }
5337
5338 #[test]
5340 fn run_restore_run_require_no_pending_before_writes_summary_then_fails() {
5341 let fixture = RestoreCliFixture::new(
5342 "canic-cli-restore-run-require-stale-pending",
5343 "restore-run.json",
5344 );
5345 let mut journal = ready_apply_journal();
5346 journal
5347 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
5348 .expect("mark pending operation");
5349 fixture.write_journal(&journal);
5350
5351 let err = fixture
5352 .run_restore_run(&[
5353 "--dry-run",
5354 "--require-no-pending-before",
5355 "2026-05-05T12:00:00Z",
5356 ])
5357 .expect_err("stale pending run should fail requirement");
5358
5359 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5360
5361 assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
5362 assert!(matches!(
5363 err,
5364 RestoreCommandError::RestoreApplyPendingStale {
5365 pending_sequence: Some(0),
5366 ..
5367 }
5368 ));
5369 }
5370
5371 #[test]
5373 fn run_restore_run_require_run_mode_writes_summary_then_fails() {
5374 let fixture =
5375 RestoreCliFixture::new("canic-cli-restore-run-require-run-mode", "restore-run.json");
5376 let journal = ready_apply_journal();
5377 fixture.write_journal(&journal);
5378
5379 let err = fixture
5380 .run_restore_run(&["--dry-run", "--require-run-mode", "execute"])
5381 .expect_err("run mode mismatch should fail requirement");
5382
5383 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5384
5385 assert_eq!(run_summary["run_mode"], "dry-run");
5386 assert!(matches!(
5387 err,
5388 RestoreCommandError::RestoreRunModeMismatch {
5389 expected,
5390 actual,
5391 ..
5392 } if expected == "execute" && actual == "dry-run"
5393 ));
5394 }
5395
5396 #[test]
5398 fn run_restore_run_require_executed_count_writes_summary_then_fails() {
5399 let root = temp_dir("canic-cli-restore-run-require-executed-count");
5400 fs::create_dir_all(&root).expect("create temp root");
5401 let journal_path = root.join("restore-apply-journal.json");
5402 let out_path = root.join("restore-run.json");
5403 let journal = ready_apply_journal();
5404
5405 fs::write(
5406 &journal_path,
5407 serde_json::to_vec(&journal).expect("serialize journal"),
5408 )
5409 .expect("write journal");
5410
5411 let err = run([
5412 OsString::from("run"),
5413 OsString::from("--journal"),
5414 OsString::from(journal_path.as_os_str()),
5415 OsString::from("--execute"),
5416 OsString::from("--dfx"),
5417 OsString::from("/bin/true"),
5418 OsString::from("--max-steps"),
5419 OsString::from("1"),
5420 OsString::from("--out"),
5421 OsString::from(out_path.as_os_str()),
5422 OsString::from("--require-executed-count"),
5423 OsString::from("2"),
5424 ])
5425 .expect_err("executed count mismatch should fail requirement");
5426
5427 let run_summary: serde_json::Value =
5428 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5429 .expect("decode run summary");
5430
5431 fs::remove_dir_all(root).expect("remove temp root");
5432 assert_eq!(run_summary["executed_operation_count"], 1);
5433 assert!(matches!(
5434 err,
5435 RestoreCommandError::RestoreRunExecutedCountMismatch {
5436 expected: 2,
5437 actual: 1,
5438 ..
5439 }
5440 ));
5441 }
5442
5443 #[test]
5445 fn run_restore_run_require_receipt_count_writes_summary_then_fails() {
5446 let fixture = RestoreCliFixture::new(
5447 "canic-cli-restore-run-require-receipt-count",
5448 "restore-run.json",
5449 );
5450 let journal = ready_apply_journal();
5451 fixture.write_journal(&journal);
5452
5453 let err = fixture
5454 .run_restore_run(&[
5455 "--execute",
5456 "--dfx",
5457 "/bin/true",
5458 "--max-steps",
5459 "1",
5460 "--require-receipt-count",
5461 "2",
5462 ])
5463 .expect_err("receipt count mismatch should fail requirement");
5464
5465 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5466
5467 assert_eq!(run_summary["operation_receipt_count"], 1);
5468 assert_eq!(
5469 run_summary["operation_receipt_summary"]["total_receipts"],
5470 1
5471 );
5472 assert!(matches!(
5473 err,
5474 RestoreCommandError::RestoreRunReceiptCountMismatch {
5475 expected: 2,
5476 actual: 1,
5477 ..
5478 }
5479 ));
5480 }
5481
5482 #[test]
5484 fn run_restore_run_require_receipt_kind_count_writes_summary_then_fails() {
5485 let fixture = RestoreCliFixture::new(
5486 "canic-cli-restore-run-require-receipt-kind-count",
5487 "restore-run.json",
5488 );
5489 let journal = ready_apply_journal();
5490 fixture.write_journal(&journal);
5491
5492 let err = fixture
5493 .run_restore_run(&[
5494 "--execute",
5495 "--dfx",
5496 "/bin/true",
5497 "--max-steps",
5498 "1",
5499 "--require-failed-receipt-count",
5500 "1",
5501 ])
5502 .expect_err("receipt kind count mismatch should fail requirement");
5503
5504 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5505
5506 assert_eq!(
5507 run_summary["operation_receipt_summary"]["command_failed"],
5508 0
5509 );
5510 assert_eq!(
5511 run_summary["operation_receipt_summary"]["command_completed"],
5512 1
5513 );
5514 assert!(matches!(
5515 err,
5516 RestoreCommandError::RestoreRunReceiptKindCountMismatch {
5517 receipt_kind: "command-failed",
5518 expected: 1,
5519 actual: 0,
5520 ..
5521 }
5522 ));
5523 }
5524
5525 #[test]
5527 fn run_restore_run_require_receipt_updated_at_writes_summary_then_fails() {
5528 let fixture = RestoreCliFixture::new(
5529 "canic-cli-restore-run-require-receipt-updated-at",
5530 "restore-run.json",
5531 );
5532 let journal = ready_apply_journal();
5533 fixture.write_journal(&journal);
5534
5535 let err = fixture
5536 .run_restore_run(&[
5537 "--execute",
5538 "--dfx",
5539 "/bin/true",
5540 "--max-steps",
5541 "1",
5542 "--updated-at",
5543 "2026-05-05T12:03:00Z",
5544 "--require-receipt-updated-at",
5545 "2026-05-05T12:04:00Z",
5546 ])
5547 .expect_err("receipt updated-at mismatch should fail requirement");
5548
5549 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5550
5551 assert_eq!(
5552 run_summary["operation_receipts"][0]["updated_at"],
5553 "2026-05-05T12:03:00Z"
5554 );
5555 assert!(matches!(
5556 err,
5557 RestoreCommandError::RestoreRunReceiptUpdatedAtMismatch {
5558 expected,
5559 actual_receipts: 1,
5560 mismatched_receipts: 1,
5561 ..
5562 } if expected == "2026-05-05T12:04:00Z"
5563 ));
5564 }
5565
5566 #[test]
5568 fn run_restore_run_require_state_updated_at_writes_summary_then_fails() {
5569 let fixture = RestoreCliFixture::new(
5570 "canic-cli-restore-run-require-state-updated-at",
5571 "restore-run.json",
5572 );
5573 let journal = ready_apply_journal();
5574 fixture.write_journal(&journal);
5575
5576 let err = fixture
5577 .run_restore_run(&[
5578 "--dry-run",
5579 "--updated-at",
5580 "2026-05-05T12:03:00Z",
5581 "--require-state-updated-at",
5582 "2026-05-05T12:04:00Z",
5583 ])
5584 .expect_err("state updated-at mismatch should fail requirement");
5585
5586 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5587
5588 assert_eq!(
5589 run_summary["requested_state_updated_at"],
5590 "2026-05-05T12:03:00Z"
5591 );
5592 assert_eq!(run_summary["operation_receipt_count"], 0);
5593 assert!(matches!(
5594 err,
5595 RestoreCommandError::RestoreRunStateUpdatedAtMismatch {
5596 expected,
5597 actual: Some(actual),
5598 ..
5599 } if expected == "2026-05-05T12:04:00Z"
5600 && actual == "2026-05-05T12:03:00Z"
5601 ));
5602 }
5603
5604 #[test]
5606 fn run_restore_run_require_batch_remaining_ready_count_writes_summary_then_fails() {
5607 let fixture = RestoreCliFixture::new(
5608 "canic-cli-restore-run-require-batch-ready-count",
5609 "restore-run.json",
5610 );
5611 let journal = ready_apply_journal();
5612 fixture.write_journal(&journal);
5613
5614 let err = fixture
5615 .run_restore_run(&[
5616 "--execute",
5617 "--dfx",
5618 "/bin/true",
5619 "--max-steps",
5620 "1",
5621 "--require-batch-remaining-ready-count",
5622 "8",
5623 ])
5624 .expect_err("batch remaining ready count mismatch should fail requirement");
5625
5626 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5627
5628 assert_eq!(run_summary["batch_summary"]["initial_ready_operations"], 8);
5629 assert_eq!(run_summary["batch_summary"]["executed_operations"], 1);
5630 assert_eq!(
5631 run_summary["batch_summary"]["remaining_ready_operations"],
5632 7
5633 );
5634 assert!(matches!(
5635 err,
5636 RestoreCommandError::RestoreRunBatchRemainingReadyCountMismatch {
5637 expected: 8,
5638 actual: 7,
5639 ..
5640 }
5641 ));
5642 }
5643
5644 #[test]
5646 fn run_restore_run_require_batch_initial_ready_count_writes_summary_then_fails() {
5647 let fixture = RestoreCliFixture::new(
5648 "canic-cli-restore-run-require-batch-initial-ready-count",
5649 "restore-run.json",
5650 );
5651 let journal = ready_apply_journal();
5652 fixture.write_journal(&journal);
5653
5654 let err = fixture
5655 .run_restore_run(&[
5656 "--execute",
5657 "--dfx",
5658 "/bin/true",
5659 "--max-steps",
5660 "1",
5661 "--require-batch-initial-ready-count",
5662 "7",
5663 ])
5664 .expect_err("batch initial ready count mismatch should fail requirement");
5665
5666 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5667
5668 assert_eq!(run_summary["batch_summary"]["initial_ready_operations"], 8);
5669 assert_eq!(
5670 run_summary["batch_summary"]["remaining_ready_operations"],
5671 7
5672 );
5673 assert!(matches!(
5674 err,
5675 RestoreCommandError::RestoreRunBatchInitialReadyCountMismatch {
5676 expected: 7,
5677 actual: 8,
5678 ..
5679 }
5680 ));
5681 }
5682
5683 #[test]
5685 fn run_restore_run_require_batch_executed_count_writes_summary_then_fails() {
5686 let fixture = RestoreCliFixture::new(
5687 "canic-cli-restore-run-require-batch-executed-count",
5688 "restore-run.json",
5689 );
5690 let journal = ready_apply_journal();
5691 fixture.write_journal(&journal);
5692
5693 let err = fixture
5694 .run_restore_run(&[
5695 "--execute",
5696 "--dfx",
5697 "/bin/true",
5698 "--max-steps",
5699 "1",
5700 "--require-batch-executed-count",
5701 "2",
5702 ])
5703 .expect_err("batch executed count mismatch should fail requirement");
5704
5705 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5706
5707 assert_eq!(run_summary["batch_summary"]["executed_operations"], 1);
5708 assert_eq!(run_summary["batch_summary"]["ready_operations_delta"], -1);
5709 assert!(matches!(
5710 err,
5711 RestoreCommandError::RestoreRunBatchExecutedCountMismatch {
5712 expected: 2,
5713 actual: 1,
5714 ..
5715 }
5716 ));
5717 }
5718
5719 #[test]
5721 fn run_restore_run_require_batch_ready_delta_writes_summary_then_fails() {
5722 let fixture = RestoreCliFixture::new(
5723 "canic-cli-restore-run-require-batch-ready-delta",
5724 "restore-run.json",
5725 );
5726 let journal = ready_apply_journal();
5727 fixture.write_journal(&journal);
5728
5729 let err = fixture
5730 .run_restore_run(&[
5731 "--execute",
5732 "--dfx",
5733 "/bin/true",
5734 "--max-steps",
5735 "1",
5736 "--require-batch-ready-delta",
5737 "0",
5738 ])
5739 .expect_err("batch ready delta mismatch should fail requirement");
5740
5741 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5742
5743 assert_eq!(run_summary["batch_summary"]["ready_operations_delta"], -1);
5744 assert_eq!(
5745 run_summary["batch_summary"]["remaining_operations_delta"],
5746 -1
5747 );
5748 assert!(matches!(
5749 err,
5750 RestoreCommandError::RestoreRunBatchReadyDeltaMismatch {
5751 expected: 0,
5752 actual: -1,
5753 ..
5754 }
5755 ));
5756 }
5757
5758 #[test]
5760 fn run_restore_run_require_batch_remaining_delta_writes_summary_then_fails() {
5761 let fixture = RestoreCliFixture::new(
5762 "canic-cli-restore-run-require-batch-remaining-delta",
5763 "restore-run.json",
5764 );
5765 let journal = ready_apply_journal();
5766 fixture.write_journal(&journal);
5767
5768 let err = fixture
5769 .run_restore_run(&[
5770 "--execute",
5771 "--dfx",
5772 "/bin/true",
5773 "--max-steps",
5774 "1",
5775 "--require-batch-remaining-delta",
5776 "0",
5777 ])
5778 .expect_err("batch remaining delta mismatch should fail requirement");
5779
5780 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5781
5782 assert_eq!(
5783 run_summary["batch_summary"]["remaining_operations_delta"],
5784 -1
5785 );
5786 assert!(matches!(
5787 err,
5788 RestoreCommandError::RestoreRunBatchRemainingDeltaMismatch {
5789 expected: 0,
5790 actual: -1,
5791 ..
5792 }
5793 ));
5794 }
5795
5796 #[test]
5798 fn run_restore_run_require_batch_stopped_by_max_steps_writes_summary_then_fails() {
5799 let fixture = RestoreCliFixture::new(
5800 "canic-cli-restore-run-require-batch-max-step-stop",
5801 "restore-run.json",
5802 );
5803 let journal = ready_apply_journal();
5804 fixture.write_journal(&journal);
5805
5806 let err = fixture
5807 .run_restore_run(&[
5808 "--execute",
5809 "--dfx",
5810 "/bin/true",
5811 "--max-steps",
5812 "1",
5813 "--require-batch-stopped-by-max-steps",
5814 "false",
5815 ])
5816 .expect_err("batch max-step mismatch should fail requirement");
5817
5818 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5819
5820 assert_eq!(run_summary["batch_summary"]["stopped_by_max_steps"], true);
5821 assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
5822 assert!(matches!(
5823 err,
5824 RestoreCommandError::RestoreRunBatchStoppedByMaxStepsMismatch {
5825 expected: false,
5826 actual: true,
5827 ..
5828 }
5829 ));
5830 }
5831
5832 #[test]
5834 fn run_restore_run_require_progress_writes_summary_then_fails() {
5835 let root = temp_dir("canic-cli-restore-run-require-progress");
5836 fs::create_dir_all(&root).expect("create temp root");
5837 let journal_path = root.join("restore-apply-journal.json");
5838 let out_path = root.join("restore-run.json");
5839 let journal = ready_apply_journal();
5840
5841 fs::write(
5842 &journal_path,
5843 serde_json::to_vec(&journal).expect("serialize journal"),
5844 )
5845 .expect("write journal");
5846
5847 let err = run([
5848 OsString::from("run"),
5849 OsString::from("--journal"),
5850 OsString::from(journal_path.as_os_str()),
5851 OsString::from("--execute"),
5852 OsString::from("--dfx"),
5853 OsString::from("/bin/true"),
5854 OsString::from("--max-steps"),
5855 OsString::from("1"),
5856 OsString::from("--out"),
5857 OsString::from(out_path.as_os_str()),
5858 OsString::from("--require-remaining-count"),
5859 OsString::from("7"),
5860 OsString::from("--require-attention-count"),
5861 OsString::from("0"),
5862 OsString::from("--require-completion-basis-points"),
5863 OsString::from("0"),
5864 ])
5865 .expect_err("completion progress mismatch should fail requirement");
5866
5867 let run_summary: serde_json::Value =
5868 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5869 .expect("decode run summary");
5870
5871 fs::remove_dir_all(root).expect("remove temp root");
5872 assert_eq!(run_summary["progress"]["remaining_operations"], 7);
5873 assert_eq!(run_summary["progress"]["attention_operations"], 0);
5874 assert_eq!(run_summary["progress"]["completion_basis_points"], 1250);
5875 assert!(matches!(
5876 err,
5877 RestoreCommandError::RestoreApplyProgressMismatch {
5878 field: "completion_basis_points",
5879 expected: 0,
5880 actual: 1250,
5881 ..
5882 }
5883 ));
5884 }
5885
5886 #[test]
5888 fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
5889 let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
5890 fs::create_dir_all(&root).expect("create temp root");
5891 let journal_path = root.join("restore-apply-journal.json");
5892 let out_path = root.join("restore-run.json");
5893 let journal = ready_apply_journal();
5894
5895 fs::write(
5896 &journal_path,
5897 serde_json::to_vec(&journal).expect("serialize journal"),
5898 )
5899 .expect("write journal");
5900
5901 let err = run([
5902 OsString::from("run"),
5903 OsString::from("--journal"),
5904 OsString::from(journal_path.as_os_str()),
5905 OsString::from("--dry-run"),
5906 OsString::from("--out"),
5907 OsString::from(out_path.as_os_str()),
5908 OsString::from("--require-stopped-reason"),
5909 OsString::from("complete"),
5910 ])
5911 .expect_err("stopped reason mismatch should fail requirement");
5912
5913 let run_summary: serde_json::Value =
5914 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5915 .expect("decode run summary");
5916
5917 fs::remove_dir_all(root).expect("remove temp root");
5918 assert_eq!(run_summary["stopped_reason"], "preview");
5919 assert!(matches!(
5920 err,
5921 RestoreCommandError::RestoreRunStoppedReasonMismatch {
5922 expected,
5923 actual,
5924 ..
5925 } if expected == "complete" && actual == "preview"
5926 ));
5927 }
5928
5929 #[test]
5931 fn run_restore_run_require_next_action_writes_summary_then_fails() {
5932 let root = temp_dir("canic-cli-restore-run-require-next-action");
5933 fs::create_dir_all(&root).expect("create temp root");
5934 let journal_path = root.join("restore-apply-journal.json");
5935 let out_path = root.join("restore-run.json");
5936 let journal = ready_apply_journal();
5937
5938 fs::write(
5939 &journal_path,
5940 serde_json::to_vec(&journal).expect("serialize journal"),
5941 )
5942 .expect("write journal");
5943
5944 let err = run([
5945 OsString::from("run"),
5946 OsString::from("--journal"),
5947 OsString::from(journal_path.as_os_str()),
5948 OsString::from("--dry-run"),
5949 OsString::from("--out"),
5950 OsString::from(out_path.as_os_str()),
5951 OsString::from("--require-next-action"),
5952 OsString::from("done"),
5953 ])
5954 .expect_err("next action mismatch should fail requirement");
5955
5956 let run_summary: serde_json::Value =
5957 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5958 .expect("decode run summary");
5959
5960 fs::remove_dir_all(root).expect("remove temp root");
5961 assert_eq!(run_summary["next_action"], "rerun");
5962 assert!(matches!(
5963 err,
5964 RestoreCommandError::RestoreRunNextActionMismatch {
5965 expected,
5966 actual,
5967 ..
5968 } if expected == "done" && actual == "rerun"
5969 ));
5970 }
5971
5972 #[test]
5974 fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
5975 let root = temp_dir("canic-cli-restore-apply-report-attention");
5976 fs::create_dir_all(&root).expect("create temp root");
5977 let journal_path = root.join("restore-apply-journal.json");
5978 let out_path = root.join("restore-apply-report.json");
5979 let mut journal = ready_apply_journal();
5980 journal
5981 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
5982 .expect("mark pending operation");
5983
5984 fs::write(
5985 &journal_path,
5986 serde_json::to_vec(&journal).expect("serialize journal"),
5987 )
5988 .expect("write journal");
5989
5990 let err = run([
5991 OsString::from("apply-report"),
5992 OsString::from("--journal"),
5993 OsString::from(journal_path.as_os_str()),
5994 OsString::from("--out"),
5995 OsString::from(out_path.as_os_str()),
5996 OsString::from("--require-no-attention"),
5997 ])
5998 .expect_err("attention report should fail requirement");
5999
6000 let report: RestoreApplyJournalReport =
6001 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
6002 .expect("decode apply report");
6003
6004 fs::remove_dir_all(root).expect("remove temp root");
6005 assert!(report.attention_required);
6006 assert_eq!(report.pending_operations, 1);
6007 assert!(matches!(
6008 err,
6009 RestoreCommandError::RestoreApplyReportNeedsAttention {
6010 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
6011 ..
6012 }
6013 ));
6014 }
6015
6016 #[test]
6018 fn run_restore_apply_status_require_complete_writes_status_then_fails() {
6019 let root = temp_dir("canic-cli-restore-apply-status-incomplete");
6020 fs::create_dir_all(&root).expect("create temp root");
6021 let journal_path = root.join("restore-apply-journal.json");
6022 let out_path = root.join("restore-apply-status.json");
6023 let journal = ready_apply_journal();
6024
6025 fs::write(
6026 &journal_path,
6027 serde_json::to_vec(&journal).expect("serialize journal"),
6028 )
6029 .expect("write journal");
6030
6031 let err = run([
6032 OsString::from("apply-status"),
6033 OsString::from("--journal"),
6034 OsString::from(journal_path.as_os_str()),
6035 OsString::from("--out"),
6036 OsString::from(out_path.as_os_str()),
6037 OsString::from("--require-complete"),
6038 ])
6039 .expect_err("incomplete journal should fail requirement");
6040
6041 assert!(out_path.exists());
6042 let status: RestoreApplyJournalStatus =
6043 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
6044 .expect("decode apply status");
6045
6046 fs::remove_dir_all(root).expect("remove temp root");
6047 assert!(!status.complete);
6048 assert_eq!(status.completed_operations, 0);
6049 assert_eq!(status.operation_count, 8);
6050 assert!(matches!(
6051 err,
6052 RestoreCommandError::RestoreApplyIncomplete {
6053 completed_operations: 0,
6054 operation_count: 8,
6055 ..
6056 }
6057 ));
6058 }
6059
6060 #[test]
6062 fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
6063 let root = temp_dir("canic-cli-restore-apply-status-failed");
6064 fs::create_dir_all(&root).expect("create temp root");
6065 let journal_path = root.join("restore-apply-journal.json");
6066 let out_path = root.join("restore-apply-status.json");
6067 let mut journal = ready_apply_journal();
6068 journal
6069 .mark_operation_failed(0, "dfx-load-failed".to_string())
6070 .expect("mark failed operation");
6071
6072 fs::write(
6073 &journal_path,
6074 serde_json::to_vec(&journal).expect("serialize journal"),
6075 )
6076 .expect("write journal");
6077
6078 let err = run([
6079 OsString::from("apply-status"),
6080 OsString::from("--journal"),
6081 OsString::from(journal_path.as_os_str()),
6082 OsString::from("--out"),
6083 OsString::from(out_path.as_os_str()),
6084 OsString::from("--require-no-failed"),
6085 ])
6086 .expect_err("failed operation should fail requirement");
6087
6088 assert!(out_path.exists());
6089 let status: RestoreApplyJournalStatus =
6090 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
6091 .expect("decode apply status");
6092
6093 fs::remove_dir_all(root).expect("remove temp root");
6094 assert_eq!(status.failed_operations, 1);
6095 assert!(matches!(
6096 err,
6097 RestoreCommandError::RestoreApplyFailed {
6098 failed_operations: 1,
6099 ..
6100 }
6101 ));
6102 }
6103
6104 #[test]
6106 fn run_restore_apply_status_require_complete_accepts_complete_journal() {
6107 let root = temp_dir("canic-cli-restore-apply-status-complete");
6108 fs::create_dir_all(&root).expect("create temp root");
6109 let journal_path = root.join("restore-apply-journal.json");
6110 let out_path = root.join("restore-apply-status.json");
6111 let mut journal = ready_apply_journal();
6112 for sequence in 0..journal.operation_count {
6113 journal
6114 .mark_operation_completed(sequence)
6115 .expect("complete operation");
6116 }
6117
6118 fs::write(
6119 &journal_path,
6120 serde_json::to_vec(&journal).expect("serialize journal"),
6121 )
6122 .expect("write journal");
6123
6124 run([
6125 OsString::from("apply-status"),
6126 OsString::from("--journal"),
6127 OsString::from(journal_path.as_os_str()),
6128 OsString::from("--out"),
6129 OsString::from(out_path.as_os_str()),
6130 OsString::from("--require-complete"),
6131 ])
6132 .expect("complete journal should pass requirement");
6133
6134 let status: RestoreApplyJournalStatus =
6135 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
6136 .expect("decode apply status");
6137
6138 fs::remove_dir_all(root).expect("remove temp root");
6139 assert!(status.complete);
6140 assert_eq!(status.completed_operations, 8);
6141 assert_eq!(status.operation_count, 8);
6142 }
6143
6144 #[test]
6146 fn run_restore_apply_next_writes_next_ready_operation() {
6147 let root = temp_dir("canic-cli-restore-apply-next");
6148 fs::create_dir_all(&root).expect("create temp root");
6149 let journal_path = root.join("restore-apply-journal.json");
6150 let out_path = root.join("restore-apply-next.json");
6151 let mut journal = ready_apply_journal();
6152 journal
6153 .mark_operation_completed(0)
6154 .expect("mark first operation complete");
6155
6156 fs::write(
6157 &journal_path,
6158 serde_json::to_vec(&journal).expect("serialize journal"),
6159 )
6160 .expect("write journal");
6161
6162 run([
6163 OsString::from("apply-next"),
6164 OsString::from("--journal"),
6165 OsString::from(journal_path.as_os_str()),
6166 OsString::from("--out"),
6167 OsString::from(out_path.as_os_str()),
6168 ])
6169 .expect("write apply next");
6170
6171 let next: RestoreApplyNextOperation =
6172 serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
6173 .expect("decode next operation");
6174 let operation = next.operation.expect("operation should be available");
6175
6176 fs::remove_dir_all(root).expect("remove temp root");
6177 assert!(next.ready);
6178 assert!(next.operation_available);
6179 assert_eq!(operation.sequence, 1);
6180 assert_eq!(
6181 operation.operation,
6182 canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
6183 );
6184 }
6185
6186 #[test]
6188 fn run_restore_apply_command_writes_next_command_preview() {
6189 let root = temp_dir("canic-cli-restore-apply-command");
6190 fs::create_dir_all(&root).expect("create temp root");
6191 let journal_path = root.join("restore-apply-journal.json");
6192 let out_path = root.join("restore-apply-command.json");
6193 let journal = ready_apply_journal();
6194
6195 fs::write(
6196 &journal_path,
6197 serde_json::to_vec(&journal).expect("serialize journal"),
6198 )
6199 .expect("write journal");
6200
6201 run([
6202 OsString::from("apply-command"),
6203 OsString::from("--journal"),
6204 OsString::from(journal_path.as_os_str()),
6205 OsString::from("--dfx"),
6206 OsString::from("/tmp/dfx"),
6207 OsString::from("--network"),
6208 OsString::from("local"),
6209 OsString::from("--out"),
6210 OsString::from(out_path.as_os_str()),
6211 ])
6212 .expect("write command preview");
6213
6214 let preview: RestoreApplyCommandPreview =
6215 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
6216 .expect("decode command preview");
6217 let command = preview.command.expect("command should be available");
6218
6219 fs::remove_dir_all(root).expect("remove temp root");
6220 assert!(preview.ready);
6221 assert!(preview.command_available);
6222 assert_eq!(command.program, "/tmp/dfx");
6223 assert_eq!(
6224 command.args,
6225 vec![
6226 "canister".to_string(),
6227 "--network".to_string(),
6228 "local".to_string(),
6229 "snapshot".to_string(),
6230 "upload".to_string(),
6231 "--dir".to_string(),
6232 "artifacts/root".to_string(),
6233 ROOT.to_string(),
6234 ]
6235 );
6236 assert!(command.mutates);
6237 }
6238
6239 #[test]
6241 fn run_restore_apply_command_require_command_writes_preview_then_fails() {
6242 let root = temp_dir("canic-cli-restore-apply-command-require");
6243 fs::create_dir_all(&root).expect("create temp root");
6244 let journal_path = root.join("restore-apply-journal.json");
6245 let out_path = root.join("restore-apply-command.json");
6246 let mut journal = ready_apply_journal();
6247
6248 for sequence in 0..journal.operation_count {
6249 journal
6250 .mark_operation_completed(sequence)
6251 .expect("mark operation completed");
6252 }
6253
6254 fs::write(
6255 &journal_path,
6256 serde_json::to_vec(&journal).expect("serialize journal"),
6257 )
6258 .expect("write journal");
6259
6260 let err = run([
6261 OsString::from("apply-command"),
6262 OsString::from("--journal"),
6263 OsString::from(journal_path.as_os_str()),
6264 OsString::from("--out"),
6265 OsString::from(out_path.as_os_str()),
6266 OsString::from("--require-command"),
6267 ])
6268 .expect_err("missing command should fail");
6269
6270 let preview: RestoreApplyCommandPreview =
6271 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
6272 .expect("decode command preview");
6273
6274 fs::remove_dir_all(root).expect("remove temp root");
6275 assert!(preview.complete);
6276 assert!(!preview.operation_available);
6277 assert!(!preview.command_available);
6278 assert!(matches!(
6279 err,
6280 RestoreCommandError::RestoreApplyCommandUnavailable {
6281 operation_available: false,
6282 complete: true,
6283 ..
6284 }
6285 ));
6286 }
6287
6288 #[test]
6290 fn run_restore_apply_claim_marks_next_operation_pending() {
6291 let root = temp_dir("canic-cli-restore-apply-claim");
6292 fs::create_dir_all(&root).expect("create temp root");
6293 let journal_path = root.join("restore-apply-journal.json");
6294 let claimed_path = root.join("restore-apply-journal.claimed.json");
6295 let journal = ready_apply_journal();
6296
6297 fs::write(
6298 &journal_path,
6299 serde_json::to_vec(&journal).expect("serialize journal"),
6300 )
6301 .expect("write journal");
6302
6303 run([
6304 OsString::from("apply-claim"),
6305 OsString::from("--journal"),
6306 OsString::from(journal_path.as_os_str()),
6307 OsString::from("--sequence"),
6308 OsString::from("0"),
6309 OsString::from("--updated-at"),
6310 OsString::from("2026-05-04T12:00:00Z"),
6311 OsString::from("--out"),
6312 OsString::from(claimed_path.as_os_str()),
6313 ])
6314 .expect("claim operation");
6315
6316 let claimed: RestoreApplyJournal =
6317 serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
6318 .expect("decode claimed journal");
6319 let status = claimed.status();
6320 let next = claimed.next_operation();
6321
6322 fs::remove_dir_all(root).expect("remove temp root");
6323 assert_eq!(claimed.pending_operations, 1);
6324 assert_eq!(claimed.ready_operations, 7);
6325 assert_eq!(
6326 claimed.operations[0].state,
6327 RestoreApplyOperationState::Pending
6328 );
6329 assert_eq!(
6330 claimed.operations[0].state_updated_at.as_deref(),
6331 Some("2026-05-04T12:00:00Z")
6332 );
6333 assert_eq!(status.next_transition_sequence, Some(0));
6334 assert_eq!(
6335 status.next_transition_state,
6336 Some(RestoreApplyOperationState::Pending)
6337 );
6338 assert_eq!(
6339 status.next_transition_updated_at.as_deref(),
6340 Some("2026-05-04T12:00:00Z")
6341 );
6342 assert_eq!(
6343 next.operation.expect("next operation").state,
6344 RestoreApplyOperationState::Pending
6345 );
6346 }
6347
6348 #[test]
6350 fn run_restore_apply_claim_rejects_sequence_mismatch() {
6351 let root = temp_dir("canic-cli-restore-apply-claim-sequence");
6352 fs::create_dir_all(&root).expect("create temp root");
6353 let journal_path = root.join("restore-apply-journal.json");
6354 let claimed_path = root.join("restore-apply-journal.claimed.json");
6355 let journal = ready_apply_journal();
6356
6357 fs::write(
6358 &journal_path,
6359 serde_json::to_vec(&journal).expect("serialize journal"),
6360 )
6361 .expect("write journal");
6362
6363 let err = run([
6364 OsString::from("apply-claim"),
6365 OsString::from("--journal"),
6366 OsString::from(journal_path.as_os_str()),
6367 OsString::from("--sequence"),
6368 OsString::from("1"),
6369 OsString::from("--out"),
6370 OsString::from(claimed_path.as_os_str()),
6371 ])
6372 .expect_err("stale sequence should fail claim");
6373
6374 assert!(!claimed_path.exists());
6375 fs::remove_dir_all(root).expect("remove temp root");
6376 assert!(matches!(
6377 err,
6378 RestoreCommandError::RestoreApplyClaimSequenceMismatch {
6379 expected: 1,
6380 actual: Some(0),
6381 }
6382 ));
6383 }
6384
6385 #[test]
6387 fn run_restore_apply_unclaim_marks_pending_operation_ready() {
6388 let root = temp_dir("canic-cli-restore-apply-unclaim");
6389 fs::create_dir_all(&root).expect("create temp root");
6390 let journal_path = root.join("restore-apply-journal.json");
6391 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
6392 let mut journal = ready_apply_journal();
6393 journal
6394 .mark_next_operation_pending()
6395 .expect("claim operation");
6396
6397 fs::write(
6398 &journal_path,
6399 serde_json::to_vec(&journal).expect("serialize journal"),
6400 )
6401 .expect("write journal");
6402
6403 run([
6404 OsString::from("apply-unclaim"),
6405 OsString::from("--journal"),
6406 OsString::from(journal_path.as_os_str()),
6407 OsString::from("--sequence"),
6408 OsString::from("0"),
6409 OsString::from("--updated-at"),
6410 OsString::from("2026-05-04T12:01:00Z"),
6411 OsString::from("--out"),
6412 OsString::from(unclaimed_path.as_os_str()),
6413 ])
6414 .expect("unclaim operation");
6415
6416 let unclaimed: RestoreApplyJournal =
6417 serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
6418 .expect("decode unclaimed journal");
6419 let status = unclaimed.status();
6420
6421 fs::remove_dir_all(root).expect("remove temp root");
6422 assert_eq!(unclaimed.pending_operations, 0);
6423 assert_eq!(unclaimed.ready_operations, 8);
6424 assert_eq!(
6425 unclaimed.operations[0].state,
6426 RestoreApplyOperationState::Ready
6427 );
6428 assert_eq!(
6429 unclaimed.operations[0].state_updated_at.as_deref(),
6430 Some("2026-05-04T12:01:00Z")
6431 );
6432 assert_eq!(status.next_ready_sequence, Some(0));
6433 assert_eq!(
6434 status.next_transition_state,
6435 Some(RestoreApplyOperationState::Ready)
6436 );
6437 assert_eq!(
6438 status.next_transition_updated_at.as_deref(),
6439 Some("2026-05-04T12:01:00Z")
6440 );
6441 }
6442
6443 #[test]
6445 fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
6446 let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
6447 fs::create_dir_all(&root).expect("create temp root");
6448 let journal_path = root.join("restore-apply-journal.json");
6449 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
6450 let mut journal = ready_apply_journal();
6451 journal
6452 .mark_next_operation_pending()
6453 .expect("claim operation");
6454
6455 fs::write(
6456 &journal_path,
6457 serde_json::to_vec(&journal).expect("serialize journal"),
6458 )
6459 .expect("write journal");
6460
6461 let err = run([
6462 OsString::from("apply-unclaim"),
6463 OsString::from("--journal"),
6464 OsString::from(journal_path.as_os_str()),
6465 OsString::from("--sequence"),
6466 OsString::from("1"),
6467 OsString::from("--out"),
6468 OsString::from(unclaimed_path.as_os_str()),
6469 ])
6470 .expect_err("stale sequence should fail unclaim");
6471
6472 assert!(!unclaimed_path.exists());
6473 fs::remove_dir_all(root).expect("remove temp root");
6474 assert!(matches!(
6475 err,
6476 RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
6477 expected: 1,
6478 actual: Some(0),
6479 }
6480 ));
6481 }
6482
6483 #[test]
6485 fn run_restore_apply_mark_completes_operation() {
6486 let root = temp_dir("canic-cli-restore-apply-mark-complete");
6487 fs::create_dir_all(&root).expect("create temp root");
6488 let journal_path = root.join("restore-apply-journal.json");
6489 let updated_path = root.join("restore-apply-journal.updated.json");
6490 let journal = ready_apply_journal();
6491
6492 fs::write(
6493 &journal_path,
6494 serde_json::to_vec(&journal).expect("serialize journal"),
6495 )
6496 .expect("write journal");
6497
6498 run([
6499 OsString::from("apply-mark"),
6500 OsString::from("--journal"),
6501 OsString::from(journal_path.as_os_str()),
6502 OsString::from("--sequence"),
6503 OsString::from("0"),
6504 OsString::from("--state"),
6505 OsString::from("completed"),
6506 OsString::from("--updated-at"),
6507 OsString::from("2026-05-04T12:02:00Z"),
6508 OsString::from("--out"),
6509 OsString::from(updated_path.as_os_str()),
6510 ])
6511 .expect("mark operation completed");
6512
6513 let updated: RestoreApplyJournal =
6514 serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
6515 .expect("decode updated journal");
6516 let status = updated.status();
6517
6518 fs::remove_dir_all(root).expect("remove temp root");
6519 assert_eq!(updated.completed_operations, 1);
6520 assert_eq!(updated.ready_operations, 7);
6521 assert_eq!(
6522 updated.operations[0].state_updated_at.as_deref(),
6523 Some("2026-05-04T12:02:00Z")
6524 );
6525 assert_eq!(status.next_ready_sequence, Some(1));
6526 }
6527
6528 #[test]
6530 fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
6531 let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
6532 fs::create_dir_all(&root).expect("create temp root");
6533 let journal_path = root.join("restore-apply-journal.json");
6534 let updated_path = root.join("restore-apply-journal.updated.json");
6535 let journal = ready_apply_journal();
6536
6537 fs::write(
6538 &journal_path,
6539 serde_json::to_vec(&journal).expect("serialize journal"),
6540 )
6541 .expect("write journal");
6542
6543 let err = run([
6544 OsString::from("apply-mark"),
6545 OsString::from("--journal"),
6546 OsString::from(journal_path.as_os_str()),
6547 OsString::from("--sequence"),
6548 OsString::from("0"),
6549 OsString::from("--state"),
6550 OsString::from("completed"),
6551 OsString::from("--out"),
6552 OsString::from(updated_path.as_os_str()),
6553 OsString::from("--require-pending"),
6554 ])
6555 .expect_err("ready operation should fail pending requirement");
6556
6557 assert!(!updated_path.exists());
6558 fs::remove_dir_all(root).expect("remove temp root");
6559 assert!(matches!(
6560 err,
6561 RestoreCommandError::RestoreApplyMarkRequiresPending {
6562 sequence: 0,
6563 state: RestoreApplyOperationState::Ready,
6564 }
6565 ));
6566 }
6567
6568 #[test]
6570 fn run_restore_apply_mark_rejects_out_of_order_operation() {
6571 let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
6572 fs::create_dir_all(&root).expect("create temp root");
6573 let journal_path = root.join("restore-apply-journal.json");
6574 let updated_path = root.join("restore-apply-journal.updated.json");
6575 let journal = ready_apply_journal();
6576
6577 fs::write(
6578 &journal_path,
6579 serde_json::to_vec(&journal).expect("serialize journal"),
6580 )
6581 .expect("write journal");
6582
6583 let err = run([
6584 OsString::from("apply-mark"),
6585 OsString::from("--journal"),
6586 OsString::from(journal_path.as_os_str()),
6587 OsString::from("--sequence"),
6588 OsString::from("1"),
6589 OsString::from("--state"),
6590 OsString::from("completed"),
6591 OsString::from("--out"),
6592 OsString::from(updated_path.as_os_str()),
6593 ])
6594 .expect_err("out-of-order operation should fail");
6595
6596 assert!(!updated_path.exists());
6597 fs::remove_dir_all(root).expect("remove temp root");
6598 assert!(matches!(
6599 err,
6600 RestoreCommandError::RestoreApplyJournal(
6601 RestoreApplyJournalError::OutOfOrderOperationTransition {
6602 requested: 1,
6603 next: 0
6604 }
6605 )
6606 ));
6607 }
6608
6609 #[test]
6611 fn run_restore_apply_mark_failed_requires_reason() {
6612 let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
6613 fs::create_dir_all(&root).expect("create temp root");
6614 let journal_path = root.join("restore-apply-journal.json");
6615 let journal = ready_apply_journal();
6616
6617 fs::write(
6618 &journal_path,
6619 serde_json::to_vec(&journal).expect("serialize journal"),
6620 )
6621 .expect("write journal");
6622
6623 let err = run([
6624 OsString::from("apply-mark"),
6625 OsString::from("--journal"),
6626 OsString::from(journal_path.as_os_str()),
6627 OsString::from("--sequence"),
6628 OsString::from("0"),
6629 OsString::from("--state"),
6630 OsString::from("failed"),
6631 ])
6632 .expect_err("failed state should require reason");
6633
6634 fs::remove_dir_all(root).expect("remove temp root");
6635 assert!(matches!(
6636 err,
6637 RestoreCommandError::RestoreApplyJournal(
6638 RestoreApplyJournalError::FailureReasonRequired(0)
6639 )
6640 ));
6641 }
6642
6643 #[test]
6645 fn run_restore_apply_dry_run_rejects_mismatched_status() {
6646 let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
6647 fs::create_dir_all(&root).expect("create temp root");
6648 let plan_path = root.join("restore-plan.json");
6649 let status_path = root.join("restore-status.json");
6650 let out_path = root.join("restore-apply-dry-run.json");
6651 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
6652 let mut status = RestoreStatus::from_plan(&plan);
6653 status.backup_id = "other-backup".to_string();
6654
6655 fs::write(
6656 &plan_path,
6657 serde_json::to_vec(&plan).expect("serialize plan"),
6658 )
6659 .expect("write plan");
6660 fs::write(
6661 &status_path,
6662 serde_json::to_vec(&status).expect("serialize status"),
6663 )
6664 .expect("write status");
6665
6666 let err = run([
6667 OsString::from("apply"),
6668 OsString::from("--plan"),
6669 OsString::from(plan_path.as_os_str()),
6670 OsString::from("--status"),
6671 OsString::from(status_path.as_os_str()),
6672 OsString::from("--dry-run"),
6673 OsString::from("--out"),
6674 OsString::from(out_path.as_os_str()),
6675 ])
6676 .expect_err("mismatched status should fail");
6677
6678 assert!(!out_path.exists());
6679 fs::remove_dir_all(root).expect("remove temp root");
6680 assert!(matches!(
6681 err,
6682 RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
6683 field: "backup_id",
6684 ..
6685 })
6686 ));
6687 }
6688
6689 fn ready_apply_journal() -> RestoreApplyJournal {
6691 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
6692 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
6693 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
6694
6695 journal.ready = true;
6696 journal.blocked_reasons = Vec::new();
6697 for operation in &mut journal.operations {
6698 operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
6699 operation.blocking_reasons = Vec::new();
6700 }
6701 journal.blocked_operations = 0;
6702 journal.ready_operations = journal.operation_count;
6703 journal.validate().expect("journal should validate");
6704 journal
6705 }
6706
6707 fn valid_manifest() -> FleetBackupManifest {
6709 FleetBackupManifest {
6710 manifest_version: 1,
6711 backup_id: "backup-test".to_string(),
6712 created_at: "2026-05-03T00:00:00Z".to_string(),
6713 tool: ToolMetadata {
6714 name: "canic".to_string(),
6715 version: "0.30.1".to_string(),
6716 },
6717 source: SourceMetadata {
6718 environment: "local".to_string(),
6719 root_canister: ROOT.to_string(),
6720 },
6721 consistency: ConsistencySection {
6722 mode: ConsistencyMode::CrashConsistent,
6723 backup_units: vec![BackupUnit {
6724 unit_id: "fleet".to_string(),
6725 kind: BackupUnitKind::SubtreeRooted,
6726 roles: vec!["root".to_string(), "app".to_string()],
6727 consistency_reason: None,
6728 dependency_closure: Vec::new(),
6729 topology_validation: "subtree-closed".to_string(),
6730 quiescence_strategy: None,
6731 }],
6732 },
6733 fleet: FleetSection {
6734 topology_hash_algorithm: "sha256".to_string(),
6735 topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
6736 discovery_topology_hash: HASH.to_string(),
6737 pre_snapshot_topology_hash: HASH.to_string(),
6738 topology_hash: HASH.to_string(),
6739 members: vec![
6740 fleet_member("root", ROOT, None, IdentityMode::Fixed),
6741 fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
6742 ],
6743 },
6744 verification: VerificationPlan::default(),
6745 }
6746 }
6747
6748 fn restore_ready_manifest() -> FleetBackupManifest {
6750 let mut manifest = valid_manifest();
6751 for member in &mut manifest.fleet.members {
6752 member.source_snapshot.module_hash = Some(HASH.to_string());
6753 member.source_snapshot.wasm_hash = Some(HASH.to_string());
6754 member.source_snapshot.checksum = Some(HASH.to_string());
6755 }
6756 manifest
6757 }
6758
6759 fn fleet_member(
6761 role: &str,
6762 canister_id: &str,
6763 parent_canister_id: Option<&str>,
6764 identity_mode: IdentityMode,
6765 ) -> FleetMember {
6766 FleetMember {
6767 role: role.to_string(),
6768 canister_id: canister_id.to_string(),
6769 parent_canister_id: parent_canister_id.map(str::to_string),
6770 subnet_canister_id: Some(ROOT.to_string()),
6771 controller_hint: None,
6772 identity_mode,
6773 restore_group: 1,
6774 verification_class: "basic".to_string(),
6775 verification_checks: vec![VerificationCheck {
6776 kind: "status".to_string(),
6777 method: None,
6778 roles: vec![role.to_string()],
6779 }],
6780 source_snapshot: SourceSnapshot {
6781 snapshot_id: format!("{role}-snapshot"),
6782 module_hash: None,
6783 wasm_hash: None,
6784 code_version: Some("v0.30.1".to_string()),
6785 artifact_path: format!("artifacts/{role}"),
6786 checksum_algorithm: "sha256".to_string(),
6787 checksum: None,
6788 },
6789 }
6790 }
6791
6792 fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
6794 layout.write_manifest(manifest).expect("write manifest");
6795
6796 let artifacts = manifest
6797 .fleet
6798 .members
6799 .iter()
6800 .map(|member| {
6801 let bytes = format!("{} artifact", member.role);
6802 let artifact_path = root.join(&member.source_snapshot.artifact_path);
6803 if let Some(parent) = artifact_path.parent() {
6804 fs::create_dir_all(parent).expect("create artifact parent");
6805 }
6806 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
6807 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
6808
6809 ArtifactJournalEntry {
6810 canister_id: member.canister_id.clone(),
6811 snapshot_id: member.source_snapshot.snapshot_id.clone(),
6812 state: ArtifactState::Durable,
6813 temp_path: None,
6814 artifact_path: member.source_snapshot.artifact_path.clone(),
6815 checksum_algorithm: checksum.algorithm,
6816 checksum: Some(checksum.hash),
6817 updated_at: "2026-05-03T00:00:00Z".to_string(),
6818 }
6819 })
6820 .collect();
6821
6822 layout
6823 .write_journal(&DownloadJournal {
6824 journal_version: 1,
6825 backup_id: manifest.backup_id.clone(),
6826 discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
6827 pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
6828 operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
6829 artifacts,
6830 })
6831 .expect("write journal");
6832 }
6833
6834 fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
6836 for member in &mut manifest.fleet.members {
6837 let bytes = format!("{} apply artifact", member.role);
6838 let artifact_path = root.join(&member.source_snapshot.artifact_path);
6839 if let Some(parent) = artifact_path.parent() {
6840 fs::create_dir_all(parent).expect("create artifact parent");
6841 }
6842 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
6843 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
6844 member.source_snapshot.checksum = Some(checksum.hash);
6845 }
6846 }
6847
6848 fn temp_dir(prefix: &str) -> PathBuf {
6850 let nanos = SystemTime::now()
6851 .duration_since(UNIX_EPOCH)
6852 .expect("system time after epoch")
6853 .as_nanos();
6854 std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
6855 }
6856}