1use canic_backup::{
2 manifest::FleetBackupManifest,
3 persistence::{BackupLayout, PersistenceError},
4 restore::{
5 RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6 RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7 RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8 RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationKindCounts,
9 RestoreApplyOperationState, RestoreApplyPendingSummary, RestoreApplyProgressSummary,
10 RestoreApplyReportOperation, RestoreApplyReportOutcome, RestoreApplyRunnerCommand,
11 RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner, RestoreStatus,
12 },
13};
14use serde::Serialize;
15use std::{
16 ffi::OsString,
17 fs,
18 io::{self, Write},
19 path::PathBuf,
20 process::Command,
21};
22use thiserror::Error as ThisError;
23
24#[derive(Debug, ThisError)]
29pub enum RestoreCommandError {
30 #[error("{0}")]
31 Usage(&'static str),
32
33 #[error("missing required option {0}")]
34 MissingOption(&'static str),
35
36 #[error("use either --manifest or --backup-dir, not both")]
37 ConflictingManifestSources,
38
39 #[error("--require-verified requires --backup-dir")]
40 RequireVerifiedNeedsBackupDir,
41
42 #[error("restore apply currently requires --dry-run")]
43 ApplyRequiresDryRun,
44
45 #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
46 RestoreRunRequiresMode,
47
48 #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
49 RestoreRunConflictingModes,
50
51 #[error("restore run command failed for operation {sequence}: status={status}")]
52 RestoreRunCommandFailed { sequence: usize, status: String },
53
54 #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
55 RestoreRunModeMismatch {
56 backup_id: String,
57 expected: String,
58 actual: String,
59 },
60
61 #[error(
62 "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
63 )]
64 RestoreRunStoppedReasonMismatch {
65 backup_id: String,
66 expected: String,
67 actual: String,
68 },
69
70 #[error(
71 "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
72 )]
73 RestoreRunNextActionMismatch {
74 backup_id: String,
75 expected: String,
76 actual: String,
77 },
78
79 #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
80 RestoreRunExecutedCountMismatch {
81 backup_id: String,
82 expected: usize,
83 actual: usize,
84 },
85
86 #[error("restore run for backup {backup_id} wrote {actual} receipts, expected {expected}")]
87 RestoreRunReceiptCountMismatch {
88 backup_id: String,
89 expected: usize,
90 actual: usize,
91 },
92
93 #[error(
94 "restore run for backup {backup_id} wrote {actual} {receipt_kind} receipts, expected {expected}"
95 )]
96 RestoreRunReceiptKindCountMismatch {
97 backup_id: String,
98 receipt_kind: &'static str,
99 expected: usize,
100 actual: usize,
101 },
102
103 #[error(
104 "restore run for backup {backup_id} wrote {actual_receipts} receipts with {mismatched_receipts} updated_at mismatches, expected {expected}"
105 )]
106 RestoreRunReceiptUpdatedAtMismatch {
107 backup_id: String,
108 expected: String,
109 actual_receipts: usize,
110 mismatched_receipts: usize,
111 },
112
113 #[error(
114 "restore run for backup {backup_id} has {actual} remaining ready operations, expected {expected}"
115 )]
116 RestoreRunBatchRemainingReadyCountMismatch {
117 backup_id: String,
118 expected: usize,
119 actual: usize,
120 },
121
122 #[error(
123 "restore run for backup {backup_id} started with {actual} ready operations, expected {expected}"
124 )]
125 RestoreRunBatchInitialReadyCountMismatch {
126 backup_id: String,
127 expected: usize,
128 actual: usize,
129 },
130
131 #[error(
132 "restore run for backup {backup_id} executed {actual} batch operations, expected {expected}"
133 )]
134 RestoreRunBatchExecutedCountMismatch {
135 backup_id: String,
136 expected: usize,
137 actual: usize,
138 },
139
140 #[error("restore run for backup {backup_id} has ready delta {actual}, expected {expected}")]
141 RestoreRunBatchReadyDeltaMismatch {
142 backup_id: String,
143 expected: isize,
144 actual: isize,
145 },
146
147 #[error("restore run for backup {backup_id} has remaining delta {actual}, expected {expected}")]
148 RestoreRunBatchRemainingDeltaMismatch {
149 backup_id: String,
150 expected: isize,
151 actual: isize,
152 },
153
154 #[error(
155 "restore run for backup {backup_id} stopped_by_max_steps={actual}, expected {expected}"
156 )]
157 RestoreRunBatchStoppedByMaxStepsMismatch {
158 backup_id: String,
159 expected: bool,
160 actual: bool,
161 },
162
163 #[error(
164 "restore run for backup {backup_id} reported requested_state_updated_at={actual:?}, expected {expected}"
165 )]
166 RestoreRunStateUpdatedAtMismatch {
167 backup_id: String,
168 expected: String,
169 actual: Option<String>,
170 },
171
172 #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
173 RestoreNotReady {
174 backup_id: String,
175 reasons: Vec<String>,
176 },
177
178 #[error(
179 "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
180 )]
181 RestoreApplyPending {
182 backup_id: String,
183 pending_operations: usize,
184 next_transition_sequence: Option<usize>,
185 },
186
187 #[error(
188 "restore apply journal for backup {backup_id} has stale or untracked pending work before {cutoff_updated_at}: pending_sequence={pending_sequence:?}, pending_updated_at={pending_updated_at:?}"
189 )]
190 RestoreApplyPendingStale {
191 backup_id: String,
192 cutoff_updated_at: String,
193 pending_sequence: Option<usize>,
194 pending_updated_at: Option<String>,
195 },
196
197 #[error(
198 "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
199 )]
200 RestoreApplyIncomplete {
201 backup_id: String,
202 completed_operations: usize,
203 operation_count: usize,
204 },
205
206 #[error(
207 "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
208 )]
209 RestoreApplyFailed {
210 backup_id: String,
211 failed_operations: usize,
212 },
213
214 #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
215 RestoreApplyNotReady {
216 backup_id: String,
217 reasons: Vec<String>,
218 },
219
220 #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
221 RestoreApplyReportNeedsAttention {
222 backup_id: String,
223 outcome: canic_backup::restore::RestoreApplyReportOutcome,
224 },
225
226 #[error(
227 "restore apply progress for backup {backup_id} has unexpected {field}: expected={expected}, actual={actual}"
228 )]
229 RestoreApplyProgressMismatch {
230 backup_id: String,
231 field: &'static str,
232 expected: usize,
233 actual: usize,
234 },
235
236 #[error(
237 "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
238 )]
239 RestoreApplyCommandUnavailable {
240 backup_id: String,
241 operation_available: bool,
242 complete: bool,
243 blocked_reasons: Vec<String>,
244 },
245
246 #[error(
247 "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
248 )]
249 RestoreApplyMarkRequiresPending {
250 sequence: usize,
251 state: RestoreApplyOperationState,
252 },
253
254 #[error(
255 "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
256 )]
257 RestoreApplyClaimSequenceMismatch {
258 expected: usize,
259 actual: Option<usize>,
260 },
261
262 #[error(
263 "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
264 )]
265 RestoreApplyUnclaimSequenceMismatch {
266 expected: usize,
267 actual: Option<usize>,
268 },
269
270 #[error("unknown option {0}")]
271 UnknownOption(String),
272
273 #[error("option {0} requires a value")]
274 MissingValue(&'static str),
275
276 #[error("option --sequence requires a non-negative integer value")]
277 InvalidSequence,
278
279 #[error("option {option} requires a positive integer value")]
280 InvalidPositiveInteger { option: &'static str },
281
282 #[error("option {option} requires an integer value")]
283 InvalidInteger { option: &'static str },
284
285 #[error("option {option} requires true or false, got {value}")]
286 InvalidBoolean { option: &'static str, value: String },
287
288 #[error("unsupported apply-mark state {0}; use completed or failed")]
289 InvalidApplyMarkState(String),
290
291 #[error(transparent)]
292 Io(#[from] std::io::Error),
293
294 #[error(transparent)]
295 Json(#[from] serde_json::Error),
296
297 #[error(transparent)]
298 Persistence(#[from] PersistenceError),
299
300 #[error(transparent)]
301 RestorePlan(#[from] RestorePlanError),
302
303 #[error(transparent)]
304 RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
305
306 #[error(transparent)]
307 RestoreApplyJournal(#[from] RestoreApplyJournalError),
308}
309
310#[derive(Clone, Debug, Eq, PartialEq)]
315pub struct RestorePlanOptions {
316 pub manifest: Option<PathBuf>,
317 pub backup_dir: Option<PathBuf>,
318 pub mapping: Option<PathBuf>,
319 pub out: Option<PathBuf>,
320 pub require_verified: bool,
321 pub require_restore_ready: bool,
322}
323
324impl RestorePlanOptions {
325 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
327 where
328 I: IntoIterator<Item = OsString>,
329 {
330 let mut manifest = None;
331 let mut backup_dir = None;
332 let mut mapping = None;
333 let mut out = None;
334 let mut require_verified = false;
335 let mut require_restore_ready = false;
336
337 let mut args = args.into_iter();
338 while let Some(arg) = args.next() {
339 let arg = arg
340 .into_string()
341 .map_err(|_| RestoreCommandError::Usage(usage()))?;
342 match arg.as_str() {
343 "--manifest" => {
344 manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
345 }
346 "--backup-dir" => {
347 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
348 }
349 "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
350 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
351 "--require-verified" => require_verified = true,
352 "--require-restore-ready" => require_restore_ready = true,
353 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
354 _ => return Err(RestoreCommandError::UnknownOption(arg)),
355 }
356 }
357
358 if manifest.is_some() && backup_dir.is_some() {
359 return Err(RestoreCommandError::ConflictingManifestSources);
360 }
361
362 if manifest.is_none() && backup_dir.is_none() {
363 return Err(RestoreCommandError::MissingOption(
364 "--manifest or --backup-dir",
365 ));
366 }
367
368 if require_verified && backup_dir.is_none() {
369 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
370 }
371
372 Ok(Self {
373 manifest,
374 backup_dir,
375 mapping,
376 out,
377 require_verified,
378 require_restore_ready,
379 })
380 }
381}
382
383#[derive(Clone, Debug, Eq, PartialEq)]
388pub struct RestoreStatusOptions {
389 pub plan: PathBuf,
390 pub out: Option<PathBuf>,
391}
392
393impl RestoreStatusOptions {
394 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
396 where
397 I: IntoIterator<Item = OsString>,
398 {
399 let mut plan = None;
400 let mut out = None;
401
402 let mut args = args.into_iter();
403 while let Some(arg) = args.next() {
404 let arg = arg
405 .into_string()
406 .map_err(|_| RestoreCommandError::Usage(usage()))?;
407 match arg.as_str() {
408 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
409 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
410 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
411 _ => return Err(RestoreCommandError::UnknownOption(arg)),
412 }
413 }
414
415 Ok(Self {
416 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
417 out,
418 })
419 }
420}
421
422#[derive(Clone, Debug, Eq, PartialEq)]
427pub struct RestoreApplyOptions {
428 pub plan: PathBuf,
429 pub status: Option<PathBuf>,
430 pub backup_dir: Option<PathBuf>,
431 pub out: Option<PathBuf>,
432 pub journal_out: Option<PathBuf>,
433 pub dry_run: bool,
434}
435
436impl RestoreApplyOptions {
437 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
439 where
440 I: IntoIterator<Item = OsString>,
441 {
442 let mut plan = None;
443 let mut status = None;
444 let mut backup_dir = None;
445 let mut out = None;
446 let mut journal_out = None;
447 let mut dry_run = false;
448
449 let mut args = args.into_iter();
450 while let Some(arg) = args.next() {
451 let arg = arg
452 .into_string()
453 .map_err(|_| RestoreCommandError::Usage(usage()))?;
454 match arg.as_str() {
455 "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
456 "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
457 "--backup-dir" => {
458 backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
459 }
460 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
461 "--journal-out" => {
462 journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
463 }
464 "--dry-run" => dry_run = true,
465 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
466 _ => return Err(RestoreCommandError::UnknownOption(arg)),
467 }
468 }
469
470 if !dry_run {
471 return Err(RestoreCommandError::ApplyRequiresDryRun);
472 }
473
474 Ok(Self {
475 plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
476 status,
477 backup_dir,
478 out,
479 journal_out,
480 dry_run,
481 })
482 }
483}
484
485#[derive(Clone, Debug, Eq, PartialEq)]
490#[expect(
491 clippy::struct_excessive_bools,
492 reason = "CLI status options mirror independent fail-closed guard flags"
493)]
494pub struct RestoreApplyStatusOptions {
495 pub journal: PathBuf,
496 pub require_ready: bool,
497 pub require_no_pending: bool,
498 pub require_no_failed: bool,
499 pub require_complete: bool,
500 pub require_remaining_count: Option<usize>,
501 pub require_attention_count: Option<usize>,
502 pub require_completion_basis_points: Option<usize>,
503 pub require_no_pending_before: Option<String>,
504 pub out: Option<PathBuf>,
505}
506
507impl RestoreApplyStatusOptions {
508 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
510 where
511 I: IntoIterator<Item = OsString>,
512 {
513 let mut journal = None;
514 let mut require_ready = false;
515 let mut require_no_pending = false;
516 let mut require_no_failed = false;
517 let mut require_complete = false;
518 let mut require_remaining_count = None;
519 let mut require_attention_count = None;
520 let mut require_completion_basis_points = None;
521 let mut require_no_pending_before = None;
522 let mut out = None;
523
524 let mut args = args.into_iter();
525 while let Some(arg) = args.next() {
526 let arg = arg
527 .into_string()
528 .map_err(|_| RestoreCommandError::Usage(usage()))?;
529 if parse_progress_requirement_option(
530 arg.as_str(),
531 &mut args,
532 &mut require_remaining_count,
533 &mut require_attention_count,
534 &mut require_completion_basis_points,
535 )? {
536 continue;
537 }
538 if parse_pending_requirement_option(
539 arg.as_str(),
540 &mut args,
541 &mut require_no_pending_before,
542 )? {
543 continue;
544 }
545 match arg.as_str() {
546 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
547 "--require-ready" => require_ready = true,
548 "--require-no-pending" => require_no_pending = true,
549 "--require-no-failed" => require_no_failed = true,
550 "--require-complete" => require_complete = true,
551 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
552 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
553 _ => return Err(RestoreCommandError::UnknownOption(arg)),
554 }
555 }
556
557 Ok(Self {
558 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
559 require_ready,
560 require_no_pending,
561 require_no_failed,
562 require_complete,
563 require_remaining_count,
564 require_attention_count,
565 require_completion_basis_points,
566 require_no_pending_before,
567 out,
568 })
569 }
570}
571
572#[derive(Clone, Debug, Eq, PartialEq)]
577pub struct RestoreApplyReportOptions {
578 pub journal: PathBuf,
579 pub require_no_attention: bool,
580 pub require_remaining_count: Option<usize>,
581 pub require_attention_count: Option<usize>,
582 pub require_completion_basis_points: Option<usize>,
583 pub require_no_pending_before: Option<String>,
584 pub out: Option<PathBuf>,
585}
586
587impl RestoreApplyReportOptions {
588 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
590 where
591 I: IntoIterator<Item = OsString>,
592 {
593 let mut journal = None;
594 let mut require_no_attention = false;
595 let mut require_remaining_count = None;
596 let mut require_attention_count = None;
597 let mut require_completion_basis_points = None;
598 let mut require_no_pending_before = None;
599 let mut out = None;
600
601 let mut args = args.into_iter();
602 while let Some(arg) = args.next() {
603 let arg = arg
604 .into_string()
605 .map_err(|_| RestoreCommandError::Usage(usage()))?;
606 if parse_progress_requirement_option(
607 arg.as_str(),
608 &mut args,
609 &mut require_remaining_count,
610 &mut require_attention_count,
611 &mut require_completion_basis_points,
612 )? {
613 continue;
614 }
615 if parse_pending_requirement_option(
616 arg.as_str(),
617 &mut args,
618 &mut require_no_pending_before,
619 )? {
620 continue;
621 }
622 match arg.as_str() {
623 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
624 "--require-no-attention" => require_no_attention = true,
625 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
626 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
627 _ => return Err(RestoreCommandError::UnknownOption(arg)),
628 }
629 }
630
631 Ok(Self {
632 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
633 require_no_attention,
634 require_remaining_count,
635 require_attention_count,
636 require_completion_basis_points,
637 require_no_pending_before,
638 out,
639 })
640 }
641}
642
643#[derive(Clone, Debug, Eq, PartialEq)]
648#[expect(
649 clippy::struct_excessive_bools,
650 reason = "CLI runner options mirror independent mode and fail-closed guard flags"
651)]
652pub struct RestoreRunOptions {
653 pub journal: PathBuf,
654 pub dfx: String,
655 pub network: Option<String>,
656 pub out: Option<PathBuf>,
657 pub dry_run: bool,
658 pub execute: bool,
659 pub unclaim_pending: bool,
660 pub max_steps: Option<usize>,
661 pub updated_at: Option<String>,
662 pub require_complete: bool,
663 pub require_no_attention: bool,
664 pub require_run_mode: Option<String>,
665 pub require_stopped_reason: Option<String>,
666 pub require_next_action: Option<String>,
667 pub require_executed_count: Option<usize>,
668 pub require_receipt_count: Option<usize>,
669 pub require_completed_receipt_count: Option<usize>,
670 pub require_failed_receipt_count: Option<usize>,
671 pub require_recovered_receipt_count: Option<usize>,
672 pub require_receipt_updated_at: Option<String>,
673 pub require_state_updated_at: Option<String>,
674 pub require_batch_initial_ready_count: Option<usize>,
675 pub require_batch_executed_count: Option<usize>,
676 pub require_batch_remaining_ready_count: Option<usize>,
677 pub require_batch_ready_delta: Option<isize>,
678 pub require_batch_remaining_delta: Option<isize>,
679 pub require_batch_stopped_by_max_steps: Option<bool>,
680 pub require_remaining_count: Option<usize>,
681 pub require_attention_count: Option<usize>,
682 pub require_completion_basis_points: Option<usize>,
683 pub require_no_pending_before: Option<String>,
684}
685
686impl RestoreRunOptions {
687 #[expect(
689 clippy::too_many_lines,
690 reason = "Restore runner options intentionally parse a broad flat CLI surface"
691 )]
692 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
693 where
694 I: IntoIterator<Item = OsString>,
695 {
696 let mut journal = None;
697 let mut dfx = "dfx".to_string();
698 let mut network = None;
699 let mut out = None;
700 let mut dry_run = false;
701 let mut execute = false;
702 let mut unclaim_pending = false;
703 let mut max_steps = None;
704 let mut updated_at = None;
705 let mut require_complete = false;
706 let mut require_no_attention = false;
707 let mut require_run_mode = None;
708 let mut require_stopped_reason = None;
709 let mut require_next_action = None;
710 let mut require_executed_count = None;
711 let mut require_receipt_count = None;
712 let mut require_completed_receipt_count = None;
713 let mut require_failed_receipt_count = None;
714 let mut require_recovered_receipt_count = None;
715 let mut require_receipt_updated_at = None;
716 let mut require_state_updated_at = None;
717 let mut require_batch_initial_ready_count = None;
718 let mut require_batch_executed_count = None;
719 let mut require_batch_remaining_ready_count = None;
720 let mut require_batch_ready_delta = None;
721 let mut require_batch_remaining_delta = None;
722 let mut require_batch_stopped_by_max_steps = None;
723 let mut require_remaining_count = None;
724 let mut require_attention_count = None;
725 let mut require_completion_basis_points = None;
726 let mut require_no_pending_before = None;
727
728 let mut args = args.into_iter();
729 while let Some(arg) = args.next() {
730 let arg = arg
731 .into_string()
732 .map_err(|_| RestoreCommandError::Usage(usage()))?;
733 if parse_progress_requirement_option(
734 arg.as_str(),
735 &mut args,
736 &mut require_remaining_count,
737 &mut require_attention_count,
738 &mut require_completion_basis_points,
739 )? {
740 continue;
741 }
742 if parse_pending_requirement_option(
743 arg.as_str(),
744 &mut args,
745 &mut require_no_pending_before,
746 )? {
747 continue;
748 }
749 if parse_run_count_requirement_option(
750 arg.as_str(),
751 &mut args,
752 &mut require_executed_count,
753 &mut require_receipt_count,
754 )? {
755 continue;
756 }
757 if parse_run_receipt_kind_requirement_option(
758 arg.as_str(),
759 &mut args,
760 &mut require_completed_receipt_count,
761 &mut require_failed_receipt_count,
762 &mut require_recovered_receipt_count,
763 )? {
764 continue;
765 }
766
767 match arg.as_str() {
768 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
769 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
770 "--network" => network = Some(next_value(&mut args, "--network")?),
771 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
772 "--dry-run" => dry_run = true,
773 "--execute" => execute = true,
774 "--unclaim-pending" => unclaim_pending = true,
775 "--max-steps" => {
776 max_steps = Some(parse_positive_integer(
777 "--max-steps",
778 next_value(&mut args, "--max-steps")?,
779 )?);
780 }
781 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
782 "--require-complete" => require_complete = true,
783 "--require-no-attention" => require_no_attention = true,
784 "--require-run-mode" => {
785 require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
786 }
787 "--require-stopped-reason" => {
788 require_stopped_reason =
789 Some(next_value(&mut args, "--require-stopped-reason")?);
790 }
791 "--require-next-action" => {
792 require_next_action = Some(next_value(&mut args, "--require-next-action")?);
793 }
794 "--require-receipt-updated-at" => {
795 require_receipt_updated_at =
796 Some(next_value(&mut args, "--require-receipt-updated-at")?);
797 }
798 "--require-state-updated-at" => {
799 require_state_updated_at =
800 Some(next_value(&mut args, "--require-state-updated-at")?);
801 }
802 "--require-batch-initial-ready-count" => {
803 require_batch_initial_ready_count = Some(parse_sequence(next_value(
804 &mut args,
805 "--require-batch-initial-ready-count",
806 )?)?);
807 }
808 "--require-batch-executed-count" => {
809 require_batch_executed_count = Some(parse_sequence(next_value(
810 &mut args,
811 "--require-batch-executed-count",
812 )?)?);
813 }
814 "--require-batch-remaining-ready-count" => {
815 require_batch_remaining_ready_count = Some(parse_sequence(next_value(
816 &mut args,
817 "--require-batch-remaining-ready-count",
818 )?)?);
819 }
820 "--require-batch-ready-delta" => {
821 require_batch_ready_delta = Some(parse_integer(
822 "--require-batch-ready-delta",
823 next_value(&mut args, "--require-batch-ready-delta")?,
824 )?);
825 }
826 "--require-batch-remaining-delta" => {
827 require_batch_remaining_delta = Some(parse_integer(
828 "--require-batch-remaining-delta",
829 next_value(&mut args, "--require-batch-remaining-delta")?,
830 )?);
831 }
832 "--require-batch-stopped-by-max-steps" => {
833 require_batch_stopped_by_max_steps = Some(parse_bool(
834 "--require-batch-stopped-by-max-steps",
835 next_value(&mut args, "--require-batch-stopped-by-max-steps")?,
836 )?);
837 }
838 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
839 _ => return Err(RestoreCommandError::UnknownOption(arg)),
840 }
841 }
842
843 validate_restore_run_mode_selection(dry_run, execute, unclaim_pending)?;
844
845 Ok(Self {
846 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
847 dfx,
848 network,
849 out,
850 dry_run,
851 execute,
852 unclaim_pending,
853 max_steps,
854 updated_at,
855 require_complete,
856 require_no_attention,
857 require_run_mode,
858 require_stopped_reason,
859 require_next_action,
860 require_executed_count,
861 require_receipt_count,
862 require_completed_receipt_count,
863 require_failed_receipt_count,
864 require_recovered_receipt_count,
865 require_receipt_updated_at,
866 require_state_updated_at,
867 require_batch_initial_ready_count,
868 require_batch_executed_count,
869 require_batch_remaining_ready_count,
870 require_batch_ready_delta,
871 require_batch_remaining_delta,
872 require_batch_stopped_by_max_steps,
873 require_remaining_count,
874 require_attention_count,
875 require_completion_basis_points,
876 require_no_pending_before,
877 })
878 }
879}
880
881fn validate_restore_run_mode_selection(
883 dry_run: bool,
884 execute: bool,
885 unclaim_pending: bool,
886) -> Result<(), RestoreCommandError> {
887 let mode_count = [dry_run, execute, unclaim_pending]
888 .into_iter()
889 .filter(|enabled| *enabled)
890 .count();
891 if mode_count > 1 {
892 return Err(RestoreCommandError::RestoreRunConflictingModes);
893 }
894
895 if mode_count == 0 {
896 return Err(RestoreCommandError::RestoreRunRequiresMode);
897 }
898
899 Ok(())
900}
901
902struct RestoreRunResult {
907 response: RestoreRunResponse,
908 error: Option<RestoreCommandError>,
909}
910
911impl RestoreRunResult {
912 const fn ok(response: RestoreRunResponse) -> Self {
914 Self {
915 response,
916 error: None,
917 }
918 }
919}
920
921const RESTORE_RUN_MODE_DRY_RUN: &str = "dry-run";
922const RESTORE_RUN_MODE_EXECUTE: &str = "execute";
923const RESTORE_RUN_MODE_UNCLAIM_PENDING: &str = "unclaim-pending";
924
925const RESTORE_RUN_STOPPED_BLOCKED: &str = "blocked";
926const RESTORE_RUN_STOPPED_COMMAND_FAILED: &str = "command-failed";
927const RESTORE_RUN_STOPPED_COMPLETE: &str = "complete";
928const RESTORE_RUN_STOPPED_MAX_STEPS: &str = "max-steps-reached";
929const RESTORE_RUN_STOPPED_PENDING: &str = "pending";
930const RESTORE_RUN_STOPPED_PREVIEW: &str = "preview";
931const RESTORE_RUN_STOPPED_READY: &str = "ready";
932const RESTORE_RUN_STOPPED_RECOVERED_PENDING: &str = "recovered-pending";
933
934const RESTORE_RUN_ACTION_DONE: &str = "done";
935const RESTORE_RUN_ACTION_FIX_BLOCKED: &str = "fix-blocked-journal";
936const RESTORE_RUN_ACTION_INSPECT_FAILED: &str = "inspect-failed-operation";
937const RESTORE_RUN_ACTION_RERUN: &str = "rerun";
938const RESTORE_RUN_ACTION_UNCLAIM_PENDING: &str = "unclaim-pending";
939
940const RESTORE_RUN_EXECUTED_COMPLETED: &str = "completed";
941const RESTORE_RUN_EXECUTED_FAILED: &str = "failed";
942const RESTORE_RUN_RECEIPT_COMPLETED: &str = "command-completed";
943const RESTORE_RUN_RECEIPT_FAILED: &str = "command-failed";
944const RESTORE_RUN_RECEIPT_RECOVERED_PENDING: &str = "pending-recovered";
945const RESTORE_RUN_RECEIPT_STATE_READY: &str = "ready";
946const RESTORE_RUN_COMMAND_EXIT_PREFIX: &str = "runner-command-exit";
947const RESTORE_RUN_RESPONSE_VERSION: u16 = 1;
948
949#[derive(Clone, Debug, Serialize)]
954#[expect(
955 clippy::struct_excessive_bools,
956 reason = "Runner response exposes stable JSON status flags for operators and CI"
957)]
958pub struct RestoreRunResponse {
959 run_version: u16,
960 backup_id: String,
961 run_mode: &'static str,
962 dry_run: bool,
963 execute: bool,
964 unclaim_pending: bool,
965 stopped_reason: &'static str,
966 next_action: &'static str,
967 #[serde(skip_serializing_if = "Option::is_none")]
968 requested_state_updated_at: Option<String>,
969 #[serde(skip_serializing_if = "Option::is_none")]
970 max_steps_reached: Option<bool>,
971 #[serde(default, skip_serializing_if = "Vec::is_empty")]
972 executed_operations: Vec<RestoreRunExecutedOperation>,
973 #[serde(default, skip_serializing_if = "Vec::is_empty")]
974 operation_receipts: Vec<RestoreRunOperationReceipt>,
975 #[serde(skip_serializing_if = "Option::is_none")]
976 operation_receipt_count: Option<usize>,
977 operation_receipt_summary: RestoreRunReceiptSummary,
978 #[serde(skip_serializing_if = "Option::is_none")]
979 executed_operation_count: Option<usize>,
980 #[serde(skip_serializing_if = "Option::is_none")]
981 recovered_operation: Option<RestoreApplyJournalOperation>,
982 batch_summary: RestoreRunBatchSummary,
983 ready: bool,
984 complete: bool,
985 attention_required: bool,
986 outcome: RestoreApplyReportOutcome,
987 operation_count: usize,
988 operation_counts: RestoreApplyOperationKindCounts,
989 operation_counts_supplied: bool,
990 progress: RestoreApplyProgressSummary,
991 pending_summary: RestoreApplyPendingSummary,
992 pending_operations: usize,
993 ready_operations: usize,
994 blocked_operations: usize,
995 completed_operations: usize,
996 failed_operations: usize,
997 blocked_reasons: Vec<String>,
998 next_transition: Option<RestoreApplyReportOperation>,
999 #[serde(skip_serializing_if = "Option::is_none")]
1000 operation_available: Option<bool>,
1001 #[serde(skip_serializing_if = "Option::is_none")]
1002 command_available: Option<bool>,
1003 #[serde(skip_serializing_if = "Option::is_none")]
1004 command: Option<RestoreApplyRunnerCommand>,
1005}
1006
1007impl RestoreRunResponse {
1008 fn from_report(
1010 backup_id: String,
1011 report: RestoreApplyJournalReport,
1012 mode: RestoreRunResponseMode,
1013 ) -> Self {
1014 Self {
1015 run_version: RESTORE_RUN_RESPONSE_VERSION,
1016 backup_id,
1017 run_mode: mode.run_mode,
1018 dry_run: mode.dry_run,
1019 execute: mode.execute,
1020 unclaim_pending: mode.unclaim_pending,
1021 stopped_reason: mode.stopped_reason,
1022 next_action: mode.next_action,
1023 requested_state_updated_at: None,
1024 max_steps_reached: None,
1025 executed_operations: Vec::new(),
1026 operation_receipts: Vec::new(),
1027 operation_receipt_count: Some(0),
1028 operation_receipt_summary: RestoreRunReceiptSummary::default(),
1029 executed_operation_count: None,
1030 recovered_operation: None,
1031 batch_summary: RestoreRunBatchSummary::from_counts(
1032 RestoreRunBatchStart::new(
1033 None,
1034 report.ready_operations,
1035 report.progress.remaining_operations,
1036 ),
1037 0,
1038 report.ready_operations,
1039 report.progress.remaining_operations,
1040 false,
1041 report.complete,
1042 ),
1043 ready: report.ready,
1044 complete: report.complete,
1045 attention_required: report.attention_required,
1046 outcome: report.outcome,
1047 operation_count: report.operation_count,
1048 operation_counts: report.operation_counts,
1049 operation_counts_supplied: report.operation_counts_supplied,
1050 progress: report.progress,
1051 pending_summary: report.pending_summary,
1052 pending_operations: report.pending_operations,
1053 ready_operations: report.ready_operations,
1054 blocked_operations: report.blocked_operations,
1055 completed_operations: report.completed_operations,
1056 failed_operations: report.failed_operations,
1057 blocked_reasons: report.blocked_reasons,
1058 next_transition: report.next_transition,
1059 operation_available: None,
1060 command_available: None,
1061 command: None,
1062 }
1063 }
1064
1065 fn set_operation_receipts(&mut self, receipts: Vec<RestoreRunOperationReceipt>) {
1067 self.operation_receipt_summary = RestoreRunReceiptSummary::from_receipts(&receipts);
1068 self.operation_receipt_count = Some(receipts.len());
1069 self.operation_receipts = receipts;
1070 }
1071
1072 fn set_requested_state_updated_at(&mut self, updated_at: Option<&String>) {
1074 self.requested_state_updated_at = updated_at.cloned();
1075 }
1076
1077 const fn set_batch_summary(
1079 &mut self,
1080 batch_start: RestoreRunBatchStart,
1081 executed_operations: usize,
1082 stopped_by_max_steps: bool,
1083 ) {
1084 self.batch_summary = RestoreRunBatchSummary::from_counts(
1085 batch_start,
1086 executed_operations,
1087 self.ready_operations,
1088 self.progress.remaining_operations,
1089 stopped_by_max_steps,
1090 self.complete,
1091 );
1092 }
1093}
1094
1095#[derive(Clone, Copy, Debug)]
1100struct RestoreRunBatchStart {
1101 requested_max_steps: Option<usize>,
1102 initial_ready_operations: usize,
1103 initial_remaining_operations: usize,
1104}
1105
1106impl RestoreRunBatchStart {
1107 const fn new(
1109 requested_max_steps: Option<usize>,
1110 initial_ready_operations: usize,
1111 initial_remaining_operations: usize,
1112 ) -> Self {
1113 Self {
1114 requested_max_steps,
1115 initial_ready_operations,
1116 initial_remaining_operations,
1117 }
1118 }
1119}
1120
1121#[derive(Clone, Debug, Serialize)]
1126struct RestoreRunBatchSummary {
1127 requested_max_steps: Option<usize>,
1128 initial_ready_operations: usize,
1129 initial_remaining_operations: usize,
1130 executed_operations: usize,
1131 remaining_ready_operations: usize,
1132 remaining_operations: usize,
1133 ready_operations_delta: isize,
1134 remaining_operations_delta: isize,
1135 stopped_by_max_steps: bool,
1136 complete: bool,
1137}
1138
1139impl RestoreRunBatchSummary {
1140 const fn from_counts(
1142 batch_start: RestoreRunBatchStart,
1143 executed_operations: usize,
1144 remaining_ready_operations: usize,
1145 remaining_operations: usize,
1146 stopped_by_max_steps: bool,
1147 complete: bool,
1148 ) -> Self {
1149 Self {
1150 requested_max_steps: batch_start.requested_max_steps,
1151 initial_ready_operations: batch_start.initial_ready_operations,
1152 initial_remaining_operations: batch_start.initial_remaining_operations,
1153 executed_operations,
1154 remaining_ready_operations,
1155 remaining_operations,
1156 ready_operations_delta: remaining_ready_operations.cast_signed()
1157 - batch_start.initial_ready_operations.cast_signed(),
1158 remaining_operations_delta: remaining_operations.cast_signed()
1159 - batch_start.initial_remaining_operations.cast_signed(),
1160 stopped_by_max_steps,
1161 complete,
1162 }
1163 }
1164}
1165
1166#[derive(Clone, Debug, Default, Serialize)]
1171struct RestoreRunReceiptSummary {
1172 total_receipts: usize,
1173 command_completed: usize,
1174 command_failed: usize,
1175 pending_recovered: usize,
1176}
1177
1178impl RestoreRunReceiptSummary {
1179 fn from_receipts(receipts: &[RestoreRunOperationReceipt]) -> Self {
1181 let mut summary = Self {
1182 total_receipts: receipts.len(),
1183 ..Self::default()
1184 };
1185
1186 for receipt in receipts {
1187 match receipt.event {
1188 RESTORE_RUN_RECEIPT_COMPLETED => summary.command_completed += 1,
1189 RESTORE_RUN_RECEIPT_FAILED => summary.command_failed += 1,
1190 RESTORE_RUN_RECEIPT_RECOVERED_PENDING => summary.pending_recovered += 1,
1191 _ => {}
1192 }
1193 }
1194
1195 summary
1196 }
1197}
1198
1199#[derive(Clone, Debug, Serialize)]
1204struct RestoreRunOperationReceipt {
1205 event: &'static str,
1206 sequence: usize,
1207 operation: RestoreApplyOperationKind,
1208 target_canister: String,
1209 state: &'static str,
1210 #[serde(skip_serializing_if = "Option::is_none")]
1211 updated_at: Option<String>,
1212 #[serde(skip_serializing_if = "Option::is_none")]
1213 command: Option<RestoreApplyRunnerCommand>,
1214 #[serde(skip_serializing_if = "Option::is_none")]
1215 status: Option<String>,
1216}
1217
1218impl RestoreRunOperationReceipt {
1219 fn completed(
1221 operation: RestoreApplyJournalOperation,
1222 command: RestoreApplyRunnerCommand,
1223 status: String,
1224 updated_at: Option<String>,
1225 ) -> Self {
1226 Self::from_operation(
1227 RESTORE_RUN_RECEIPT_COMPLETED,
1228 operation,
1229 RESTORE_RUN_EXECUTED_COMPLETED,
1230 updated_at,
1231 Some(command),
1232 Some(status),
1233 )
1234 }
1235
1236 fn failed(
1238 operation: RestoreApplyJournalOperation,
1239 command: RestoreApplyRunnerCommand,
1240 status: String,
1241 updated_at: Option<String>,
1242 ) -> Self {
1243 Self::from_operation(
1244 RESTORE_RUN_RECEIPT_FAILED,
1245 operation,
1246 RESTORE_RUN_EXECUTED_FAILED,
1247 updated_at,
1248 Some(command),
1249 Some(status),
1250 )
1251 }
1252
1253 fn recovered_pending(
1255 operation: RestoreApplyJournalOperation,
1256 updated_at: Option<String>,
1257 ) -> Self {
1258 Self::from_operation(
1259 RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
1260 operation,
1261 RESTORE_RUN_RECEIPT_STATE_READY,
1262 updated_at,
1263 None,
1264 None,
1265 )
1266 }
1267
1268 fn from_operation(
1270 event: &'static str,
1271 operation: RestoreApplyJournalOperation,
1272 state: &'static str,
1273 updated_at: Option<String>,
1274 command: Option<RestoreApplyRunnerCommand>,
1275 status: Option<String>,
1276 ) -> Self {
1277 Self {
1278 event,
1279 sequence: operation.sequence,
1280 operation: operation.operation,
1281 target_canister: operation.target_canister,
1282 state,
1283 updated_at,
1284 command,
1285 status,
1286 }
1287 }
1288}
1289
1290#[derive(Clone, Debug, Serialize)]
1295struct RestoreRunExecutedOperation {
1296 sequence: usize,
1297 operation: RestoreApplyOperationKind,
1298 target_canister: String,
1299 command: RestoreApplyRunnerCommand,
1300 status: String,
1301 state: &'static str,
1302}
1303
1304impl RestoreRunExecutedOperation {
1305 fn completed(
1307 operation: RestoreApplyJournalOperation,
1308 command: RestoreApplyRunnerCommand,
1309 status: String,
1310 ) -> Self {
1311 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_COMPLETED)
1312 }
1313
1314 fn failed(
1316 operation: RestoreApplyJournalOperation,
1317 command: RestoreApplyRunnerCommand,
1318 status: String,
1319 ) -> Self {
1320 Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_FAILED)
1321 }
1322
1323 fn from_operation(
1325 operation: RestoreApplyJournalOperation,
1326 command: RestoreApplyRunnerCommand,
1327 status: String,
1328 state: &'static str,
1329 ) -> Self {
1330 Self {
1331 sequence: operation.sequence,
1332 operation: operation.operation,
1333 target_canister: operation.target_canister,
1334 command,
1335 status,
1336 state,
1337 }
1338 }
1339}
1340
1341struct RestoreRunResponseMode {
1346 run_mode: &'static str,
1347 dry_run: bool,
1348 execute: bool,
1349 unclaim_pending: bool,
1350 stopped_reason: &'static str,
1351 next_action: &'static str,
1352}
1353
1354impl RestoreRunResponseMode {
1355 const fn new(
1357 run_mode: &'static str,
1358 dry_run: bool,
1359 execute: bool,
1360 unclaim_pending: bool,
1361 stopped_reason: &'static str,
1362 next_action: &'static str,
1363 ) -> Self {
1364 Self {
1365 run_mode,
1366 dry_run,
1367 execute,
1368 unclaim_pending,
1369 stopped_reason,
1370 next_action,
1371 }
1372 }
1373
1374 const fn dry_run(stopped_reason: &'static str, next_action: &'static str) -> Self {
1376 Self::new(
1377 RESTORE_RUN_MODE_DRY_RUN,
1378 true,
1379 false,
1380 false,
1381 stopped_reason,
1382 next_action,
1383 )
1384 }
1385
1386 const fn execute(stopped_reason: &'static str, next_action: &'static str) -> Self {
1388 Self::new(
1389 RESTORE_RUN_MODE_EXECUTE,
1390 false,
1391 true,
1392 false,
1393 stopped_reason,
1394 next_action,
1395 )
1396 }
1397
1398 const fn unclaim_pending(next_action: &'static str) -> Self {
1400 Self::new(
1401 RESTORE_RUN_MODE_UNCLAIM_PENDING,
1402 false,
1403 false,
1404 true,
1405 RESTORE_RUN_STOPPED_RECOVERED_PENDING,
1406 next_action,
1407 )
1408 }
1409}
1410
1411#[derive(Clone, Debug, Eq, PartialEq)]
1416pub struct RestoreApplyNextOptions {
1417 pub journal: PathBuf,
1418 pub out: Option<PathBuf>,
1419}
1420
1421impl RestoreApplyNextOptions {
1422 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1424 where
1425 I: IntoIterator<Item = OsString>,
1426 {
1427 let mut journal = None;
1428 let mut out = None;
1429
1430 let mut args = args.into_iter();
1431 while let Some(arg) = args.next() {
1432 let arg = arg
1433 .into_string()
1434 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1435 match arg.as_str() {
1436 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1437 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1438 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1439 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1440 }
1441 }
1442
1443 Ok(Self {
1444 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1445 out,
1446 })
1447 }
1448}
1449
1450#[derive(Clone, Debug, Eq, PartialEq)]
1455pub struct RestoreApplyCommandOptions {
1456 pub journal: PathBuf,
1457 pub dfx: String,
1458 pub network: Option<String>,
1459 pub out: Option<PathBuf>,
1460 pub require_command: bool,
1461}
1462
1463impl RestoreApplyCommandOptions {
1464 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1466 where
1467 I: IntoIterator<Item = OsString>,
1468 {
1469 let mut journal = None;
1470 let mut dfx = "dfx".to_string();
1471 let mut network = None;
1472 let mut out = None;
1473 let mut require_command = false;
1474
1475 let mut args = args.into_iter();
1476 while let Some(arg) = args.next() {
1477 let arg = arg
1478 .into_string()
1479 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1480 match arg.as_str() {
1481 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1482 "--dfx" => dfx = next_value(&mut args, "--dfx")?,
1483 "--network" => network = Some(next_value(&mut args, "--network")?),
1484 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1485 "--require-command" => require_command = true,
1486 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1487 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1488 }
1489 }
1490
1491 Ok(Self {
1492 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1493 dfx,
1494 network,
1495 out,
1496 require_command,
1497 })
1498 }
1499}
1500
1501#[derive(Clone, Debug, Eq, PartialEq)]
1506pub struct RestoreApplyClaimOptions {
1507 pub journal: PathBuf,
1508 pub sequence: Option<usize>,
1509 pub updated_at: Option<String>,
1510 pub out: Option<PathBuf>,
1511}
1512
1513impl RestoreApplyClaimOptions {
1514 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1516 where
1517 I: IntoIterator<Item = OsString>,
1518 {
1519 let mut journal = None;
1520 let mut sequence = None;
1521 let mut updated_at = None;
1522 let mut out = None;
1523
1524 let mut args = args.into_iter();
1525 while let Some(arg) = args.next() {
1526 let arg = arg
1527 .into_string()
1528 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1529 match arg.as_str() {
1530 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1531 "--sequence" => {
1532 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1533 }
1534 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1535 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1536 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1537 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1538 }
1539 }
1540
1541 Ok(Self {
1542 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1543 sequence,
1544 updated_at,
1545 out,
1546 })
1547 }
1548}
1549
1550#[derive(Clone, Debug, Eq, PartialEq)]
1555pub struct RestoreApplyUnclaimOptions {
1556 pub journal: PathBuf,
1557 pub sequence: Option<usize>,
1558 pub updated_at: Option<String>,
1559 pub out: Option<PathBuf>,
1560}
1561
1562impl RestoreApplyUnclaimOptions {
1563 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1565 where
1566 I: IntoIterator<Item = OsString>,
1567 {
1568 let mut journal = None;
1569 let mut sequence = None;
1570 let mut updated_at = None;
1571 let mut out = None;
1572
1573 let mut args = args.into_iter();
1574 while let Some(arg) = args.next() {
1575 let arg = arg
1576 .into_string()
1577 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1578 match arg.as_str() {
1579 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1580 "--sequence" => {
1581 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1582 }
1583 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1584 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1585 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1586 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1587 }
1588 }
1589
1590 Ok(Self {
1591 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1592 sequence,
1593 updated_at,
1594 out,
1595 })
1596 }
1597}
1598
1599#[derive(Clone, Debug, Eq, PartialEq)]
1604pub struct RestoreApplyMarkOptions {
1605 pub journal: PathBuf,
1606 pub sequence: usize,
1607 pub state: RestoreApplyMarkState,
1608 pub reason: Option<String>,
1609 pub updated_at: Option<String>,
1610 pub out: Option<PathBuf>,
1611 pub require_pending: bool,
1612}
1613
1614impl RestoreApplyMarkOptions {
1615 pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1617 where
1618 I: IntoIterator<Item = OsString>,
1619 {
1620 let mut journal = None;
1621 let mut sequence = None;
1622 let mut state = None;
1623 let mut reason = None;
1624 let mut updated_at = None;
1625 let mut out = None;
1626 let mut require_pending = false;
1627
1628 let mut args = args.into_iter();
1629 while let Some(arg) = args.next() {
1630 let arg = arg
1631 .into_string()
1632 .map_err(|_| RestoreCommandError::Usage(usage()))?;
1633 match arg.as_str() {
1634 "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1635 "--sequence" => {
1636 sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1637 }
1638 "--state" => {
1639 state = Some(RestoreApplyMarkState::parse(next_value(
1640 &mut args, "--state",
1641 )?)?);
1642 }
1643 "--reason" => reason = Some(next_value(&mut args, "--reason")?),
1644 "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1645 "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1646 "--require-pending" => require_pending = true,
1647 "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1648 _ => return Err(RestoreCommandError::UnknownOption(arg)),
1649 }
1650 }
1651
1652 Ok(Self {
1653 journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1654 sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
1655 state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
1656 reason,
1657 updated_at,
1658 out,
1659 require_pending,
1660 })
1661 }
1662}
1663
1664#[derive(Clone, Debug, Eq, PartialEq)]
1669pub enum RestoreApplyMarkState {
1670 Completed,
1671 Failed,
1672}
1673
1674impl RestoreApplyMarkState {
1675 fn parse(value: String) -> Result<Self, RestoreCommandError> {
1677 match value.as_str() {
1678 "completed" => Ok(Self::Completed),
1679 "failed" => Ok(Self::Failed),
1680 _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
1681 }
1682 }
1683}
1684
1685pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
1687where
1688 I: IntoIterator<Item = OsString>,
1689{
1690 let mut args = args.into_iter();
1691 let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
1692 return Err(RestoreCommandError::Usage(usage()));
1693 };
1694
1695 match command.as_str() {
1696 "plan" => {
1697 let options = RestorePlanOptions::parse(args)?;
1698 let plan = plan_restore(&options)?;
1699 write_plan(&options, &plan)?;
1700 enforce_restore_plan_requirements(&options, &plan)?;
1701 Ok(())
1702 }
1703 "status" => {
1704 let options = RestoreStatusOptions::parse(args)?;
1705 let status = restore_status(&options)?;
1706 write_status(&options, &status)?;
1707 Ok(())
1708 }
1709 "apply" => {
1710 let options = RestoreApplyOptions::parse(args)?;
1711 let dry_run = restore_apply_dry_run(&options)?;
1712 write_apply_dry_run(&options, &dry_run)?;
1713 write_apply_journal_if_requested(&options, &dry_run)?;
1714 Ok(())
1715 }
1716 "apply-status" => {
1717 let options = RestoreApplyStatusOptions::parse(args)?;
1718 let status = restore_apply_status(&options)?;
1719 write_apply_status(&options, &status)?;
1720 enforce_apply_status_requirements(&options, &status)?;
1721 Ok(())
1722 }
1723 "apply-report" => {
1724 let options = RestoreApplyReportOptions::parse(args)?;
1725 let report = restore_apply_report(&options)?;
1726 write_apply_report(&options, &report)?;
1727 enforce_apply_report_requirements(&options, &report)?;
1728 Ok(())
1729 }
1730 "run" => {
1731 let options = RestoreRunOptions::parse(args)?;
1732 let run = if options.execute {
1733 restore_run_execute_result(&options)?
1734 } else if options.unclaim_pending {
1735 RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1736 } else {
1737 RestoreRunResult::ok(restore_run_dry_run(&options)?)
1738 };
1739 write_restore_run(&options, &run.response)?;
1740 if let Some(error) = run.error {
1741 return Err(error);
1742 }
1743 enforce_restore_run_requirements(&options, &run.response)?;
1744 Ok(())
1745 }
1746 "apply-next" => {
1747 let options = RestoreApplyNextOptions::parse(args)?;
1748 let next = restore_apply_next(&options)?;
1749 write_apply_next(&options, &next)?;
1750 Ok(())
1751 }
1752 "apply-command" => {
1753 let options = RestoreApplyCommandOptions::parse(args)?;
1754 let preview = restore_apply_command(&options)?;
1755 write_apply_command(&options, &preview)?;
1756 enforce_apply_command_requirements(&options, &preview)?;
1757 Ok(())
1758 }
1759 "apply-claim" => {
1760 let options = RestoreApplyClaimOptions::parse(args)?;
1761 let journal = restore_apply_claim(&options)?;
1762 write_apply_claim(&options, &journal)?;
1763 Ok(())
1764 }
1765 "apply-unclaim" => {
1766 let options = RestoreApplyUnclaimOptions::parse(args)?;
1767 let journal = restore_apply_unclaim(&options)?;
1768 write_apply_unclaim(&options, &journal)?;
1769 Ok(())
1770 }
1771 "apply-mark" => {
1772 let options = RestoreApplyMarkOptions::parse(args)?;
1773 let journal = restore_apply_mark(&options)?;
1774 write_apply_mark(&options, &journal)?;
1775 Ok(())
1776 }
1777 "help" | "--help" | "-h" => Err(RestoreCommandError::Usage(usage())),
1778 _ => Err(RestoreCommandError::UnknownOption(command)),
1779 }
1780}
1781
1782pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1784 verify_backup_layout_if_required(options)?;
1785
1786 let manifest = read_manifest_source(options)?;
1787 let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1788
1789 RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1790}
1791
1792pub fn restore_status(
1794 options: &RestoreStatusOptions,
1795) -> Result<RestoreStatus, RestoreCommandError> {
1796 let plan = read_plan(&options.plan)?;
1797 Ok(RestoreStatus::from_plan(&plan))
1798}
1799
1800pub fn restore_apply_dry_run(
1802 options: &RestoreApplyOptions,
1803) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1804 let plan = read_plan(&options.plan)?;
1805 let status = options.status.as_ref().map(read_status).transpose()?;
1806 if let Some(backup_dir) = &options.backup_dir {
1807 return RestoreApplyDryRun::try_from_plan_with_artifacts(
1808 &plan,
1809 status.as_ref(),
1810 backup_dir,
1811 )
1812 .map_err(RestoreCommandError::from);
1813 }
1814
1815 RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1816}
1817
1818pub fn restore_apply_status(
1820 options: &RestoreApplyStatusOptions,
1821) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1822 let journal = read_apply_journal(&options.journal)?;
1823 Ok(journal.status())
1824}
1825
1826pub fn restore_apply_report(
1828 options: &RestoreApplyReportOptions,
1829) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1830 let journal = read_apply_journal(&options.journal)?;
1831 Ok(journal.report())
1832}
1833
1834pub fn restore_run_dry_run(
1836 options: &RestoreRunOptions,
1837) -> Result<RestoreRunResponse, RestoreCommandError> {
1838 let journal = read_apply_journal(&options.journal)?;
1839 let report = journal.report();
1840 let initial_ready_operations = report.ready_operations;
1841 let initial_remaining_operations = report.progress.remaining_operations;
1842 let preview = journal.next_command_preview_with_config(&restore_run_command_config(options));
1843 let stopped_reason = restore_run_stopped_reason(&report, false, false);
1844 let next_action = restore_run_next_action(&report, false);
1845
1846 let mut response = RestoreRunResponse::from_report(
1847 journal.backup_id,
1848 report,
1849 RestoreRunResponseMode::dry_run(stopped_reason, next_action),
1850 );
1851 response.set_requested_state_updated_at(options.updated_at.as_ref());
1852 response.set_batch_summary(
1853 RestoreRunBatchStart::new(
1854 options.max_steps,
1855 initial_ready_operations,
1856 initial_remaining_operations,
1857 ),
1858 0,
1859 false,
1860 );
1861 response.operation_available = Some(preview.operation_available);
1862 response.command_available = Some(preview.command_available);
1863 response.command = preview.command;
1864 Ok(response)
1865}
1866
1867pub fn restore_run_unclaim_pending(
1869 options: &RestoreRunOptions,
1870) -> Result<RestoreRunResponse, RestoreCommandError> {
1871 let mut journal = read_apply_journal(&options.journal)?;
1872 let initial_report = journal.report();
1873 let initial_ready_operations = initial_report.ready_operations;
1874 let initial_remaining_operations = initial_report.progress.remaining_operations;
1875 let recovered_operation = journal
1876 .next_transition_operation()
1877 .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1878 .cloned()
1879 .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1880
1881 let recovered_updated_at = state_updated_at(options.updated_at.as_ref());
1882 journal.mark_next_operation_ready_at(Some(recovered_updated_at.clone()))?;
1883 write_apply_journal_file(&options.journal, &journal)?;
1884
1885 let report = journal.report();
1886 let next_action = restore_run_next_action(&report, true);
1887 let mut response = RestoreRunResponse::from_report(
1888 journal.backup_id,
1889 report,
1890 RestoreRunResponseMode::unclaim_pending(next_action),
1891 );
1892 response.set_requested_state_updated_at(options.updated_at.as_ref());
1893 response.set_batch_summary(
1894 RestoreRunBatchStart::new(
1895 options.max_steps,
1896 initial_ready_operations,
1897 initial_remaining_operations,
1898 ),
1899 0,
1900 false,
1901 );
1902 response.set_operation_receipts(vec![RestoreRunOperationReceipt::recovered_pending(
1903 recovered_operation.clone(),
1904 Some(recovered_updated_at),
1905 )]);
1906 response.recovered_operation = Some(recovered_operation);
1907 Ok(response)
1908}
1909
1910pub fn restore_run_execute(
1912 options: &RestoreRunOptions,
1913) -> Result<RestoreRunResponse, RestoreCommandError> {
1914 let run = restore_run_execute_result(options)?;
1915 if let Some(error) = run.error {
1916 return Err(error);
1917 }
1918
1919 Ok(run.response)
1920}
1921
1922fn restore_run_execute_result(
1924 options: &RestoreRunOptions,
1925) -> Result<RestoreRunResult, RestoreCommandError> {
1926 let mut journal = read_apply_journal(&options.journal)?;
1927 let initial_report = journal.report();
1928 let batch_start = RestoreRunBatchStart::new(
1929 options.max_steps,
1930 initial_report.ready_operations,
1931 initial_report.progress.remaining_operations,
1932 );
1933 let mut executed_operations = Vec::new();
1934 let mut operation_receipts = Vec::new();
1935 let config = restore_run_command_config(options);
1936
1937 loop {
1938 let report = journal.report();
1939 let max_steps_reached =
1940 restore_run_max_steps_reached(options, executed_operations.len(), &report);
1941 if report.complete || max_steps_reached {
1942 return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1943 &journal,
1944 executed_operations,
1945 operation_receipts,
1946 max_steps_reached,
1947 options.updated_at.as_ref(),
1948 batch_start,
1949 )));
1950 }
1951
1952 enforce_restore_run_executable(&journal, &report)?;
1953 let preview = journal.next_command_preview_with_config(&config);
1954 enforce_restore_run_command_available(&preview)?;
1955
1956 let operation = preview
1957 .operation
1958 .clone()
1959 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1960 let command = preview
1961 .command
1962 .clone()
1963 .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1964 let sequence = operation.sequence;
1965
1966 enforce_apply_claim_sequence(sequence, &journal)?;
1967 journal.mark_operation_pending_at(
1968 sequence,
1969 Some(state_updated_at(options.updated_at.as_ref())),
1970 )?;
1971 write_apply_journal_file(&options.journal, &journal)?;
1972
1973 let status = Command::new(&command.program)
1974 .args(&command.args)
1975 .status()?;
1976 let status_label = exit_status_label(status);
1977 if status.success() {
1978 let completed_updated_at = state_updated_at(options.updated_at.as_ref());
1979 journal.mark_operation_completed_at(sequence, Some(completed_updated_at.clone()))?;
1980 write_apply_journal_file(&options.journal, &journal)?;
1981 executed_operations.push(RestoreRunExecutedOperation::completed(
1982 operation.clone(),
1983 command.clone(),
1984 status_label.clone(),
1985 ));
1986 operation_receipts.push(RestoreRunOperationReceipt::completed(
1987 operation,
1988 command,
1989 status_label,
1990 Some(completed_updated_at),
1991 ));
1992 continue;
1993 }
1994
1995 let failed_updated_at = state_updated_at(options.updated_at.as_ref());
1996 journal.mark_operation_failed_at(
1997 sequence,
1998 format!("{RESTORE_RUN_COMMAND_EXIT_PREFIX}-{status_label}"),
1999 Some(failed_updated_at.clone()),
2000 )?;
2001 write_apply_journal_file(&options.journal, &journal)?;
2002 executed_operations.push(RestoreRunExecutedOperation::failed(
2003 operation.clone(),
2004 command.clone(),
2005 status_label.clone(),
2006 ));
2007 operation_receipts.push(RestoreRunOperationReceipt::failed(
2008 operation,
2009 command,
2010 status_label.clone(),
2011 Some(failed_updated_at),
2012 ));
2013 let response = restore_run_execute_summary(
2014 &journal,
2015 executed_operations,
2016 operation_receipts,
2017 false,
2018 options.updated_at.as_ref(),
2019 batch_start,
2020 );
2021 return Ok(RestoreRunResult {
2022 response,
2023 error: Some(RestoreCommandError::RestoreRunCommandFailed {
2024 sequence,
2025 status: status_label,
2026 }),
2027 });
2028 }
2029}
2030
2031fn restore_run_command_config(options: &RestoreRunOptions) -> RestoreApplyCommandConfig {
2033 restore_command_config(&options.dfx, options.network.as_deref())
2034}
2035
2036fn restore_apply_command_config(options: &RestoreApplyCommandOptions) -> RestoreApplyCommandConfig {
2038 restore_command_config(&options.dfx, options.network.as_deref())
2039}
2040
2041fn restore_command_config(program: &str, network: Option<&str>) -> RestoreApplyCommandConfig {
2043 RestoreApplyCommandConfig {
2044 program: program.to_string(),
2045 network: network.map(str::to_string),
2046 }
2047}
2048
2049fn restore_run_max_steps_reached(
2051 options: &RestoreRunOptions,
2052 executed_operation_count: usize,
2053 report: &RestoreApplyJournalReport,
2054) -> bool {
2055 options.max_steps == Some(executed_operation_count) && !report.complete
2056}
2057
2058fn restore_run_execute_summary(
2060 journal: &RestoreApplyJournal,
2061 executed_operations: Vec<RestoreRunExecutedOperation>,
2062 operation_receipts: Vec<RestoreRunOperationReceipt>,
2063 max_steps_reached: bool,
2064 requested_state_updated_at: Option<&String>,
2065 batch_start: RestoreRunBatchStart,
2066) -> RestoreRunResponse {
2067 let report = journal.report();
2068 let executed_operation_count = executed_operations.len();
2069 let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
2070 let next_action = restore_run_next_action(&report, false);
2071
2072 let mut response = RestoreRunResponse::from_report(
2073 journal.backup_id.clone(),
2074 report,
2075 RestoreRunResponseMode::execute(stopped_reason, next_action),
2076 );
2077 response.set_requested_state_updated_at(requested_state_updated_at);
2078 response.set_batch_summary(batch_start, executed_operation_count, max_steps_reached);
2079 response.max_steps_reached = Some(max_steps_reached);
2080 response.executed_operation_count = Some(executed_operation_count);
2081 response.executed_operations = executed_operations;
2082 response.set_operation_receipts(operation_receipts);
2083 response
2084}
2085
2086const fn restore_run_stopped_reason(
2088 report: &RestoreApplyJournalReport,
2089 max_steps_reached: bool,
2090 executed: bool,
2091) -> &'static str {
2092 if report.complete {
2093 return RESTORE_RUN_STOPPED_COMPLETE;
2094 }
2095 if report.failed_operations > 0 {
2096 return RESTORE_RUN_STOPPED_COMMAND_FAILED;
2097 }
2098 if report.pending_operations > 0 {
2099 return RESTORE_RUN_STOPPED_PENDING;
2100 }
2101 if !report.ready || report.blocked_operations > 0 {
2102 return RESTORE_RUN_STOPPED_BLOCKED;
2103 }
2104 if max_steps_reached {
2105 return RESTORE_RUN_STOPPED_MAX_STEPS;
2106 }
2107 if executed {
2108 return RESTORE_RUN_STOPPED_READY;
2109 }
2110 RESTORE_RUN_STOPPED_PREVIEW
2111}
2112
2113const fn restore_run_next_action(
2115 report: &RestoreApplyJournalReport,
2116 recovered_pending: bool,
2117) -> &'static str {
2118 if report.complete {
2119 return RESTORE_RUN_ACTION_DONE;
2120 }
2121 if report.failed_operations > 0 {
2122 return RESTORE_RUN_ACTION_INSPECT_FAILED;
2123 }
2124 if report.pending_operations > 0 {
2125 return RESTORE_RUN_ACTION_UNCLAIM_PENDING;
2126 }
2127 if !report.ready || report.blocked_operations > 0 {
2128 return RESTORE_RUN_ACTION_FIX_BLOCKED;
2129 }
2130 if recovered_pending {
2131 return RESTORE_RUN_ACTION_RERUN;
2132 }
2133 RESTORE_RUN_ACTION_RERUN
2134}
2135
2136fn enforce_restore_run_executable(
2138 journal: &RestoreApplyJournal,
2139 report: &RestoreApplyJournalReport,
2140) -> Result<(), RestoreCommandError> {
2141 if report.pending_operations > 0 {
2142 return Err(RestoreCommandError::RestoreApplyPending {
2143 backup_id: report.backup_id.clone(),
2144 pending_operations: report.pending_operations,
2145 next_transition_sequence: report
2146 .next_transition
2147 .as_ref()
2148 .map(|operation| operation.sequence),
2149 });
2150 }
2151
2152 if report.failed_operations > 0 {
2153 return Err(RestoreCommandError::RestoreApplyFailed {
2154 backup_id: report.backup_id.clone(),
2155 failed_operations: report.failed_operations,
2156 });
2157 }
2158
2159 if report.ready {
2160 return Ok(());
2161 }
2162
2163 Err(RestoreCommandError::RestoreApplyNotReady {
2164 backup_id: journal.backup_id.clone(),
2165 reasons: report.blocked_reasons.clone(),
2166 })
2167}
2168
2169fn enforce_restore_run_command_available(
2171 preview: &RestoreApplyCommandPreview,
2172) -> Result<(), RestoreCommandError> {
2173 if preview.command_available {
2174 return Ok(());
2175 }
2176
2177 Err(restore_command_unavailable_error(preview))
2178}
2179
2180fn restore_command_unavailable_error(preview: &RestoreApplyCommandPreview) -> RestoreCommandError {
2182 RestoreCommandError::RestoreApplyCommandUnavailable {
2183 backup_id: preview.backup_id.clone(),
2184 operation_available: preview.operation_available,
2185 complete: preview.complete,
2186 blocked_reasons: preview.blocked_reasons.clone(),
2187 }
2188}
2189
2190fn exit_status_label(status: std::process::ExitStatus) -> String {
2192 status
2193 .code()
2194 .map_or_else(|| "signal".to_string(), |code| code.to_string())
2195}
2196
2197fn enforce_restore_run_requirements(
2199 options: &RestoreRunOptions,
2200 run: &RestoreRunResponse,
2201) -> Result<(), RestoreCommandError> {
2202 if options.require_complete && !run.complete {
2203 return Err(RestoreCommandError::RestoreApplyIncomplete {
2204 backup_id: run.backup_id.clone(),
2205 completed_operations: run.completed_operations,
2206 operation_count: run.operation_count,
2207 });
2208 }
2209
2210 if options.require_no_attention && run.attention_required {
2211 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
2212 backup_id: run.backup_id.clone(),
2213 outcome: run.outcome.clone(),
2214 });
2215 }
2216
2217 if let Some(expected) = &options.require_run_mode
2218 && run.run_mode != expected
2219 {
2220 return Err(RestoreCommandError::RestoreRunModeMismatch {
2221 backup_id: run.backup_id.clone(),
2222 expected: expected.clone(),
2223 actual: run.run_mode.to_string(),
2224 });
2225 }
2226
2227 if let Some(expected) = &options.require_stopped_reason
2228 && run.stopped_reason != expected
2229 {
2230 return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
2231 backup_id: run.backup_id.clone(),
2232 expected: expected.clone(),
2233 actual: run.stopped_reason.to_string(),
2234 });
2235 }
2236
2237 if let Some(expected) = &options.require_next_action
2238 && run.next_action != expected
2239 {
2240 return Err(RestoreCommandError::RestoreRunNextActionMismatch {
2241 backup_id: run.backup_id.clone(),
2242 expected: expected.clone(),
2243 actual: run.next_action.to_string(),
2244 });
2245 }
2246
2247 if let Some(expected) = options.require_executed_count {
2248 let actual = run.executed_operation_count.unwrap_or(0);
2249 if actual != expected {
2250 return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
2251 backup_id: run.backup_id.clone(),
2252 expected,
2253 actual,
2254 });
2255 }
2256 }
2257
2258 enforce_restore_run_receipt_requirements(options, run)?;
2259 enforce_restore_run_batch_requirements(options, run)?;
2260
2261 enforce_progress_requirements(
2262 &run.backup_id,
2263 &run.progress,
2264 options.require_remaining_count,
2265 options.require_attention_count,
2266 options.require_completion_basis_points,
2267 )?;
2268 enforce_pending_before_requirement(
2269 &run.backup_id,
2270 &run.pending_summary,
2271 options.require_no_pending_before.as_deref(),
2272 )?;
2273
2274 Ok(())
2275}
2276
2277fn enforce_restore_run_batch_requirements(
2279 options: &RestoreRunOptions,
2280 run: &RestoreRunResponse,
2281) -> Result<(), RestoreCommandError> {
2282 if let Some(expected) = options.require_batch_initial_ready_count {
2283 let actual = run.batch_summary.initial_ready_operations;
2284 if actual != expected {
2285 return Err(
2286 RestoreCommandError::RestoreRunBatchInitialReadyCountMismatch {
2287 backup_id: run.backup_id.clone(),
2288 expected,
2289 actual,
2290 },
2291 );
2292 }
2293 }
2294
2295 if let Some(expected) = options.require_batch_executed_count {
2296 let actual = run.batch_summary.executed_operations;
2297 if actual != expected {
2298 return Err(RestoreCommandError::RestoreRunBatchExecutedCountMismatch {
2299 backup_id: run.backup_id.clone(),
2300 expected,
2301 actual,
2302 });
2303 }
2304 }
2305
2306 if let Some(expected) = options.require_batch_remaining_ready_count {
2307 let actual = run.batch_summary.remaining_ready_operations;
2308 if actual != expected {
2309 return Err(
2310 RestoreCommandError::RestoreRunBatchRemainingReadyCountMismatch {
2311 backup_id: run.backup_id.clone(),
2312 expected,
2313 actual,
2314 },
2315 );
2316 }
2317 }
2318
2319 if let Some(expected) = options.require_batch_ready_delta {
2320 let actual = run.batch_summary.ready_operations_delta;
2321 if actual != expected {
2322 return Err(RestoreCommandError::RestoreRunBatchReadyDeltaMismatch {
2323 backup_id: run.backup_id.clone(),
2324 expected,
2325 actual,
2326 });
2327 }
2328 }
2329
2330 if let Some(expected) = options.require_batch_remaining_delta {
2331 let actual = run.batch_summary.remaining_operations_delta;
2332 if actual != expected {
2333 return Err(RestoreCommandError::RestoreRunBatchRemainingDeltaMismatch {
2334 backup_id: run.backup_id.clone(),
2335 expected,
2336 actual,
2337 });
2338 }
2339 }
2340
2341 if let Some(expected) = options.require_batch_stopped_by_max_steps {
2342 let actual = run.batch_summary.stopped_by_max_steps;
2343 if actual != expected {
2344 return Err(
2345 RestoreCommandError::RestoreRunBatchStoppedByMaxStepsMismatch {
2346 backup_id: run.backup_id.clone(),
2347 expected,
2348 actual,
2349 },
2350 );
2351 }
2352 }
2353
2354 Ok(())
2355}
2356
2357fn enforce_restore_run_receipt_requirements(
2359 options: &RestoreRunOptions,
2360 run: &RestoreRunResponse,
2361) -> Result<(), RestoreCommandError> {
2362 if let Some(expected) = options.require_receipt_count {
2363 let actual = run.operation_receipt_count.unwrap_or(0);
2364 if actual != expected {
2365 return Err(RestoreCommandError::RestoreRunReceiptCountMismatch {
2366 backup_id: run.backup_id.clone(),
2367 expected,
2368 actual,
2369 });
2370 }
2371 }
2372
2373 enforce_restore_run_receipt_kind_requirement(
2374 &run.backup_id,
2375 RESTORE_RUN_RECEIPT_COMPLETED,
2376 options.require_completed_receipt_count,
2377 run.operation_receipt_summary.command_completed,
2378 )?;
2379 enforce_restore_run_receipt_kind_requirement(
2380 &run.backup_id,
2381 RESTORE_RUN_RECEIPT_FAILED,
2382 options.require_failed_receipt_count,
2383 run.operation_receipt_summary.command_failed,
2384 )?;
2385 enforce_restore_run_receipt_kind_requirement(
2386 &run.backup_id,
2387 RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
2388 options.require_recovered_receipt_count,
2389 run.operation_receipt_summary.pending_recovered,
2390 )?;
2391 enforce_restore_run_receipt_updated_at_requirement(
2392 &run.backup_id,
2393 &run.operation_receipts,
2394 options.require_receipt_updated_at.as_deref(),
2395 )?;
2396 enforce_restore_run_state_updated_at_requirement(
2397 &run.backup_id,
2398 run.requested_state_updated_at.as_deref(),
2399 options.require_state_updated_at.as_deref(),
2400 )?;
2401
2402 Ok(())
2403}
2404
2405fn enforce_restore_run_state_updated_at_requirement(
2407 backup_id: &str,
2408 actual: Option<&str>,
2409 expected: Option<&str>,
2410) -> Result<(), RestoreCommandError> {
2411 if let Some(expected) = expected
2412 && actual != Some(expected)
2413 {
2414 return Err(RestoreCommandError::RestoreRunStateUpdatedAtMismatch {
2415 backup_id: backup_id.to_string(),
2416 expected: expected.to_string(),
2417 actual: actual.map(str::to_string),
2418 });
2419 }
2420
2421 Ok(())
2422}
2423
2424fn enforce_restore_run_receipt_updated_at_requirement(
2426 backup_id: &str,
2427 receipts: &[RestoreRunOperationReceipt],
2428 expected: Option<&str>,
2429) -> Result<(), RestoreCommandError> {
2430 let Some(expected) = expected else {
2431 return Ok(());
2432 };
2433
2434 let actual_receipts = receipts.len();
2435 let mismatched_receipts = receipts
2436 .iter()
2437 .filter(|receipt| receipt.updated_at.as_deref() != Some(expected))
2438 .count();
2439 if actual_receipts == 0 || mismatched_receipts > 0 {
2440 return Err(RestoreCommandError::RestoreRunReceiptUpdatedAtMismatch {
2441 backup_id: backup_id.to_string(),
2442 expected: expected.to_string(),
2443 actual_receipts,
2444 mismatched_receipts,
2445 });
2446 }
2447
2448 Ok(())
2449}
2450
2451fn enforce_restore_run_receipt_kind_requirement(
2453 backup_id: &str,
2454 receipt_kind: &'static str,
2455 expected: Option<usize>,
2456 actual: usize,
2457) -> Result<(), RestoreCommandError> {
2458 if let Some(expected) = expected
2459 && actual != expected
2460 {
2461 return Err(RestoreCommandError::RestoreRunReceiptKindCountMismatch {
2462 backup_id: backup_id.to_string(),
2463 receipt_kind,
2464 expected,
2465 actual,
2466 });
2467 }
2468
2469 Ok(())
2470}
2471
2472fn enforce_progress_requirements(
2474 backup_id: &str,
2475 progress: &RestoreApplyProgressSummary,
2476 require_remaining_count: Option<usize>,
2477 require_attention_count: Option<usize>,
2478 require_completion_basis_points: Option<usize>,
2479) -> Result<(), RestoreCommandError> {
2480 if let Some(expected) = require_remaining_count
2481 && progress.remaining_operations != expected
2482 {
2483 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2484 backup_id: backup_id.to_string(),
2485 field: "remaining_operations",
2486 expected,
2487 actual: progress.remaining_operations,
2488 });
2489 }
2490
2491 if let Some(expected) = require_attention_count
2492 && progress.attention_operations != expected
2493 {
2494 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2495 backup_id: backup_id.to_string(),
2496 field: "attention_operations",
2497 expected,
2498 actual: progress.attention_operations,
2499 });
2500 }
2501
2502 if let Some(expected) = require_completion_basis_points
2503 && progress.completion_basis_points != expected
2504 {
2505 return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2506 backup_id: backup_id.to_string(),
2507 field: "completion_basis_points",
2508 expected,
2509 actual: progress.completion_basis_points,
2510 });
2511 }
2512
2513 Ok(())
2514}
2515
2516fn enforce_pending_before_requirement(
2518 backup_id: &str,
2519 pending: &RestoreApplyPendingSummary,
2520 require_no_pending_before: Option<&str>,
2521) -> Result<(), RestoreCommandError> {
2522 let Some(cutoff_updated_at) = require_no_pending_before else {
2523 return Ok(());
2524 };
2525
2526 if pending.pending_operations == 0 {
2527 return Ok(());
2528 }
2529
2530 if pending.pending_updated_at_known
2531 && pending
2532 .pending_updated_at
2533 .as_deref()
2534 .is_some_and(|updated_at| updated_at >= cutoff_updated_at)
2535 {
2536 return Ok(());
2537 }
2538
2539 Err(RestoreCommandError::RestoreApplyPendingStale {
2540 backup_id: backup_id.to_string(),
2541 cutoff_updated_at: cutoff_updated_at.to_string(),
2542 pending_sequence: pending.pending_sequence,
2543 pending_updated_at: pending.pending_updated_at.clone(),
2544 })
2545}
2546
2547fn enforce_apply_report_requirements(
2549 options: &RestoreApplyReportOptions,
2550 report: &RestoreApplyJournalReport,
2551) -> Result<(), RestoreCommandError> {
2552 if options.require_no_attention && report.attention_required {
2553 return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
2554 backup_id: report.backup_id.clone(),
2555 outcome: report.outcome.clone(),
2556 });
2557 }
2558
2559 enforce_progress_requirements(
2560 &report.backup_id,
2561 &report.progress,
2562 options.require_remaining_count,
2563 options.require_attention_count,
2564 options.require_completion_basis_points,
2565 )?;
2566 enforce_pending_before_requirement(
2567 &report.backup_id,
2568 &report.pending_summary,
2569 options.require_no_pending_before.as_deref(),
2570 )
2571}
2572
2573fn enforce_apply_status_requirements(
2575 options: &RestoreApplyStatusOptions,
2576 status: &RestoreApplyJournalStatus,
2577) -> Result<(), RestoreCommandError> {
2578 if options.require_ready && !status.ready {
2579 return Err(RestoreCommandError::RestoreApplyNotReady {
2580 backup_id: status.backup_id.clone(),
2581 reasons: status.blocked_reasons.clone(),
2582 });
2583 }
2584
2585 if options.require_no_pending && status.pending_operations > 0 {
2586 return Err(RestoreCommandError::RestoreApplyPending {
2587 backup_id: status.backup_id.clone(),
2588 pending_operations: status.pending_operations,
2589 next_transition_sequence: status.next_transition_sequence,
2590 });
2591 }
2592
2593 if options.require_no_failed && status.failed_operations > 0 {
2594 return Err(RestoreCommandError::RestoreApplyFailed {
2595 backup_id: status.backup_id.clone(),
2596 failed_operations: status.failed_operations,
2597 });
2598 }
2599
2600 if options.require_complete && !status.complete {
2601 return Err(RestoreCommandError::RestoreApplyIncomplete {
2602 backup_id: status.backup_id.clone(),
2603 completed_operations: status.completed_operations,
2604 operation_count: status.operation_count,
2605 });
2606 }
2607
2608 enforce_progress_requirements(
2609 &status.backup_id,
2610 &status.progress,
2611 options.require_remaining_count,
2612 options.require_attention_count,
2613 options.require_completion_basis_points,
2614 )?;
2615 enforce_pending_before_requirement(
2616 &status.backup_id,
2617 &status.pending_summary,
2618 options.require_no_pending_before.as_deref(),
2619 )?;
2620
2621 Ok(())
2622}
2623
2624pub fn restore_apply_next(
2626 options: &RestoreApplyNextOptions,
2627) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
2628 let journal = read_apply_journal(&options.journal)?;
2629 Ok(journal.next_operation())
2630}
2631
2632pub fn restore_apply_command(
2634 options: &RestoreApplyCommandOptions,
2635) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
2636 let journal = read_apply_journal(&options.journal)?;
2637 Ok(journal.next_command_preview_with_config(&restore_apply_command_config(options)))
2638}
2639
2640fn enforce_apply_command_requirements(
2642 options: &RestoreApplyCommandOptions,
2643 preview: &RestoreApplyCommandPreview,
2644) -> Result<(), RestoreCommandError> {
2645 if !options.require_command || preview.command_available {
2646 return Ok(());
2647 }
2648
2649 Err(restore_command_unavailable_error(preview))
2650}
2651
2652pub fn restore_apply_claim(
2654 options: &RestoreApplyClaimOptions,
2655) -> Result<RestoreApplyJournal, RestoreCommandError> {
2656 let mut journal = read_apply_journal(&options.journal)?;
2657 let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
2658
2659 if let Some(sequence) = options.sequence {
2660 enforce_apply_claim_sequence(sequence, &journal)?;
2661 journal.mark_operation_pending_at(sequence, updated_at)?;
2662 return Ok(journal);
2663 }
2664
2665 journal.mark_next_operation_pending_at(updated_at)?;
2666 Ok(journal)
2667}
2668
2669fn enforce_apply_claim_sequence(
2671 expected: usize,
2672 journal: &RestoreApplyJournal,
2673) -> Result<(), RestoreCommandError> {
2674 let actual = journal
2675 .next_transition_operation()
2676 .map(|operation| operation.sequence);
2677
2678 if actual == Some(expected) {
2679 return Ok(());
2680 }
2681
2682 Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
2683}
2684
2685pub fn restore_apply_unclaim(
2687 options: &RestoreApplyUnclaimOptions,
2688) -> Result<RestoreApplyJournal, RestoreCommandError> {
2689 let mut journal = read_apply_journal(&options.journal)?;
2690 if let Some(sequence) = options.sequence {
2691 enforce_apply_unclaim_sequence(sequence, &journal)?;
2692 }
2693
2694 journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
2695 Ok(journal)
2696}
2697
2698fn enforce_apply_unclaim_sequence(
2700 expected: usize,
2701 journal: &RestoreApplyJournal,
2702) -> Result<(), RestoreCommandError> {
2703 let actual = journal
2704 .next_transition_operation()
2705 .map(|operation| operation.sequence);
2706
2707 if actual == Some(expected) {
2708 return Ok(());
2709 }
2710
2711 Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
2712}
2713
2714pub fn restore_apply_mark(
2716 options: &RestoreApplyMarkOptions,
2717) -> Result<RestoreApplyJournal, RestoreCommandError> {
2718 let mut journal = read_apply_journal(&options.journal)?;
2719 enforce_apply_mark_pending_requirement(options, &journal)?;
2720
2721 match options.state {
2722 RestoreApplyMarkState::Completed => {
2723 journal.mark_operation_completed_at(
2724 options.sequence,
2725 Some(state_updated_at(options.updated_at.as_ref())),
2726 )?;
2727 }
2728 RestoreApplyMarkState::Failed => {
2729 let reason =
2730 options
2731 .reason
2732 .clone()
2733 .ok_or(RestoreApplyJournalError::FailureReasonRequired(
2734 options.sequence,
2735 ))?;
2736 journal.mark_operation_failed_at(
2737 options.sequence,
2738 reason,
2739 Some(state_updated_at(options.updated_at.as_ref())),
2740 )?;
2741 }
2742 }
2743
2744 Ok(journal)
2745}
2746
2747fn enforce_apply_mark_pending_requirement(
2749 options: &RestoreApplyMarkOptions,
2750 journal: &RestoreApplyJournal,
2751) -> Result<(), RestoreCommandError> {
2752 if !options.require_pending {
2753 return Ok(());
2754 }
2755
2756 let state = journal
2757 .operations
2758 .iter()
2759 .find(|operation| operation.sequence == options.sequence)
2760 .map(|operation| operation.state.clone())
2761 .ok_or(RestoreApplyJournalError::OperationNotFound(
2762 options.sequence,
2763 ))?;
2764
2765 if state == RestoreApplyOperationState::Pending {
2766 return Ok(());
2767 }
2768
2769 Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
2770 sequence: options.sequence,
2771 state,
2772 })
2773}
2774
2775fn enforce_restore_plan_requirements(
2777 options: &RestorePlanOptions,
2778 plan: &RestorePlan,
2779) -> Result<(), RestoreCommandError> {
2780 if !options.require_restore_ready || plan.readiness_summary.ready {
2781 return Ok(());
2782 }
2783
2784 Err(RestoreCommandError::RestoreNotReady {
2785 backup_id: plan.backup_id.clone(),
2786 reasons: plan.readiness_summary.reasons.clone(),
2787 })
2788}
2789
2790fn verify_backup_layout_if_required(
2792 options: &RestorePlanOptions,
2793) -> Result<(), RestoreCommandError> {
2794 if !options.require_verified {
2795 return Ok(());
2796 }
2797
2798 let Some(dir) = &options.backup_dir else {
2799 return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
2800 };
2801
2802 BackupLayout::new(dir.clone()).verify_integrity()?;
2803 Ok(())
2804}
2805
2806fn read_manifest_source(
2808 options: &RestorePlanOptions,
2809) -> Result<FleetBackupManifest, RestoreCommandError> {
2810 if let Some(path) = &options.manifest {
2811 return read_manifest(path);
2812 }
2813
2814 let Some(dir) = &options.backup_dir else {
2815 return Err(RestoreCommandError::MissingOption(
2816 "--manifest or --backup-dir",
2817 ));
2818 };
2819
2820 BackupLayout::new(dir.clone())
2821 .read_manifest()
2822 .map_err(RestoreCommandError::from)
2823}
2824
2825fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
2827 let data = fs::read_to_string(path)?;
2828 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2829}
2830
2831fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
2833 let data = fs::read_to_string(path)?;
2834 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2835}
2836
2837fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
2839 let data = fs::read_to_string(path)?;
2840 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2841}
2842
2843fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
2845 let data = fs::read_to_string(path)?;
2846 serde_json::from_str(&data).map_err(RestoreCommandError::from)
2847}
2848
2849fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
2851 let data = fs::read_to_string(path)?;
2852 let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
2853 journal.validate()?;
2854 Ok(journal)
2855}
2856
2857fn parse_progress_requirement_option<I>(
2859 arg: &str,
2860 args: &mut I,
2861 require_remaining_count: &mut Option<usize>,
2862 require_attention_count: &mut Option<usize>,
2863 require_completion_basis_points: &mut Option<usize>,
2864) -> Result<bool, RestoreCommandError>
2865where
2866 I: Iterator<Item = OsString>,
2867{
2868 match arg {
2869 "--require-remaining-count" => {
2870 *require_remaining_count = Some(parse_sequence(next_value(
2871 args,
2872 "--require-remaining-count",
2873 )?)?);
2874 Ok(true)
2875 }
2876 "--require-attention-count" => {
2877 *require_attention_count = Some(parse_sequence(next_value(
2878 args,
2879 "--require-attention-count",
2880 )?)?);
2881 Ok(true)
2882 }
2883 "--require-completion-basis-points" => {
2884 *require_completion_basis_points = Some(parse_sequence(next_value(
2885 args,
2886 "--require-completion-basis-points",
2887 )?)?);
2888 Ok(true)
2889 }
2890 _ => Ok(false),
2891 }
2892}
2893
2894fn parse_pending_requirement_option<I>(
2896 arg: &str,
2897 args: &mut I,
2898 require_no_pending_before: &mut Option<String>,
2899) -> Result<bool, RestoreCommandError>
2900where
2901 I: Iterator<Item = OsString>,
2902{
2903 match arg {
2904 "--require-no-pending-before" => {
2905 *require_no_pending_before = Some(next_value(args, "--require-no-pending-before")?);
2906 Ok(true)
2907 }
2908 _ => Ok(false),
2909 }
2910}
2911
2912fn parse_run_count_requirement_option<I>(
2914 arg: &str,
2915 args: &mut I,
2916 require_executed_count: &mut Option<usize>,
2917 require_receipt_count: &mut Option<usize>,
2918) -> Result<bool, RestoreCommandError>
2919where
2920 I: Iterator<Item = OsString>,
2921{
2922 match arg {
2923 "--require-executed-count" => {
2924 *require_executed_count = Some(parse_sequence(next_value(
2925 args,
2926 "--require-executed-count",
2927 )?)?);
2928 Ok(true)
2929 }
2930 "--require-receipt-count" => {
2931 *require_receipt_count = Some(parse_sequence(next_value(
2932 args,
2933 "--require-receipt-count",
2934 )?)?);
2935 Ok(true)
2936 }
2937 _ => Ok(false),
2938 }
2939}
2940
2941fn parse_run_receipt_kind_requirement_option<I>(
2943 arg: &str,
2944 args: &mut I,
2945 require_completed_receipt_count: &mut Option<usize>,
2946 require_failed_receipt_count: &mut Option<usize>,
2947 require_recovered_receipt_count: &mut Option<usize>,
2948) -> Result<bool, RestoreCommandError>
2949where
2950 I: Iterator<Item = OsString>,
2951{
2952 match arg {
2953 "--require-completed-receipt-count" => {
2954 *require_completed_receipt_count = Some(parse_sequence(next_value(
2955 args,
2956 "--require-completed-receipt-count",
2957 )?)?);
2958 Ok(true)
2959 }
2960 "--require-failed-receipt-count" => {
2961 *require_failed_receipt_count = Some(parse_sequence(next_value(
2962 args,
2963 "--require-failed-receipt-count",
2964 )?)?);
2965 Ok(true)
2966 }
2967 "--require-recovered-receipt-count" => {
2968 *require_recovered_receipt_count = Some(parse_sequence(next_value(
2969 args,
2970 "--require-recovered-receipt-count",
2971 )?)?);
2972 Ok(true)
2973 }
2974 _ => Ok(false),
2975 }
2976}
2977
2978fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
2980 value
2981 .parse::<usize>()
2982 .map_err(|_| RestoreCommandError::InvalidSequence)
2983}
2984
2985fn parse_integer(option: &'static str, value: String) -> Result<isize, RestoreCommandError> {
2987 value
2988 .parse::<isize>()
2989 .map_err(|_| RestoreCommandError::InvalidInteger { option })
2990}
2991
2992fn parse_positive_integer(
2994 option: &'static str,
2995 value: String,
2996) -> Result<usize, RestoreCommandError> {
2997 let parsed = parse_sequence(value)?;
2998 if parsed == 0 {
2999 return Err(RestoreCommandError::InvalidPositiveInteger { option });
3000 }
3001
3002 Ok(parsed)
3003}
3004
3005fn parse_bool(option: &'static str, value: String) -> Result<bool, RestoreCommandError> {
3007 match value.as_str() {
3008 "true" => Ok(true),
3009 "false" => Ok(false),
3010 _ => Err(RestoreCommandError::InvalidBoolean { option, value }),
3011 }
3012}
3013
3014fn state_updated_at(updated_at: Option<&String>) -> String {
3016 updated_at.cloned().unwrap_or_else(timestamp_placeholder)
3017}
3018
3019fn timestamp_placeholder() -> String {
3021 "unknown".to_string()
3022}
3023
3024fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
3026 if let Some(path) = &options.out {
3027 let data = serde_json::to_vec_pretty(plan)?;
3028 fs::write(path, data)?;
3029 return Ok(());
3030 }
3031
3032 let stdout = io::stdout();
3033 let mut handle = stdout.lock();
3034 serde_json::to_writer_pretty(&mut handle, plan)?;
3035 writeln!(handle)?;
3036 Ok(())
3037}
3038
3039fn write_status(
3041 options: &RestoreStatusOptions,
3042 status: &RestoreStatus,
3043) -> Result<(), RestoreCommandError> {
3044 if let Some(path) = &options.out {
3045 let data = serde_json::to_vec_pretty(status)?;
3046 fs::write(path, data)?;
3047 return Ok(());
3048 }
3049
3050 let stdout = io::stdout();
3051 let mut handle = stdout.lock();
3052 serde_json::to_writer_pretty(&mut handle, status)?;
3053 writeln!(handle)?;
3054 Ok(())
3055}
3056
3057fn write_apply_dry_run(
3059 options: &RestoreApplyOptions,
3060 dry_run: &RestoreApplyDryRun,
3061) -> Result<(), RestoreCommandError> {
3062 if let Some(path) = &options.out {
3063 let data = serde_json::to_vec_pretty(dry_run)?;
3064 fs::write(path, data)?;
3065 return Ok(());
3066 }
3067
3068 let stdout = io::stdout();
3069 let mut handle = stdout.lock();
3070 serde_json::to_writer_pretty(&mut handle, dry_run)?;
3071 writeln!(handle)?;
3072 Ok(())
3073}
3074
3075fn write_apply_journal_if_requested(
3077 options: &RestoreApplyOptions,
3078 dry_run: &RestoreApplyDryRun,
3079) -> Result<(), RestoreCommandError> {
3080 let Some(path) = &options.journal_out else {
3081 return Ok(());
3082 };
3083
3084 let journal = RestoreApplyJournal::from_dry_run(dry_run);
3085 let data = serde_json::to_vec_pretty(&journal)?;
3086 fs::write(path, data)?;
3087 Ok(())
3088}
3089
3090fn write_apply_status(
3092 options: &RestoreApplyStatusOptions,
3093 status: &RestoreApplyJournalStatus,
3094) -> Result<(), RestoreCommandError> {
3095 if let Some(path) = &options.out {
3096 let data = serde_json::to_vec_pretty(status)?;
3097 fs::write(path, data)?;
3098 return Ok(());
3099 }
3100
3101 let stdout = io::stdout();
3102 let mut handle = stdout.lock();
3103 serde_json::to_writer_pretty(&mut handle, status)?;
3104 writeln!(handle)?;
3105 Ok(())
3106}
3107
3108fn write_apply_report(
3110 options: &RestoreApplyReportOptions,
3111 report: &RestoreApplyJournalReport,
3112) -> Result<(), RestoreCommandError> {
3113 if let Some(path) = &options.out {
3114 let data = serde_json::to_vec_pretty(report)?;
3115 fs::write(path, data)?;
3116 return Ok(());
3117 }
3118
3119 let stdout = io::stdout();
3120 let mut handle = stdout.lock();
3121 serde_json::to_writer_pretty(&mut handle, report)?;
3122 writeln!(handle)?;
3123 Ok(())
3124}
3125
3126fn write_restore_run(
3128 options: &RestoreRunOptions,
3129 run: &RestoreRunResponse,
3130) -> Result<(), RestoreCommandError> {
3131 if let Some(path) = &options.out {
3132 let data = serde_json::to_vec_pretty(run)?;
3133 fs::write(path, data)?;
3134 return Ok(());
3135 }
3136
3137 let stdout = io::stdout();
3138 let mut handle = stdout.lock();
3139 serde_json::to_writer_pretty(&mut handle, run)?;
3140 writeln!(handle)?;
3141 Ok(())
3142}
3143
3144fn write_apply_journal_file(
3146 path: &PathBuf,
3147 journal: &RestoreApplyJournal,
3148) -> Result<(), RestoreCommandError> {
3149 let data = serde_json::to_vec_pretty(journal)?;
3150 fs::write(path, data)?;
3151 Ok(())
3152}
3153
3154fn write_apply_next(
3156 options: &RestoreApplyNextOptions,
3157 next: &RestoreApplyNextOperation,
3158) -> Result<(), RestoreCommandError> {
3159 if let Some(path) = &options.out {
3160 let data = serde_json::to_vec_pretty(next)?;
3161 fs::write(path, data)?;
3162 return Ok(());
3163 }
3164
3165 let stdout = io::stdout();
3166 let mut handle = stdout.lock();
3167 serde_json::to_writer_pretty(&mut handle, next)?;
3168 writeln!(handle)?;
3169 Ok(())
3170}
3171
3172fn write_apply_command(
3174 options: &RestoreApplyCommandOptions,
3175 preview: &RestoreApplyCommandPreview,
3176) -> Result<(), RestoreCommandError> {
3177 if let Some(path) = &options.out {
3178 let data = serde_json::to_vec_pretty(preview)?;
3179 fs::write(path, data)?;
3180 return Ok(());
3181 }
3182
3183 let stdout = io::stdout();
3184 let mut handle = stdout.lock();
3185 serde_json::to_writer_pretty(&mut handle, preview)?;
3186 writeln!(handle)?;
3187 Ok(())
3188}
3189
3190fn write_apply_claim(
3192 options: &RestoreApplyClaimOptions,
3193 journal: &RestoreApplyJournal,
3194) -> Result<(), RestoreCommandError> {
3195 if let Some(path) = &options.out {
3196 let data = serde_json::to_vec_pretty(journal)?;
3197 fs::write(path, data)?;
3198 return Ok(());
3199 }
3200
3201 let stdout = io::stdout();
3202 let mut handle = stdout.lock();
3203 serde_json::to_writer_pretty(&mut handle, journal)?;
3204 writeln!(handle)?;
3205 Ok(())
3206}
3207
3208fn write_apply_unclaim(
3210 options: &RestoreApplyUnclaimOptions,
3211 journal: &RestoreApplyJournal,
3212) -> Result<(), RestoreCommandError> {
3213 if let Some(path) = &options.out {
3214 let data = serde_json::to_vec_pretty(journal)?;
3215 fs::write(path, data)?;
3216 return Ok(());
3217 }
3218
3219 let stdout = io::stdout();
3220 let mut handle = stdout.lock();
3221 serde_json::to_writer_pretty(&mut handle, journal)?;
3222 writeln!(handle)?;
3223 Ok(())
3224}
3225
3226fn write_apply_mark(
3228 options: &RestoreApplyMarkOptions,
3229 journal: &RestoreApplyJournal,
3230) -> Result<(), RestoreCommandError> {
3231 if let Some(path) = &options.out {
3232 let data = serde_json::to_vec_pretty(journal)?;
3233 fs::write(path, data)?;
3234 return Ok(());
3235 }
3236
3237 let stdout = io::stdout();
3238 let mut handle = stdout.lock();
3239 serde_json::to_writer_pretty(&mut handle, journal)?;
3240 writeln!(handle)?;
3241 Ok(())
3242}
3243
3244fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
3246where
3247 I: Iterator<Item = OsString>,
3248{
3249 args.next()
3250 .and_then(|value| value.into_string().ok())
3251 .ok_or(RestoreCommandError::MissingValue(option))
3252}
3253
3254const fn usage() -> &'static str {
3256 "usage: canic restore plan (--manifest <file> | --backup-dir <dir>) [--mapping <file>] [--out <file>] [--require-verified] [--require-restore-ready]\n canic restore status --plan <file> [--out <file>]\n canic restore apply --plan <file> [--status <file>] [--backup-dir <dir>] --dry-run [--out <file>] [--journal-out <file>]\n canic restore apply-status --journal <file> [--out <file>] [--require-ready] [--require-no-pending] [--require-no-failed] [--require-complete] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n canic restore apply-report --journal <file> [--out <file>] [--require-no-attention] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n canic restore run --journal <file> (--dry-run | --execute | --unclaim-pending) [--dfx <path>] [--network <name>] [--max-steps <n>] [--updated-at <text>] [--out <file>] [--require-complete] [--require-no-attention] [--require-run-mode <text>] [--require-stopped-reason <text>] [--require-next-action <text>] [--require-executed-count <n>] [--require-receipt-count <n>] [--require-completed-receipt-count <n>] [--require-failed-receipt-count <n>] [--require-recovered-receipt-count <n>] [--require-receipt-updated-at <text>] [--require-state-updated-at <text>] [--require-batch-initial-ready-count <n>] [--require-batch-executed-count <n>] [--require-batch-remaining-ready-count <n>] [--require-batch-ready-delta <n>] [--require-batch-remaining-delta <n>] [--require-batch-stopped-by-max-steps true|false] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n canic restore apply-next --journal <file> [--out <file>]\n canic restore apply-command --journal <file> [--dfx <path>] [--network <name>] [--out <file>] [--require-command]\n canic restore apply-claim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-unclaim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n canic restore apply-mark --journal <file> --sequence <n> --state completed|failed [--reason <text>] [--updated-at <text>] [--out <file>] [--require-pending]"
3257}
3258
3259#[cfg(test)]
3260mod tests {
3261 use super::*;
3262 use canic_backup::restore::RestoreApplyOperationState;
3263 use canic_backup::{
3264 artifacts::ArtifactChecksum,
3265 journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
3266 manifest::{
3267 BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
3268 FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
3269 VerificationCheck, VerificationPlan,
3270 },
3271 };
3272 use serde_json::json;
3273 use std::{
3274 path::Path,
3275 time::{SystemTime, UNIX_EPOCH},
3276 };
3277
3278 const ROOT: &str = "aaaaa-aa";
3279 const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
3280 const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
3281 const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
3282
3283 struct RestoreCliFixture {
3288 root: PathBuf,
3289 journal_path: PathBuf,
3290 out_path: PathBuf,
3291 }
3292
3293 impl RestoreCliFixture {
3294 fn new(prefix: &str, out_file: &str) -> Self {
3296 let root = temp_dir(prefix);
3297 fs::create_dir_all(&root).expect("create temp root");
3298
3299 Self {
3300 journal_path: root.join("restore-apply-journal.json"),
3301 out_path: root.join(out_file),
3302 root,
3303 }
3304 }
3305
3306 fn write_journal(&self, journal: &RestoreApplyJournal) {
3308 fs::write(
3309 &self.journal_path,
3310 serde_json::to_vec(journal).expect("serialize journal"),
3311 )
3312 .expect("write journal");
3313 }
3314
3315 fn run_apply_status(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
3317 self.run_journal_command("apply-status", extra)
3318 }
3319
3320 fn run_apply_report(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
3322 self.run_journal_command("apply-report", extra)
3323 }
3324
3325 fn run_restore_run(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
3327 self.run_journal_command("run", extra)
3328 }
3329
3330 fn read_out<T>(&self, label: &str) -> T
3332 where
3333 T: serde::de::DeserializeOwned,
3334 {
3335 serde_json::from_slice(&fs::read(&self.out_path).expect(label)).expect(label)
3336 }
3337
3338 fn run_journal_command(
3340 &self,
3341 command: &str,
3342 extra: &[&str],
3343 ) -> Result<(), RestoreCommandError> {
3344 let mut args = vec![
3345 OsString::from(command),
3346 OsString::from("--journal"),
3347 OsString::from(self.journal_path.as_os_str()),
3348 OsString::from("--out"),
3349 OsString::from(self.out_path.as_os_str()),
3350 ];
3351 args.extend(extra.iter().map(OsString::from));
3352 run(args)
3353 }
3354 }
3355
3356 impl Drop for RestoreCliFixture {
3357 fn drop(&mut self) {
3359 let _ = fs::remove_dir_all(&self.root);
3360 }
3361 }
3362
3363 fn assert_batch_summary(summary: &serde_json::Value, expected: serde_json::Value) {
3365 assert_eq!(summary, &expected);
3366 }
3367
3368 fn assert_completed_execute_batch_summary(run_summary: &serde_json::Value) {
3370 assert_batch_summary(
3371 &run_summary["batch_summary"],
3372 json!({
3373 "requested_max_steps": 1,
3374 "initial_ready_operations": 8,
3375 "initial_remaining_operations": 8,
3376 "executed_operations": 1,
3377 "remaining_ready_operations": 7,
3378 "remaining_operations": 7,
3379 "ready_operations_delta": -1,
3380 "remaining_operations_delta": -1,
3381 "stopped_by_max_steps": true,
3382 "complete": false,
3383 }),
3384 );
3385 }
3386
3387 #[test]
3389 fn parses_restore_plan_options() {
3390 let options = RestorePlanOptions::parse([
3391 OsString::from("--manifest"),
3392 OsString::from("manifest.json"),
3393 OsString::from("--mapping"),
3394 OsString::from("mapping.json"),
3395 OsString::from("--out"),
3396 OsString::from("plan.json"),
3397 OsString::from("--require-restore-ready"),
3398 ])
3399 .expect("parse options");
3400
3401 assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
3402 assert_eq!(options.backup_dir, None);
3403 assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
3404 assert_eq!(options.out, Some(PathBuf::from("plan.json")));
3405 assert!(!options.require_verified);
3406 assert!(options.require_restore_ready);
3407 }
3408
3409 #[test]
3411 fn parses_verified_restore_plan_options() {
3412 let options = RestorePlanOptions::parse([
3413 OsString::from("--backup-dir"),
3414 OsString::from("backups/run"),
3415 OsString::from("--require-verified"),
3416 ])
3417 .expect("parse verified options");
3418
3419 assert_eq!(options.manifest, None);
3420 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
3421 assert_eq!(options.mapping, None);
3422 assert_eq!(options.out, None);
3423 assert!(options.require_verified);
3424 assert!(!options.require_restore_ready);
3425 }
3426
3427 #[test]
3429 fn parses_restore_status_options() {
3430 let options = RestoreStatusOptions::parse([
3431 OsString::from("--plan"),
3432 OsString::from("restore-plan.json"),
3433 OsString::from("--out"),
3434 OsString::from("restore-status.json"),
3435 ])
3436 .expect("parse status options");
3437
3438 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
3439 assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
3440 }
3441
3442 #[test]
3444 fn parses_restore_apply_dry_run_options() {
3445 let options = RestoreApplyOptions::parse([
3446 OsString::from("--plan"),
3447 OsString::from("restore-plan.json"),
3448 OsString::from("--status"),
3449 OsString::from("restore-status.json"),
3450 OsString::from("--backup-dir"),
3451 OsString::from("backups/run"),
3452 OsString::from("--dry-run"),
3453 OsString::from("--out"),
3454 OsString::from("restore-apply-dry-run.json"),
3455 OsString::from("--journal-out"),
3456 OsString::from("restore-apply-journal.json"),
3457 ])
3458 .expect("parse apply options");
3459
3460 assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
3461 assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
3462 assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
3463 assert_eq!(
3464 options.out,
3465 Some(PathBuf::from("restore-apply-dry-run.json"))
3466 );
3467 assert_eq!(
3468 options.journal_out,
3469 Some(PathBuf::from("restore-apply-journal.json"))
3470 );
3471 assert!(options.dry_run);
3472 }
3473
3474 #[test]
3476 fn parses_restore_apply_status_options() {
3477 let options = RestoreApplyStatusOptions::parse([
3478 OsString::from("--journal"),
3479 OsString::from("restore-apply-journal.json"),
3480 OsString::from("--out"),
3481 OsString::from("restore-apply-status.json"),
3482 OsString::from("--require-ready"),
3483 OsString::from("--require-no-pending"),
3484 OsString::from("--require-no-failed"),
3485 OsString::from("--require-complete"),
3486 OsString::from("--require-remaining-count"),
3487 OsString::from("7"),
3488 OsString::from("--require-attention-count"),
3489 OsString::from("0"),
3490 OsString::from("--require-completion-basis-points"),
3491 OsString::from("1250"),
3492 OsString::from("--require-no-pending-before"),
3493 OsString::from("2026-05-05T12:00:00Z"),
3494 ])
3495 .expect("parse apply-status options");
3496
3497 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3498 assert!(options.require_ready);
3499 assert!(options.require_no_pending);
3500 assert!(options.require_no_failed);
3501 assert!(options.require_complete);
3502 assert_eq!(options.require_remaining_count, Some(7));
3503 assert_eq!(options.require_attention_count, Some(0));
3504 assert_eq!(options.require_completion_basis_points, Some(1250));
3505 assert_eq!(
3506 options.require_no_pending_before.as_deref(),
3507 Some("2026-05-05T12:00:00Z")
3508 );
3509 assert_eq!(
3510 options.out,
3511 Some(PathBuf::from("restore-apply-status.json"))
3512 );
3513 }
3514
3515 #[test]
3517 fn parses_restore_apply_report_options() {
3518 let options = RestoreApplyReportOptions::parse([
3519 OsString::from("--journal"),
3520 OsString::from("restore-apply-journal.json"),
3521 OsString::from("--out"),
3522 OsString::from("restore-apply-report.json"),
3523 OsString::from("--require-no-attention"),
3524 OsString::from("--require-remaining-count"),
3525 OsString::from("8"),
3526 OsString::from("--require-attention-count"),
3527 OsString::from("0"),
3528 OsString::from("--require-completion-basis-points"),
3529 OsString::from("0"),
3530 OsString::from("--require-no-pending-before"),
3531 OsString::from("2026-05-05T12:00:00Z"),
3532 ])
3533 .expect("parse apply-report options");
3534
3535 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3536 assert!(options.require_no_attention);
3537 assert_eq!(options.require_remaining_count, Some(8));
3538 assert_eq!(options.require_attention_count, Some(0));
3539 assert_eq!(options.require_completion_basis_points, Some(0));
3540 assert_eq!(
3541 options.require_no_pending_before.as_deref(),
3542 Some("2026-05-05T12:00:00Z")
3543 );
3544 assert_eq!(
3545 options.out,
3546 Some(PathBuf::from("restore-apply-report.json"))
3547 );
3548 }
3549
3550 #[test]
3552 fn parses_restore_run_dry_run_options() {
3553 let options = RestoreRunOptions::parse([
3554 OsString::from("--journal"),
3555 OsString::from("restore-apply-journal.json"),
3556 OsString::from("--dry-run"),
3557 OsString::from("--dfx"),
3558 OsString::from("/tmp/dfx"),
3559 OsString::from("--network"),
3560 OsString::from("local"),
3561 OsString::from("--out"),
3562 OsString::from("restore-run-dry-run.json"),
3563 OsString::from("--max-steps"),
3564 OsString::from("1"),
3565 OsString::from("--updated-at"),
3566 OsString::from("2026-05-05T12:03:00Z"),
3567 OsString::from("--require-complete"),
3568 OsString::from("--require-no-attention"),
3569 OsString::from("--require-run-mode"),
3570 OsString::from("dry-run"),
3571 OsString::from("--require-stopped-reason"),
3572 OsString::from("preview"),
3573 OsString::from("--require-next-action"),
3574 OsString::from("rerun"),
3575 OsString::from("--require-executed-count"),
3576 OsString::from("0"),
3577 OsString::from("--require-receipt-count"),
3578 OsString::from("0"),
3579 OsString::from("--require-completed-receipt-count"),
3580 OsString::from("0"),
3581 OsString::from("--require-failed-receipt-count"),
3582 OsString::from("0"),
3583 OsString::from("--require-recovered-receipt-count"),
3584 OsString::from("0"),
3585 OsString::from("--require-receipt-updated-at"),
3586 OsString::from("2026-05-05T12:03:00Z"),
3587 OsString::from("--require-state-updated-at"),
3588 OsString::from("2026-05-05T12:03:00Z"),
3589 OsString::from("--require-batch-initial-ready-count"),
3590 OsString::from("8"),
3591 OsString::from("--require-batch-executed-count"),
3592 OsString::from("0"),
3593 OsString::from("--require-batch-remaining-ready-count"),
3594 OsString::from("8"),
3595 OsString::from("--require-batch-ready-delta"),
3596 OsString::from("0"),
3597 OsString::from("--require-batch-remaining-delta"),
3598 OsString::from("0"),
3599 OsString::from("--require-batch-stopped-by-max-steps"),
3600 OsString::from("false"),
3601 OsString::from("--require-remaining-count"),
3602 OsString::from("8"),
3603 OsString::from("--require-attention-count"),
3604 OsString::from("0"),
3605 OsString::from("--require-completion-basis-points"),
3606 OsString::from("0"),
3607 OsString::from("--require-no-pending-before"),
3608 OsString::from("2026-05-05T12:00:00Z"),
3609 ])
3610 .expect("parse restore run options");
3611
3612 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3613 assert_eq!(options.dfx, "/tmp/dfx");
3614 assert_eq!(options.network.as_deref(), Some("local"));
3615 assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
3616 assert!(options.dry_run);
3617 assert!(!options.execute);
3618 assert!(!options.unclaim_pending);
3619 assert_eq!(options.max_steps, Some(1));
3620 assert_eq!(options.updated_at.as_deref(), Some("2026-05-05T12:03:00Z"));
3621 assert!(options.require_complete);
3622 assert!(options.require_no_attention);
3623 assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
3624 assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
3625 assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
3626 assert_eq!(options.require_executed_count, Some(0));
3627 assert_eq!(options.require_receipt_count, Some(0));
3628 assert_eq!(options.require_completed_receipt_count, Some(0));
3629 assert_eq!(options.require_failed_receipt_count, Some(0));
3630 assert_eq!(options.require_recovered_receipt_count, Some(0));
3631 assert_eq!(
3632 options.require_receipt_updated_at.as_deref(),
3633 Some("2026-05-05T12:03:00Z")
3634 );
3635 assert_eq!(
3636 options.require_state_updated_at.as_deref(),
3637 Some("2026-05-05T12:03:00Z")
3638 );
3639 assert_eq!(options.require_batch_initial_ready_count, Some(8));
3640 assert_eq!(options.require_batch_executed_count, Some(0));
3641 assert_eq!(options.require_batch_remaining_ready_count, Some(8));
3642 assert_eq!(options.require_batch_ready_delta, Some(0));
3643 assert_eq!(options.require_batch_remaining_delta, Some(0));
3644 assert_eq!(options.require_batch_stopped_by_max_steps, Some(false));
3645 assert_eq!(options.require_remaining_count, Some(8));
3646 assert_eq!(options.require_attention_count, Some(0));
3647 assert_eq!(options.require_completion_basis_points, Some(0));
3648 assert_eq!(
3649 options.require_no_pending_before.as_deref(),
3650 Some("2026-05-05T12:00:00Z")
3651 );
3652 }
3653
3654 #[test]
3656 fn parses_restore_run_execute_options() {
3657 let options = RestoreRunOptions::parse([
3658 OsString::from("--journal"),
3659 OsString::from("restore-apply-journal.json"),
3660 OsString::from("--execute"),
3661 OsString::from("--dfx"),
3662 OsString::from("/bin/true"),
3663 OsString::from("--max-steps"),
3664 OsString::from("4"),
3665 ])
3666 .expect("parse restore run execute options");
3667
3668 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3669 assert_eq!(options.dfx, "/bin/true");
3670 assert_eq!(options.network, None);
3671 assert_eq!(options.out, None);
3672 assert!(!options.dry_run);
3673 assert!(options.execute);
3674 assert!(!options.unclaim_pending);
3675 assert_eq!(options.max_steps, Some(4));
3676 assert_eq!(options.updated_at, None);
3677 assert!(!options.require_complete);
3678 assert!(!options.require_no_attention);
3679 assert_eq!(options.require_run_mode, None);
3680 assert_eq!(options.require_stopped_reason, None);
3681 assert_eq!(options.require_next_action, None);
3682 assert_eq!(options.require_executed_count, None);
3683 assert_eq!(options.require_receipt_count, None);
3684 assert_eq!(options.require_completed_receipt_count, None);
3685 assert_eq!(options.require_failed_receipt_count, None);
3686 assert_eq!(options.require_recovered_receipt_count, None);
3687 assert_eq!(options.require_receipt_updated_at, None);
3688 assert_eq!(options.require_state_updated_at, None);
3689 assert_eq!(options.require_batch_initial_ready_count, None);
3690 assert_eq!(options.require_batch_executed_count, None);
3691 assert_eq!(options.require_batch_remaining_ready_count, None);
3692 assert_eq!(options.require_batch_ready_delta, None);
3693 assert_eq!(options.require_batch_remaining_delta, None);
3694 assert_eq!(options.require_batch_stopped_by_max_steps, None);
3695 }
3696
3697 #[test]
3699 fn parses_restore_run_unclaim_pending_options() {
3700 let options = RestoreRunOptions::parse([
3701 OsString::from("--journal"),
3702 OsString::from("restore-apply-journal.json"),
3703 OsString::from("--unclaim-pending"),
3704 OsString::from("--out"),
3705 OsString::from("restore-run.json"),
3706 ])
3707 .expect("parse restore run unclaim options");
3708
3709 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3710 assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
3711 assert!(!options.dry_run);
3712 assert!(!options.execute);
3713 assert!(options.unclaim_pending);
3714 }
3715
3716 #[test]
3718 fn parses_restore_apply_next_options() {
3719 let options = RestoreApplyNextOptions::parse([
3720 OsString::from("--journal"),
3721 OsString::from("restore-apply-journal.json"),
3722 OsString::from("--out"),
3723 OsString::from("restore-apply-next.json"),
3724 ])
3725 .expect("parse apply-next options");
3726
3727 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3728 assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
3729 }
3730
3731 #[test]
3733 fn parses_restore_apply_command_options() {
3734 let options = RestoreApplyCommandOptions::parse([
3735 OsString::from("--journal"),
3736 OsString::from("restore-apply-journal.json"),
3737 OsString::from("--dfx"),
3738 OsString::from("/tmp/dfx"),
3739 OsString::from("--network"),
3740 OsString::from("local"),
3741 OsString::from("--out"),
3742 OsString::from("restore-apply-command.json"),
3743 OsString::from("--require-command"),
3744 ])
3745 .expect("parse apply-command options");
3746
3747 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3748 assert_eq!(options.dfx, "/tmp/dfx");
3749 assert_eq!(options.network.as_deref(), Some("local"));
3750 assert!(options.require_command);
3751 assert_eq!(
3752 options.out,
3753 Some(PathBuf::from("restore-apply-command.json"))
3754 );
3755 }
3756
3757 #[test]
3759 fn parses_restore_apply_claim_options() {
3760 let options = RestoreApplyClaimOptions::parse([
3761 OsString::from("--journal"),
3762 OsString::from("restore-apply-journal.json"),
3763 OsString::from("--sequence"),
3764 OsString::from("0"),
3765 OsString::from("--updated-at"),
3766 OsString::from("2026-05-04T12:00:00Z"),
3767 OsString::from("--out"),
3768 OsString::from("restore-apply-journal.claimed.json"),
3769 ])
3770 .expect("parse apply-claim options");
3771
3772 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3773 assert_eq!(options.sequence, Some(0));
3774 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
3775 assert_eq!(
3776 options.out,
3777 Some(PathBuf::from("restore-apply-journal.claimed.json"))
3778 );
3779 }
3780
3781 #[test]
3783 fn parses_restore_apply_unclaim_options() {
3784 let options = RestoreApplyUnclaimOptions::parse([
3785 OsString::from("--journal"),
3786 OsString::from("restore-apply-journal.json"),
3787 OsString::from("--sequence"),
3788 OsString::from("0"),
3789 OsString::from("--updated-at"),
3790 OsString::from("2026-05-04T12:01:00Z"),
3791 OsString::from("--out"),
3792 OsString::from("restore-apply-journal.unclaimed.json"),
3793 ])
3794 .expect("parse apply-unclaim options");
3795
3796 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3797 assert_eq!(options.sequence, Some(0));
3798 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
3799 assert_eq!(
3800 options.out,
3801 Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
3802 );
3803 }
3804
3805 #[test]
3807 fn parses_restore_apply_mark_options() {
3808 let options = RestoreApplyMarkOptions::parse([
3809 OsString::from("--journal"),
3810 OsString::from("restore-apply-journal.json"),
3811 OsString::from("--sequence"),
3812 OsString::from("4"),
3813 OsString::from("--state"),
3814 OsString::from("failed"),
3815 OsString::from("--reason"),
3816 OsString::from("dfx-load-failed"),
3817 OsString::from("--updated-at"),
3818 OsString::from("2026-05-04T12:02:00Z"),
3819 OsString::from("--out"),
3820 OsString::from("restore-apply-journal.updated.json"),
3821 OsString::from("--require-pending"),
3822 ])
3823 .expect("parse apply-mark options");
3824
3825 assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3826 assert_eq!(options.sequence, 4);
3827 assert_eq!(options.state, RestoreApplyMarkState::Failed);
3828 assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
3829 assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
3830 assert!(options.require_pending);
3831 assert_eq!(
3832 options.out,
3833 Some(PathBuf::from("restore-apply-journal.updated.json"))
3834 );
3835 }
3836
3837 #[test]
3839 fn restore_apply_requires_dry_run() {
3840 let err = RestoreApplyOptions::parse([
3841 OsString::from("--plan"),
3842 OsString::from("restore-plan.json"),
3843 ])
3844 .expect_err("apply without dry-run should fail");
3845
3846 assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
3847 }
3848
3849 #[test]
3851 fn restore_run_requires_mode() {
3852 let err = RestoreRunOptions::parse([
3853 OsString::from("--journal"),
3854 OsString::from("restore-apply-journal.json"),
3855 ])
3856 .expect_err("restore run without dry-run should fail");
3857
3858 assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
3859 }
3860
3861 #[test]
3863 fn restore_run_rejects_conflicting_modes() {
3864 let err = RestoreRunOptions::parse([
3865 OsString::from("--journal"),
3866 OsString::from("restore-apply-journal.json"),
3867 OsString::from("--dry-run"),
3868 OsString::from("--execute"),
3869 OsString::from("--unclaim-pending"),
3870 ])
3871 .expect_err("restore run should reject conflicting modes");
3872
3873 assert!(matches!(
3874 err,
3875 RestoreCommandError::RestoreRunConflictingModes
3876 ));
3877 }
3878
3879 #[test]
3881 fn restore_run_rejects_zero_max_steps() {
3882 let err = RestoreRunOptions::parse([
3883 OsString::from("--journal"),
3884 OsString::from("restore-apply-journal.json"),
3885 OsString::from("--execute"),
3886 OsString::from("--max-steps"),
3887 OsString::from("0"),
3888 ])
3889 .expect_err("restore run should reject zero max steps");
3890
3891 assert!(matches!(
3892 err,
3893 RestoreCommandError::InvalidPositiveInteger {
3894 option: "--max-steps"
3895 }
3896 ));
3897 }
3898
3899 #[test]
3901 fn restore_run_rejects_invalid_batch_bool() {
3902 let err = RestoreRunOptions::parse([
3903 OsString::from("--journal"),
3904 OsString::from("restore-apply-journal.json"),
3905 OsString::from("--dry-run"),
3906 OsString::from("--require-batch-stopped-by-max-steps"),
3907 OsString::from("maybe"),
3908 ])
3909 .expect_err("restore run should reject invalid boolean gates");
3910
3911 assert!(matches!(
3912 err,
3913 RestoreCommandError::InvalidBoolean {
3914 option: "--require-batch-stopped-by-max-steps",
3915 value,
3916 } if value == "maybe"
3917 ));
3918 }
3919
3920 #[test]
3922 fn restore_run_rejects_invalid_batch_delta() {
3923 let err = RestoreRunOptions::parse([
3924 OsString::from("--journal"),
3925 OsString::from("restore-apply-journal.json"),
3926 OsString::from("--dry-run"),
3927 OsString::from("--require-batch-ready-delta"),
3928 OsString::from("not-an-int"),
3929 ])
3930 .expect_err("restore run should reject invalid signed integer gates");
3931
3932 assert!(matches!(
3933 err,
3934 RestoreCommandError::InvalidInteger {
3935 option: "--require-batch-ready-delta"
3936 }
3937 ));
3938 }
3939
3940 #[test]
3942 fn plan_restore_reads_manifest_from_backup_dir() {
3943 let root = temp_dir("canic-cli-restore-plan-layout");
3944 let layout = BackupLayout::new(root.clone());
3945 layout
3946 .write_manifest(&valid_manifest())
3947 .expect("write manifest");
3948
3949 let options = RestorePlanOptions {
3950 manifest: None,
3951 backup_dir: Some(root.clone()),
3952 mapping: None,
3953 out: None,
3954 require_verified: false,
3955 require_restore_ready: false,
3956 };
3957
3958 let plan = plan_restore(&options).expect("plan restore");
3959
3960 fs::remove_dir_all(root).expect("remove temp root");
3961 assert_eq!(plan.backup_id, "backup-test");
3962 assert_eq!(plan.member_count, 2);
3963 }
3964
3965 #[test]
3967 fn parse_rejects_conflicting_manifest_sources() {
3968 let err = RestorePlanOptions::parse([
3969 OsString::from("--manifest"),
3970 OsString::from("manifest.json"),
3971 OsString::from("--backup-dir"),
3972 OsString::from("backups/run"),
3973 ])
3974 .expect_err("conflicting sources should fail");
3975
3976 assert!(matches!(
3977 err,
3978 RestoreCommandError::ConflictingManifestSources
3979 ));
3980 }
3981
3982 #[test]
3984 fn parse_rejects_require_verified_with_manifest_source() {
3985 let err = RestorePlanOptions::parse([
3986 OsString::from("--manifest"),
3987 OsString::from("manifest.json"),
3988 OsString::from("--require-verified"),
3989 ])
3990 .expect_err("verification should require a backup layout");
3991
3992 assert!(matches!(
3993 err,
3994 RestoreCommandError::RequireVerifiedNeedsBackupDir
3995 ));
3996 }
3997
3998 #[test]
4000 fn plan_restore_requires_verified_backup_layout() {
4001 let root = temp_dir("canic-cli-restore-plan-verified");
4002 let layout = BackupLayout::new(root.clone());
4003 let manifest = valid_manifest();
4004 write_verified_layout(&root, &layout, &manifest);
4005
4006 let options = RestorePlanOptions {
4007 manifest: None,
4008 backup_dir: Some(root.clone()),
4009 mapping: None,
4010 out: None,
4011 require_verified: true,
4012 require_restore_ready: false,
4013 };
4014
4015 let plan = plan_restore(&options).expect("plan verified restore");
4016
4017 fs::remove_dir_all(root).expect("remove temp root");
4018 assert_eq!(plan.backup_id, "backup-test");
4019 assert_eq!(plan.member_count, 2);
4020 }
4021
4022 #[test]
4024 fn plan_restore_rejects_unverified_backup_layout() {
4025 let root = temp_dir("canic-cli-restore-plan-unverified");
4026 let layout = BackupLayout::new(root.clone());
4027 layout
4028 .write_manifest(&valid_manifest())
4029 .expect("write manifest");
4030
4031 let options = RestorePlanOptions {
4032 manifest: None,
4033 backup_dir: Some(root.clone()),
4034 mapping: None,
4035 out: None,
4036 require_verified: true,
4037 require_restore_ready: false,
4038 };
4039
4040 let err = plan_restore(&options).expect_err("missing journal should fail");
4041
4042 fs::remove_dir_all(root).expect("remove temp root");
4043 assert!(matches!(err, RestoreCommandError::Persistence(_)));
4044 }
4045
4046 #[test]
4048 fn plan_restore_reads_manifest_and_mapping() {
4049 let root = temp_dir("canic-cli-restore-plan");
4050 fs::create_dir_all(&root).expect("create temp root");
4051 let manifest_path = root.join("manifest.json");
4052 let mapping_path = root.join("mapping.json");
4053
4054 fs::write(
4055 &manifest_path,
4056 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
4057 )
4058 .expect("write manifest");
4059 fs::write(
4060 &mapping_path,
4061 json!({
4062 "members": [
4063 {"source_canister": ROOT, "target_canister": ROOT},
4064 {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
4065 ]
4066 })
4067 .to_string(),
4068 )
4069 .expect("write mapping");
4070
4071 let options = RestorePlanOptions {
4072 manifest: Some(manifest_path),
4073 backup_dir: None,
4074 mapping: Some(mapping_path),
4075 out: None,
4076 require_verified: false,
4077 require_restore_ready: false,
4078 };
4079
4080 let plan = plan_restore(&options).expect("plan restore");
4081
4082 fs::remove_dir_all(root).expect("remove temp root");
4083 let members = plan.ordered_members();
4084 assert_eq!(members.len(), 2);
4085 assert_eq!(members[0].source_canister, ROOT);
4086 assert_eq!(members[1].target_canister, MAPPED_CHILD);
4087 }
4088
4089 #[test]
4091 fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
4092 let root = temp_dir("canic-cli-restore-plan-require-ready");
4093 fs::create_dir_all(&root).expect("create temp root");
4094 let manifest_path = root.join("manifest.json");
4095 let out_path = root.join("plan.json");
4096
4097 fs::write(
4098 &manifest_path,
4099 serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
4100 )
4101 .expect("write manifest");
4102
4103 let err = run([
4104 OsString::from("plan"),
4105 OsString::from("--manifest"),
4106 OsString::from(manifest_path.as_os_str()),
4107 OsString::from("--out"),
4108 OsString::from(out_path.as_os_str()),
4109 OsString::from("--require-restore-ready"),
4110 ])
4111 .expect_err("restore readiness should be enforced");
4112
4113 assert!(out_path.exists());
4114 let plan: RestorePlan =
4115 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
4116
4117 fs::remove_dir_all(root).expect("remove temp root");
4118 assert!(!plan.readiness_summary.ready);
4119 assert!(matches!(
4120 err,
4121 RestoreCommandError::RestoreNotReady {
4122 reasons,
4123 ..
4124 } if reasons == [
4125 "missing-module-hash",
4126 "missing-wasm-hash",
4127 "missing-snapshot-checksum"
4128 ]
4129 ));
4130 }
4131
4132 #[test]
4134 fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
4135 let root = temp_dir("canic-cli-restore-plan-ready");
4136 fs::create_dir_all(&root).expect("create temp root");
4137 let manifest_path = root.join("manifest.json");
4138 let out_path = root.join("plan.json");
4139
4140 fs::write(
4141 &manifest_path,
4142 serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
4143 )
4144 .expect("write manifest");
4145
4146 run([
4147 OsString::from("plan"),
4148 OsString::from("--manifest"),
4149 OsString::from(manifest_path.as_os_str()),
4150 OsString::from("--out"),
4151 OsString::from(out_path.as_os_str()),
4152 OsString::from("--require-restore-ready"),
4153 ])
4154 .expect("restore-ready plan should pass");
4155
4156 let plan: RestorePlan =
4157 serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
4158
4159 fs::remove_dir_all(root).expect("remove temp root");
4160 assert!(plan.readiness_summary.ready);
4161 assert!(plan.readiness_summary.reasons.is_empty());
4162 }
4163
4164 #[test]
4166 fn run_restore_status_writes_planned_status() {
4167 let root = temp_dir("canic-cli-restore-status");
4168 fs::create_dir_all(&root).expect("create temp root");
4169 let plan_path = root.join("restore-plan.json");
4170 let out_path = root.join("restore-status.json");
4171 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4172
4173 fs::write(
4174 &plan_path,
4175 serde_json::to_vec(&plan).expect("serialize plan"),
4176 )
4177 .expect("write plan");
4178
4179 run([
4180 OsString::from("status"),
4181 OsString::from("--plan"),
4182 OsString::from(plan_path.as_os_str()),
4183 OsString::from("--out"),
4184 OsString::from(out_path.as_os_str()),
4185 ])
4186 .expect("write restore status");
4187
4188 let status: RestoreStatus =
4189 serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
4190 .expect("decode restore status");
4191 let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
4192
4193 fs::remove_dir_all(root).expect("remove temp root");
4194 assert_eq!(status.status_version, 1);
4195 assert_eq!(status.backup_id.as_str(), "backup-test");
4196 assert!(status.ready);
4197 assert!(status.readiness_reasons.is_empty());
4198 assert_eq!(status.member_count, 2);
4199 assert_eq!(status.phase_count, 1);
4200 assert_eq!(status.planned_snapshot_uploads, 2);
4201 assert_eq!(status.planned_snapshot_loads, 2);
4202 assert_eq!(status.planned_code_reinstalls, 2);
4203 assert_eq!(status.planned_verification_checks, 2);
4204 assert_eq!(status.planned_operations, 8);
4205 assert_eq!(status.phases[0].members[0].source_canister, ROOT);
4206 assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
4207 }
4208
4209 #[test]
4211 fn run_restore_apply_dry_run_writes_operations() {
4212 let root = temp_dir("canic-cli-restore-apply-dry-run");
4213 fs::create_dir_all(&root).expect("create temp root");
4214 let plan_path = root.join("restore-plan.json");
4215 let status_path = root.join("restore-status.json");
4216 let out_path = root.join("restore-apply-dry-run.json");
4217 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4218 let status = RestoreStatus::from_plan(&plan);
4219
4220 fs::write(
4221 &plan_path,
4222 serde_json::to_vec(&plan).expect("serialize plan"),
4223 )
4224 .expect("write plan");
4225 fs::write(
4226 &status_path,
4227 serde_json::to_vec(&status).expect("serialize status"),
4228 )
4229 .expect("write status");
4230
4231 run([
4232 OsString::from("apply"),
4233 OsString::from("--plan"),
4234 OsString::from(plan_path.as_os_str()),
4235 OsString::from("--status"),
4236 OsString::from(status_path.as_os_str()),
4237 OsString::from("--dry-run"),
4238 OsString::from("--out"),
4239 OsString::from(out_path.as_os_str()),
4240 ])
4241 .expect("write apply dry-run");
4242
4243 let dry_run: RestoreApplyDryRun =
4244 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4245 .expect("decode dry-run");
4246 let dry_run_json: serde_json::Value =
4247 serde_json::to_value(&dry_run).expect("encode dry-run");
4248
4249 fs::remove_dir_all(root).expect("remove temp root");
4250 assert_eq!(dry_run.dry_run_version, 1);
4251 assert_eq!(dry_run.backup_id.as_str(), "backup-test");
4252 assert!(dry_run.ready);
4253 assert!(dry_run.status_supplied);
4254 assert_eq!(dry_run.member_count, 2);
4255 assert_eq!(dry_run.phase_count, 1);
4256 assert_eq!(dry_run.planned_snapshot_uploads, 2);
4257 assert_eq!(dry_run.planned_operations, 8);
4258 assert_eq!(dry_run.rendered_operations, 8);
4259 assert_eq!(dry_run_json["operation_counts"]["snapshot_uploads"], 2);
4260 assert_eq!(dry_run_json["operation_counts"]["snapshot_loads"], 2);
4261 assert_eq!(dry_run_json["operation_counts"]["code_reinstalls"], 2);
4262 assert_eq!(dry_run_json["operation_counts"]["member_verifications"], 2);
4263 assert_eq!(dry_run_json["operation_counts"]["fleet_verifications"], 0);
4264 assert_eq!(
4265 dry_run_json["operation_counts"]["verification_operations"],
4266 2
4267 );
4268 assert_eq!(
4269 dry_run_json["phases"][0]["operations"][0]["operation"],
4270 "upload-snapshot"
4271 );
4272 assert_eq!(
4273 dry_run_json["phases"][0]["operations"][3]["operation"],
4274 "verify-member"
4275 );
4276 assert_eq!(
4277 dry_run_json["phases"][0]["operations"][3]["verification_kind"],
4278 "status"
4279 );
4280 assert_eq!(
4281 dry_run_json["phases"][0]["operations"][3]["verification_method"],
4282 serde_json::Value::Null
4283 );
4284 }
4285
4286 #[test]
4288 fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
4289 let root = temp_dir("canic-cli-restore-apply-artifacts");
4290 fs::create_dir_all(&root).expect("create temp root");
4291 let plan_path = root.join("restore-plan.json");
4292 let out_path = root.join("restore-apply-dry-run.json");
4293 let journal_path = root.join("restore-apply-journal.json");
4294 let status_path = root.join("restore-apply-status.json");
4295 let mut manifest = restore_ready_manifest();
4296 write_manifest_artifacts(&root, &mut manifest);
4297 let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
4298
4299 fs::write(
4300 &plan_path,
4301 serde_json::to_vec(&plan).expect("serialize plan"),
4302 )
4303 .expect("write plan");
4304
4305 run([
4306 OsString::from("apply"),
4307 OsString::from("--plan"),
4308 OsString::from(plan_path.as_os_str()),
4309 OsString::from("--backup-dir"),
4310 OsString::from(root.as_os_str()),
4311 OsString::from("--dry-run"),
4312 OsString::from("--out"),
4313 OsString::from(out_path.as_os_str()),
4314 OsString::from("--journal-out"),
4315 OsString::from(journal_path.as_os_str()),
4316 ])
4317 .expect("write apply dry-run");
4318 run([
4319 OsString::from("apply-status"),
4320 OsString::from("--journal"),
4321 OsString::from(journal_path.as_os_str()),
4322 OsString::from("--out"),
4323 OsString::from(status_path.as_os_str()),
4324 ])
4325 .expect("write apply status");
4326
4327 let dry_run: RestoreApplyDryRun =
4328 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4329 .expect("decode dry-run");
4330 let validation = dry_run
4331 .artifact_validation
4332 .expect("artifact validation should be present");
4333 let journal_json: serde_json::Value =
4334 serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
4335 .expect("decode journal");
4336 let status_json: serde_json::Value =
4337 serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
4338 .expect("decode apply status");
4339
4340 fs::remove_dir_all(root).expect("remove temp root");
4341 assert_eq!(validation.checked_members, 2);
4342 assert!(validation.artifacts_present);
4343 assert!(validation.checksums_verified);
4344 assert_eq!(validation.members_with_expected_checksums, 2);
4345 assert_eq!(journal_json["ready"], true);
4346 assert_eq!(journal_json["operation_count"], 8);
4347 assert_eq!(journal_json["operation_counts"]["snapshot_uploads"], 2);
4348 assert_eq!(journal_json["operation_counts"]["snapshot_loads"], 2);
4349 assert_eq!(journal_json["operation_counts"]["code_reinstalls"], 2);
4350 assert_eq!(journal_json["operation_counts"]["member_verifications"], 2);
4351 assert_eq!(journal_json["operation_counts"]["fleet_verifications"], 0);
4352 assert_eq!(
4353 journal_json["operation_counts"]["verification_operations"],
4354 2
4355 );
4356 assert_eq!(journal_json["ready_operations"], 8);
4357 assert_eq!(journal_json["blocked_operations"], 0);
4358 assert_eq!(journal_json["operations"][0]["state"], "ready");
4359 assert_eq!(status_json["ready"], true);
4360 assert_eq!(status_json["operation_count"], 8);
4361 assert_eq!(status_json["operation_counts"]["snapshot_uploads"], 2);
4362 assert_eq!(status_json["operation_counts"]["snapshot_loads"], 2);
4363 assert_eq!(status_json["operation_counts"]["code_reinstalls"], 2);
4364 assert_eq!(status_json["operation_counts"]["member_verifications"], 2);
4365 assert_eq!(status_json["operation_counts"]["fleet_verifications"], 0);
4366 assert_eq!(
4367 status_json["operation_counts"]["verification_operations"],
4368 2
4369 );
4370 assert_eq!(status_json["operation_counts_supplied"], true);
4371 assert_eq!(status_json["progress"]["operation_count"], 8);
4372 assert_eq!(status_json["progress"]["completed_operations"], 0);
4373 assert_eq!(status_json["progress"]["remaining_operations"], 8);
4374 assert_eq!(status_json["progress"]["transitionable_operations"], 8);
4375 assert_eq!(status_json["progress"]["attention_operations"], 0);
4376 assert_eq!(status_json["progress"]["completion_basis_points"], 0);
4377 assert_eq!(status_json["next_ready_sequence"], 0);
4378 assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
4379 }
4380
4381 #[test]
4383 fn run_restore_apply_status_rejects_invalid_journal() {
4384 let root = temp_dir("canic-cli-restore-apply-status-invalid");
4385 fs::create_dir_all(&root).expect("create temp root");
4386 let journal_path = root.join("restore-apply-journal.json");
4387 let out_path = root.join("restore-apply-status.json");
4388 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4389 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4390 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4391 journal.operation_count += 1;
4392
4393 fs::write(
4394 &journal_path,
4395 serde_json::to_vec(&journal).expect("serialize journal"),
4396 )
4397 .expect("write journal");
4398
4399 let err = run([
4400 OsString::from("apply-status"),
4401 OsString::from("--journal"),
4402 OsString::from(journal_path.as_os_str()),
4403 OsString::from("--out"),
4404 OsString::from(out_path.as_os_str()),
4405 ])
4406 .expect_err("invalid journal should fail");
4407
4408 assert!(!out_path.exists());
4409 fs::remove_dir_all(root).expect("remove temp root");
4410 assert!(matches!(
4411 err,
4412 RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
4413 field: "operation_count",
4414 ..
4415 })
4416 ));
4417 }
4418
4419 #[test]
4421 fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
4422 let fixture = RestoreCliFixture::new(
4423 "canic-cli-restore-apply-status-pending",
4424 "restore-apply-status.json",
4425 );
4426 let mut journal = ready_apply_journal();
4427 journal
4428 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4429 .expect("claim operation");
4430 fixture.write_journal(&journal);
4431
4432 let err = fixture
4433 .run_apply_status(&["--require-no-pending"])
4434 .expect_err("pending operation should fail requirement");
4435
4436 assert!(fixture.out_path.exists());
4437 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4438
4439 assert_eq!(status.pending_operations, 1);
4440 assert_eq!(status.next_transition_sequence, Some(0));
4441 assert_eq!(status.pending_summary.pending_operations, 1);
4442 assert_eq!(status.pending_summary.pending_sequence, Some(0));
4443 assert_eq!(
4444 status.pending_summary.pending_updated_at.as_deref(),
4445 Some("2026-05-04T12:00:00Z")
4446 );
4447 assert!(status.pending_summary.pending_updated_at_known);
4448 assert_eq!(
4449 status.next_transition_updated_at.as_deref(),
4450 Some("2026-05-04T12:00:00Z")
4451 );
4452 assert!(matches!(
4453 err,
4454 RestoreCommandError::RestoreApplyPending {
4455 pending_operations: 1,
4456 next_transition_sequence: Some(0),
4457 ..
4458 }
4459 ));
4460 }
4461
4462 #[test]
4464 fn run_restore_apply_status_require_no_pending_before_writes_status_then_fails() {
4465 let fixture = RestoreCliFixture::new(
4466 "canic-cli-restore-apply-status-stale-pending",
4467 "restore-apply-status.json",
4468 );
4469 let mut journal = ready_apply_journal();
4470 journal
4471 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4472 .expect("claim operation");
4473 fixture.write_journal(&journal);
4474
4475 let err = fixture
4476 .run_apply_status(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
4477 .expect_err("stale pending operation should fail requirement");
4478
4479 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4480
4481 assert_eq!(status.pending_summary.pending_sequence, Some(0));
4482 assert_eq!(
4483 status.pending_summary.pending_updated_at.as_deref(),
4484 Some("2026-05-04T12:00:00Z")
4485 );
4486 assert!(matches!(
4487 err,
4488 RestoreCommandError::RestoreApplyPendingStale {
4489 cutoff_updated_at,
4490 pending_sequence: Some(0),
4491 pending_updated_at,
4492 ..
4493 } if cutoff_updated_at == "2026-05-05T12:00:00Z"
4494 && pending_updated_at.as_deref() == Some("2026-05-04T12:00:00Z")
4495 ));
4496 }
4497
4498 #[test]
4500 fn run_restore_apply_status_require_progress_writes_status_then_fails() {
4501 let fixture = RestoreCliFixture::new(
4502 "canic-cli-restore-apply-status-progress",
4503 "restore-apply-status.json",
4504 );
4505 let journal = ready_apply_journal();
4506 fixture.write_journal(&journal);
4507
4508 let err = fixture
4509 .run_apply_status(&[
4510 "--require-remaining-count",
4511 "7",
4512 "--require-attention-count",
4513 "0",
4514 "--require-completion-basis-points",
4515 "0",
4516 ])
4517 .expect_err("remaining progress mismatch should fail requirement");
4518
4519 let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
4520
4521 assert_eq!(status.progress.remaining_operations, 8);
4522 assert_eq!(status.progress.attention_operations, 0);
4523 assert_eq!(status.progress.completion_basis_points, 0);
4524 assert!(matches!(
4525 err,
4526 RestoreCommandError::RestoreApplyProgressMismatch {
4527 field: "remaining_operations",
4528 expected: 7,
4529 actual: 8,
4530 ..
4531 }
4532 ));
4533 }
4534
4535 #[test]
4537 fn run_restore_apply_status_require_ready_writes_status_then_fails() {
4538 let root = temp_dir("canic-cli-restore-apply-status-ready");
4539 fs::create_dir_all(&root).expect("create temp root");
4540 let journal_path = root.join("restore-apply-journal.json");
4541 let out_path = root.join("restore-apply-status.json");
4542 let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
4543 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4544 let journal = RestoreApplyJournal::from_dry_run(&dry_run);
4545
4546 fs::write(
4547 &journal_path,
4548 serde_json::to_vec(&journal).expect("serialize journal"),
4549 )
4550 .expect("write journal");
4551
4552 let err = run([
4553 OsString::from("apply-status"),
4554 OsString::from("--journal"),
4555 OsString::from(journal_path.as_os_str()),
4556 OsString::from("--out"),
4557 OsString::from(out_path.as_os_str()),
4558 OsString::from("--require-ready"),
4559 ])
4560 .expect_err("unready journal should fail requirement");
4561
4562 let status: RestoreApplyJournalStatus =
4563 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
4564 .expect("decode apply status");
4565
4566 fs::remove_dir_all(root).expect("remove temp root");
4567 assert!(!status.ready);
4568 assert_eq!(status.blocked_operations, status.operation_count);
4569 assert!(
4570 status
4571 .blocked_reasons
4572 .contains(&"missing-snapshot-checksum".to_string())
4573 );
4574 assert!(matches!(
4575 err,
4576 RestoreCommandError::RestoreApplyNotReady { reasons, .. }
4577 if reasons.contains(&"missing-snapshot-checksum".to_string())
4578 ));
4579 }
4580
4581 #[test]
4583 fn run_restore_apply_report_writes_attention_summary() {
4584 let root = temp_dir("canic-cli-restore-apply-report");
4585 fs::create_dir_all(&root).expect("create temp root");
4586 let journal_path = root.join("restore-apply-journal.json");
4587 let out_path = root.join("restore-apply-report.json");
4588 let mut journal = ready_apply_journal();
4589 journal
4590 .mark_operation_failed_at(
4591 0,
4592 "dfx-upload-failed".to_string(),
4593 Some("2026-05-05T12:00:00Z".to_string()),
4594 )
4595 .expect("mark failed operation");
4596 journal
4597 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4598 .expect("mark pending operation");
4599
4600 fs::write(
4601 &journal_path,
4602 serde_json::to_vec(&journal).expect("serialize journal"),
4603 )
4604 .expect("write journal");
4605
4606 run([
4607 OsString::from("apply-report"),
4608 OsString::from("--journal"),
4609 OsString::from(journal_path.as_os_str()),
4610 OsString::from("--out"),
4611 OsString::from(out_path.as_os_str()),
4612 ])
4613 .expect("write apply report");
4614
4615 let report: RestoreApplyJournalReport =
4616 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
4617 .expect("decode apply report");
4618 let report_json: serde_json::Value =
4619 serde_json::to_value(&report).expect("encode apply report");
4620
4621 fs::remove_dir_all(root).expect("remove temp root");
4622 assert_eq!(report.backup_id, "backup-test");
4623 assert!(report.attention_required);
4624 assert_eq!(report.failed_operations, 1);
4625 assert_eq!(report.pending_operations, 1);
4626 assert_eq!(report.operation_counts.snapshot_uploads, 2);
4627 assert_eq!(report.operation_counts.snapshot_loads, 2);
4628 assert_eq!(report.operation_counts.code_reinstalls, 2);
4629 assert_eq!(report.operation_counts.member_verifications, 2);
4630 assert_eq!(report.operation_counts.fleet_verifications, 0);
4631 assert_eq!(report.operation_counts.verification_operations, 2);
4632 assert!(report.operation_counts_supplied);
4633 assert_eq!(report.progress.operation_count, 8);
4634 assert_eq!(report.progress.completed_operations, 0);
4635 assert_eq!(report.progress.remaining_operations, 8);
4636 assert_eq!(report.progress.transitionable_operations, 7);
4637 assert_eq!(report.progress.attention_operations, 2);
4638 assert_eq!(report.progress.completion_basis_points, 0);
4639 assert_eq!(report.pending_summary.pending_operations, 1);
4640 assert_eq!(report.pending_summary.pending_sequence, Some(1));
4641 assert_eq!(
4642 report.pending_summary.pending_updated_at.as_deref(),
4643 Some("2026-05-05T12:01:00Z")
4644 );
4645 assert!(report.pending_summary.pending_updated_at_known);
4646 assert_eq!(report.failed.len(), 1);
4647 assert_eq!(report.pending.len(), 1);
4648 assert_eq!(report.failed[0].sequence, 0);
4649 assert_eq!(report.pending[0].sequence, 1);
4650 assert_eq!(
4651 report.next_transition.as_ref().map(|op| op.sequence),
4652 Some(1)
4653 );
4654 assert_eq!(report_json["outcome"], "failed");
4655 assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
4656 }
4657
4658 #[test]
4660 fn run_restore_apply_report_require_progress_writes_report_then_fails() {
4661 let fixture = RestoreCliFixture::new(
4662 "canic-cli-restore-apply-report-progress",
4663 "restore-apply-report.json",
4664 );
4665 let journal = ready_apply_journal();
4666 fixture.write_journal(&journal);
4667
4668 let err = fixture
4669 .run_apply_report(&[
4670 "--require-remaining-count",
4671 "8",
4672 "--require-attention-count",
4673 "1",
4674 "--require-completion-basis-points",
4675 "0",
4676 ])
4677 .expect_err("attention progress mismatch should fail requirement");
4678
4679 let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4680
4681 assert_eq!(report.progress.remaining_operations, 8);
4682 assert_eq!(report.progress.attention_operations, 0);
4683 assert_eq!(report.progress.completion_basis_points, 0);
4684 assert!(matches!(
4685 err,
4686 RestoreCommandError::RestoreApplyProgressMismatch {
4687 field: "attention_operations",
4688 expected: 1,
4689 actual: 0,
4690 ..
4691 }
4692 ));
4693 }
4694
4695 #[test]
4697 fn run_restore_apply_report_require_no_pending_before_writes_report_then_fails() {
4698 let fixture = RestoreCliFixture::new(
4699 "canic-cli-restore-apply-report-stale-pending",
4700 "restore-apply-report.json",
4701 );
4702 let mut journal = ready_apply_journal();
4703 journal
4704 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4705 .expect("mark pending operation");
4706 fixture.write_journal(&journal);
4707
4708 let err = fixture
4709 .run_apply_report(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
4710 .expect_err("stale pending report should fail requirement");
4711
4712 let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4713
4714 assert_eq!(report.pending_summary.pending_sequence, Some(0));
4715 assert!(matches!(
4716 err,
4717 RestoreCommandError::RestoreApplyPendingStale {
4718 pending_sequence: Some(0),
4719 ..
4720 }
4721 ));
4722 }
4723
4724 #[test]
4726 fn run_restore_run_dry_run_writes_native_runner_preview() {
4727 let root = temp_dir("canic-cli-restore-run-dry-run");
4728 fs::create_dir_all(&root).expect("create temp root");
4729 let journal_path = root.join("restore-apply-journal.json");
4730 let out_path = root.join("restore-run-dry-run.json");
4731 let journal = ready_apply_journal();
4732
4733 fs::write(
4734 &journal_path,
4735 serde_json::to_vec(&journal).expect("serialize journal"),
4736 )
4737 .expect("write journal");
4738
4739 run([
4740 OsString::from("run"),
4741 OsString::from("--journal"),
4742 OsString::from(journal_path.as_os_str()),
4743 OsString::from("--dry-run"),
4744 OsString::from("--dfx"),
4745 OsString::from("/tmp/dfx"),
4746 OsString::from("--network"),
4747 OsString::from("local"),
4748 OsString::from("--updated-at"),
4749 OsString::from("2026-05-05T12:00:00Z"),
4750 OsString::from("--out"),
4751 OsString::from(out_path.as_os_str()),
4752 OsString::from("--require-state-updated-at"),
4753 OsString::from("2026-05-05T12:00:00Z"),
4754 ])
4755 .expect("write restore run dry-run");
4756
4757 let dry_run: serde_json::Value =
4758 serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4759 .expect("decode dry-run");
4760
4761 fs::remove_dir_all(root).expect("remove temp root");
4762 assert_eq!(dry_run["run_version"], 1);
4763 assert_eq!(dry_run["backup_id"], "backup-test");
4764 assert_eq!(dry_run["run_mode"], "dry-run");
4765 assert_eq!(dry_run["dry_run"], true);
4766 assert_eq!(
4767 dry_run["requested_state_updated_at"],
4768 "2026-05-05T12:00:00Z"
4769 );
4770 assert_eq!(dry_run["ready"], true);
4771 assert_eq!(dry_run["complete"], false);
4772 assert_eq!(dry_run["attention_required"], false);
4773 assert_eq!(dry_run["operation_counts"]["snapshot_uploads"], 2);
4774 assert_eq!(dry_run["operation_counts"]["snapshot_loads"], 2);
4775 assert_eq!(dry_run["operation_counts"]["code_reinstalls"], 2);
4776 assert_eq!(dry_run["operation_counts"]["member_verifications"], 2);
4777 assert_eq!(dry_run["operation_counts"]["fleet_verifications"], 0);
4778 assert_eq!(dry_run["operation_counts"]["verification_operations"], 2);
4779 assert_eq!(dry_run["operation_counts_supplied"], true);
4780 assert_eq!(dry_run["progress"]["operation_count"], 8);
4781 assert_eq!(dry_run["progress"]["completed_operations"], 0);
4782 assert_eq!(dry_run["progress"]["remaining_operations"], 8);
4783 assert_eq!(dry_run["progress"]["transitionable_operations"], 8);
4784 assert_eq!(dry_run["progress"]["attention_operations"], 0);
4785 assert_eq!(dry_run["progress"]["completion_basis_points"], 0);
4786 assert_eq!(dry_run["pending_summary"]["pending_operations"], 0);
4787 assert_eq!(
4788 dry_run["pending_summary"]["pending_operation_available"],
4789 false
4790 );
4791 assert_eq!(dry_run["operation_receipt_count"], 0);
4792 assert_eq!(dry_run["operation_receipt_summary"]["total_receipts"], 0);
4793 assert_eq!(dry_run["operation_receipt_summary"]["command_completed"], 0);
4794 assert_eq!(dry_run["operation_receipt_summary"]["command_failed"], 0);
4795 assert_eq!(dry_run["operation_receipt_summary"]["pending_recovered"], 0);
4796 assert_batch_summary(
4797 &dry_run["batch_summary"],
4798 json!({
4799 "requested_max_steps": null,
4800 "initial_ready_operations": 8,
4801 "initial_remaining_operations": 8,
4802 "executed_operations": 0,
4803 "remaining_ready_operations": 8,
4804 "remaining_operations": 8,
4805 "ready_operations_delta": 0,
4806 "remaining_operations_delta": 0,
4807 "stopped_by_max_steps": false,
4808 "complete": false,
4809 }),
4810 );
4811 assert_eq!(dry_run["stopped_reason"], "preview");
4812 assert_eq!(dry_run["next_action"], "rerun");
4813 assert_eq!(dry_run["operation_available"], true);
4814 assert_eq!(dry_run["command_available"], true);
4815 assert_eq!(dry_run["next_transition"]["sequence"], 0);
4816 assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
4817 assert_eq!(
4818 dry_run["command"]["args"],
4819 json!([
4820 "canister",
4821 "--network",
4822 "local",
4823 "snapshot",
4824 "upload",
4825 "--dir",
4826 "artifacts/root",
4827 ROOT
4828 ])
4829 );
4830 assert_eq!(dry_run["command"]["mutates"], true);
4831 }
4832
4833 #[test]
4835 fn run_restore_run_unclaim_pending_marks_operation_ready() {
4836 let root = temp_dir("canic-cli-restore-run-unclaim-pending");
4837 fs::create_dir_all(&root).expect("create temp root");
4838 let journal_path = root.join("restore-apply-journal.json");
4839 let out_path = root.join("restore-run.json");
4840 let mut journal = ready_apply_journal();
4841 journal
4842 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4843 .expect("mark pending operation");
4844
4845 fs::write(
4846 &journal_path,
4847 serde_json::to_vec(&journal).expect("serialize journal"),
4848 )
4849 .expect("write journal");
4850
4851 run([
4852 OsString::from("run"),
4853 OsString::from("--journal"),
4854 OsString::from(journal_path.as_os_str()),
4855 OsString::from("--unclaim-pending"),
4856 OsString::from("--updated-at"),
4857 OsString::from("2026-05-05T12:02:00Z"),
4858 OsString::from("--out"),
4859 OsString::from(out_path.as_os_str()),
4860 ])
4861 .expect("unclaim pending operation");
4862
4863 let run_summary: serde_json::Value =
4864 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4865 .expect("decode run summary");
4866 let updated: RestoreApplyJournal =
4867 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4868 .expect("decode updated journal");
4869
4870 fs::remove_dir_all(root).expect("remove temp root");
4871 assert_eq!(run_summary["run_mode"], "unclaim-pending");
4872 assert_eq!(run_summary["unclaim_pending"], true);
4873 assert_eq!(run_summary["stopped_reason"], "recovered-pending");
4874 assert_eq!(run_summary["next_action"], "rerun");
4875 assert_eq!(
4876 run_summary["requested_state_updated_at"],
4877 "2026-05-05T12:02:00Z"
4878 );
4879 assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
4880 assert_eq!(run_summary["recovered_operation"]["state"], "pending");
4881 assert_eq!(run_summary["operation_receipt_count"], 1);
4882 assert_eq!(
4883 run_summary["operation_receipt_summary"]["total_receipts"],
4884 1
4885 );
4886 assert_batch_summary(
4887 &run_summary["batch_summary"],
4888 json!({
4889 "requested_max_steps": null,
4890 "initial_ready_operations": 7,
4891 "initial_remaining_operations": 8,
4892 "executed_operations": 0,
4893 "remaining_ready_operations": 8,
4894 "remaining_operations": 8,
4895 "ready_operations_delta": 1,
4896 "remaining_operations_delta": 0,
4897 "stopped_by_max_steps": false,
4898 "complete": false,
4899 }),
4900 );
4901 assert_eq!(
4902 run_summary["operation_receipt_summary"]["command_completed"],
4903 0
4904 );
4905 assert_eq!(
4906 run_summary["operation_receipt_summary"]["command_failed"],
4907 0
4908 );
4909 assert_eq!(
4910 run_summary["operation_receipt_summary"]["pending_recovered"],
4911 1
4912 );
4913 assert_eq!(
4914 run_summary["operation_receipts"][0]["event"],
4915 "pending-recovered"
4916 );
4917 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
4918 assert_eq!(run_summary["operation_receipts"][0]["state"], "ready");
4919 assert_eq!(
4920 run_summary["operation_receipts"][0]["updated_at"],
4921 "2026-05-05T12:02:00Z"
4922 );
4923 assert_eq!(run_summary["pending_operations"], 0);
4924 assert_eq!(run_summary["ready_operations"], 8);
4925 assert_eq!(run_summary["attention_required"], false);
4926 assert_eq!(updated.pending_operations, 0);
4927 assert_eq!(updated.ready_operations, 8);
4928 assert_eq!(
4929 updated.operations[0].state,
4930 RestoreApplyOperationState::Ready
4931 );
4932 assert_eq!(
4933 updated.operations[0].state_updated_at.as_deref(),
4934 Some("2026-05-05T12:02:00Z")
4935 );
4936 }
4937
4938 #[test]
4940 fn run_restore_run_execute_marks_completed_operation() {
4941 let root = temp_dir("canic-cli-restore-run-execute");
4942 fs::create_dir_all(&root).expect("create temp root");
4943 let journal_path = root.join("restore-apply-journal.json");
4944 let out_path = root.join("restore-run.json");
4945 let journal = ready_apply_journal();
4946
4947 fs::write(
4948 &journal_path,
4949 serde_json::to_vec(&journal).expect("serialize journal"),
4950 )
4951 .expect("write journal");
4952
4953 run([
4954 OsString::from("run"),
4955 OsString::from("--journal"),
4956 OsString::from(journal_path.as_os_str()),
4957 OsString::from("--execute"),
4958 OsString::from("--dfx"),
4959 OsString::from("/bin/true"),
4960 OsString::from("--max-steps"),
4961 OsString::from("1"),
4962 OsString::from("--updated-at"),
4963 OsString::from("2026-05-05T12:03:00Z"),
4964 OsString::from("--out"),
4965 OsString::from(out_path.as_os_str()),
4966 OsString::from("--require-receipt-updated-at"),
4967 OsString::from("2026-05-05T12:03:00Z"),
4968 ])
4969 .expect("execute one restore run step");
4970
4971 let run_summary: serde_json::Value =
4972 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4973 .expect("decode run summary");
4974 let updated: RestoreApplyJournal =
4975 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4976 .expect("decode updated journal");
4977
4978 fs::remove_dir_all(root).expect("remove temp root");
4979 assert_eq!(run_summary["run_mode"], "execute");
4980 assert_eq!(run_summary["execute"], true);
4981 assert_eq!(run_summary["dry_run"], false);
4982 assert_eq!(run_summary["max_steps_reached"], true);
4983 assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
4984 assert_eq!(run_summary["next_action"], "rerun");
4985 assert_eq!(
4986 run_summary["requested_state_updated_at"],
4987 "2026-05-05T12:03:00Z"
4988 );
4989 assert_eq!(run_summary["executed_operation_count"], 1);
4990 assert_completed_execute_batch_summary(&run_summary);
4991 assert_eq!(run_summary["operation_receipt_count"], 1);
4992 assert_eq!(
4993 run_summary["operation_receipt_summary"]["total_receipts"],
4994 1
4995 );
4996 assert_eq!(
4997 run_summary["operation_receipt_summary"]["command_completed"],
4998 1
4999 );
5000 assert_eq!(
5001 run_summary["operation_receipt_summary"]["command_failed"],
5002 0
5003 );
5004 assert_eq!(
5005 run_summary["operation_receipt_summary"]["pending_recovered"],
5006 0
5007 );
5008 assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
5009 assert_eq!(
5010 run_summary["executed_operations"][0]["command"]["program"],
5011 "/bin/true"
5012 );
5013 assert_eq!(
5014 run_summary["operation_receipts"][0]["event"],
5015 "command-completed"
5016 );
5017 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
5018 assert_eq!(run_summary["operation_receipts"][0]["state"], "completed");
5019 assert_eq!(
5020 run_summary["operation_receipts"][0]["command"]["program"],
5021 "/bin/true"
5022 );
5023 assert_eq!(run_summary["operation_receipts"][0]["status"], "0");
5024 assert_eq!(
5025 run_summary["operation_receipts"][0]["updated_at"],
5026 "2026-05-05T12:03:00Z"
5027 );
5028 assert_eq!(updated.completed_operations, 1);
5029 assert_eq!(updated.pending_operations, 0);
5030 assert_eq!(updated.failed_operations, 0);
5031 assert_eq!(
5032 updated.operations[0].state,
5033 RestoreApplyOperationState::Completed
5034 );
5035 assert_eq!(
5036 updated.operations[0].state_updated_at.as_deref(),
5037 Some("2026-05-05T12:03:00Z")
5038 );
5039 }
5040
5041 #[test]
5043 fn run_restore_run_require_complete_writes_summary_then_fails() {
5044 let root = temp_dir("canic-cli-restore-run-require-complete");
5045 fs::create_dir_all(&root).expect("create temp root");
5046 let journal_path = root.join("restore-apply-journal.json");
5047 let out_path = root.join("restore-run.json");
5048 let journal = ready_apply_journal();
5049
5050 fs::write(
5051 &journal_path,
5052 serde_json::to_vec(&journal).expect("serialize journal"),
5053 )
5054 .expect("write journal");
5055
5056 let err = run([
5057 OsString::from("run"),
5058 OsString::from("--journal"),
5059 OsString::from(journal_path.as_os_str()),
5060 OsString::from("--execute"),
5061 OsString::from("--dfx"),
5062 OsString::from("/bin/true"),
5063 OsString::from("--max-steps"),
5064 OsString::from("1"),
5065 OsString::from("--out"),
5066 OsString::from(out_path.as_os_str()),
5067 OsString::from("--require-complete"),
5068 ])
5069 .expect_err("incomplete run should fail requirement");
5070
5071 let run_summary: serde_json::Value =
5072 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5073 .expect("decode run summary");
5074
5075 fs::remove_dir_all(root).expect("remove temp root");
5076 assert_eq!(run_summary["executed_operation_count"], 1);
5077 assert_eq!(run_summary["complete"], false);
5078 assert!(matches!(
5079 err,
5080 RestoreCommandError::RestoreApplyIncomplete {
5081 completed_operations: 1,
5082 operation_count: 8,
5083 ..
5084 }
5085 ));
5086 }
5087
5088 #[test]
5090 fn run_restore_run_execute_marks_failed_operation() {
5091 let root = temp_dir("canic-cli-restore-run-execute-failed");
5092 fs::create_dir_all(&root).expect("create temp root");
5093 let journal_path = root.join("restore-apply-journal.json");
5094 let out_path = root.join("restore-run.json");
5095 let journal = ready_apply_journal();
5096
5097 fs::write(
5098 &journal_path,
5099 serde_json::to_vec(&journal).expect("serialize journal"),
5100 )
5101 .expect("write journal");
5102
5103 let err = run([
5104 OsString::from("run"),
5105 OsString::from("--journal"),
5106 OsString::from(journal_path.as_os_str()),
5107 OsString::from("--execute"),
5108 OsString::from("--dfx"),
5109 OsString::from("/bin/false"),
5110 OsString::from("--max-steps"),
5111 OsString::from("1"),
5112 OsString::from("--updated-at"),
5113 OsString::from("2026-05-05T12:04:00Z"),
5114 OsString::from("--out"),
5115 OsString::from(out_path.as_os_str()),
5116 ])
5117 .expect_err("failing runner command should fail");
5118
5119 let run_summary: serde_json::Value =
5120 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5121 .expect("decode run summary");
5122 let updated: RestoreApplyJournal =
5123 serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
5124 .expect("decode updated journal");
5125
5126 fs::remove_dir_all(root).expect("remove temp root");
5127 assert!(matches!(
5128 err,
5129 RestoreCommandError::RestoreRunCommandFailed {
5130 sequence: 0,
5131 status,
5132 } if status == "1"
5133 ));
5134 assert_eq!(updated.failed_operations, 1);
5135 assert_eq!(updated.pending_operations, 0);
5136 assert_eq!(
5137 updated.operations[0].state,
5138 RestoreApplyOperationState::Failed
5139 );
5140 assert_eq!(run_summary["execute"], true);
5141 assert_eq!(run_summary["attention_required"], true);
5142 assert_eq!(run_summary["outcome"], "failed");
5143 assert_eq!(run_summary["stopped_reason"], "command-failed");
5144 assert_eq!(run_summary["next_action"], "inspect-failed-operation");
5145 assert_eq!(
5146 run_summary["requested_state_updated_at"],
5147 "2026-05-05T12:04:00Z"
5148 );
5149 assert_eq!(run_summary["executed_operation_count"], 1);
5150 assert_eq!(run_summary["operation_receipt_count"], 1);
5151 assert_eq!(
5152 run_summary["operation_receipt_summary"]["total_receipts"],
5153 1
5154 );
5155 assert_eq!(
5156 run_summary["operation_receipt_summary"]["command_completed"],
5157 0
5158 );
5159 assert_eq!(
5160 run_summary["operation_receipt_summary"]["command_failed"],
5161 1
5162 );
5163 assert_eq!(
5164 run_summary["operation_receipt_summary"]["pending_recovered"],
5165 0
5166 );
5167 assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
5168 assert_eq!(run_summary["executed_operations"][0]["status"], "1");
5169 assert_eq!(
5170 run_summary["operation_receipts"][0]["event"],
5171 "command-failed"
5172 );
5173 assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
5174 assert_eq!(run_summary["operation_receipts"][0]["state"], "failed");
5175 assert_eq!(
5176 run_summary["operation_receipts"][0]["command"]["program"],
5177 "/bin/false"
5178 );
5179 assert_eq!(run_summary["operation_receipts"][0]["status"], "1");
5180 assert_eq!(
5181 run_summary["operation_receipts"][0]["updated_at"],
5182 "2026-05-05T12:04:00Z"
5183 );
5184 assert_eq!(
5185 updated.operations[0].state_updated_at.as_deref(),
5186 Some("2026-05-05T12:04:00Z")
5187 );
5188 assert_eq!(
5189 updated.operations[0].blocking_reasons,
5190 vec!["runner-command-exit-1".to_string()]
5191 );
5192 }
5193
5194 #[test]
5196 fn run_restore_run_require_no_attention_writes_summary_then_fails() {
5197 let fixture = RestoreCliFixture::new(
5198 "canic-cli-restore-run-require-attention",
5199 "restore-run.json",
5200 );
5201 let mut journal = ready_apply_journal();
5202 journal
5203 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
5204 .expect("mark pending operation");
5205 fixture.write_journal(&journal);
5206
5207 let err = fixture
5208 .run_restore_run(&["--dry-run", "--require-no-attention"])
5209 .expect_err("attention run should fail requirement");
5210
5211 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5212
5213 assert_eq!(run_summary["attention_required"], true);
5214 assert_eq!(run_summary["outcome"], "pending");
5215 assert_eq!(run_summary["stopped_reason"], "pending");
5216 assert_eq!(run_summary["next_action"], "unclaim-pending");
5217 assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
5218 assert_eq!(
5219 run_summary["pending_summary"]["pending_updated_at"],
5220 "2026-05-05T12:01:00Z"
5221 );
5222 assert!(matches!(
5223 err,
5224 RestoreCommandError::RestoreApplyReportNeedsAttention {
5225 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
5226 ..
5227 }
5228 ));
5229 }
5230
5231 #[test]
5233 fn run_restore_run_require_no_pending_before_writes_summary_then_fails() {
5234 let fixture = RestoreCliFixture::new(
5235 "canic-cli-restore-run-require-stale-pending",
5236 "restore-run.json",
5237 );
5238 let mut journal = ready_apply_journal();
5239 journal
5240 .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
5241 .expect("mark pending operation");
5242 fixture.write_journal(&journal);
5243
5244 let err = fixture
5245 .run_restore_run(&[
5246 "--dry-run",
5247 "--require-no-pending-before",
5248 "2026-05-05T12:00:00Z",
5249 ])
5250 .expect_err("stale pending run should fail requirement");
5251
5252 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5253
5254 assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
5255 assert!(matches!(
5256 err,
5257 RestoreCommandError::RestoreApplyPendingStale {
5258 pending_sequence: Some(0),
5259 ..
5260 }
5261 ));
5262 }
5263
5264 #[test]
5266 fn run_restore_run_require_run_mode_writes_summary_then_fails() {
5267 let fixture =
5268 RestoreCliFixture::new("canic-cli-restore-run-require-run-mode", "restore-run.json");
5269 let journal = ready_apply_journal();
5270 fixture.write_journal(&journal);
5271
5272 let err = fixture
5273 .run_restore_run(&["--dry-run", "--require-run-mode", "execute"])
5274 .expect_err("run mode mismatch should fail requirement");
5275
5276 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5277
5278 assert_eq!(run_summary["run_mode"], "dry-run");
5279 assert!(matches!(
5280 err,
5281 RestoreCommandError::RestoreRunModeMismatch {
5282 expected,
5283 actual,
5284 ..
5285 } if expected == "execute" && actual == "dry-run"
5286 ));
5287 }
5288
5289 #[test]
5291 fn run_restore_run_require_executed_count_writes_summary_then_fails() {
5292 let root = temp_dir("canic-cli-restore-run-require-executed-count");
5293 fs::create_dir_all(&root).expect("create temp root");
5294 let journal_path = root.join("restore-apply-journal.json");
5295 let out_path = root.join("restore-run.json");
5296 let journal = ready_apply_journal();
5297
5298 fs::write(
5299 &journal_path,
5300 serde_json::to_vec(&journal).expect("serialize journal"),
5301 )
5302 .expect("write journal");
5303
5304 let err = run([
5305 OsString::from("run"),
5306 OsString::from("--journal"),
5307 OsString::from(journal_path.as_os_str()),
5308 OsString::from("--execute"),
5309 OsString::from("--dfx"),
5310 OsString::from("/bin/true"),
5311 OsString::from("--max-steps"),
5312 OsString::from("1"),
5313 OsString::from("--out"),
5314 OsString::from(out_path.as_os_str()),
5315 OsString::from("--require-executed-count"),
5316 OsString::from("2"),
5317 ])
5318 .expect_err("executed count mismatch should fail requirement");
5319
5320 let run_summary: serde_json::Value =
5321 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5322 .expect("decode run summary");
5323
5324 fs::remove_dir_all(root).expect("remove temp root");
5325 assert_eq!(run_summary["executed_operation_count"], 1);
5326 assert!(matches!(
5327 err,
5328 RestoreCommandError::RestoreRunExecutedCountMismatch {
5329 expected: 2,
5330 actual: 1,
5331 ..
5332 }
5333 ));
5334 }
5335
5336 #[test]
5338 fn run_restore_run_require_receipt_count_writes_summary_then_fails() {
5339 let fixture = RestoreCliFixture::new(
5340 "canic-cli-restore-run-require-receipt-count",
5341 "restore-run.json",
5342 );
5343 let journal = ready_apply_journal();
5344 fixture.write_journal(&journal);
5345
5346 let err = fixture
5347 .run_restore_run(&[
5348 "--execute",
5349 "--dfx",
5350 "/bin/true",
5351 "--max-steps",
5352 "1",
5353 "--require-receipt-count",
5354 "2",
5355 ])
5356 .expect_err("receipt count mismatch should fail requirement");
5357
5358 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5359
5360 assert_eq!(run_summary["operation_receipt_count"], 1);
5361 assert_eq!(
5362 run_summary["operation_receipt_summary"]["total_receipts"],
5363 1
5364 );
5365 assert!(matches!(
5366 err,
5367 RestoreCommandError::RestoreRunReceiptCountMismatch {
5368 expected: 2,
5369 actual: 1,
5370 ..
5371 }
5372 ));
5373 }
5374
5375 #[test]
5377 fn run_restore_run_require_receipt_kind_count_writes_summary_then_fails() {
5378 let fixture = RestoreCliFixture::new(
5379 "canic-cli-restore-run-require-receipt-kind-count",
5380 "restore-run.json",
5381 );
5382 let journal = ready_apply_journal();
5383 fixture.write_journal(&journal);
5384
5385 let err = fixture
5386 .run_restore_run(&[
5387 "--execute",
5388 "--dfx",
5389 "/bin/true",
5390 "--max-steps",
5391 "1",
5392 "--require-failed-receipt-count",
5393 "1",
5394 ])
5395 .expect_err("receipt kind count mismatch should fail requirement");
5396
5397 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5398
5399 assert_eq!(
5400 run_summary["operation_receipt_summary"]["command_failed"],
5401 0
5402 );
5403 assert_eq!(
5404 run_summary["operation_receipt_summary"]["command_completed"],
5405 1
5406 );
5407 assert!(matches!(
5408 err,
5409 RestoreCommandError::RestoreRunReceiptKindCountMismatch {
5410 receipt_kind: "command-failed",
5411 expected: 1,
5412 actual: 0,
5413 ..
5414 }
5415 ));
5416 }
5417
5418 #[test]
5420 fn run_restore_run_require_receipt_updated_at_writes_summary_then_fails() {
5421 let fixture = RestoreCliFixture::new(
5422 "canic-cli-restore-run-require-receipt-updated-at",
5423 "restore-run.json",
5424 );
5425 let journal = ready_apply_journal();
5426 fixture.write_journal(&journal);
5427
5428 let err = fixture
5429 .run_restore_run(&[
5430 "--execute",
5431 "--dfx",
5432 "/bin/true",
5433 "--max-steps",
5434 "1",
5435 "--updated-at",
5436 "2026-05-05T12:03:00Z",
5437 "--require-receipt-updated-at",
5438 "2026-05-05T12:04:00Z",
5439 ])
5440 .expect_err("receipt updated-at mismatch should fail requirement");
5441
5442 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5443
5444 assert_eq!(
5445 run_summary["operation_receipts"][0]["updated_at"],
5446 "2026-05-05T12:03:00Z"
5447 );
5448 assert!(matches!(
5449 err,
5450 RestoreCommandError::RestoreRunReceiptUpdatedAtMismatch {
5451 expected,
5452 actual_receipts: 1,
5453 mismatched_receipts: 1,
5454 ..
5455 } if expected == "2026-05-05T12:04:00Z"
5456 ));
5457 }
5458
5459 #[test]
5461 fn run_restore_run_require_state_updated_at_writes_summary_then_fails() {
5462 let fixture = RestoreCliFixture::new(
5463 "canic-cli-restore-run-require-state-updated-at",
5464 "restore-run.json",
5465 );
5466 let journal = ready_apply_journal();
5467 fixture.write_journal(&journal);
5468
5469 let err = fixture
5470 .run_restore_run(&[
5471 "--dry-run",
5472 "--updated-at",
5473 "2026-05-05T12:03:00Z",
5474 "--require-state-updated-at",
5475 "2026-05-05T12:04:00Z",
5476 ])
5477 .expect_err("state updated-at mismatch should fail requirement");
5478
5479 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5480
5481 assert_eq!(
5482 run_summary["requested_state_updated_at"],
5483 "2026-05-05T12:03:00Z"
5484 );
5485 assert_eq!(run_summary["operation_receipt_count"], 0);
5486 assert!(matches!(
5487 err,
5488 RestoreCommandError::RestoreRunStateUpdatedAtMismatch {
5489 expected,
5490 actual: Some(actual),
5491 ..
5492 } if expected == "2026-05-05T12:04:00Z"
5493 && actual == "2026-05-05T12:03:00Z"
5494 ));
5495 }
5496
5497 #[test]
5499 fn run_restore_run_require_batch_remaining_ready_count_writes_summary_then_fails() {
5500 let fixture = RestoreCliFixture::new(
5501 "canic-cli-restore-run-require-batch-ready-count",
5502 "restore-run.json",
5503 );
5504 let journal = ready_apply_journal();
5505 fixture.write_journal(&journal);
5506
5507 let err = fixture
5508 .run_restore_run(&[
5509 "--execute",
5510 "--dfx",
5511 "/bin/true",
5512 "--max-steps",
5513 "1",
5514 "--require-batch-remaining-ready-count",
5515 "8",
5516 ])
5517 .expect_err("batch remaining ready count mismatch should fail requirement");
5518
5519 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5520
5521 assert_eq!(run_summary["batch_summary"]["initial_ready_operations"], 8);
5522 assert_eq!(run_summary["batch_summary"]["executed_operations"], 1);
5523 assert_eq!(
5524 run_summary["batch_summary"]["remaining_ready_operations"],
5525 7
5526 );
5527 assert!(matches!(
5528 err,
5529 RestoreCommandError::RestoreRunBatchRemainingReadyCountMismatch {
5530 expected: 8,
5531 actual: 7,
5532 ..
5533 }
5534 ));
5535 }
5536
5537 #[test]
5539 fn run_restore_run_require_batch_initial_ready_count_writes_summary_then_fails() {
5540 let fixture = RestoreCliFixture::new(
5541 "canic-cli-restore-run-require-batch-initial-ready-count",
5542 "restore-run.json",
5543 );
5544 let journal = ready_apply_journal();
5545 fixture.write_journal(&journal);
5546
5547 let err = fixture
5548 .run_restore_run(&[
5549 "--execute",
5550 "--dfx",
5551 "/bin/true",
5552 "--max-steps",
5553 "1",
5554 "--require-batch-initial-ready-count",
5555 "7",
5556 ])
5557 .expect_err("batch initial ready count mismatch should fail requirement");
5558
5559 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5560
5561 assert_eq!(run_summary["batch_summary"]["initial_ready_operations"], 8);
5562 assert_eq!(
5563 run_summary["batch_summary"]["remaining_ready_operations"],
5564 7
5565 );
5566 assert!(matches!(
5567 err,
5568 RestoreCommandError::RestoreRunBatchInitialReadyCountMismatch {
5569 expected: 7,
5570 actual: 8,
5571 ..
5572 }
5573 ));
5574 }
5575
5576 #[test]
5578 fn run_restore_run_require_batch_executed_count_writes_summary_then_fails() {
5579 let fixture = RestoreCliFixture::new(
5580 "canic-cli-restore-run-require-batch-executed-count",
5581 "restore-run.json",
5582 );
5583 let journal = ready_apply_journal();
5584 fixture.write_journal(&journal);
5585
5586 let err = fixture
5587 .run_restore_run(&[
5588 "--execute",
5589 "--dfx",
5590 "/bin/true",
5591 "--max-steps",
5592 "1",
5593 "--require-batch-executed-count",
5594 "2",
5595 ])
5596 .expect_err("batch executed count mismatch should fail requirement");
5597
5598 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5599
5600 assert_eq!(run_summary["batch_summary"]["executed_operations"], 1);
5601 assert_eq!(run_summary["batch_summary"]["ready_operations_delta"], -1);
5602 assert!(matches!(
5603 err,
5604 RestoreCommandError::RestoreRunBatchExecutedCountMismatch {
5605 expected: 2,
5606 actual: 1,
5607 ..
5608 }
5609 ));
5610 }
5611
5612 #[test]
5614 fn run_restore_run_require_batch_ready_delta_writes_summary_then_fails() {
5615 let fixture = RestoreCliFixture::new(
5616 "canic-cli-restore-run-require-batch-ready-delta",
5617 "restore-run.json",
5618 );
5619 let journal = ready_apply_journal();
5620 fixture.write_journal(&journal);
5621
5622 let err = fixture
5623 .run_restore_run(&[
5624 "--execute",
5625 "--dfx",
5626 "/bin/true",
5627 "--max-steps",
5628 "1",
5629 "--require-batch-ready-delta",
5630 "0",
5631 ])
5632 .expect_err("batch ready delta mismatch should fail requirement");
5633
5634 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5635
5636 assert_eq!(run_summary["batch_summary"]["ready_operations_delta"], -1);
5637 assert_eq!(
5638 run_summary["batch_summary"]["remaining_operations_delta"],
5639 -1
5640 );
5641 assert!(matches!(
5642 err,
5643 RestoreCommandError::RestoreRunBatchReadyDeltaMismatch {
5644 expected: 0,
5645 actual: -1,
5646 ..
5647 }
5648 ));
5649 }
5650
5651 #[test]
5653 fn run_restore_run_require_batch_remaining_delta_writes_summary_then_fails() {
5654 let fixture = RestoreCliFixture::new(
5655 "canic-cli-restore-run-require-batch-remaining-delta",
5656 "restore-run.json",
5657 );
5658 let journal = ready_apply_journal();
5659 fixture.write_journal(&journal);
5660
5661 let err = fixture
5662 .run_restore_run(&[
5663 "--execute",
5664 "--dfx",
5665 "/bin/true",
5666 "--max-steps",
5667 "1",
5668 "--require-batch-remaining-delta",
5669 "0",
5670 ])
5671 .expect_err("batch remaining delta mismatch should fail requirement");
5672
5673 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5674
5675 assert_eq!(
5676 run_summary["batch_summary"]["remaining_operations_delta"],
5677 -1
5678 );
5679 assert!(matches!(
5680 err,
5681 RestoreCommandError::RestoreRunBatchRemainingDeltaMismatch {
5682 expected: 0,
5683 actual: -1,
5684 ..
5685 }
5686 ));
5687 }
5688
5689 #[test]
5691 fn run_restore_run_require_batch_stopped_by_max_steps_writes_summary_then_fails() {
5692 let fixture = RestoreCliFixture::new(
5693 "canic-cli-restore-run-require-batch-max-step-stop",
5694 "restore-run.json",
5695 );
5696 let journal = ready_apply_journal();
5697 fixture.write_journal(&journal);
5698
5699 let err = fixture
5700 .run_restore_run(&[
5701 "--execute",
5702 "--dfx",
5703 "/bin/true",
5704 "--max-steps",
5705 "1",
5706 "--require-batch-stopped-by-max-steps",
5707 "false",
5708 ])
5709 .expect_err("batch max-step mismatch should fail requirement");
5710
5711 let run_summary: serde_json::Value = fixture.read_out("read run summary");
5712
5713 assert_eq!(run_summary["batch_summary"]["stopped_by_max_steps"], true);
5714 assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
5715 assert!(matches!(
5716 err,
5717 RestoreCommandError::RestoreRunBatchStoppedByMaxStepsMismatch {
5718 expected: false,
5719 actual: true,
5720 ..
5721 }
5722 ));
5723 }
5724
5725 #[test]
5727 fn run_restore_run_require_progress_writes_summary_then_fails() {
5728 let root = temp_dir("canic-cli-restore-run-require-progress");
5729 fs::create_dir_all(&root).expect("create temp root");
5730 let journal_path = root.join("restore-apply-journal.json");
5731 let out_path = root.join("restore-run.json");
5732 let journal = ready_apply_journal();
5733
5734 fs::write(
5735 &journal_path,
5736 serde_json::to_vec(&journal).expect("serialize journal"),
5737 )
5738 .expect("write journal");
5739
5740 let err = run([
5741 OsString::from("run"),
5742 OsString::from("--journal"),
5743 OsString::from(journal_path.as_os_str()),
5744 OsString::from("--execute"),
5745 OsString::from("--dfx"),
5746 OsString::from("/bin/true"),
5747 OsString::from("--max-steps"),
5748 OsString::from("1"),
5749 OsString::from("--out"),
5750 OsString::from(out_path.as_os_str()),
5751 OsString::from("--require-remaining-count"),
5752 OsString::from("7"),
5753 OsString::from("--require-attention-count"),
5754 OsString::from("0"),
5755 OsString::from("--require-completion-basis-points"),
5756 OsString::from("0"),
5757 ])
5758 .expect_err("completion progress mismatch should fail requirement");
5759
5760 let run_summary: serde_json::Value =
5761 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5762 .expect("decode run summary");
5763
5764 fs::remove_dir_all(root).expect("remove temp root");
5765 assert_eq!(run_summary["progress"]["remaining_operations"], 7);
5766 assert_eq!(run_summary["progress"]["attention_operations"], 0);
5767 assert_eq!(run_summary["progress"]["completion_basis_points"], 1250);
5768 assert!(matches!(
5769 err,
5770 RestoreCommandError::RestoreApplyProgressMismatch {
5771 field: "completion_basis_points",
5772 expected: 0,
5773 actual: 1250,
5774 ..
5775 }
5776 ));
5777 }
5778
5779 #[test]
5781 fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
5782 let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
5783 fs::create_dir_all(&root).expect("create temp root");
5784 let journal_path = root.join("restore-apply-journal.json");
5785 let out_path = root.join("restore-run.json");
5786 let journal = ready_apply_journal();
5787
5788 fs::write(
5789 &journal_path,
5790 serde_json::to_vec(&journal).expect("serialize journal"),
5791 )
5792 .expect("write journal");
5793
5794 let err = run([
5795 OsString::from("run"),
5796 OsString::from("--journal"),
5797 OsString::from(journal_path.as_os_str()),
5798 OsString::from("--dry-run"),
5799 OsString::from("--out"),
5800 OsString::from(out_path.as_os_str()),
5801 OsString::from("--require-stopped-reason"),
5802 OsString::from("complete"),
5803 ])
5804 .expect_err("stopped reason mismatch should fail requirement");
5805
5806 let run_summary: serde_json::Value =
5807 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5808 .expect("decode run summary");
5809
5810 fs::remove_dir_all(root).expect("remove temp root");
5811 assert_eq!(run_summary["stopped_reason"], "preview");
5812 assert!(matches!(
5813 err,
5814 RestoreCommandError::RestoreRunStoppedReasonMismatch {
5815 expected,
5816 actual,
5817 ..
5818 } if expected == "complete" && actual == "preview"
5819 ));
5820 }
5821
5822 #[test]
5824 fn run_restore_run_require_next_action_writes_summary_then_fails() {
5825 let root = temp_dir("canic-cli-restore-run-require-next-action");
5826 fs::create_dir_all(&root).expect("create temp root");
5827 let journal_path = root.join("restore-apply-journal.json");
5828 let out_path = root.join("restore-run.json");
5829 let journal = ready_apply_journal();
5830
5831 fs::write(
5832 &journal_path,
5833 serde_json::to_vec(&journal).expect("serialize journal"),
5834 )
5835 .expect("write journal");
5836
5837 let err = run([
5838 OsString::from("run"),
5839 OsString::from("--journal"),
5840 OsString::from(journal_path.as_os_str()),
5841 OsString::from("--dry-run"),
5842 OsString::from("--out"),
5843 OsString::from(out_path.as_os_str()),
5844 OsString::from("--require-next-action"),
5845 OsString::from("done"),
5846 ])
5847 .expect_err("next action mismatch should fail requirement");
5848
5849 let run_summary: serde_json::Value =
5850 serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
5851 .expect("decode run summary");
5852
5853 fs::remove_dir_all(root).expect("remove temp root");
5854 assert_eq!(run_summary["next_action"], "rerun");
5855 assert!(matches!(
5856 err,
5857 RestoreCommandError::RestoreRunNextActionMismatch {
5858 expected,
5859 actual,
5860 ..
5861 } if expected == "done" && actual == "rerun"
5862 ));
5863 }
5864
5865 #[test]
5867 fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
5868 let root = temp_dir("canic-cli-restore-apply-report-attention");
5869 fs::create_dir_all(&root).expect("create temp root");
5870 let journal_path = root.join("restore-apply-journal.json");
5871 let out_path = root.join("restore-apply-report.json");
5872 let mut journal = ready_apply_journal();
5873 journal
5874 .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
5875 .expect("mark pending operation");
5876
5877 fs::write(
5878 &journal_path,
5879 serde_json::to_vec(&journal).expect("serialize journal"),
5880 )
5881 .expect("write journal");
5882
5883 let err = run([
5884 OsString::from("apply-report"),
5885 OsString::from("--journal"),
5886 OsString::from(journal_path.as_os_str()),
5887 OsString::from("--out"),
5888 OsString::from(out_path.as_os_str()),
5889 OsString::from("--require-no-attention"),
5890 ])
5891 .expect_err("attention report should fail requirement");
5892
5893 let report: RestoreApplyJournalReport =
5894 serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
5895 .expect("decode apply report");
5896
5897 fs::remove_dir_all(root).expect("remove temp root");
5898 assert!(report.attention_required);
5899 assert_eq!(report.pending_operations, 1);
5900 assert!(matches!(
5901 err,
5902 RestoreCommandError::RestoreApplyReportNeedsAttention {
5903 outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
5904 ..
5905 }
5906 ));
5907 }
5908
5909 #[test]
5911 fn run_restore_apply_status_require_complete_writes_status_then_fails() {
5912 let root = temp_dir("canic-cli-restore-apply-status-incomplete");
5913 fs::create_dir_all(&root).expect("create temp root");
5914 let journal_path = root.join("restore-apply-journal.json");
5915 let out_path = root.join("restore-apply-status.json");
5916 let journal = ready_apply_journal();
5917
5918 fs::write(
5919 &journal_path,
5920 serde_json::to_vec(&journal).expect("serialize journal"),
5921 )
5922 .expect("write journal");
5923
5924 let err = run([
5925 OsString::from("apply-status"),
5926 OsString::from("--journal"),
5927 OsString::from(journal_path.as_os_str()),
5928 OsString::from("--out"),
5929 OsString::from(out_path.as_os_str()),
5930 OsString::from("--require-complete"),
5931 ])
5932 .expect_err("incomplete journal should fail requirement");
5933
5934 assert!(out_path.exists());
5935 let status: RestoreApplyJournalStatus =
5936 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
5937 .expect("decode apply status");
5938
5939 fs::remove_dir_all(root).expect("remove temp root");
5940 assert!(!status.complete);
5941 assert_eq!(status.completed_operations, 0);
5942 assert_eq!(status.operation_count, 8);
5943 assert!(matches!(
5944 err,
5945 RestoreCommandError::RestoreApplyIncomplete {
5946 completed_operations: 0,
5947 operation_count: 8,
5948 ..
5949 }
5950 ));
5951 }
5952
5953 #[test]
5955 fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
5956 let root = temp_dir("canic-cli-restore-apply-status-failed");
5957 fs::create_dir_all(&root).expect("create temp root");
5958 let journal_path = root.join("restore-apply-journal.json");
5959 let out_path = root.join("restore-apply-status.json");
5960 let mut journal = ready_apply_journal();
5961 journal
5962 .mark_operation_failed(0, "dfx-load-failed".to_string())
5963 .expect("mark failed operation");
5964
5965 fs::write(
5966 &journal_path,
5967 serde_json::to_vec(&journal).expect("serialize journal"),
5968 )
5969 .expect("write journal");
5970
5971 let err = run([
5972 OsString::from("apply-status"),
5973 OsString::from("--journal"),
5974 OsString::from(journal_path.as_os_str()),
5975 OsString::from("--out"),
5976 OsString::from(out_path.as_os_str()),
5977 OsString::from("--require-no-failed"),
5978 ])
5979 .expect_err("failed operation should fail requirement");
5980
5981 assert!(out_path.exists());
5982 let status: RestoreApplyJournalStatus =
5983 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
5984 .expect("decode apply status");
5985
5986 fs::remove_dir_all(root).expect("remove temp root");
5987 assert_eq!(status.failed_operations, 1);
5988 assert!(matches!(
5989 err,
5990 RestoreCommandError::RestoreApplyFailed {
5991 failed_operations: 1,
5992 ..
5993 }
5994 ));
5995 }
5996
5997 #[test]
5999 fn run_restore_apply_status_require_complete_accepts_complete_journal() {
6000 let root = temp_dir("canic-cli-restore-apply-status-complete");
6001 fs::create_dir_all(&root).expect("create temp root");
6002 let journal_path = root.join("restore-apply-journal.json");
6003 let out_path = root.join("restore-apply-status.json");
6004 let mut journal = ready_apply_journal();
6005 for sequence in 0..journal.operation_count {
6006 journal
6007 .mark_operation_completed(sequence)
6008 .expect("complete operation");
6009 }
6010
6011 fs::write(
6012 &journal_path,
6013 serde_json::to_vec(&journal).expect("serialize journal"),
6014 )
6015 .expect("write journal");
6016
6017 run([
6018 OsString::from("apply-status"),
6019 OsString::from("--journal"),
6020 OsString::from(journal_path.as_os_str()),
6021 OsString::from("--out"),
6022 OsString::from(out_path.as_os_str()),
6023 OsString::from("--require-complete"),
6024 ])
6025 .expect("complete journal should pass requirement");
6026
6027 let status: RestoreApplyJournalStatus =
6028 serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
6029 .expect("decode apply status");
6030
6031 fs::remove_dir_all(root).expect("remove temp root");
6032 assert!(status.complete);
6033 assert_eq!(status.completed_operations, 8);
6034 assert_eq!(status.operation_count, 8);
6035 }
6036
6037 #[test]
6039 fn run_restore_apply_next_writes_next_ready_operation() {
6040 let root = temp_dir("canic-cli-restore-apply-next");
6041 fs::create_dir_all(&root).expect("create temp root");
6042 let journal_path = root.join("restore-apply-journal.json");
6043 let out_path = root.join("restore-apply-next.json");
6044 let mut journal = ready_apply_journal();
6045 journal
6046 .mark_operation_completed(0)
6047 .expect("mark first operation complete");
6048
6049 fs::write(
6050 &journal_path,
6051 serde_json::to_vec(&journal).expect("serialize journal"),
6052 )
6053 .expect("write journal");
6054
6055 run([
6056 OsString::from("apply-next"),
6057 OsString::from("--journal"),
6058 OsString::from(journal_path.as_os_str()),
6059 OsString::from("--out"),
6060 OsString::from(out_path.as_os_str()),
6061 ])
6062 .expect("write apply next");
6063
6064 let next: RestoreApplyNextOperation =
6065 serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
6066 .expect("decode next operation");
6067 let operation = next.operation.expect("operation should be available");
6068
6069 fs::remove_dir_all(root).expect("remove temp root");
6070 assert!(next.ready);
6071 assert!(next.operation_available);
6072 assert_eq!(operation.sequence, 1);
6073 assert_eq!(
6074 operation.operation,
6075 canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
6076 );
6077 }
6078
6079 #[test]
6081 fn run_restore_apply_command_writes_next_command_preview() {
6082 let root = temp_dir("canic-cli-restore-apply-command");
6083 fs::create_dir_all(&root).expect("create temp root");
6084 let journal_path = root.join("restore-apply-journal.json");
6085 let out_path = root.join("restore-apply-command.json");
6086 let journal = ready_apply_journal();
6087
6088 fs::write(
6089 &journal_path,
6090 serde_json::to_vec(&journal).expect("serialize journal"),
6091 )
6092 .expect("write journal");
6093
6094 run([
6095 OsString::from("apply-command"),
6096 OsString::from("--journal"),
6097 OsString::from(journal_path.as_os_str()),
6098 OsString::from("--dfx"),
6099 OsString::from("/tmp/dfx"),
6100 OsString::from("--network"),
6101 OsString::from("local"),
6102 OsString::from("--out"),
6103 OsString::from(out_path.as_os_str()),
6104 ])
6105 .expect("write command preview");
6106
6107 let preview: RestoreApplyCommandPreview =
6108 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
6109 .expect("decode command preview");
6110 let command = preview.command.expect("command should be available");
6111
6112 fs::remove_dir_all(root).expect("remove temp root");
6113 assert!(preview.ready);
6114 assert!(preview.command_available);
6115 assert_eq!(command.program, "/tmp/dfx");
6116 assert_eq!(
6117 command.args,
6118 vec![
6119 "canister".to_string(),
6120 "--network".to_string(),
6121 "local".to_string(),
6122 "snapshot".to_string(),
6123 "upload".to_string(),
6124 "--dir".to_string(),
6125 "artifacts/root".to_string(),
6126 ROOT.to_string(),
6127 ]
6128 );
6129 assert!(command.mutates);
6130 }
6131
6132 #[test]
6134 fn run_restore_apply_command_require_command_writes_preview_then_fails() {
6135 let root = temp_dir("canic-cli-restore-apply-command-require");
6136 fs::create_dir_all(&root).expect("create temp root");
6137 let journal_path = root.join("restore-apply-journal.json");
6138 let out_path = root.join("restore-apply-command.json");
6139 let mut journal = ready_apply_journal();
6140
6141 for sequence in 0..journal.operation_count {
6142 journal
6143 .mark_operation_completed(sequence)
6144 .expect("mark operation completed");
6145 }
6146
6147 fs::write(
6148 &journal_path,
6149 serde_json::to_vec(&journal).expect("serialize journal"),
6150 )
6151 .expect("write journal");
6152
6153 let err = run([
6154 OsString::from("apply-command"),
6155 OsString::from("--journal"),
6156 OsString::from(journal_path.as_os_str()),
6157 OsString::from("--out"),
6158 OsString::from(out_path.as_os_str()),
6159 OsString::from("--require-command"),
6160 ])
6161 .expect_err("missing command should fail");
6162
6163 let preview: RestoreApplyCommandPreview =
6164 serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
6165 .expect("decode command preview");
6166
6167 fs::remove_dir_all(root).expect("remove temp root");
6168 assert!(preview.complete);
6169 assert!(!preview.operation_available);
6170 assert!(!preview.command_available);
6171 assert!(matches!(
6172 err,
6173 RestoreCommandError::RestoreApplyCommandUnavailable {
6174 operation_available: false,
6175 complete: true,
6176 ..
6177 }
6178 ));
6179 }
6180
6181 #[test]
6183 fn run_restore_apply_claim_marks_next_operation_pending() {
6184 let root = temp_dir("canic-cli-restore-apply-claim");
6185 fs::create_dir_all(&root).expect("create temp root");
6186 let journal_path = root.join("restore-apply-journal.json");
6187 let claimed_path = root.join("restore-apply-journal.claimed.json");
6188 let journal = ready_apply_journal();
6189
6190 fs::write(
6191 &journal_path,
6192 serde_json::to_vec(&journal).expect("serialize journal"),
6193 )
6194 .expect("write journal");
6195
6196 run([
6197 OsString::from("apply-claim"),
6198 OsString::from("--journal"),
6199 OsString::from(journal_path.as_os_str()),
6200 OsString::from("--sequence"),
6201 OsString::from("0"),
6202 OsString::from("--updated-at"),
6203 OsString::from("2026-05-04T12:00:00Z"),
6204 OsString::from("--out"),
6205 OsString::from(claimed_path.as_os_str()),
6206 ])
6207 .expect("claim operation");
6208
6209 let claimed: RestoreApplyJournal =
6210 serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
6211 .expect("decode claimed journal");
6212 let status = claimed.status();
6213 let next = claimed.next_operation();
6214
6215 fs::remove_dir_all(root).expect("remove temp root");
6216 assert_eq!(claimed.pending_operations, 1);
6217 assert_eq!(claimed.ready_operations, 7);
6218 assert_eq!(
6219 claimed.operations[0].state,
6220 RestoreApplyOperationState::Pending
6221 );
6222 assert_eq!(
6223 claimed.operations[0].state_updated_at.as_deref(),
6224 Some("2026-05-04T12:00:00Z")
6225 );
6226 assert_eq!(status.next_transition_sequence, Some(0));
6227 assert_eq!(
6228 status.next_transition_state,
6229 Some(RestoreApplyOperationState::Pending)
6230 );
6231 assert_eq!(
6232 status.next_transition_updated_at.as_deref(),
6233 Some("2026-05-04T12:00:00Z")
6234 );
6235 assert_eq!(
6236 next.operation.expect("next operation").state,
6237 RestoreApplyOperationState::Pending
6238 );
6239 }
6240
6241 #[test]
6243 fn run_restore_apply_claim_rejects_sequence_mismatch() {
6244 let root = temp_dir("canic-cli-restore-apply-claim-sequence");
6245 fs::create_dir_all(&root).expect("create temp root");
6246 let journal_path = root.join("restore-apply-journal.json");
6247 let claimed_path = root.join("restore-apply-journal.claimed.json");
6248 let journal = ready_apply_journal();
6249
6250 fs::write(
6251 &journal_path,
6252 serde_json::to_vec(&journal).expect("serialize journal"),
6253 )
6254 .expect("write journal");
6255
6256 let err = run([
6257 OsString::from("apply-claim"),
6258 OsString::from("--journal"),
6259 OsString::from(journal_path.as_os_str()),
6260 OsString::from("--sequence"),
6261 OsString::from("1"),
6262 OsString::from("--out"),
6263 OsString::from(claimed_path.as_os_str()),
6264 ])
6265 .expect_err("stale sequence should fail claim");
6266
6267 assert!(!claimed_path.exists());
6268 fs::remove_dir_all(root).expect("remove temp root");
6269 assert!(matches!(
6270 err,
6271 RestoreCommandError::RestoreApplyClaimSequenceMismatch {
6272 expected: 1,
6273 actual: Some(0),
6274 }
6275 ));
6276 }
6277
6278 #[test]
6280 fn run_restore_apply_unclaim_marks_pending_operation_ready() {
6281 let root = temp_dir("canic-cli-restore-apply-unclaim");
6282 fs::create_dir_all(&root).expect("create temp root");
6283 let journal_path = root.join("restore-apply-journal.json");
6284 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
6285 let mut journal = ready_apply_journal();
6286 journal
6287 .mark_next_operation_pending()
6288 .expect("claim operation");
6289
6290 fs::write(
6291 &journal_path,
6292 serde_json::to_vec(&journal).expect("serialize journal"),
6293 )
6294 .expect("write journal");
6295
6296 run([
6297 OsString::from("apply-unclaim"),
6298 OsString::from("--journal"),
6299 OsString::from(journal_path.as_os_str()),
6300 OsString::from("--sequence"),
6301 OsString::from("0"),
6302 OsString::from("--updated-at"),
6303 OsString::from("2026-05-04T12:01:00Z"),
6304 OsString::from("--out"),
6305 OsString::from(unclaimed_path.as_os_str()),
6306 ])
6307 .expect("unclaim operation");
6308
6309 let unclaimed: RestoreApplyJournal =
6310 serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
6311 .expect("decode unclaimed journal");
6312 let status = unclaimed.status();
6313
6314 fs::remove_dir_all(root).expect("remove temp root");
6315 assert_eq!(unclaimed.pending_operations, 0);
6316 assert_eq!(unclaimed.ready_operations, 8);
6317 assert_eq!(
6318 unclaimed.operations[0].state,
6319 RestoreApplyOperationState::Ready
6320 );
6321 assert_eq!(
6322 unclaimed.operations[0].state_updated_at.as_deref(),
6323 Some("2026-05-04T12:01:00Z")
6324 );
6325 assert_eq!(status.next_ready_sequence, Some(0));
6326 assert_eq!(
6327 status.next_transition_state,
6328 Some(RestoreApplyOperationState::Ready)
6329 );
6330 assert_eq!(
6331 status.next_transition_updated_at.as_deref(),
6332 Some("2026-05-04T12:01:00Z")
6333 );
6334 }
6335
6336 #[test]
6338 fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
6339 let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
6340 fs::create_dir_all(&root).expect("create temp root");
6341 let journal_path = root.join("restore-apply-journal.json");
6342 let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
6343 let mut journal = ready_apply_journal();
6344 journal
6345 .mark_next_operation_pending()
6346 .expect("claim operation");
6347
6348 fs::write(
6349 &journal_path,
6350 serde_json::to_vec(&journal).expect("serialize journal"),
6351 )
6352 .expect("write journal");
6353
6354 let err = run([
6355 OsString::from("apply-unclaim"),
6356 OsString::from("--journal"),
6357 OsString::from(journal_path.as_os_str()),
6358 OsString::from("--sequence"),
6359 OsString::from("1"),
6360 OsString::from("--out"),
6361 OsString::from(unclaimed_path.as_os_str()),
6362 ])
6363 .expect_err("stale sequence should fail unclaim");
6364
6365 assert!(!unclaimed_path.exists());
6366 fs::remove_dir_all(root).expect("remove temp root");
6367 assert!(matches!(
6368 err,
6369 RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
6370 expected: 1,
6371 actual: Some(0),
6372 }
6373 ));
6374 }
6375
6376 #[test]
6378 fn run_restore_apply_mark_completes_operation() {
6379 let root = temp_dir("canic-cli-restore-apply-mark-complete");
6380 fs::create_dir_all(&root).expect("create temp root");
6381 let journal_path = root.join("restore-apply-journal.json");
6382 let updated_path = root.join("restore-apply-journal.updated.json");
6383 let journal = ready_apply_journal();
6384
6385 fs::write(
6386 &journal_path,
6387 serde_json::to_vec(&journal).expect("serialize journal"),
6388 )
6389 .expect("write journal");
6390
6391 run([
6392 OsString::from("apply-mark"),
6393 OsString::from("--journal"),
6394 OsString::from(journal_path.as_os_str()),
6395 OsString::from("--sequence"),
6396 OsString::from("0"),
6397 OsString::from("--state"),
6398 OsString::from("completed"),
6399 OsString::from("--updated-at"),
6400 OsString::from("2026-05-04T12:02:00Z"),
6401 OsString::from("--out"),
6402 OsString::from(updated_path.as_os_str()),
6403 ])
6404 .expect("mark operation completed");
6405
6406 let updated: RestoreApplyJournal =
6407 serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
6408 .expect("decode updated journal");
6409 let status = updated.status();
6410
6411 fs::remove_dir_all(root).expect("remove temp root");
6412 assert_eq!(updated.completed_operations, 1);
6413 assert_eq!(updated.ready_operations, 7);
6414 assert_eq!(
6415 updated.operations[0].state_updated_at.as_deref(),
6416 Some("2026-05-04T12:02:00Z")
6417 );
6418 assert_eq!(status.next_ready_sequence, Some(1));
6419 }
6420
6421 #[test]
6423 fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
6424 let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
6425 fs::create_dir_all(&root).expect("create temp root");
6426 let journal_path = root.join("restore-apply-journal.json");
6427 let updated_path = root.join("restore-apply-journal.updated.json");
6428 let journal = ready_apply_journal();
6429
6430 fs::write(
6431 &journal_path,
6432 serde_json::to_vec(&journal).expect("serialize journal"),
6433 )
6434 .expect("write journal");
6435
6436 let err = run([
6437 OsString::from("apply-mark"),
6438 OsString::from("--journal"),
6439 OsString::from(journal_path.as_os_str()),
6440 OsString::from("--sequence"),
6441 OsString::from("0"),
6442 OsString::from("--state"),
6443 OsString::from("completed"),
6444 OsString::from("--out"),
6445 OsString::from(updated_path.as_os_str()),
6446 OsString::from("--require-pending"),
6447 ])
6448 .expect_err("ready operation should fail pending requirement");
6449
6450 assert!(!updated_path.exists());
6451 fs::remove_dir_all(root).expect("remove temp root");
6452 assert!(matches!(
6453 err,
6454 RestoreCommandError::RestoreApplyMarkRequiresPending {
6455 sequence: 0,
6456 state: RestoreApplyOperationState::Ready,
6457 }
6458 ));
6459 }
6460
6461 #[test]
6463 fn run_restore_apply_mark_rejects_out_of_order_operation() {
6464 let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
6465 fs::create_dir_all(&root).expect("create temp root");
6466 let journal_path = root.join("restore-apply-journal.json");
6467 let updated_path = root.join("restore-apply-journal.updated.json");
6468 let journal = ready_apply_journal();
6469
6470 fs::write(
6471 &journal_path,
6472 serde_json::to_vec(&journal).expect("serialize journal"),
6473 )
6474 .expect("write journal");
6475
6476 let err = run([
6477 OsString::from("apply-mark"),
6478 OsString::from("--journal"),
6479 OsString::from(journal_path.as_os_str()),
6480 OsString::from("--sequence"),
6481 OsString::from("1"),
6482 OsString::from("--state"),
6483 OsString::from("completed"),
6484 OsString::from("--out"),
6485 OsString::from(updated_path.as_os_str()),
6486 ])
6487 .expect_err("out-of-order operation should fail");
6488
6489 assert!(!updated_path.exists());
6490 fs::remove_dir_all(root).expect("remove temp root");
6491 assert!(matches!(
6492 err,
6493 RestoreCommandError::RestoreApplyJournal(
6494 RestoreApplyJournalError::OutOfOrderOperationTransition {
6495 requested: 1,
6496 next: 0
6497 }
6498 )
6499 ));
6500 }
6501
6502 #[test]
6504 fn run_restore_apply_mark_failed_requires_reason() {
6505 let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
6506 fs::create_dir_all(&root).expect("create temp root");
6507 let journal_path = root.join("restore-apply-journal.json");
6508 let journal = ready_apply_journal();
6509
6510 fs::write(
6511 &journal_path,
6512 serde_json::to_vec(&journal).expect("serialize journal"),
6513 )
6514 .expect("write journal");
6515
6516 let err = run([
6517 OsString::from("apply-mark"),
6518 OsString::from("--journal"),
6519 OsString::from(journal_path.as_os_str()),
6520 OsString::from("--sequence"),
6521 OsString::from("0"),
6522 OsString::from("--state"),
6523 OsString::from("failed"),
6524 ])
6525 .expect_err("failed state should require reason");
6526
6527 fs::remove_dir_all(root).expect("remove temp root");
6528 assert!(matches!(
6529 err,
6530 RestoreCommandError::RestoreApplyJournal(
6531 RestoreApplyJournalError::FailureReasonRequired(0)
6532 )
6533 ));
6534 }
6535
6536 #[test]
6538 fn run_restore_apply_dry_run_rejects_mismatched_status() {
6539 let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
6540 fs::create_dir_all(&root).expect("create temp root");
6541 let plan_path = root.join("restore-plan.json");
6542 let status_path = root.join("restore-status.json");
6543 let out_path = root.join("restore-apply-dry-run.json");
6544 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
6545 let mut status = RestoreStatus::from_plan(&plan);
6546 status.backup_id = "other-backup".to_string();
6547
6548 fs::write(
6549 &plan_path,
6550 serde_json::to_vec(&plan).expect("serialize plan"),
6551 )
6552 .expect("write plan");
6553 fs::write(
6554 &status_path,
6555 serde_json::to_vec(&status).expect("serialize status"),
6556 )
6557 .expect("write status");
6558
6559 let err = run([
6560 OsString::from("apply"),
6561 OsString::from("--plan"),
6562 OsString::from(plan_path.as_os_str()),
6563 OsString::from("--status"),
6564 OsString::from(status_path.as_os_str()),
6565 OsString::from("--dry-run"),
6566 OsString::from("--out"),
6567 OsString::from(out_path.as_os_str()),
6568 ])
6569 .expect_err("mismatched status should fail");
6570
6571 assert!(!out_path.exists());
6572 fs::remove_dir_all(root).expect("remove temp root");
6573 assert!(matches!(
6574 err,
6575 RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
6576 field: "backup_id",
6577 ..
6578 })
6579 ));
6580 }
6581
6582 fn ready_apply_journal() -> RestoreApplyJournal {
6584 let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
6585 let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
6586 let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
6587
6588 journal.ready = true;
6589 journal.blocked_reasons = Vec::new();
6590 for operation in &mut journal.operations {
6591 operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
6592 operation.blocking_reasons = Vec::new();
6593 }
6594 journal.blocked_operations = 0;
6595 journal.ready_operations = journal.operation_count;
6596 journal.validate().expect("journal should validate");
6597 journal
6598 }
6599
6600 fn valid_manifest() -> FleetBackupManifest {
6602 FleetBackupManifest {
6603 manifest_version: 1,
6604 backup_id: "backup-test".to_string(),
6605 created_at: "2026-05-03T00:00:00Z".to_string(),
6606 tool: ToolMetadata {
6607 name: "canic".to_string(),
6608 version: "0.30.1".to_string(),
6609 },
6610 source: SourceMetadata {
6611 environment: "local".to_string(),
6612 root_canister: ROOT.to_string(),
6613 },
6614 consistency: ConsistencySection {
6615 mode: ConsistencyMode::CrashConsistent,
6616 backup_units: vec![BackupUnit {
6617 unit_id: "fleet".to_string(),
6618 kind: BackupUnitKind::SubtreeRooted,
6619 roles: vec!["root".to_string(), "app".to_string()],
6620 consistency_reason: None,
6621 dependency_closure: Vec::new(),
6622 topology_validation: "subtree-closed".to_string(),
6623 quiescence_strategy: None,
6624 }],
6625 },
6626 fleet: FleetSection {
6627 topology_hash_algorithm: "sha256".to_string(),
6628 topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
6629 discovery_topology_hash: HASH.to_string(),
6630 pre_snapshot_topology_hash: HASH.to_string(),
6631 topology_hash: HASH.to_string(),
6632 members: vec![
6633 fleet_member("root", ROOT, None, IdentityMode::Fixed),
6634 fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
6635 ],
6636 },
6637 verification: VerificationPlan::default(),
6638 }
6639 }
6640
6641 fn restore_ready_manifest() -> FleetBackupManifest {
6643 let mut manifest = valid_manifest();
6644 for member in &mut manifest.fleet.members {
6645 member.source_snapshot.module_hash = Some(HASH.to_string());
6646 member.source_snapshot.wasm_hash = Some(HASH.to_string());
6647 member.source_snapshot.checksum = Some(HASH.to_string());
6648 }
6649 manifest
6650 }
6651
6652 fn fleet_member(
6654 role: &str,
6655 canister_id: &str,
6656 parent_canister_id: Option<&str>,
6657 identity_mode: IdentityMode,
6658 ) -> FleetMember {
6659 FleetMember {
6660 role: role.to_string(),
6661 canister_id: canister_id.to_string(),
6662 parent_canister_id: parent_canister_id.map(str::to_string),
6663 subnet_canister_id: Some(ROOT.to_string()),
6664 controller_hint: None,
6665 identity_mode,
6666 restore_group: 1,
6667 verification_class: "basic".to_string(),
6668 verification_checks: vec![VerificationCheck {
6669 kind: "status".to_string(),
6670 method: None,
6671 roles: vec![role.to_string()],
6672 }],
6673 source_snapshot: SourceSnapshot {
6674 snapshot_id: format!("{role}-snapshot"),
6675 module_hash: None,
6676 wasm_hash: None,
6677 code_version: Some("v0.30.1".to_string()),
6678 artifact_path: format!("artifacts/{role}"),
6679 checksum_algorithm: "sha256".to_string(),
6680 checksum: None,
6681 },
6682 }
6683 }
6684
6685 fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
6687 layout.write_manifest(manifest).expect("write manifest");
6688
6689 let artifacts = manifest
6690 .fleet
6691 .members
6692 .iter()
6693 .map(|member| {
6694 let bytes = format!("{} artifact", member.role);
6695 let artifact_path = root.join(&member.source_snapshot.artifact_path);
6696 if let Some(parent) = artifact_path.parent() {
6697 fs::create_dir_all(parent).expect("create artifact parent");
6698 }
6699 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
6700 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
6701
6702 ArtifactJournalEntry {
6703 canister_id: member.canister_id.clone(),
6704 snapshot_id: member.source_snapshot.snapshot_id.clone(),
6705 state: ArtifactState::Durable,
6706 temp_path: None,
6707 artifact_path: member.source_snapshot.artifact_path.clone(),
6708 checksum_algorithm: checksum.algorithm,
6709 checksum: Some(checksum.hash),
6710 updated_at: "2026-05-03T00:00:00Z".to_string(),
6711 }
6712 })
6713 .collect();
6714
6715 layout
6716 .write_journal(&DownloadJournal {
6717 journal_version: 1,
6718 backup_id: manifest.backup_id.clone(),
6719 discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
6720 pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
6721 operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
6722 artifacts,
6723 })
6724 .expect("write journal");
6725 }
6726
6727 fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
6729 for member in &mut manifest.fleet.members {
6730 let bytes = format!("{} apply artifact", member.role);
6731 let artifact_path = root.join(&member.source_snapshot.artifact_path);
6732 if let Some(parent) = artifact_path.parent() {
6733 fs::create_dir_all(parent).expect("create artifact parent");
6734 }
6735 fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
6736 let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
6737 member.source_snapshot.checksum = Some(checksum.hash);
6738 }
6739 }
6740
6741 fn temp_dir(prefix: &str) -> PathBuf {
6743 let nanos = SystemTime::now()
6744 .duration_since(UNIX_EPOCH)
6745 .expect("system time after epoch")
6746 .as_nanos();
6747 std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
6748 }
6749}