Skip to main content

canic_cli/restore/
mod.rs

1use canic_backup::{
2    manifest::FleetBackupManifest,
3    persistence::{BackupLayout, PersistenceError},
4    restore::{
5        RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6        RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7        RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8        RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationState,
9        RestoreApplyReportOperation, RestoreApplyReportOutcome, RestoreApplyRunnerCommand,
10        RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner, RestoreStatus,
11    },
12};
13use serde::Serialize;
14use std::{
15    ffi::OsString,
16    fs,
17    io::{self, Write},
18    path::PathBuf,
19    process::Command,
20};
21use thiserror::Error as ThisError;
22
23///
24/// RestoreCommandError
25///
26
27#[derive(Debug, ThisError)]
28pub enum RestoreCommandError {
29    #[error("{0}")]
30    Usage(&'static str),
31
32    #[error("missing required option {0}")]
33    MissingOption(&'static str),
34
35    #[error("use either --manifest or --backup-dir, not both")]
36    ConflictingManifestSources,
37
38    #[error("--require-verified requires --backup-dir")]
39    RequireVerifiedNeedsBackupDir,
40
41    #[error("restore apply currently requires --dry-run")]
42    ApplyRequiresDryRun,
43
44    #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
45    RestoreRunRequiresMode,
46
47    #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
48    RestoreRunConflictingModes,
49
50    #[error("restore run command failed for operation {sequence}: status={status}")]
51    RestoreRunCommandFailed { sequence: usize, status: String },
52
53    #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
54    RestoreRunModeMismatch {
55        backup_id: String,
56        expected: String,
57        actual: String,
58    },
59
60    #[error(
61        "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
62    )]
63    RestoreRunStoppedReasonMismatch {
64        backup_id: String,
65        expected: String,
66        actual: String,
67    },
68
69    #[error(
70        "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
71    )]
72    RestoreRunNextActionMismatch {
73        backup_id: String,
74        expected: String,
75        actual: String,
76    },
77
78    #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
79    RestoreRunExecutedCountMismatch {
80        backup_id: String,
81        expected: usize,
82        actual: usize,
83    },
84
85    #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
86    RestoreNotReady {
87        backup_id: String,
88        reasons: Vec<String>,
89    },
90
91    #[error(
92        "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
93    )]
94    RestoreApplyPending {
95        backup_id: String,
96        pending_operations: usize,
97        next_transition_sequence: Option<usize>,
98    },
99
100    #[error(
101        "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
102    )]
103    RestoreApplyIncomplete {
104        backup_id: String,
105        completed_operations: usize,
106        operation_count: usize,
107    },
108
109    #[error(
110        "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
111    )]
112    RestoreApplyFailed {
113        backup_id: String,
114        failed_operations: usize,
115    },
116
117    #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
118    RestoreApplyNotReady {
119        backup_id: String,
120        reasons: Vec<String>,
121    },
122
123    #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
124    RestoreApplyReportNeedsAttention {
125        backup_id: String,
126        outcome: canic_backup::restore::RestoreApplyReportOutcome,
127    },
128
129    #[error(
130        "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
131    )]
132    RestoreApplyCommandUnavailable {
133        backup_id: String,
134        operation_available: bool,
135        complete: bool,
136        blocked_reasons: Vec<String>,
137    },
138
139    #[error(
140        "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
141    )]
142    RestoreApplyMarkRequiresPending {
143        sequence: usize,
144        state: RestoreApplyOperationState,
145    },
146
147    #[error(
148        "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
149    )]
150    RestoreApplyClaimSequenceMismatch {
151        expected: usize,
152        actual: Option<usize>,
153    },
154
155    #[error(
156        "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
157    )]
158    RestoreApplyUnclaimSequenceMismatch {
159        expected: usize,
160        actual: Option<usize>,
161    },
162
163    #[error("unknown option {0}")]
164    UnknownOption(String),
165
166    #[error("option {0} requires a value")]
167    MissingValue(&'static str),
168
169    #[error("option --sequence requires a non-negative integer value")]
170    InvalidSequence,
171
172    #[error("unsupported apply-mark state {0}; use completed or failed")]
173    InvalidApplyMarkState(String),
174
175    #[error(transparent)]
176    Io(#[from] std::io::Error),
177
178    #[error(transparent)]
179    Json(#[from] serde_json::Error),
180
181    #[error(transparent)]
182    Persistence(#[from] PersistenceError),
183
184    #[error(transparent)]
185    RestorePlan(#[from] RestorePlanError),
186
187    #[error(transparent)]
188    RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
189
190    #[error(transparent)]
191    RestoreApplyJournal(#[from] RestoreApplyJournalError),
192}
193
194///
195/// RestorePlanOptions
196///
197
198#[derive(Clone, Debug, Eq, PartialEq)]
199pub struct RestorePlanOptions {
200    pub manifest: Option<PathBuf>,
201    pub backup_dir: Option<PathBuf>,
202    pub mapping: Option<PathBuf>,
203    pub out: Option<PathBuf>,
204    pub require_verified: bool,
205    pub require_restore_ready: bool,
206}
207
208impl RestorePlanOptions {
209    /// Parse restore planning options from CLI arguments.
210    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
211    where
212        I: IntoIterator<Item = OsString>,
213    {
214        let mut manifest = None;
215        let mut backup_dir = None;
216        let mut mapping = None;
217        let mut out = None;
218        let mut require_verified = false;
219        let mut require_restore_ready = false;
220
221        let mut args = args.into_iter();
222        while let Some(arg) = args.next() {
223            let arg = arg
224                .into_string()
225                .map_err(|_| RestoreCommandError::Usage(usage()))?;
226            match arg.as_str() {
227                "--manifest" => {
228                    manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
229                }
230                "--backup-dir" => {
231                    backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
232                }
233                "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
234                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
235                "--require-verified" => require_verified = true,
236                "--require-restore-ready" => require_restore_ready = true,
237                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
238                _ => return Err(RestoreCommandError::UnknownOption(arg)),
239            }
240        }
241
242        if manifest.is_some() && backup_dir.is_some() {
243            return Err(RestoreCommandError::ConflictingManifestSources);
244        }
245
246        if manifest.is_none() && backup_dir.is_none() {
247            return Err(RestoreCommandError::MissingOption(
248                "--manifest or --backup-dir",
249            ));
250        }
251
252        if require_verified && backup_dir.is_none() {
253            return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
254        }
255
256        Ok(Self {
257            manifest,
258            backup_dir,
259            mapping,
260            out,
261            require_verified,
262            require_restore_ready,
263        })
264    }
265}
266
267///
268/// RestoreStatusOptions
269///
270
271#[derive(Clone, Debug, Eq, PartialEq)]
272pub struct RestoreStatusOptions {
273    pub plan: PathBuf,
274    pub out: Option<PathBuf>,
275}
276
277impl RestoreStatusOptions {
278    /// Parse restore status options from CLI arguments.
279    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
280    where
281        I: IntoIterator<Item = OsString>,
282    {
283        let mut plan = None;
284        let mut out = None;
285
286        let mut args = args.into_iter();
287        while let Some(arg) = args.next() {
288            let arg = arg
289                .into_string()
290                .map_err(|_| RestoreCommandError::Usage(usage()))?;
291            match arg.as_str() {
292                "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
293                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
294                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
295                _ => return Err(RestoreCommandError::UnknownOption(arg)),
296            }
297        }
298
299        Ok(Self {
300            plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
301            out,
302        })
303    }
304}
305
306///
307/// RestoreApplyOptions
308///
309
310#[derive(Clone, Debug, Eq, PartialEq)]
311pub struct RestoreApplyOptions {
312    pub plan: PathBuf,
313    pub status: Option<PathBuf>,
314    pub backup_dir: Option<PathBuf>,
315    pub out: Option<PathBuf>,
316    pub journal_out: Option<PathBuf>,
317    pub dry_run: bool,
318}
319
320impl RestoreApplyOptions {
321    /// Parse restore apply options from CLI arguments.
322    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
323    where
324        I: IntoIterator<Item = OsString>,
325    {
326        let mut plan = None;
327        let mut status = None;
328        let mut backup_dir = None;
329        let mut out = None;
330        let mut journal_out = None;
331        let mut dry_run = false;
332
333        let mut args = args.into_iter();
334        while let Some(arg) = args.next() {
335            let arg = arg
336                .into_string()
337                .map_err(|_| RestoreCommandError::Usage(usage()))?;
338            match arg.as_str() {
339                "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
340                "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
341                "--backup-dir" => {
342                    backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
343                }
344                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
345                "--journal-out" => {
346                    journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
347                }
348                "--dry-run" => dry_run = true,
349                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
350                _ => return Err(RestoreCommandError::UnknownOption(arg)),
351            }
352        }
353
354        if !dry_run {
355            return Err(RestoreCommandError::ApplyRequiresDryRun);
356        }
357
358        Ok(Self {
359            plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
360            status,
361            backup_dir,
362            out,
363            journal_out,
364            dry_run,
365        })
366    }
367}
368
369///
370/// RestoreApplyStatusOptions
371///
372
373#[derive(Clone, Debug, Eq, PartialEq)]
374#[expect(
375    clippy::struct_excessive_bools,
376    reason = "CLI status options mirror independent fail-closed guard flags"
377)]
378pub struct RestoreApplyStatusOptions {
379    pub journal: PathBuf,
380    pub require_ready: bool,
381    pub require_no_pending: bool,
382    pub require_no_failed: bool,
383    pub require_complete: bool,
384    pub out: Option<PathBuf>,
385}
386
387impl RestoreApplyStatusOptions {
388    /// Parse restore apply-status options from CLI arguments.
389    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
390    where
391        I: IntoIterator<Item = OsString>,
392    {
393        let mut journal = None;
394        let mut require_ready = false;
395        let mut require_no_pending = false;
396        let mut require_no_failed = false;
397        let mut require_complete = false;
398        let mut out = None;
399
400        let mut args = args.into_iter();
401        while let Some(arg) = args.next() {
402            let arg = arg
403                .into_string()
404                .map_err(|_| RestoreCommandError::Usage(usage()))?;
405            match arg.as_str() {
406                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
407                "--require-ready" => require_ready = true,
408                "--require-no-pending" => require_no_pending = true,
409                "--require-no-failed" => require_no_failed = true,
410                "--require-complete" => require_complete = true,
411                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
412                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
413                _ => return Err(RestoreCommandError::UnknownOption(arg)),
414            }
415        }
416
417        Ok(Self {
418            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
419            require_ready,
420            require_no_pending,
421            require_no_failed,
422            require_complete,
423            out,
424        })
425    }
426}
427
428///
429/// RestoreApplyReportOptions
430///
431
432#[derive(Clone, Debug, Eq, PartialEq)]
433pub struct RestoreApplyReportOptions {
434    pub journal: PathBuf,
435    pub require_no_attention: bool,
436    pub out: Option<PathBuf>,
437}
438
439impl RestoreApplyReportOptions {
440    /// Parse restore apply-report options from CLI arguments.
441    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
442    where
443        I: IntoIterator<Item = OsString>,
444    {
445        let mut journal = None;
446        let mut require_no_attention = false;
447        let mut out = None;
448
449        let mut args = args.into_iter();
450        while let Some(arg) = args.next() {
451            let arg = arg
452                .into_string()
453                .map_err(|_| RestoreCommandError::Usage(usage()))?;
454            match arg.as_str() {
455                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
456                "--require-no-attention" => require_no_attention = true,
457                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
458                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
459                _ => return Err(RestoreCommandError::UnknownOption(arg)),
460            }
461        }
462
463        Ok(Self {
464            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
465            require_no_attention,
466            out,
467        })
468    }
469}
470
471///
472/// RestoreRunOptions
473///
474
475#[derive(Clone, Debug, Eq, PartialEq)]
476#[expect(
477    clippy::struct_excessive_bools,
478    reason = "CLI runner options mirror independent mode and fail-closed guard flags"
479)]
480pub struct RestoreRunOptions {
481    pub journal: PathBuf,
482    pub dfx: String,
483    pub network: Option<String>,
484    pub out: Option<PathBuf>,
485    pub dry_run: bool,
486    pub execute: bool,
487    pub unclaim_pending: bool,
488    pub max_steps: Option<usize>,
489    pub require_complete: bool,
490    pub require_no_attention: bool,
491    pub require_run_mode: Option<String>,
492    pub require_stopped_reason: Option<String>,
493    pub require_next_action: Option<String>,
494    pub require_executed_count: Option<usize>,
495}
496
497impl RestoreRunOptions {
498    /// Parse restore run options from CLI arguments.
499    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
500    where
501        I: IntoIterator<Item = OsString>,
502    {
503        let mut journal = None;
504        let mut dfx = "dfx".to_string();
505        let mut network = None;
506        let mut out = None;
507        let mut dry_run = false;
508        let mut execute = false;
509        let mut unclaim_pending = false;
510        let mut max_steps = None;
511        let mut require_complete = false;
512        let mut require_no_attention = false;
513        let mut require_run_mode = None;
514        let mut require_stopped_reason = None;
515        let mut require_next_action = None;
516        let mut require_executed_count = None;
517
518        let mut args = args.into_iter();
519        while let Some(arg) = args.next() {
520            let arg = arg
521                .into_string()
522                .map_err(|_| RestoreCommandError::Usage(usage()))?;
523            match arg.as_str() {
524                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
525                "--dfx" => dfx = next_value(&mut args, "--dfx")?,
526                "--network" => network = Some(next_value(&mut args, "--network")?),
527                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
528                "--dry-run" => dry_run = true,
529                "--execute" => execute = true,
530                "--unclaim-pending" => unclaim_pending = true,
531                "--max-steps" => {
532                    max_steps = Some(parse_sequence(next_value(&mut args, "--max-steps")?)?);
533                }
534                "--require-complete" => require_complete = true,
535                "--require-no-attention" => require_no_attention = true,
536                "--require-run-mode" => {
537                    require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
538                }
539                "--require-stopped-reason" => {
540                    require_stopped_reason =
541                        Some(next_value(&mut args, "--require-stopped-reason")?);
542                }
543                "--require-next-action" => {
544                    require_next_action = Some(next_value(&mut args, "--require-next-action")?);
545                }
546                "--require-executed-count" => {
547                    require_executed_count = Some(parse_sequence(next_value(
548                        &mut args,
549                        "--require-executed-count",
550                    )?)?);
551                }
552                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
553                _ => return Err(RestoreCommandError::UnknownOption(arg)),
554            }
555        }
556
557        let mode_count = [dry_run, execute, unclaim_pending]
558            .into_iter()
559            .filter(|enabled| *enabled)
560            .count();
561        if mode_count > 1 {
562            return Err(RestoreCommandError::RestoreRunConflictingModes);
563        }
564
565        if mode_count == 0 {
566            return Err(RestoreCommandError::RestoreRunRequiresMode);
567        }
568
569        Ok(Self {
570            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
571            dfx,
572            network,
573            out,
574            dry_run,
575            execute,
576            unclaim_pending,
577            max_steps,
578            require_complete,
579            require_no_attention,
580            require_run_mode,
581            require_stopped_reason,
582            require_next_action,
583            require_executed_count,
584        })
585    }
586}
587
588///
589/// RestoreRunResult
590///
591
592struct RestoreRunResult {
593    response: RestoreRunResponse,
594    error: Option<RestoreCommandError>,
595}
596
597impl RestoreRunResult {
598    // Build a successful runner response with no deferred error.
599    const fn ok(response: RestoreRunResponse) -> Self {
600        Self {
601            response,
602            error: None,
603        }
604    }
605}
606
607///
608/// RestoreRunResponse
609///
610
611#[derive(Clone, Debug, Serialize)]
612#[expect(
613    clippy::struct_excessive_bools,
614    reason = "Runner response exposes stable JSON status flags for operators and CI"
615)]
616pub struct RestoreRunResponse {
617    run_version: u16,
618    backup_id: String,
619    run_mode: &'static str,
620    dry_run: bool,
621    execute: bool,
622    unclaim_pending: bool,
623    stopped_reason: &'static str,
624    next_action: &'static str,
625    #[serde(skip_serializing_if = "Option::is_none")]
626    max_steps_reached: Option<bool>,
627    #[serde(default, skip_serializing_if = "Vec::is_empty")]
628    executed_operations: Vec<RestoreRunExecutedOperation>,
629    #[serde(skip_serializing_if = "Option::is_none")]
630    executed_operation_count: Option<usize>,
631    #[serde(skip_serializing_if = "Option::is_none")]
632    recovered_operation: Option<RestoreApplyJournalOperation>,
633    ready: bool,
634    complete: bool,
635    attention_required: bool,
636    outcome: RestoreApplyReportOutcome,
637    operation_count: usize,
638    pending_operations: usize,
639    ready_operations: usize,
640    blocked_operations: usize,
641    completed_operations: usize,
642    failed_operations: usize,
643    blocked_reasons: Vec<String>,
644    next_transition: Option<RestoreApplyReportOperation>,
645    #[serde(skip_serializing_if = "Option::is_none")]
646    operation_available: Option<bool>,
647    #[serde(skip_serializing_if = "Option::is_none")]
648    command_available: Option<bool>,
649    #[serde(skip_serializing_if = "Option::is_none")]
650    command: Option<RestoreApplyRunnerCommand>,
651}
652
653///
654/// RestoreRunExecutedOperation
655///
656
657#[derive(Clone, Debug, Serialize)]
658struct RestoreRunExecutedOperation {
659    sequence: usize,
660    operation: RestoreApplyOperationKind,
661    target_canister: String,
662    command: RestoreApplyRunnerCommand,
663    status: String,
664    state: &'static str,
665}
666
667///
668/// RestoreRunResponseMode
669///
670
671struct RestoreRunResponseMode {
672    run_mode: &'static str,
673    dry_run: bool,
674    execute: bool,
675    unclaim_pending: bool,
676    stopped_reason: &'static str,
677    next_action: &'static str,
678}
679
680///
681/// RestoreApplyNextOptions
682///
683
684#[derive(Clone, Debug, Eq, PartialEq)]
685pub struct RestoreApplyNextOptions {
686    pub journal: PathBuf,
687    pub out: Option<PathBuf>,
688}
689
690impl RestoreApplyNextOptions {
691    /// Parse restore apply-next options from CLI arguments.
692    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
693    where
694        I: IntoIterator<Item = OsString>,
695    {
696        let mut journal = None;
697        let mut out = None;
698
699        let mut args = args.into_iter();
700        while let Some(arg) = args.next() {
701            let arg = arg
702                .into_string()
703                .map_err(|_| RestoreCommandError::Usage(usage()))?;
704            match arg.as_str() {
705                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
706                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
707                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
708                _ => return Err(RestoreCommandError::UnknownOption(arg)),
709            }
710        }
711
712        Ok(Self {
713            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
714            out,
715        })
716    }
717}
718
719///
720/// RestoreApplyCommandOptions
721///
722
723#[derive(Clone, Debug, Eq, PartialEq)]
724pub struct RestoreApplyCommandOptions {
725    pub journal: PathBuf,
726    pub dfx: String,
727    pub network: Option<String>,
728    pub out: Option<PathBuf>,
729    pub require_command: bool,
730}
731
732impl RestoreApplyCommandOptions {
733    /// Parse restore apply-command options from CLI arguments.
734    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
735    where
736        I: IntoIterator<Item = OsString>,
737    {
738        let mut journal = None;
739        let mut dfx = "dfx".to_string();
740        let mut network = None;
741        let mut out = None;
742        let mut require_command = false;
743
744        let mut args = args.into_iter();
745        while let Some(arg) = args.next() {
746            let arg = arg
747                .into_string()
748                .map_err(|_| RestoreCommandError::Usage(usage()))?;
749            match arg.as_str() {
750                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
751                "--dfx" => dfx = next_value(&mut args, "--dfx")?,
752                "--network" => network = Some(next_value(&mut args, "--network")?),
753                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
754                "--require-command" => require_command = true,
755                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
756                _ => return Err(RestoreCommandError::UnknownOption(arg)),
757            }
758        }
759
760        Ok(Self {
761            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
762            dfx,
763            network,
764            out,
765            require_command,
766        })
767    }
768}
769
770///
771/// RestoreApplyClaimOptions
772///
773
774#[derive(Clone, Debug, Eq, PartialEq)]
775pub struct RestoreApplyClaimOptions {
776    pub journal: PathBuf,
777    pub sequence: Option<usize>,
778    pub updated_at: Option<String>,
779    pub out: Option<PathBuf>,
780}
781
782impl RestoreApplyClaimOptions {
783    /// Parse restore apply-claim options from CLI arguments.
784    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
785    where
786        I: IntoIterator<Item = OsString>,
787    {
788        let mut journal = None;
789        let mut sequence = None;
790        let mut updated_at = None;
791        let mut out = None;
792
793        let mut args = args.into_iter();
794        while let Some(arg) = args.next() {
795            let arg = arg
796                .into_string()
797                .map_err(|_| RestoreCommandError::Usage(usage()))?;
798            match arg.as_str() {
799                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
800                "--sequence" => {
801                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
802                }
803                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
804                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
805                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
806                _ => return Err(RestoreCommandError::UnknownOption(arg)),
807            }
808        }
809
810        Ok(Self {
811            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
812            sequence,
813            updated_at,
814            out,
815        })
816    }
817}
818
819///
820/// RestoreApplyMarkOptions
821///
822
823#[derive(Clone, Debug, Eq, PartialEq)]
824pub struct RestoreApplyUnclaimOptions {
825    pub journal: PathBuf,
826    pub sequence: Option<usize>,
827    pub updated_at: Option<String>,
828    pub out: Option<PathBuf>,
829}
830
831impl RestoreApplyUnclaimOptions {
832    /// Parse restore apply-unclaim options from CLI arguments.
833    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
834    where
835        I: IntoIterator<Item = OsString>,
836    {
837        let mut journal = None;
838        let mut sequence = None;
839        let mut updated_at = None;
840        let mut out = None;
841
842        let mut args = args.into_iter();
843        while let Some(arg) = args.next() {
844            let arg = arg
845                .into_string()
846                .map_err(|_| RestoreCommandError::Usage(usage()))?;
847            match arg.as_str() {
848                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
849                "--sequence" => {
850                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
851                }
852                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
853                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
854                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
855                _ => return Err(RestoreCommandError::UnknownOption(arg)),
856            }
857        }
858
859        Ok(Self {
860            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
861            sequence,
862            updated_at,
863            out,
864        })
865    }
866}
867
868///
869/// RestoreApplyMarkOptions
870///
871
872#[derive(Clone, Debug, Eq, PartialEq)]
873pub struct RestoreApplyMarkOptions {
874    pub journal: PathBuf,
875    pub sequence: usize,
876    pub state: RestoreApplyMarkState,
877    pub reason: Option<String>,
878    pub updated_at: Option<String>,
879    pub out: Option<PathBuf>,
880    pub require_pending: bool,
881}
882
883impl RestoreApplyMarkOptions {
884    /// Parse restore apply-mark options from CLI arguments.
885    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
886    where
887        I: IntoIterator<Item = OsString>,
888    {
889        let mut journal = None;
890        let mut sequence = None;
891        let mut state = None;
892        let mut reason = None;
893        let mut updated_at = None;
894        let mut out = None;
895        let mut require_pending = false;
896
897        let mut args = args.into_iter();
898        while let Some(arg) = args.next() {
899            let arg = arg
900                .into_string()
901                .map_err(|_| RestoreCommandError::Usage(usage()))?;
902            match arg.as_str() {
903                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
904                "--sequence" => {
905                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
906                }
907                "--state" => {
908                    state = Some(RestoreApplyMarkState::parse(next_value(
909                        &mut args, "--state",
910                    )?)?);
911                }
912                "--reason" => reason = Some(next_value(&mut args, "--reason")?),
913                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
914                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
915                "--require-pending" => require_pending = true,
916                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
917                _ => return Err(RestoreCommandError::UnknownOption(arg)),
918            }
919        }
920
921        Ok(Self {
922            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
923            sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
924            state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
925            reason,
926            updated_at,
927            out,
928            require_pending,
929        })
930    }
931}
932
933///
934/// RestoreApplyMarkState
935///
936
937#[derive(Clone, Debug, Eq, PartialEq)]
938pub enum RestoreApplyMarkState {
939    Completed,
940    Failed,
941}
942
943impl RestoreApplyMarkState {
944    // Parse the restricted operation states accepted by apply-mark.
945    fn parse(value: String) -> Result<Self, RestoreCommandError> {
946        match value.as_str() {
947            "completed" => Ok(Self::Completed),
948            "failed" => Ok(Self::Failed),
949            _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
950        }
951    }
952}
953
954/// Run a restore subcommand.
955pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
956where
957    I: IntoIterator<Item = OsString>,
958{
959    let mut args = args.into_iter();
960    let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
961        return Err(RestoreCommandError::Usage(usage()));
962    };
963
964    match command.as_str() {
965        "plan" => {
966            let options = RestorePlanOptions::parse(args)?;
967            let plan = plan_restore(&options)?;
968            write_plan(&options, &plan)?;
969            enforce_restore_plan_requirements(&options, &plan)?;
970            Ok(())
971        }
972        "status" => {
973            let options = RestoreStatusOptions::parse(args)?;
974            let status = restore_status(&options)?;
975            write_status(&options, &status)?;
976            Ok(())
977        }
978        "apply" => {
979            let options = RestoreApplyOptions::parse(args)?;
980            let dry_run = restore_apply_dry_run(&options)?;
981            write_apply_dry_run(&options, &dry_run)?;
982            write_apply_journal_if_requested(&options, &dry_run)?;
983            Ok(())
984        }
985        "apply-status" => {
986            let options = RestoreApplyStatusOptions::parse(args)?;
987            let status = restore_apply_status(&options)?;
988            write_apply_status(&options, &status)?;
989            enforce_apply_status_requirements(&options, &status)?;
990            Ok(())
991        }
992        "apply-report" => {
993            let options = RestoreApplyReportOptions::parse(args)?;
994            let report = restore_apply_report(&options)?;
995            write_apply_report(&options, &report)?;
996            enforce_apply_report_requirements(&options, &report)?;
997            Ok(())
998        }
999        "run" => {
1000            let options = RestoreRunOptions::parse(args)?;
1001            let run = if options.execute {
1002                restore_run_execute_result(&options)?
1003            } else if options.unclaim_pending {
1004                RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1005            } else {
1006                RestoreRunResult::ok(restore_run_dry_run(&options)?)
1007            };
1008            write_restore_run(&options, &run.response)?;
1009            if let Some(error) = run.error {
1010                return Err(error);
1011            }
1012            enforce_restore_run_requirements(&options, &run.response)?;
1013            Ok(())
1014        }
1015        "apply-next" => {
1016            let options = RestoreApplyNextOptions::parse(args)?;
1017            let next = restore_apply_next(&options)?;
1018            write_apply_next(&options, &next)?;
1019            Ok(())
1020        }
1021        "apply-command" => {
1022            let options = RestoreApplyCommandOptions::parse(args)?;
1023            let preview = restore_apply_command(&options)?;
1024            write_apply_command(&options, &preview)?;
1025            enforce_apply_command_requirements(&options, &preview)?;
1026            Ok(())
1027        }
1028        "apply-claim" => {
1029            let options = RestoreApplyClaimOptions::parse(args)?;
1030            let journal = restore_apply_claim(&options)?;
1031            write_apply_claim(&options, &journal)?;
1032            Ok(())
1033        }
1034        "apply-unclaim" => {
1035            let options = RestoreApplyUnclaimOptions::parse(args)?;
1036            let journal = restore_apply_unclaim(&options)?;
1037            write_apply_unclaim(&options, &journal)?;
1038            Ok(())
1039        }
1040        "apply-mark" => {
1041            let options = RestoreApplyMarkOptions::parse(args)?;
1042            let journal = restore_apply_mark(&options)?;
1043            write_apply_mark(&options, &journal)?;
1044            Ok(())
1045        }
1046        "help" | "--help" | "-h" => Err(RestoreCommandError::Usage(usage())),
1047        _ => Err(RestoreCommandError::UnknownOption(command)),
1048    }
1049}
1050
1051/// Build a no-mutation restore plan from a manifest and optional mapping.
1052pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1053    verify_backup_layout_if_required(options)?;
1054
1055    let manifest = read_manifest_source(options)?;
1056    let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1057
1058    RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1059}
1060
1061/// Build the initial no-mutation restore status from a restore plan.
1062pub fn restore_status(
1063    options: &RestoreStatusOptions,
1064) -> Result<RestoreStatus, RestoreCommandError> {
1065    let plan = read_plan(&options.plan)?;
1066    Ok(RestoreStatus::from_plan(&plan))
1067}
1068
1069/// Build a no-mutation restore apply dry-run from a restore plan.
1070pub fn restore_apply_dry_run(
1071    options: &RestoreApplyOptions,
1072) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1073    let plan = read_plan(&options.plan)?;
1074    let status = options.status.as_ref().map(read_status).transpose()?;
1075    if let Some(backup_dir) = &options.backup_dir {
1076        return RestoreApplyDryRun::try_from_plan_with_artifacts(
1077            &plan,
1078            status.as_ref(),
1079            backup_dir,
1080        )
1081        .map_err(RestoreCommandError::from);
1082    }
1083
1084    RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1085}
1086
1087/// Build a compact restore apply status from a journal file.
1088pub fn restore_apply_status(
1089    options: &RestoreApplyStatusOptions,
1090) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1091    let journal = read_apply_journal(&options.journal)?;
1092    Ok(journal.status())
1093}
1094
1095/// Build an operator-oriented restore apply report from a journal file.
1096pub fn restore_apply_report(
1097    options: &RestoreApplyReportOptions,
1098) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1099    let journal = read_apply_journal(&options.journal)?;
1100    Ok(journal.report())
1101}
1102
1103/// Build a no-mutation native restore runner preview from a journal file.
1104pub fn restore_run_dry_run(
1105    options: &RestoreRunOptions,
1106) -> Result<RestoreRunResponse, RestoreCommandError> {
1107    let journal = read_apply_journal(&options.journal)?;
1108    let report = journal.report();
1109    let preview = journal.next_command_preview_with_config(&RestoreApplyCommandConfig {
1110        program: options.dfx.clone(),
1111        network: options.network.clone(),
1112    });
1113    let stopped_reason = restore_run_stopped_reason(&report, false, false);
1114    let next_action = restore_run_next_action(&report, false);
1115
1116    let mut response = restore_run_response_from_report(
1117        journal.backup_id,
1118        report,
1119        RestoreRunResponseMode {
1120            run_mode: "dry-run",
1121            dry_run: true,
1122            execute: false,
1123            unclaim_pending: false,
1124            stopped_reason,
1125            next_action,
1126        },
1127    );
1128    response.operation_available = Some(preview.operation_available);
1129    response.command_available = Some(preview.command_available);
1130    response.command = preview.command;
1131    Ok(response)
1132}
1133
1134/// Recover an interrupted restore runner by unclaiming the pending operation.
1135pub fn restore_run_unclaim_pending(
1136    options: &RestoreRunOptions,
1137) -> Result<RestoreRunResponse, RestoreCommandError> {
1138    let mut journal = read_apply_journal(&options.journal)?;
1139    let recovered_operation = journal
1140        .next_transition_operation()
1141        .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1142        .cloned()
1143        .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1144
1145    journal.mark_next_operation_ready_at(Some(timestamp_placeholder()))?;
1146    write_apply_journal_file(&options.journal, &journal)?;
1147
1148    let report = journal.report();
1149    let next_action = restore_run_next_action(&report, true);
1150    let mut response = restore_run_response_from_report(
1151        journal.backup_id,
1152        report,
1153        RestoreRunResponseMode {
1154            run_mode: "unclaim-pending",
1155            dry_run: false,
1156            execute: false,
1157            unclaim_pending: true,
1158            stopped_reason: "recovered-pending",
1159            next_action,
1160        },
1161    );
1162    response.recovered_operation = Some(recovered_operation);
1163    Ok(response)
1164}
1165
1166/// Execute ready restore apply journal operations through generated runner commands.
1167pub fn restore_run_execute(
1168    options: &RestoreRunOptions,
1169) -> Result<RestoreRunResponse, RestoreCommandError> {
1170    let run = restore_run_execute_result(options)?;
1171    if let Some(error) = run.error {
1172        return Err(error);
1173    }
1174
1175    Ok(run.response)
1176}
1177
1178// Execute ready restore apply operations and retain any deferred runner error.
1179fn restore_run_execute_result(
1180    options: &RestoreRunOptions,
1181) -> Result<RestoreRunResult, RestoreCommandError> {
1182    let mut journal = read_apply_journal(&options.journal)?;
1183    let mut executed_operations = Vec::new();
1184    let config = RestoreApplyCommandConfig {
1185        program: options.dfx.clone(),
1186        network: options.network.clone(),
1187    };
1188
1189    loop {
1190        let report = journal.report();
1191        if report.complete || options.max_steps == Some(executed_operations.len()) {
1192            let max_steps_reached =
1193                options.max_steps == Some(executed_operations.len()) && !report.complete;
1194            return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1195                &journal,
1196                executed_operations,
1197                max_steps_reached,
1198            )));
1199        }
1200
1201        enforce_restore_run_executable(&journal, &report)?;
1202        let preview = journal.next_command_preview_with_config(&config);
1203        enforce_restore_run_command_available(&preview)?;
1204
1205        let operation = preview
1206            .operation
1207            .clone()
1208            .ok_or_else(|| restore_run_command_unavailable_error(&preview))?;
1209        let command = preview
1210            .command
1211            .clone()
1212            .ok_or_else(|| restore_run_command_unavailable_error(&preview))?;
1213        let sequence = operation.sequence;
1214
1215        enforce_apply_claim_sequence(sequence, &journal)?;
1216        journal.mark_operation_pending_at(sequence, Some(timestamp_placeholder()))?;
1217        write_apply_journal_file(&options.journal, &journal)?;
1218
1219        let status = Command::new(&command.program)
1220            .args(&command.args)
1221            .status()?;
1222        let status_label = exit_status_label(status);
1223        if status.success() {
1224            journal.mark_operation_completed_at(sequence, Some(timestamp_placeholder()))?;
1225            write_apply_journal_file(&options.journal, &journal)?;
1226            executed_operations.push(RestoreRunExecutedOperation {
1227                sequence,
1228                operation: operation.operation,
1229                target_canister: operation.target_canister,
1230                command,
1231                status: status_label,
1232                state: "completed",
1233            });
1234            continue;
1235        }
1236
1237        journal.mark_operation_failed_at(
1238            sequence,
1239            format!("runner-command-exit-{status_label}"),
1240            Some(timestamp_placeholder()),
1241        )?;
1242        write_apply_journal_file(&options.journal, &journal)?;
1243        executed_operations.push(RestoreRunExecutedOperation {
1244            sequence,
1245            operation: operation.operation,
1246            target_canister: operation.target_canister,
1247            command,
1248            status: status_label.clone(),
1249            state: "failed",
1250        });
1251        let response = restore_run_execute_summary(&journal, executed_operations, false);
1252        return Ok(RestoreRunResult {
1253            response,
1254            error: Some(RestoreCommandError::RestoreRunCommandFailed {
1255                sequence,
1256                status: status_label,
1257            }),
1258        });
1259    }
1260}
1261
1262// Build the final native runner execution summary.
1263fn restore_run_execute_summary(
1264    journal: &RestoreApplyJournal,
1265    executed_operations: Vec<RestoreRunExecutedOperation>,
1266    max_steps_reached: bool,
1267) -> RestoreRunResponse {
1268    let report = journal.report();
1269    let executed_operation_count = executed_operations.len();
1270    let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
1271    let next_action = restore_run_next_action(&report, false);
1272
1273    let mut response = restore_run_response_from_report(
1274        journal.backup_id.clone(),
1275        report,
1276        RestoreRunResponseMode {
1277            run_mode: "execute",
1278            dry_run: false,
1279            execute: true,
1280            unclaim_pending: false,
1281            stopped_reason,
1282            next_action,
1283        },
1284    );
1285    response.max_steps_reached = Some(max_steps_reached);
1286    response.executed_operation_count = Some(executed_operation_count);
1287    response.executed_operations = executed_operations;
1288    response
1289}
1290
1291// Build the shared native runner response fields from an apply journal report.
1292fn restore_run_response_from_report(
1293    backup_id: String,
1294    report: RestoreApplyJournalReport,
1295    mode: RestoreRunResponseMode,
1296) -> RestoreRunResponse {
1297    RestoreRunResponse {
1298        run_version: 1,
1299        backup_id,
1300        run_mode: mode.run_mode,
1301        dry_run: mode.dry_run,
1302        execute: mode.execute,
1303        unclaim_pending: mode.unclaim_pending,
1304        stopped_reason: mode.stopped_reason,
1305        next_action: mode.next_action,
1306        max_steps_reached: None,
1307        executed_operations: Vec::new(),
1308        executed_operation_count: None,
1309        recovered_operation: None,
1310        ready: report.ready,
1311        complete: report.complete,
1312        attention_required: report.attention_required,
1313        outcome: report.outcome,
1314        operation_count: report.operation_count,
1315        pending_operations: report.pending_operations,
1316        ready_operations: report.ready_operations,
1317        blocked_operations: report.blocked_operations,
1318        completed_operations: report.completed_operations,
1319        failed_operations: report.failed_operations,
1320        blocked_reasons: report.blocked_reasons,
1321        next_transition: report.next_transition,
1322        operation_available: None,
1323        command_available: None,
1324        command: None,
1325    }
1326}
1327
1328// Classify why the native runner stopped for operator summaries.
1329const fn restore_run_stopped_reason(
1330    report: &RestoreApplyJournalReport,
1331    max_steps_reached: bool,
1332    executed: bool,
1333) -> &'static str {
1334    if report.complete {
1335        return "complete";
1336    }
1337    if report.failed_operations > 0 {
1338        return "command-failed";
1339    }
1340    if report.pending_operations > 0 {
1341        return "pending";
1342    }
1343    if !report.ready || report.blocked_operations > 0 {
1344        return "blocked";
1345    }
1346    if max_steps_reached {
1347        return "max-steps-reached";
1348    }
1349    if executed {
1350        return "ready";
1351    }
1352    "preview"
1353}
1354
1355// Recommend the next operator action for the native runner summary.
1356const fn restore_run_next_action(
1357    report: &RestoreApplyJournalReport,
1358    recovered_pending: bool,
1359) -> &'static str {
1360    if report.complete {
1361        return "done";
1362    }
1363    if report.failed_operations > 0 {
1364        return "inspect-failed-operation";
1365    }
1366    if report.pending_operations > 0 {
1367        return "unclaim-pending";
1368    }
1369    if !report.ready || report.blocked_operations > 0 {
1370        return "fix-blocked-journal";
1371    }
1372    if recovered_pending {
1373        return "rerun";
1374    }
1375    "rerun"
1376}
1377
1378// Ensure the journal can be advanced by the native restore runner.
1379fn enforce_restore_run_executable(
1380    journal: &RestoreApplyJournal,
1381    report: &RestoreApplyJournalReport,
1382) -> Result<(), RestoreCommandError> {
1383    if report.pending_operations > 0 {
1384        return Err(RestoreCommandError::RestoreApplyPending {
1385            backup_id: report.backup_id.clone(),
1386            pending_operations: report.pending_operations,
1387            next_transition_sequence: report
1388                .next_transition
1389                .as_ref()
1390                .map(|operation| operation.sequence),
1391        });
1392    }
1393
1394    if report.failed_operations > 0 {
1395        return Err(RestoreCommandError::RestoreApplyFailed {
1396            backup_id: report.backup_id.clone(),
1397            failed_operations: report.failed_operations,
1398        });
1399    }
1400
1401    if report.ready {
1402        return Ok(());
1403    }
1404
1405    Err(RestoreCommandError::RestoreApplyNotReady {
1406        backup_id: journal.backup_id.clone(),
1407        reasons: report.blocked_reasons.clone(),
1408    })
1409}
1410
1411// Convert an unavailable native runner command into the shared fail-closed error.
1412fn enforce_restore_run_command_available(
1413    preview: &RestoreApplyCommandPreview,
1414) -> Result<(), RestoreCommandError> {
1415    if preview.command_available {
1416        return Ok(());
1417    }
1418
1419    Err(restore_run_command_unavailable_error(preview))
1420}
1421
1422// Build a restore runner command-unavailable error from a preview.
1423fn restore_run_command_unavailable_error(
1424    preview: &RestoreApplyCommandPreview,
1425) -> RestoreCommandError {
1426    RestoreCommandError::RestoreApplyCommandUnavailable {
1427        backup_id: preview.backup_id.clone(),
1428        operation_available: preview.operation_available,
1429        complete: preview.complete,
1430        blocked_reasons: preview.blocked_reasons.clone(),
1431    }
1432}
1433
1434// Render process exit status without relying on platform-specific internals.
1435fn exit_status_label(status: std::process::ExitStatus) -> String {
1436    status
1437        .code()
1438        .map_or_else(|| "signal".to_string(), |code| code.to_string())
1439}
1440
1441// Enforce caller-requested native runner requirements after output is emitted.
1442fn enforce_restore_run_requirements(
1443    options: &RestoreRunOptions,
1444    run: &RestoreRunResponse,
1445) -> Result<(), RestoreCommandError> {
1446    if options.require_complete && !run.complete {
1447        return Err(RestoreCommandError::RestoreApplyIncomplete {
1448            backup_id: run.backup_id.clone(),
1449            completed_operations: run.completed_operations,
1450            operation_count: run.operation_count,
1451        });
1452    }
1453
1454    if options.require_no_attention && run.attention_required {
1455        return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1456            backup_id: run.backup_id.clone(),
1457            outcome: run.outcome.clone(),
1458        });
1459    }
1460
1461    if let Some(expected) = &options.require_run_mode
1462        && run.run_mode != expected
1463    {
1464        return Err(RestoreCommandError::RestoreRunModeMismatch {
1465            backup_id: run.backup_id.clone(),
1466            expected: expected.clone(),
1467            actual: run.run_mode.to_string(),
1468        });
1469    }
1470
1471    if let Some(expected) = &options.require_stopped_reason
1472        && run.stopped_reason != expected
1473    {
1474        return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
1475            backup_id: run.backup_id.clone(),
1476            expected: expected.clone(),
1477            actual: run.stopped_reason.to_string(),
1478        });
1479    }
1480
1481    if let Some(expected) = &options.require_next_action
1482        && run.next_action != expected
1483    {
1484        return Err(RestoreCommandError::RestoreRunNextActionMismatch {
1485            backup_id: run.backup_id.clone(),
1486            expected: expected.clone(),
1487            actual: run.next_action.to_string(),
1488        });
1489    }
1490
1491    if let Some(expected) = options.require_executed_count {
1492        let actual = run.executed_operation_count.unwrap_or(0);
1493        if actual != expected {
1494            return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
1495                backup_id: run.backup_id.clone(),
1496                expected,
1497                actual,
1498            });
1499        }
1500    }
1501
1502    Ok(())
1503}
1504
1505// Enforce caller-requested apply report requirements after report output is emitted.
1506fn enforce_apply_report_requirements(
1507    options: &RestoreApplyReportOptions,
1508    report: &RestoreApplyJournalReport,
1509) -> Result<(), RestoreCommandError> {
1510    if !options.require_no_attention || !report.attention_required {
1511        return Ok(());
1512    }
1513
1514    Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1515        backup_id: report.backup_id.clone(),
1516        outcome: report.outcome.clone(),
1517    })
1518}
1519
1520// Enforce caller-requested apply journal requirements after status is emitted.
1521fn enforce_apply_status_requirements(
1522    options: &RestoreApplyStatusOptions,
1523    status: &RestoreApplyJournalStatus,
1524) -> Result<(), RestoreCommandError> {
1525    if options.require_ready && !status.ready {
1526        return Err(RestoreCommandError::RestoreApplyNotReady {
1527            backup_id: status.backup_id.clone(),
1528            reasons: status.blocked_reasons.clone(),
1529        });
1530    }
1531
1532    if options.require_no_pending && status.pending_operations > 0 {
1533        return Err(RestoreCommandError::RestoreApplyPending {
1534            backup_id: status.backup_id.clone(),
1535            pending_operations: status.pending_operations,
1536            next_transition_sequence: status.next_transition_sequence,
1537        });
1538    }
1539
1540    if options.require_no_failed && status.failed_operations > 0 {
1541        return Err(RestoreCommandError::RestoreApplyFailed {
1542            backup_id: status.backup_id.clone(),
1543            failed_operations: status.failed_operations,
1544        });
1545    }
1546
1547    if options.require_complete && !status.complete {
1548        return Err(RestoreCommandError::RestoreApplyIncomplete {
1549            backup_id: status.backup_id.clone(),
1550            completed_operations: status.completed_operations,
1551            operation_count: status.operation_count,
1552        });
1553    }
1554
1555    Ok(())
1556}
1557
1558/// Build the next restore apply operation response from a journal file.
1559pub fn restore_apply_next(
1560    options: &RestoreApplyNextOptions,
1561) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
1562    let journal = read_apply_journal(&options.journal)?;
1563    Ok(journal.next_operation())
1564}
1565
1566/// Build the next restore apply command preview from a journal file.
1567pub fn restore_apply_command(
1568    options: &RestoreApplyCommandOptions,
1569) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
1570    let journal = read_apply_journal(&options.journal)?;
1571    Ok(
1572        journal.next_command_preview_with_config(&RestoreApplyCommandConfig {
1573            program: options.dfx.clone(),
1574            network: options.network.clone(),
1575        }),
1576    )
1577}
1578
1579// Enforce caller-requested command preview requirements after preview output is emitted.
1580fn enforce_apply_command_requirements(
1581    options: &RestoreApplyCommandOptions,
1582    preview: &RestoreApplyCommandPreview,
1583) -> Result<(), RestoreCommandError> {
1584    if !options.require_command || preview.command_available {
1585        return Ok(());
1586    }
1587
1588    Err(RestoreCommandError::RestoreApplyCommandUnavailable {
1589        backup_id: preview.backup_id.clone(),
1590        operation_available: preview.operation_available,
1591        complete: preview.complete,
1592        blocked_reasons: preview.blocked_reasons.clone(),
1593    })
1594}
1595
1596/// Mark the next restore apply journal operation pending.
1597pub fn restore_apply_claim(
1598    options: &RestoreApplyClaimOptions,
1599) -> Result<RestoreApplyJournal, RestoreCommandError> {
1600    let mut journal = read_apply_journal(&options.journal)?;
1601    let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
1602
1603    if let Some(sequence) = options.sequence {
1604        enforce_apply_claim_sequence(sequence, &journal)?;
1605        journal.mark_operation_pending_at(sequence, updated_at)?;
1606        return Ok(journal);
1607    }
1608
1609    journal.mark_next_operation_pending_at(updated_at)?;
1610    Ok(journal)
1611}
1612
1613// Ensure a runner claim still matches the operation it previewed.
1614fn enforce_apply_claim_sequence(
1615    expected: usize,
1616    journal: &RestoreApplyJournal,
1617) -> Result<(), RestoreCommandError> {
1618    let actual = journal
1619        .next_transition_operation()
1620        .map(|operation| operation.sequence);
1621
1622    if actual == Some(expected) {
1623        return Ok(());
1624    }
1625
1626    Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
1627}
1628
1629/// Mark the current pending restore apply journal operation ready again.
1630pub fn restore_apply_unclaim(
1631    options: &RestoreApplyUnclaimOptions,
1632) -> Result<RestoreApplyJournal, RestoreCommandError> {
1633    let mut journal = read_apply_journal(&options.journal)?;
1634    if let Some(sequence) = options.sequence {
1635        enforce_apply_unclaim_sequence(sequence, &journal)?;
1636    }
1637
1638    journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
1639    Ok(journal)
1640}
1641
1642// Ensure a runner unclaim still matches the pending operation it recovered.
1643fn enforce_apply_unclaim_sequence(
1644    expected: usize,
1645    journal: &RestoreApplyJournal,
1646) -> Result<(), RestoreCommandError> {
1647    let actual = journal
1648        .next_transition_operation()
1649        .map(|operation| operation.sequence);
1650
1651    if actual == Some(expected) {
1652        return Ok(());
1653    }
1654
1655    Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
1656}
1657
1658/// Mark one restore apply journal operation completed or failed.
1659pub fn restore_apply_mark(
1660    options: &RestoreApplyMarkOptions,
1661) -> Result<RestoreApplyJournal, RestoreCommandError> {
1662    let mut journal = read_apply_journal(&options.journal)?;
1663    enforce_apply_mark_pending_requirement(options, &journal)?;
1664
1665    match options.state {
1666        RestoreApplyMarkState::Completed => {
1667            journal.mark_operation_completed_at(
1668                options.sequence,
1669                Some(state_updated_at(options.updated_at.as_ref())),
1670            )?;
1671        }
1672        RestoreApplyMarkState::Failed => {
1673            let reason =
1674                options
1675                    .reason
1676                    .clone()
1677                    .ok_or(RestoreApplyJournalError::FailureReasonRequired(
1678                        options.sequence,
1679                    ))?;
1680            journal.mark_operation_failed_at(
1681                options.sequence,
1682                reason,
1683                Some(state_updated_at(options.updated_at.as_ref())),
1684            )?;
1685        }
1686    }
1687
1688    Ok(journal)
1689}
1690
1691// Enforce that apply-mark only records an already claimed operation when requested.
1692fn enforce_apply_mark_pending_requirement(
1693    options: &RestoreApplyMarkOptions,
1694    journal: &RestoreApplyJournal,
1695) -> Result<(), RestoreCommandError> {
1696    if !options.require_pending {
1697        return Ok(());
1698    }
1699
1700    let state = journal
1701        .operations
1702        .iter()
1703        .find(|operation| operation.sequence == options.sequence)
1704        .map(|operation| operation.state.clone())
1705        .ok_or(RestoreApplyJournalError::OperationNotFound(
1706            options.sequence,
1707        ))?;
1708
1709    if state == RestoreApplyOperationState::Pending {
1710        return Ok(());
1711    }
1712
1713    Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
1714        sequence: options.sequence,
1715        state,
1716    })
1717}
1718
1719// Enforce caller-requested restore plan requirements after the plan is emitted.
1720fn enforce_restore_plan_requirements(
1721    options: &RestorePlanOptions,
1722    plan: &RestorePlan,
1723) -> Result<(), RestoreCommandError> {
1724    if !options.require_restore_ready || plan.readiness_summary.ready {
1725        return Ok(());
1726    }
1727
1728    Err(RestoreCommandError::RestoreNotReady {
1729        backup_id: plan.backup_id.clone(),
1730        reasons: plan.readiness_summary.reasons.clone(),
1731    })
1732}
1733
1734// Verify backup layout integrity before restore planning when requested.
1735fn verify_backup_layout_if_required(
1736    options: &RestorePlanOptions,
1737) -> Result<(), RestoreCommandError> {
1738    if !options.require_verified {
1739        return Ok(());
1740    }
1741
1742    let Some(dir) = &options.backup_dir else {
1743        return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
1744    };
1745
1746    BackupLayout::new(dir.clone()).verify_integrity()?;
1747    Ok(())
1748}
1749
1750// Read the manifest from a direct path or canonical backup layout.
1751fn read_manifest_source(
1752    options: &RestorePlanOptions,
1753) -> Result<FleetBackupManifest, RestoreCommandError> {
1754    if let Some(path) = &options.manifest {
1755        return read_manifest(path);
1756    }
1757
1758    let Some(dir) = &options.backup_dir else {
1759        return Err(RestoreCommandError::MissingOption(
1760            "--manifest or --backup-dir",
1761        ));
1762    };
1763
1764    BackupLayout::new(dir.clone())
1765        .read_manifest()
1766        .map_err(RestoreCommandError::from)
1767}
1768
1769// Read and decode a fleet backup manifest from disk.
1770fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
1771    let data = fs::read_to_string(path)?;
1772    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1773}
1774
1775// Read and decode an optional source-to-target restore mapping from disk.
1776fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
1777    let data = fs::read_to_string(path)?;
1778    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1779}
1780
1781// Read and decode a restore plan from disk.
1782fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
1783    let data = fs::read_to_string(path)?;
1784    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1785}
1786
1787// Read and decode a restore status from disk.
1788fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
1789    let data = fs::read_to_string(path)?;
1790    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1791}
1792
1793// Read and decode a restore apply journal from disk.
1794fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
1795    let data = fs::read_to_string(path)?;
1796    let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
1797    journal.validate()?;
1798    Ok(journal)
1799}
1800
1801// Parse a restore apply journal operation sequence value.
1802fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
1803    value
1804        .parse::<usize>()
1805        .map_err(|_| RestoreCommandError::InvalidSequence)
1806}
1807
1808// Return the caller-supplied journal update marker or the current placeholder.
1809fn state_updated_at(updated_at: Option<&String>) -> String {
1810    updated_at.cloned().unwrap_or_else(timestamp_placeholder)
1811}
1812
1813// Return a placeholder timestamp until the CLI owns a clock abstraction.
1814fn timestamp_placeholder() -> String {
1815    "unknown".to_string()
1816}
1817
1818// Write the computed plan to stdout or a requested output file.
1819fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
1820    if let Some(path) = &options.out {
1821        let data = serde_json::to_vec_pretty(plan)?;
1822        fs::write(path, data)?;
1823        return Ok(());
1824    }
1825
1826    let stdout = io::stdout();
1827    let mut handle = stdout.lock();
1828    serde_json::to_writer_pretty(&mut handle, plan)?;
1829    writeln!(handle)?;
1830    Ok(())
1831}
1832
1833// Write the computed status to stdout or a requested output file.
1834fn write_status(
1835    options: &RestoreStatusOptions,
1836    status: &RestoreStatus,
1837) -> Result<(), RestoreCommandError> {
1838    if let Some(path) = &options.out {
1839        let data = serde_json::to_vec_pretty(status)?;
1840        fs::write(path, data)?;
1841        return Ok(());
1842    }
1843
1844    let stdout = io::stdout();
1845    let mut handle = stdout.lock();
1846    serde_json::to_writer_pretty(&mut handle, status)?;
1847    writeln!(handle)?;
1848    Ok(())
1849}
1850
1851// Write the computed apply dry-run to stdout or a requested output file.
1852fn write_apply_dry_run(
1853    options: &RestoreApplyOptions,
1854    dry_run: &RestoreApplyDryRun,
1855) -> Result<(), RestoreCommandError> {
1856    if let Some(path) = &options.out {
1857        let data = serde_json::to_vec_pretty(dry_run)?;
1858        fs::write(path, data)?;
1859        return Ok(());
1860    }
1861
1862    let stdout = io::stdout();
1863    let mut handle = stdout.lock();
1864    serde_json::to_writer_pretty(&mut handle, dry_run)?;
1865    writeln!(handle)?;
1866    Ok(())
1867}
1868
1869// Write the initial apply journal when the caller requests one.
1870fn write_apply_journal_if_requested(
1871    options: &RestoreApplyOptions,
1872    dry_run: &RestoreApplyDryRun,
1873) -> Result<(), RestoreCommandError> {
1874    let Some(path) = &options.journal_out else {
1875        return Ok(());
1876    };
1877
1878    let journal = RestoreApplyJournal::from_dry_run(dry_run);
1879    let data = serde_json::to_vec_pretty(&journal)?;
1880    fs::write(path, data)?;
1881    Ok(())
1882}
1883
1884// Write the computed apply journal status to stdout or a requested output file.
1885fn write_apply_status(
1886    options: &RestoreApplyStatusOptions,
1887    status: &RestoreApplyJournalStatus,
1888) -> Result<(), RestoreCommandError> {
1889    if let Some(path) = &options.out {
1890        let data = serde_json::to_vec_pretty(status)?;
1891        fs::write(path, data)?;
1892        return Ok(());
1893    }
1894
1895    let stdout = io::stdout();
1896    let mut handle = stdout.lock();
1897    serde_json::to_writer_pretty(&mut handle, status)?;
1898    writeln!(handle)?;
1899    Ok(())
1900}
1901
1902// Write the computed apply journal report to stdout or a requested output file.
1903fn write_apply_report(
1904    options: &RestoreApplyReportOptions,
1905    report: &RestoreApplyJournalReport,
1906) -> Result<(), RestoreCommandError> {
1907    if let Some(path) = &options.out {
1908        let data = serde_json::to_vec_pretty(report)?;
1909        fs::write(path, data)?;
1910        return Ok(());
1911    }
1912
1913    let stdout = io::stdout();
1914    let mut handle = stdout.lock();
1915    serde_json::to_writer_pretty(&mut handle, report)?;
1916    writeln!(handle)?;
1917    Ok(())
1918}
1919
1920// Write the restore runner response to stdout or a requested output file.
1921fn write_restore_run(
1922    options: &RestoreRunOptions,
1923    run: &RestoreRunResponse,
1924) -> Result<(), RestoreCommandError> {
1925    if let Some(path) = &options.out {
1926        let data = serde_json::to_vec_pretty(run)?;
1927        fs::write(path, data)?;
1928        return Ok(());
1929    }
1930
1931    let stdout = io::stdout();
1932    let mut handle = stdout.lock();
1933    serde_json::to_writer_pretty(&mut handle, run)?;
1934    writeln!(handle)?;
1935    Ok(())
1936}
1937
1938// Persist the restore apply journal to its canonical runner path.
1939fn write_apply_journal_file(
1940    path: &PathBuf,
1941    journal: &RestoreApplyJournal,
1942) -> Result<(), RestoreCommandError> {
1943    let data = serde_json::to_vec_pretty(journal)?;
1944    fs::write(path, data)?;
1945    Ok(())
1946}
1947
1948// Write the computed apply next-operation response to stdout or a requested output file.
1949fn write_apply_next(
1950    options: &RestoreApplyNextOptions,
1951    next: &RestoreApplyNextOperation,
1952) -> Result<(), RestoreCommandError> {
1953    if let Some(path) = &options.out {
1954        let data = serde_json::to_vec_pretty(next)?;
1955        fs::write(path, data)?;
1956        return Ok(());
1957    }
1958
1959    let stdout = io::stdout();
1960    let mut handle = stdout.lock();
1961    serde_json::to_writer_pretty(&mut handle, next)?;
1962    writeln!(handle)?;
1963    Ok(())
1964}
1965
1966// Write the computed apply command preview to stdout or a requested output file.
1967fn write_apply_command(
1968    options: &RestoreApplyCommandOptions,
1969    preview: &RestoreApplyCommandPreview,
1970) -> Result<(), RestoreCommandError> {
1971    if let Some(path) = &options.out {
1972        let data = serde_json::to_vec_pretty(preview)?;
1973        fs::write(path, data)?;
1974        return Ok(());
1975    }
1976
1977    let stdout = io::stdout();
1978    let mut handle = stdout.lock();
1979    serde_json::to_writer_pretty(&mut handle, preview)?;
1980    writeln!(handle)?;
1981    Ok(())
1982}
1983
1984// Write the claimed apply journal to stdout or a requested output file.
1985fn write_apply_claim(
1986    options: &RestoreApplyClaimOptions,
1987    journal: &RestoreApplyJournal,
1988) -> Result<(), RestoreCommandError> {
1989    if let Some(path) = &options.out {
1990        let data = serde_json::to_vec_pretty(journal)?;
1991        fs::write(path, data)?;
1992        return Ok(());
1993    }
1994
1995    let stdout = io::stdout();
1996    let mut handle = stdout.lock();
1997    serde_json::to_writer_pretty(&mut handle, journal)?;
1998    writeln!(handle)?;
1999    Ok(())
2000}
2001
2002// Write the unclaimed apply journal to stdout or a requested output file.
2003fn write_apply_unclaim(
2004    options: &RestoreApplyUnclaimOptions,
2005    journal: &RestoreApplyJournal,
2006) -> Result<(), RestoreCommandError> {
2007    if let Some(path) = &options.out {
2008        let data = serde_json::to_vec_pretty(journal)?;
2009        fs::write(path, data)?;
2010        return Ok(());
2011    }
2012
2013    let stdout = io::stdout();
2014    let mut handle = stdout.lock();
2015    serde_json::to_writer_pretty(&mut handle, journal)?;
2016    writeln!(handle)?;
2017    Ok(())
2018}
2019
2020// Write the updated apply journal to stdout or a requested output file.
2021fn write_apply_mark(
2022    options: &RestoreApplyMarkOptions,
2023    journal: &RestoreApplyJournal,
2024) -> Result<(), RestoreCommandError> {
2025    if let Some(path) = &options.out {
2026        let data = serde_json::to_vec_pretty(journal)?;
2027        fs::write(path, data)?;
2028        return Ok(());
2029    }
2030
2031    let stdout = io::stdout();
2032    let mut handle = stdout.lock();
2033    serde_json::to_writer_pretty(&mut handle, journal)?;
2034    writeln!(handle)?;
2035    Ok(())
2036}
2037
2038// Read the next required option value.
2039fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
2040where
2041    I: Iterator<Item = OsString>,
2042{
2043    args.next()
2044        .and_then(|value| value.into_string().ok())
2045        .ok_or(RestoreCommandError::MissingValue(option))
2046}
2047
2048// Return restore command usage text.
2049const fn usage() -> &'static str {
2050    "usage: canic restore plan (--manifest <file> | --backup-dir <dir>) [--mapping <file>] [--out <file>] [--require-verified] [--require-restore-ready]\n       canic restore status --plan <file> [--out <file>]\n       canic restore apply --plan <file> [--status <file>] [--backup-dir <dir>] --dry-run [--out <file>] [--journal-out <file>]\n       canic restore apply-status --journal <file> [--out <file>] [--require-ready] [--require-no-pending] [--require-no-failed] [--require-complete]\n       canic restore apply-report --journal <file> [--out <file>] [--require-no-attention]\n       canic restore run --journal <file> (--dry-run | --execute | --unclaim-pending) [--dfx <path>] [--network <name>] [--max-steps <n>] [--out <file>] [--require-complete] [--require-no-attention] [--require-run-mode <text>] [--require-stopped-reason <text>] [--require-next-action <text>] [--require-executed-count <n>]\n       canic restore apply-next --journal <file> [--out <file>]\n       canic restore apply-command --journal <file> [--dfx <path>] [--network <name>] [--out <file>] [--require-command]\n       canic restore apply-claim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n       canic restore apply-unclaim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n       canic restore apply-mark --journal <file> --sequence <n> --state completed|failed [--reason <text>] [--updated-at <text>] [--out <file>] [--require-pending]"
2051}
2052
2053#[cfg(test)]
2054mod tests {
2055    use super::*;
2056    use canic_backup::restore::RestoreApplyOperationState;
2057    use canic_backup::{
2058        artifacts::ArtifactChecksum,
2059        journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
2060        manifest::{
2061            BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
2062            FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
2063            VerificationCheck, VerificationPlan,
2064        },
2065    };
2066    use serde_json::json;
2067    use std::{
2068        path::Path,
2069        time::{SystemTime, UNIX_EPOCH},
2070    };
2071
2072    const ROOT: &str = "aaaaa-aa";
2073    const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2074    const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2075    const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2076
2077    // Ensure restore plan options parse the intended no-mutation command.
2078    #[test]
2079    fn parses_restore_plan_options() {
2080        let options = RestorePlanOptions::parse([
2081            OsString::from("--manifest"),
2082            OsString::from("manifest.json"),
2083            OsString::from("--mapping"),
2084            OsString::from("mapping.json"),
2085            OsString::from("--out"),
2086            OsString::from("plan.json"),
2087            OsString::from("--require-restore-ready"),
2088        ])
2089        .expect("parse options");
2090
2091        assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
2092        assert_eq!(options.backup_dir, None);
2093        assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
2094        assert_eq!(options.out, Some(PathBuf::from("plan.json")));
2095        assert!(!options.require_verified);
2096        assert!(options.require_restore_ready);
2097    }
2098
2099    // Ensure verified restore plan options parse with the canonical backup source.
2100    #[test]
2101    fn parses_verified_restore_plan_options() {
2102        let options = RestorePlanOptions::parse([
2103            OsString::from("--backup-dir"),
2104            OsString::from("backups/run"),
2105            OsString::from("--require-verified"),
2106        ])
2107        .expect("parse verified options");
2108
2109        assert_eq!(options.manifest, None);
2110        assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2111        assert_eq!(options.mapping, None);
2112        assert_eq!(options.out, None);
2113        assert!(options.require_verified);
2114        assert!(!options.require_restore_ready);
2115    }
2116
2117    // Ensure restore status options parse the intended no-mutation command.
2118    #[test]
2119    fn parses_restore_status_options() {
2120        let options = RestoreStatusOptions::parse([
2121            OsString::from("--plan"),
2122            OsString::from("restore-plan.json"),
2123            OsString::from("--out"),
2124            OsString::from("restore-status.json"),
2125        ])
2126        .expect("parse status options");
2127
2128        assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2129        assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
2130    }
2131
2132    // Ensure restore apply options require the explicit dry-run mode.
2133    #[test]
2134    fn parses_restore_apply_dry_run_options() {
2135        let options = RestoreApplyOptions::parse([
2136            OsString::from("--plan"),
2137            OsString::from("restore-plan.json"),
2138            OsString::from("--status"),
2139            OsString::from("restore-status.json"),
2140            OsString::from("--backup-dir"),
2141            OsString::from("backups/run"),
2142            OsString::from("--dry-run"),
2143            OsString::from("--out"),
2144            OsString::from("restore-apply-dry-run.json"),
2145            OsString::from("--journal-out"),
2146            OsString::from("restore-apply-journal.json"),
2147        ])
2148        .expect("parse apply options");
2149
2150        assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2151        assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
2152        assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2153        assert_eq!(
2154            options.out,
2155            Some(PathBuf::from("restore-apply-dry-run.json"))
2156        );
2157        assert_eq!(
2158            options.journal_out,
2159            Some(PathBuf::from("restore-apply-journal.json"))
2160        );
2161        assert!(options.dry_run);
2162    }
2163
2164    // Ensure restore apply-status options parse the intended journal command.
2165    #[test]
2166    fn parses_restore_apply_status_options() {
2167        let options = RestoreApplyStatusOptions::parse([
2168            OsString::from("--journal"),
2169            OsString::from("restore-apply-journal.json"),
2170            OsString::from("--out"),
2171            OsString::from("restore-apply-status.json"),
2172            OsString::from("--require-ready"),
2173            OsString::from("--require-no-pending"),
2174            OsString::from("--require-no-failed"),
2175            OsString::from("--require-complete"),
2176        ])
2177        .expect("parse apply-status options");
2178
2179        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2180        assert!(options.require_ready);
2181        assert!(options.require_no_pending);
2182        assert!(options.require_no_failed);
2183        assert!(options.require_complete);
2184        assert_eq!(
2185            options.out,
2186            Some(PathBuf::from("restore-apply-status.json"))
2187        );
2188    }
2189
2190    // Ensure restore apply-report options parse the intended journal command.
2191    #[test]
2192    fn parses_restore_apply_report_options() {
2193        let options = RestoreApplyReportOptions::parse([
2194            OsString::from("--journal"),
2195            OsString::from("restore-apply-journal.json"),
2196            OsString::from("--out"),
2197            OsString::from("restore-apply-report.json"),
2198            OsString::from("--require-no-attention"),
2199        ])
2200        .expect("parse apply-report options");
2201
2202        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2203        assert!(options.require_no_attention);
2204        assert_eq!(
2205            options.out,
2206            Some(PathBuf::from("restore-apply-report.json"))
2207        );
2208    }
2209
2210    // Ensure restore run options parse the native runner dry-run command.
2211    #[test]
2212    fn parses_restore_run_dry_run_options() {
2213        let options = RestoreRunOptions::parse([
2214            OsString::from("--journal"),
2215            OsString::from("restore-apply-journal.json"),
2216            OsString::from("--dry-run"),
2217            OsString::from("--dfx"),
2218            OsString::from("/tmp/dfx"),
2219            OsString::from("--network"),
2220            OsString::from("local"),
2221            OsString::from("--out"),
2222            OsString::from("restore-run-dry-run.json"),
2223            OsString::from("--max-steps"),
2224            OsString::from("1"),
2225            OsString::from("--require-complete"),
2226            OsString::from("--require-no-attention"),
2227            OsString::from("--require-run-mode"),
2228            OsString::from("dry-run"),
2229            OsString::from("--require-stopped-reason"),
2230            OsString::from("preview"),
2231            OsString::from("--require-next-action"),
2232            OsString::from("rerun"),
2233            OsString::from("--require-executed-count"),
2234            OsString::from("0"),
2235        ])
2236        .expect("parse restore run options");
2237
2238        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2239        assert_eq!(options.dfx, "/tmp/dfx");
2240        assert_eq!(options.network.as_deref(), Some("local"));
2241        assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
2242        assert!(options.dry_run);
2243        assert!(!options.execute);
2244        assert!(!options.unclaim_pending);
2245        assert_eq!(options.max_steps, Some(1));
2246        assert!(options.require_complete);
2247        assert!(options.require_no_attention);
2248        assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
2249        assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
2250        assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
2251        assert_eq!(options.require_executed_count, Some(0));
2252    }
2253
2254    // Ensure restore run options parse the native execute command.
2255    #[test]
2256    fn parses_restore_run_execute_options() {
2257        let options = RestoreRunOptions::parse([
2258            OsString::from("--journal"),
2259            OsString::from("restore-apply-journal.json"),
2260            OsString::from("--execute"),
2261            OsString::from("--dfx"),
2262            OsString::from("/bin/true"),
2263            OsString::from("--max-steps"),
2264            OsString::from("4"),
2265        ])
2266        .expect("parse restore run execute options");
2267
2268        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2269        assert_eq!(options.dfx, "/bin/true");
2270        assert_eq!(options.network, None);
2271        assert_eq!(options.out, None);
2272        assert!(!options.dry_run);
2273        assert!(options.execute);
2274        assert!(!options.unclaim_pending);
2275        assert_eq!(options.max_steps, Some(4));
2276        assert!(!options.require_complete);
2277        assert!(!options.require_no_attention);
2278        assert_eq!(options.require_run_mode, None);
2279        assert_eq!(options.require_stopped_reason, None);
2280        assert_eq!(options.require_next_action, None);
2281        assert_eq!(options.require_executed_count, None);
2282    }
2283
2284    // Ensure restore run options parse the native pending-operation recovery mode.
2285    #[test]
2286    fn parses_restore_run_unclaim_pending_options() {
2287        let options = RestoreRunOptions::parse([
2288            OsString::from("--journal"),
2289            OsString::from("restore-apply-journal.json"),
2290            OsString::from("--unclaim-pending"),
2291            OsString::from("--out"),
2292            OsString::from("restore-run.json"),
2293        ])
2294        .expect("parse restore run unclaim options");
2295
2296        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2297        assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
2298        assert!(!options.dry_run);
2299        assert!(!options.execute);
2300        assert!(options.unclaim_pending);
2301    }
2302
2303    // Ensure restore apply-next options parse the intended journal command.
2304    #[test]
2305    fn parses_restore_apply_next_options() {
2306        let options = RestoreApplyNextOptions::parse([
2307            OsString::from("--journal"),
2308            OsString::from("restore-apply-journal.json"),
2309            OsString::from("--out"),
2310            OsString::from("restore-apply-next.json"),
2311        ])
2312        .expect("parse apply-next options");
2313
2314        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2315        assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
2316    }
2317
2318    // Ensure restore apply-command options parse the intended preview command.
2319    #[test]
2320    fn parses_restore_apply_command_options() {
2321        let options = RestoreApplyCommandOptions::parse([
2322            OsString::from("--journal"),
2323            OsString::from("restore-apply-journal.json"),
2324            OsString::from("--dfx"),
2325            OsString::from("/tmp/dfx"),
2326            OsString::from("--network"),
2327            OsString::from("local"),
2328            OsString::from("--out"),
2329            OsString::from("restore-apply-command.json"),
2330            OsString::from("--require-command"),
2331        ])
2332        .expect("parse apply-command options");
2333
2334        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2335        assert_eq!(options.dfx, "/tmp/dfx");
2336        assert_eq!(options.network.as_deref(), Some("local"));
2337        assert!(options.require_command);
2338        assert_eq!(
2339            options.out,
2340            Some(PathBuf::from("restore-apply-command.json"))
2341        );
2342    }
2343
2344    // Ensure restore apply-claim options parse the intended journal command.
2345    #[test]
2346    fn parses_restore_apply_claim_options() {
2347        let options = RestoreApplyClaimOptions::parse([
2348            OsString::from("--journal"),
2349            OsString::from("restore-apply-journal.json"),
2350            OsString::from("--sequence"),
2351            OsString::from("0"),
2352            OsString::from("--updated-at"),
2353            OsString::from("2026-05-04T12:00:00Z"),
2354            OsString::from("--out"),
2355            OsString::from("restore-apply-journal.claimed.json"),
2356        ])
2357        .expect("parse apply-claim options");
2358
2359        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2360        assert_eq!(options.sequence, Some(0));
2361        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
2362        assert_eq!(
2363            options.out,
2364            Some(PathBuf::from("restore-apply-journal.claimed.json"))
2365        );
2366    }
2367
2368    // Ensure restore apply-unclaim options parse the intended journal command.
2369    #[test]
2370    fn parses_restore_apply_unclaim_options() {
2371        let options = RestoreApplyUnclaimOptions::parse([
2372            OsString::from("--journal"),
2373            OsString::from("restore-apply-journal.json"),
2374            OsString::from("--sequence"),
2375            OsString::from("0"),
2376            OsString::from("--updated-at"),
2377            OsString::from("2026-05-04T12:01:00Z"),
2378            OsString::from("--out"),
2379            OsString::from("restore-apply-journal.unclaimed.json"),
2380        ])
2381        .expect("parse apply-unclaim options");
2382
2383        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2384        assert_eq!(options.sequence, Some(0));
2385        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
2386        assert_eq!(
2387            options.out,
2388            Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
2389        );
2390    }
2391
2392    // Ensure restore apply-mark options parse the intended journal update command.
2393    #[test]
2394    fn parses_restore_apply_mark_options() {
2395        let options = RestoreApplyMarkOptions::parse([
2396            OsString::from("--journal"),
2397            OsString::from("restore-apply-journal.json"),
2398            OsString::from("--sequence"),
2399            OsString::from("4"),
2400            OsString::from("--state"),
2401            OsString::from("failed"),
2402            OsString::from("--reason"),
2403            OsString::from("dfx-load-failed"),
2404            OsString::from("--updated-at"),
2405            OsString::from("2026-05-04T12:02:00Z"),
2406            OsString::from("--out"),
2407            OsString::from("restore-apply-journal.updated.json"),
2408            OsString::from("--require-pending"),
2409        ])
2410        .expect("parse apply-mark options");
2411
2412        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2413        assert_eq!(options.sequence, 4);
2414        assert_eq!(options.state, RestoreApplyMarkState::Failed);
2415        assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
2416        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
2417        assert!(options.require_pending);
2418        assert_eq!(
2419            options.out,
2420            Some(PathBuf::from("restore-apply-journal.updated.json"))
2421        );
2422    }
2423
2424    // Ensure restore apply refuses non-dry-run execution while apply is scaffolded.
2425    #[test]
2426    fn restore_apply_requires_dry_run() {
2427        let err = RestoreApplyOptions::parse([
2428            OsString::from("--plan"),
2429            OsString::from("restore-plan.json"),
2430        ])
2431        .expect_err("apply without dry-run should fail");
2432
2433        assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
2434    }
2435
2436    // Ensure restore run refuses mutation while native execution is scaffolded.
2437    #[test]
2438    fn restore_run_requires_mode() {
2439        let err = RestoreRunOptions::parse([
2440            OsString::from("--journal"),
2441            OsString::from("restore-apply-journal.json"),
2442        ])
2443        .expect_err("restore run without dry-run should fail");
2444
2445        assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
2446    }
2447
2448    // Ensure restore run rejects ambiguous execution modes.
2449    #[test]
2450    fn restore_run_rejects_conflicting_modes() {
2451        let err = RestoreRunOptions::parse([
2452            OsString::from("--journal"),
2453            OsString::from("restore-apply-journal.json"),
2454            OsString::from("--dry-run"),
2455            OsString::from("--execute"),
2456            OsString::from("--unclaim-pending"),
2457        ])
2458        .expect_err("restore run should reject conflicting modes");
2459
2460        assert!(matches!(
2461            err,
2462            RestoreCommandError::RestoreRunConflictingModes
2463        ));
2464    }
2465
2466    // Ensure backup-dir restore planning reads the canonical layout manifest.
2467    #[test]
2468    fn plan_restore_reads_manifest_from_backup_dir() {
2469        let root = temp_dir("canic-cli-restore-plan-layout");
2470        let layout = BackupLayout::new(root.clone());
2471        layout
2472            .write_manifest(&valid_manifest())
2473            .expect("write manifest");
2474
2475        let options = RestorePlanOptions {
2476            manifest: None,
2477            backup_dir: Some(root.clone()),
2478            mapping: None,
2479            out: None,
2480            require_verified: false,
2481            require_restore_ready: false,
2482        };
2483
2484        let plan = plan_restore(&options).expect("plan restore");
2485
2486        fs::remove_dir_all(root).expect("remove temp root");
2487        assert_eq!(plan.backup_id, "backup-test");
2488        assert_eq!(plan.member_count, 2);
2489    }
2490
2491    // Ensure restore planning has exactly one manifest source.
2492    #[test]
2493    fn parse_rejects_conflicting_manifest_sources() {
2494        let err = RestorePlanOptions::parse([
2495            OsString::from("--manifest"),
2496            OsString::from("manifest.json"),
2497            OsString::from("--backup-dir"),
2498            OsString::from("backups/run"),
2499        ])
2500        .expect_err("conflicting sources should fail");
2501
2502        assert!(matches!(
2503            err,
2504            RestoreCommandError::ConflictingManifestSources
2505        ));
2506    }
2507
2508    // Ensure verified planning requires the canonical backup layout source.
2509    #[test]
2510    fn parse_rejects_require_verified_with_manifest_source() {
2511        let err = RestorePlanOptions::parse([
2512            OsString::from("--manifest"),
2513            OsString::from("manifest.json"),
2514            OsString::from("--require-verified"),
2515        ])
2516        .expect_err("verification should require a backup layout");
2517
2518        assert!(matches!(
2519            err,
2520            RestoreCommandError::RequireVerifiedNeedsBackupDir
2521        ));
2522    }
2523
2524    // Ensure restore planning can require manifest, journal, and artifact integrity.
2525    #[test]
2526    fn plan_restore_requires_verified_backup_layout() {
2527        let root = temp_dir("canic-cli-restore-plan-verified");
2528        let layout = BackupLayout::new(root.clone());
2529        let manifest = valid_manifest();
2530        write_verified_layout(&root, &layout, &manifest);
2531
2532        let options = RestorePlanOptions {
2533            manifest: None,
2534            backup_dir: Some(root.clone()),
2535            mapping: None,
2536            out: None,
2537            require_verified: true,
2538            require_restore_ready: false,
2539        };
2540
2541        let plan = plan_restore(&options).expect("plan verified restore");
2542
2543        fs::remove_dir_all(root).expect("remove temp root");
2544        assert_eq!(plan.backup_id, "backup-test");
2545        assert_eq!(plan.member_count, 2);
2546    }
2547
2548    // Ensure required verification fails before planning when the layout is incomplete.
2549    #[test]
2550    fn plan_restore_rejects_unverified_backup_layout() {
2551        let root = temp_dir("canic-cli-restore-plan-unverified");
2552        let layout = BackupLayout::new(root.clone());
2553        layout
2554            .write_manifest(&valid_manifest())
2555            .expect("write manifest");
2556
2557        let options = RestorePlanOptions {
2558            manifest: None,
2559            backup_dir: Some(root.clone()),
2560            mapping: None,
2561            out: None,
2562            require_verified: true,
2563            require_restore_ready: false,
2564        };
2565
2566        let err = plan_restore(&options).expect_err("missing journal should fail");
2567
2568        fs::remove_dir_all(root).expect("remove temp root");
2569        assert!(matches!(err, RestoreCommandError::Persistence(_)));
2570    }
2571
2572    // Ensure the CLI planning path validates manifests and applies mappings.
2573    #[test]
2574    fn plan_restore_reads_manifest_and_mapping() {
2575        let root = temp_dir("canic-cli-restore-plan");
2576        fs::create_dir_all(&root).expect("create temp root");
2577        let manifest_path = root.join("manifest.json");
2578        let mapping_path = root.join("mapping.json");
2579
2580        fs::write(
2581            &manifest_path,
2582            serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2583        )
2584        .expect("write manifest");
2585        fs::write(
2586            &mapping_path,
2587            json!({
2588                "members": [
2589                    {"source_canister": ROOT, "target_canister": ROOT},
2590                    {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
2591                ]
2592            })
2593            .to_string(),
2594        )
2595        .expect("write mapping");
2596
2597        let options = RestorePlanOptions {
2598            manifest: Some(manifest_path),
2599            backup_dir: None,
2600            mapping: Some(mapping_path),
2601            out: None,
2602            require_verified: false,
2603            require_restore_ready: false,
2604        };
2605
2606        let plan = plan_restore(&options).expect("plan restore");
2607
2608        fs::remove_dir_all(root).expect("remove temp root");
2609        let members = plan.ordered_members();
2610        assert_eq!(members.len(), 2);
2611        assert_eq!(members[0].source_canister, ROOT);
2612        assert_eq!(members[1].target_canister, MAPPED_CHILD);
2613    }
2614
2615    // Ensure restore-readiness gating happens after writing the plan artifact.
2616    #[test]
2617    fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
2618        let root = temp_dir("canic-cli-restore-plan-require-ready");
2619        fs::create_dir_all(&root).expect("create temp root");
2620        let manifest_path = root.join("manifest.json");
2621        let out_path = root.join("plan.json");
2622
2623        fs::write(
2624            &manifest_path,
2625            serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2626        )
2627        .expect("write manifest");
2628
2629        let err = run([
2630            OsString::from("plan"),
2631            OsString::from("--manifest"),
2632            OsString::from(manifest_path.as_os_str()),
2633            OsString::from("--out"),
2634            OsString::from(out_path.as_os_str()),
2635            OsString::from("--require-restore-ready"),
2636        ])
2637        .expect_err("restore readiness should be enforced");
2638
2639        assert!(out_path.exists());
2640        let plan: RestorePlan =
2641            serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2642
2643        fs::remove_dir_all(root).expect("remove temp root");
2644        assert!(!plan.readiness_summary.ready);
2645        assert!(matches!(
2646            err,
2647            RestoreCommandError::RestoreNotReady {
2648                reasons,
2649                ..
2650            } if reasons == [
2651                "missing-module-hash",
2652                "missing-wasm-hash",
2653                "missing-snapshot-checksum"
2654            ]
2655        ));
2656    }
2657
2658    // Ensure restore-readiness gating accepts plans with complete provenance.
2659    #[test]
2660    fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
2661        let root = temp_dir("canic-cli-restore-plan-ready");
2662        fs::create_dir_all(&root).expect("create temp root");
2663        let manifest_path = root.join("manifest.json");
2664        let out_path = root.join("plan.json");
2665
2666        fs::write(
2667            &manifest_path,
2668            serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
2669        )
2670        .expect("write manifest");
2671
2672        run([
2673            OsString::from("plan"),
2674            OsString::from("--manifest"),
2675            OsString::from(manifest_path.as_os_str()),
2676            OsString::from("--out"),
2677            OsString::from(out_path.as_os_str()),
2678            OsString::from("--require-restore-ready"),
2679        ])
2680        .expect("restore-ready plan should pass");
2681
2682        let plan: RestorePlan =
2683            serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2684
2685        fs::remove_dir_all(root).expect("remove temp root");
2686        assert!(plan.readiness_summary.ready);
2687        assert!(plan.readiness_summary.reasons.is_empty());
2688    }
2689
2690    // Ensure restore status writes the initial planned execution journal.
2691    #[test]
2692    fn run_restore_status_writes_planned_status() {
2693        let root = temp_dir("canic-cli-restore-status");
2694        fs::create_dir_all(&root).expect("create temp root");
2695        let plan_path = root.join("restore-plan.json");
2696        let out_path = root.join("restore-status.json");
2697        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2698
2699        fs::write(
2700            &plan_path,
2701            serde_json::to_vec(&plan).expect("serialize plan"),
2702        )
2703        .expect("write plan");
2704
2705        run([
2706            OsString::from("status"),
2707            OsString::from("--plan"),
2708            OsString::from(plan_path.as_os_str()),
2709            OsString::from("--out"),
2710            OsString::from(out_path.as_os_str()),
2711        ])
2712        .expect("write restore status");
2713
2714        let status: RestoreStatus =
2715            serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
2716                .expect("decode restore status");
2717        let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
2718
2719        fs::remove_dir_all(root).expect("remove temp root");
2720        assert_eq!(status.status_version, 1);
2721        assert_eq!(status.backup_id.as_str(), "backup-test");
2722        assert!(status.ready);
2723        assert!(status.readiness_reasons.is_empty());
2724        assert_eq!(status.member_count, 2);
2725        assert_eq!(status.phase_count, 1);
2726        assert_eq!(status.planned_snapshot_loads, 2);
2727        assert_eq!(status.planned_code_reinstalls, 2);
2728        assert_eq!(status.planned_verification_checks, 2);
2729        assert_eq!(status.phases[0].members[0].source_canister, ROOT);
2730        assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
2731    }
2732
2733    // Ensure restore apply dry-run writes ordered operations from plan and status.
2734    #[test]
2735    fn run_restore_apply_dry_run_writes_operations() {
2736        let root = temp_dir("canic-cli-restore-apply-dry-run");
2737        fs::create_dir_all(&root).expect("create temp root");
2738        let plan_path = root.join("restore-plan.json");
2739        let status_path = root.join("restore-status.json");
2740        let out_path = root.join("restore-apply-dry-run.json");
2741        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2742        let status = RestoreStatus::from_plan(&plan);
2743
2744        fs::write(
2745            &plan_path,
2746            serde_json::to_vec(&plan).expect("serialize plan"),
2747        )
2748        .expect("write plan");
2749        fs::write(
2750            &status_path,
2751            serde_json::to_vec(&status).expect("serialize status"),
2752        )
2753        .expect("write status");
2754
2755        run([
2756            OsString::from("apply"),
2757            OsString::from("--plan"),
2758            OsString::from(plan_path.as_os_str()),
2759            OsString::from("--status"),
2760            OsString::from(status_path.as_os_str()),
2761            OsString::from("--dry-run"),
2762            OsString::from("--out"),
2763            OsString::from(out_path.as_os_str()),
2764        ])
2765        .expect("write apply dry-run");
2766
2767        let dry_run: RestoreApplyDryRun =
2768            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2769                .expect("decode dry-run");
2770        let dry_run_json: serde_json::Value =
2771            serde_json::to_value(&dry_run).expect("encode dry-run");
2772
2773        fs::remove_dir_all(root).expect("remove temp root");
2774        assert_eq!(dry_run.dry_run_version, 1);
2775        assert_eq!(dry_run.backup_id.as_str(), "backup-test");
2776        assert!(dry_run.ready);
2777        assert!(dry_run.status_supplied);
2778        assert_eq!(dry_run.member_count, 2);
2779        assert_eq!(dry_run.phase_count, 1);
2780        assert_eq!(dry_run.rendered_operations, 8);
2781        assert_eq!(
2782            dry_run_json["phases"][0]["operations"][0]["operation"],
2783            "upload-snapshot"
2784        );
2785        assert_eq!(
2786            dry_run_json["phases"][0]["operations"][3]["operation"],
2787            "verify-member"
2788        );
2789        assert_eq!(
2790            dry_run_json["phases"][0]["operations"][3]["verification_kind"],
2791            "status"
2792        );
2793        assert_eq!(
2794            dry_run_json["phases"][0]["operations"][3]["verification_method"],
2795            serde_json::Value::Null
2796        );
2797    }
2798
2799    // Ensure restore apply dry-run can validate artifacts under a backup directory.
2800    #[test]
2801    fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
2802        let root = temp_dir("canic-cli-restore-apply-artifacts");
2803        fs::create_dir_all(&root).expect("create temp root");
2804        let plan_path = root.join("restore-plan.json");
2805        let out_path = root.join("restore-apply-dry-run.json");
2806        let journal_path = root.join("restore-apply-journal.json");
2807        let status_path = root.join("restore-apply-status.json");
2808        let mut manifest = restore_ready_manifest();
2809        write_manifest_artifacts(&root, &mut manifest);
2810        let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
2811
2812        fs::write(
2813            &plan_path,
2814            serde_json::to_vec(&plan).expect("serialize plan"),
2815        )
2816        .expect("write plan");
2817
2818        run([
2819            OsString::from("apply"),
2820            OsString::from("--plan"),
2821            OsString::from(plan_path.as_os_str()),
2822            OsString::from("--backup-dir"),
2823            OsString::from(root.as_os_str()),
2824            OsString::from("--dry-run"),
2825            OsString::from("--out"),
2826            OsString::from(out_path.as_os_str()),
2827            OsString::from("--journal-out"),
2828            OsString::from(journal_path.as_os_str()),
2829        ])
2830        .expect("write apply dry-run");
2831        run([
2832            OsString::from("apply-status"),
2833            OsString::from("--journal"),
2834            OsString::from(journal_path.as_os_str()),
2835            OsString::from("--out"),
2836            OsString::from(status_path.as_os_str()),
2837        ])
2838        .expect("write apply status");
2839
2840        let dry_run: RestoreApplyDryRun =
2841            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2842                .expect("decode dry-run");
2843        let validation = dry_run
2844            .artifact_validation
2845            .expect("artifact validation should be present");
2846        let journal_json: serde_json::Value =
2847            serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
2848                .expect("decode journal");
2849        let status_json: serde_json::Value =
2850            serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
2851                .expect("decode apply status");
2852
2853        fs::remove_dir_all(root).expect("remove temp root");
2854        assert_eq!(validation.checked_members, 2);
2855        assert!(validation.artifacts_present);
2856        assert!(validation.checksums_verified);
2857        assert_eq!(validation.members_with_expected_checksums, 2);
2858        assert_eq!(journal_json["ready"], true);
2859        assert_eq!(journal_json["operation_count"], 8);
2860        assert_eq!(journal_json["ready_operations"], 8);
2861        assert_eq!(journal_json["blocked_operations"], 0);
2862        assert_eq!(journal_json["operations"][0]["state"], "ready");
2863        assert_eq!(status_json["ready"], true);
2864        assert_eq!(status_json["operation_count"], 8);
2865        assert_eq!(status_json["next_ready_sequence"], 0);
2866        assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
2867    }
2868
2869    // Ensure apply-status rejects structurally inconsistent journals.
2870    #[test]
2871    fn run_restore_apply_status_rejects_invalid_journal() {
2872        let root = temp_dir("canic-cli-restore-apply-status-invalid");
2873        fs::create_dir_all(&root).expect("create temp root");
2874        let journal_path = root.join("restore-apply-journal.json");
2875        let out_path = root.join("restore-apply-status.json");
2876        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2877        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
2878        let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
2879        journal.operation_count += 1;
2880
2881        fs::write(
2882            &journal_path,
2883            serde_json::to_vec(&journal).expect("serialize journal"),
2884        )
2885        .expect("write journal");
2886
2887        let err = run([
2888            OsString::from("apply-status"),
2889            OsString::from("--journal"),
2890            OsString::from(journal_path.as_os_str()),
2891            OsString::from("--out"),
2892            OsString::from(out_path.as_os_str()),
2893        ])
2894        .expect_err("invalid journal should fail");
2895
2896        assert!(!out_path.exists());
2897        fs::remove_dir_all(root).expect("remove temp root");
2898        assert!(matches!(
2899            err,
2900            RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
2901                field: "operation_count",
2902                ..
2903            })
2904        ));
2905    }
2906
2907    // Ensure apply-status can fail closed after writing status for pending work.
2908    #[test]
2909    fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
2910        let root = temp_dir("canic-cli-restore-apply-status-pending");
2911        fs::create_dir_all(&root).expect("create temp root");
2912        let journal_path = root.join("restore-apply-journal.json");
2913        let out_path = root.join("restore-apply-status.json");
2914        let mut journal = ready_apply_journal();
2915        journal
2916            .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
2917            .expect("claim operation");
2918
2919        fs::write(
2920            &journal_path,
2921            serde_json::to_vec(&journal).expect("serialize journal"),
2922        )
2923        .expect("write journal");
2924
2925        let err = run([
2926            OsString::from("apply-status"),
2927            OsString::from("--journal"),
2928            OsString::from(journal_path.as_os_str()),
2929            OsString::from("--out"),
2930            OsString::from(out_path.as_os_str()),
2931            OsString::from("--require-no-pending"),
2932        ])
2933        .expect_err("pending operation should fail requirement");
2934
2935        assert!(out_path.exists());
2936        let status: RestoreApplyJournalStatus =
2937            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
2938                .expect("decode apply status");
2939
2940        fs::remove_dir_all(root).expect("remove temp root");
2941        assert_eq!(status.pending_operations, 1);
2942        assert_eq!(status.next_transition_sequence, Some(0));
2943        assert_eq!(
2944            status.next_transition_updated_at.as_deref(),
2945            Some("2026-05-04T12:00:00Z")
2946        );
2947        assert!(matches!(
2948            err,
2949            RestoreCommandError::RestoreApplyPending {
2950                pending_operations: 1,
2951                next_transition_sequence: Some(0),
2952                ..
2953            }
2954        ));
2955    }
2956
2957    // Ensure apply-status can fail closed after writing status for unready work.
2958    #[test]
2959    fn run_restore_apply_status_require_ready_writes_status_then_fails() {
2960        let root = temp_dir("canic-cli-restore-apply-status-ready");
2961        fs::create_dir_all(&root).expect("create temp root");
2962        let journal_path = root.join("restore-apply-journal.json");
2963        let out_path = root.join("restore-apply-status.json");
2964        let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
2965        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
2966        let journal = RestoreApplyJournal::from_dry_run(&dry_run);
2967
2968        fs::write(
2969            &journal_path,
2970            serde_json::to_vec(&journal).expect("serialize journal"),
2971        )
2972        .expect("write journal");
2973
2974        let err = run([
2975            OsString::from("apply-status"),
2976            OsString::from("--journal"),
2977            OsString::from(journal_path.as_os_str()),
2978            OsString::from("--out"),
2979            OsString::from(out_path.as_os_str()),
2980            OsString::from("--require-ready"),
2981        ])
2982        .expect_err("unready journal should fail requirement");
2983
2984        let status: RestoreApplyJournalStatus =
2985            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
2986                .expect("decode apply status");
2987
2988        fs::remove_dir_all(root).expect("remove temp root");
2989        assert!(!status.ready);
2990        assert_eq!(status.blocked_operations, status.operation_count);
2991        assert!(
2992            status
2993                .blocked_reasons
2994                .contains(&"missing-snapshot-checksum".to_string())
2995        );
2996        assert!(matches!(
2997            err,
2998            RestoreCommandError::RestoreApplyNotReady { reasons, .. }
2999                if reasons.contains(&"missing-snapshot-checksum".to_string())
3000        ));
3001    }
3002
3003    // Ensure apply-report writes the operator-focused journal summary.
3004    #[test]
3005    fn run_restore_apply_report_writes_attention_summary() {
3006        let root = temp_dir("canic-cli-restore-apply-report");
3007        fs::create_dir_all(&root).expect("create temp root");
3008        let journal_path = root.join("restore-apply-journal.json");
3009        let out_path = root.join("restore-apply-report.json");
3010        let mut journal = ready_apply_journal();
3011        journal
3012            .mark_operation_failed_at(
3013                0,
3014                "dfx-upload-failed".to_string(),
3015                Some("2026-05-05T12:00:00Z".to_string()),
3016            )
3017            .expect("mark failed operation");
3018        journal
3019            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3020            .expect("mark pending operation");
3021
3022        fs::write(
3023            &journal_path,
3024            serde_json::to_vec(&journal).expect("serialize journal"),
3025        )
3026        .expect("write journal");
3027
3028        run([
3029            OsString::from("apply-report"),
3030            OsString::from("--journal"),
3031            OsString::from(journal_path.as_os_str()),
3032            OsString::from("--out"),
3033            OsString::from(out_path.as_os_str()),
3034        ])
3035        .expect("write apply report");
3036
3037        let report: RestoreApplyJournalReport =
3038            serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3039                .expect("decode apply report");
3040        let report_json: serde_json::Value =
3041            serde_json::to_value(&report).expect("encode apply report");
3042
3043        fs::remove_dir_all(root).expect("remove temp root");
3044        assert_eq!(report.backup_id, "backup-test");
3045        assert!(report.attention_required);
3046        assert_eq!(report.failed_operations, 1);
3047        assert_eq!(report.pending_operations, 1);
3048        assert_eq!(report.failed.len(), 1);
3049        assert_eq!(report.pending.len(), 1);
3050        assert_eq!(report.failed[0].sequence, 0);
3051        assert_eq!(report.pending[0].sequence, 1);
3052        assert_eq!(
3053            report.next_transition.as_ref().map(|op| op.sequence),
3054            Some(1)
3055        );
3056        assert_eq!(report_json["outcome"], "failed");
3057        assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
3058    }
3059
3060    // Ensure restore run writes a native no-mutation runner preview.
3061    #[test]
3062    fn run_restore_run_dry_run_writes_native_runner_preview() {
3063        let root = temp_dir("canic-cli-restore-run-dry-run");
3064        fs::create_dir_all(&root).expect("create temp root");
3065        let journal_path = root.join("restore-apply-journal.json");
3066        let out_path = root.join("restore-run-dry-run.json");
3067        let journal = ready_apply_journal();
3068
3069        fs::write(
3070            &journal_path,
3071            serde_json::to_vec(&journal).expect("serialize journal"),
3072        )
3073        .expect("write journal");
3074
3075        run([
3076            OsString::from("run"),
3077            OsString::from("--journal"),
3078            OsString::from(journal_path.as_os_str()),
3079            OsString::from("--dry-run"),
3080            OsString::from("--dfx"),
3081            OsString::from("/tmp/dfx"),
3082            OsString::from("--network"),
3083            OsString::from("local"),
3084            OsString::from("--out"),
3085            OsString::from(out_path.as_os_str()),
3086        ])
3087        .expect("write restore run dry-run");
3088
3089        let dry_run: serde_json::Value =
3090            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3091                .expect("decode dry-run");
3092
3093        fs::remove_dir_all(root).expect("remove temp root");
3094        assert_eq!(dry_run["run_version"], 1);
3095        assert_eq!(dry_run["backup_id"], "backup-test");
3096        assert_eq!(dry_run["run_mode"], "dry-run");
3097        assert_eq!(dry_run["dry_run"], true);
3098        assert_eq!(dry_run["ready"], true);
3099        assert_eq!(dry_run["complete"], false);
3100        assert_eq!(dry_run["attention_required"], false);
3101        assert_eq!(dry_run["stopped_reason"], "preview");
3102        assert_eq!(dry_run["next_action"], "rerun");
3103        assert_eq!(dry_run["operation_available"], true);
3104        assert_eq!(dry_run["command_available"], true);
3105        assert_eq!(dry_run["next_transition"]["sequence"], 0);
3106        assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
3107        assert_eq!(
3108            dry_run["command"]["args"],
3109            json!([
3110                "canister",
3111                "--network",
3112                "local",
3113                "snapshot",
3114                "upload",
3115                "--dir",
3116                "artifacts/root",
3117                ROOT
3118            ])
3119        );
3120        assert_eq!(dry_run["command"]["mutates"], true);
3121    }
3122
3123    // Ensure restore run can recover one interrupted pending operation.
3124    #[test]
3125    fn run_restore_run_unclaim_pending_marks_operation_ready() {
3126        let root = temp_dir("canic-cli-restore-run-unclaim-pending");
3127        fs::create_dir_all(&root).expect("create temp root");
3128        let journal_path = root.join("restore-apply-journal.json");
3129        let out_path = root.join("restore-run.json");
3130        let mut journal = ready_apply_journal();
3131        journal
3132            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3133            .expect("mark pending operation");
3134
3135        fs::write(
3136            &journal_path,
3137            serde_json::to_vec(&journal).expect("serialize journal"),
3138        )
3139        .expect("write journal");
3140
3141        run([
3142            OsString::from("run"),
3143            OsString::from("--journal"),
3144            OsString::from(journal_path.as_os_str()),
3145            OsString::from("--unclaim-pending"),
3146            OsString::from("--out"),
3147            OsString::from(out_path.as_os_str()),
3148        ])
3149        .expect("unclaim pending operation");
3150
3151        let run_summary: serde_json::Value =
3152            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3153                .expect("decode run summary");
3154        let updated: RestoreApplyJournal =
3155            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3156                .expect("decode updated journal");
3157
3158        fs::remove_dir_all(root).expect("remove temp root");
3159        assert_eq!(run_summary["run_mode"], "unclaim-pending");
3160        assert_eq!(run_summary["unclaim_pending"], true);
3161        assert_eq!(run_summary["stopped_reason"], "recovered-pending");
3162        assert_eq!(run_summary["next_action"], "rerun");
3163        assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
3164        assert_eq!(run_summary["recovered_operation"]["state"], "pending");
3165        assert_eq!(run_summary["pending_operations"], 0);
3166        assert_eq!(run_summary["ready_operations"], 8);
3167        assert_eq!(run_summary["attention_required"], false);
3168        assert_eq!(updated.pending_operations, 0);
3169        assert_eq!(updated.ready_operations, 8);
3170        assert_eq!(
3171            updated.operations[0].state,
3172            RestoreApplyOperationState::Ready
3173        );
3174    }
3175
3176    // Ensure restore run execute claims and completes one generated command.
3177    #[test]
3178    fn run_restore_run_execute_marks_completed_operation() {
3179        let root = temp_dir("canic-cli-restore-run-execute");
3180        fs::create_dir_all(&root).expect("create temp root");
3181        let journal_path = root.join("restore-apply-journal.json");
3182        let out_path = root.join("restore-run.json");
3183        let journal = ready_apply_journal();
3184
3185        fs::write(
3186            &journal_path,
3187            serde_json::to_vec(&journal).expect("serialize journal"),
3188        )
3189        .expect("write journal");
3190
3191        run([
3192            OsString::from("run"),
3193            OsString::from("--journal"),
3194            OsString::from(journal_path.as_os_str()),
3195            OsString::from("--execute"),
3196            OsString::from("--dfx"),
3197            OsString::from("/bin/true"),
3198            OsString::from("--max-steps"),
3199            OsString::from("1"),
3200            OsString::from("--out"),
3201            OsString::from(out_path.as_os_str()),
3202        ])
3203        .expect("execute one restore run step");
3204
3205        let run_summary: serde_json::Value =
3206            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3207                .expect("decode run summary");
3208        let updated: RestoreApplyJournal =
3209            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3210                .expect("decode updated journal");
3211
3212        fs::remove_dir_all(root).expect("remove temp root");
3213        assert_eq!(run_summary["run_mode"], "execute");
3214        assert_eq!(run_summary["execute"], true);
3215        assert_eq!(run_summary["dry_run"], false);
3216        assert_eq!(run_summary["max_steps_reached"], true);
3217        assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
3218        assert_eq!(run_summary["next_action"], "rerun");
3219        assert_eq!(run_summary["executed_operation_count"], 1);
3220        assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
3221        assert_eq!(
3222            run_summary["executed_operations"][0]["command"]["program"],
3223            "/bin/true"
3224        );
3225        assert_eq!(updated.completed_operations, 1);
3226        assert_eq!(updated.pending_operations, 0);
3227        assert_eq!(updated.failed_operations, 0);
3228        assert_eq!(
3229            updated.operations[0].state,
3230            RestoreApplyOperationState::Completed
3231        );
3232    }
3233
3234    // Ensure restore run can fail closed after writing an incomplete summary.
3235    #[test]
3236    fn run_restore_run_require_complete_writes_summary_then_fails() {
3237        let root = temp_dir("canic-cli-restore-run-require-complete");
3238        fs::create_dir_all(&root).expect("create temp root");
3239        let journal_path = root.join("restore-apply-journal.json");
3240        let out_path = root.join("restore-run.json");
3241        let journal = ready_apply_journal();
3242
3243        fs::write(
3244            &journal_path,
3245            serde_json::to_vec(&journal).expect("serialize journal"),
3246        )
3247        .expect("write journal");
3248
3249        let err = run([
3250            OsString::from("run"),
3251            OsString::from("--journal"),
3252            OsString::from(journal_path.as_os_str()),
3253            OsString::from("--execute"),
3254            OsString::from("--dfx"),
3255            OsString::from("/bin/true"),
3256            OsString::from("--max-steps"),
3257            OsString::from("1"),
3258            OsString::from("--out"),
3259            OsString::from(out_path.as_os_str()),
3260            OsString::from("--require-complete"),
3261        ])
3262        .expect_err("incomplete run should fail requirement");
3263
3264        let run_summary: serde_json::Value =
3265            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3266                .expect("decode run summary");
3267
3268        fs::remove_dir_all(root).expect("remove temp root");
3269        assert_eq!(run_summary["executed_operation_count"], 1);
3270        assert_eq!(run_summary["complete"], false);
3271        assert!(matches!(
3272            err,
3273            RestoreCommandError::RestoreApplyIncomplete {
3274                completed_operations: 1,
3275                operation_count: 8,
3276                ..
3277            }
3278        ));
3279    }
3280
3281    // Ensure restore run execute records failed command exits in the journal.
3282    #[test]
3283    fn run_restore_run_execute_marks_failed_operation() {
3284        let root = temp_dir("canic-cli-restore-run-execute-failed");
3285        fs::create_dir_all(&root).expect("create temp root");
3286        let journal_path = root.join("restore-apply-journal.json");
3287        let out_path = root.join("restore-run.json");
3288        let journal = ready_apply_journal();
3289
3290        fs::write(
3291            &journal_path,
3292            serde_json::to_vec(&journal).expect("serialize journal"),
3293        )
3294        .expect("write journal");
3295
3296        let err = run([
3297            OsString::from("run"),
3298            OsString::from("--journal"),
3299            OsString::from(journal_path.as_os_str()),
3300            OsString::from("--execute"),
3301            OsString::from("--dfx"),
3302            OsString::from("/bin/false"),
3303            OsString::from("--max-steps"),
3304            OsString::from("1"),
3305            OsString::from("--out"),
3306            OsString::from(out_path.as_os_str()),
3307        ])
3308        .expect_err("failing runner command should fail");
3309
3310        let run_summary: serde_json::Value =
3311            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3312                .expect("decode run summary");
3313        let updated: RestoreApplyJournal =
3314            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3315                .expect("decode updated journal");
3316
3317        fs::remove_dir_all(root).expect("remove temp root");
3318        assert!(matches!(
3319            err,
3320            RestoreCommandError::RestoreRunCommandFailed {
3321                sequence: 0,
3322                status,
3323            } if status == "1"
3324        ));
3325        assert_eq!(updated.failed_operations, 1);
3326        assert_eq!(updated.pending_operations, 0);
3327        assert_eq!(
3328            updated.operations[0].state,
3329            RestoreApplyOperationState::Failed
3330        );
3331        assert_eq!(run_summary["execute"], true);
3332        assert_eq!(run_summary["attention_required"], true);
3333        assert_eq!(run_summary["outcome"], "failed");
3334        assert_eq!(run_summary["stopped_reason"], "command-failed");
3335        assert_eq!(run_summary["next_action"], "inspect-failed-operation");
3336        assert_eq!(run_summary["executed_operation_count"], 1);
3337        assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
3338        assert_eq!(run_summary["executed_operations"][0]["status"], "1");
3339        assert_eq!(
3340            updated.operations[0].blocking_reasons,
3341            vec!["runner-command-exit-1".to_string()]
3342        );
3343    }
3344
3345    // Ensure restore run can fail closed after writing an attention summary.
3346    #[test]
3347    fn run_restore_run_require_no_attention_writes_summary_then_fails() {
3348        let root = temp_dir("canic-cli-restore-run-require-attention");
3349        fs::create_dir_all(&root).expect("create temp root");
3350        let journal_path = root.join("restore-apply-journal.json");
3351        let out_path = root.join("restore-run.json");
3352        let mut journal = ready_apply_journal();
3353        journal
3354            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3355            .expect("mark pending operation");
3356
3357        fs::write(
3358            &journal_path,
3359            serde_json::to_vec(&journal).expect("serialize journal"),
3360        )
3361        .expect("write journal");
3362
3363        let err = run([
3364            OsString::from("run"),
3365            OsString::from("--journal"),
3366            OsString::from(journal_path.as_os_str()),
3367            OsString::from("--dry-run"),
3368            OsString::from("--out"),
3369            OsString::from(out_path.as_os_str()),
3370            OsString::from("--require-no-attention"),
3371        ])
3372        .expect_err("attention run should fail requirement");
3373
3374        let run_summary: serde_json::Value =
3375            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3376                .expect("decode run summary");
3377
3378        fs::remove_dir_all(root).expect("remove temp root");
3379        assert_eq!(run_summary["attention_required"], true);
3380        assert_eq!(run_summary["outcome"], "pending");
3381        assert_eq!(run_summary["stopped_reason"], "pending");
3382        assert_eq!(run_summary["next_action"], "unclaim-pending");
3383        assert!(matches!(
3384            err,
3385            RestoreCommandError::RestoreApplyReportNeedsAttention {
3386                outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3387                ..
3388            }
3389        ));
3390    }
3391
3392    // Ensure restore run can fail closed on an unexpected run mode.
3393    #[test]
3394    fn run_restore_run_require_run_mode_writes_summary_then_fails() {
3395        let root = temp_dir("canic-cli-restore-run-require-run-mode");
3396        fs::create_dir_all(&root).expect("create temp root");
3397        let journal_path = root.join("restore-apply-journal.json");
3398        let out_path = root.join("restore-run.json");
3399        let journal = ready_apply_journal();
3400
3401        fs::write(
3402            &journal_path,
3403            serde_json::to_vec(&journal).expect("serialize journal"),
3404        )
3405        .expect("write journal");
3406
3407        let err = run([
3408            OsString::from("run"),
3409            OsString::from("--journal"),
3410            OsString::from(journal_path.as_os_str()),
3411            OsString::from("--dry-run"),
3412            OsString::from("--out"),
3413            OsString::from(out_path.as_os_str()),
3414            OsString::from("--require-run-mode"),
3415            OsString::from("execute"),
3416        ])
3417        .expect_err("run mode mismatch should fail requirement");
3418
3419        let run_summary: serde_json::Value =
3420            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3421                .expect("decode run summary");
3422
3423        fs::remove_dir_all(root).expect("remove temp root");
3424        assert_eq!(run_summary["run_mode"], "dry-run");
3425        assert!(matches!(
3426            err,
3427            RestoreCommandError::RestoreRunModeMismatch {
3428                expected,
3429                actual,
3430                ..
3431            } if expected == "execute" && actual == "dry-run"
3432        ));
3433    }
3434
3435    // Ensure restore run can fail closed on an unexpected executed operation count.
3436    #[test]
3437    fn run_restore_run_require_executed_count_writes_summary_then_fails() {
3438        let root = temp_dir("canic-cli-restore-run-require-executed-count");
3439        fs::create_dir_all(&root).expect("create temp root");
3440        let journal_path = root.join("restore-apply-journal.json");
3441        let out_path = root.join("restore-run.json");
3442        let journal = ready_apply_journal();
3443
3444        fs::write(
3445            &journal_path,
3446            serde_json::to_vec(&journal).expect("serialize journal"),
3447        )
3448        .expect("write journal");
3449
3450        let err = run([
3451            OsString::from("run"),
3452            OsString::from("--journal"),
3453            OsString::from(journal_path.as_os_str()),
3454            OsString::from("--execute"),
3455            OsString::from("--dfx"),
3456            OsString::from("/bin/true"),
3457            OsString::from("--max-steps"),
3458            OsString::from("1"),
3459            OsString::from("--out"),
3460            OsString::from(out_path.as_os_str()),
3461            OsString::from("--require-executed-count"),
3462            OsString::from("2"),
3463        ])
3464        .expect_err("executed count mismatch should fail requirement");
3465
3466        let run_summary: serde_json::Value =
3467            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3468                .expect("decode run summary");
3469
3470        fs::remove_dir_all(root).expect("remove temp root");
3471        assert_eq!(run_summary["executed_operation_count"], 1);
3472        assert!(matches!(
3473            err,
3474            RestoreCommandError::RestoreRunExecutedCountMismatch {
3475                expected: 2,
3476                actual: 1,
3477                ..
3478            }
3479        ));
3480    }
3481
3482    // Ensure restore run can fail closed on an unexpected stopped reason.
3483    #[test]
3484    fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
3485        let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
3486        fs::create_dir_all(&root).expect("create temp root");
3487        let journal_path = root.join("restore-apply-journal.json");
3488        let out_path = root.join("restore-run.json");
3489        let journal = ready_apply_journal();
3490
3491        fs::write(
3492            &journal_path,
3493            serde_json::to_vec(&journal).expect("serialize journal"),
3494        )
3495        .expect("write journal");
3496
3497        let err = run([
3498            OsString::from("run"),
3499            OsString::from("--journal"),
3500            OsString::from(journal_path.as_os_str()),
3501            OsString::from("--dry-run"),
3502            OsString::from("--out"),
3503            OsString::from(out_path.as_os_str()),
3504            OsString::from("--require-stopped-reason"),
3505            OsString::from("complete"),
3506        ])
3507        .expect_err("stopped reason mismatch should fail requirement");
3508
3509        let run_summary: serde_json::Value =
3510            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3511                .expect("decode run summary");
3512
3513        fs::remove_dir_all(root).expect("remove temp root");
3514        assert_eq!(run_summary["stopped_reason"], "preview");
3515        assert!(matches!(
3516            err,
3517            RestoreCommandError::RestoreRunStoppedReasonMismatch {
3518                expected,
3519                actual,
3520                ..
3521            } if expected == "complete" && actual == "preview"
3522        ));
3523    }
3524
3525    // Ensure restore run can fail closed on an unexpected next action.
3526    #[test]
3527    fn run_restore_run_require_next_action_writes_summary_then_fails() {
3528        let root = temp_dir("canic-cli-restore-run-require-next-action");
3529        fs::create_dir_all(&root).expect("create temp root");
3530        let journal_path = root.join("restore-apply-journal.json");
3531        let out_path = root.join("restore-run.json");
3532        let journal = ready_apply_journal();
3533
3534        fs::write(
3535            &journal_path,
3536            serde_json::to_vec(&journal).expect("serialize journal"),
3537        )
3538        .expect("write journal");
3539
3540        let err = run([
3541            OsString::from("run"),
3542            OsString::from("--journal"),
3543            OsString::from(journal_path.as_os_str()),
3544            OsString::from("--dry-run"),
3545            OsString::from("--out"),
3546            OsString::from(out_path.as_os_str()),
3547            OsString::from("--require-next-action"),
3548            OsString::from("done"),
3549        ])
3550        .expect_err("next action mismatch should fail requirement");
3551
3552        let run_summary: serde_json::Value =
3553            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3554                .expect("decode run summary");
3555
3556        fs::remove_dir_all(root).expect("remove temp root");
3557        assert_eq!(run_summary["next_action"], "rerun");
3558        assert!(matches!(
3559            err,
3560            RestoreCommandError::RestoreRunNextActionMismatch {
3561                expected,
3562                actual,
3563                ..
3564            } if expected == "done" && actual == "rerun"
3565        ));
3566    }
3567
3568    // Ensure apply-report can fail closed after writing an attention report.
3569    #[test]
3570    fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
3571        let root = temp_dir("canic-cli-restore-apply-report-attention");
3572        fs::create_dir_all(&root).expect("create temp root");
3573        let journal_path = root.join("restore-apply-journal.json");
3574        let out_path = root.join("restore-apply-report.json");
3575        let mut journal = ready_apply_journal();
3576        journal
3577            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3578            .expect("mark pending operation");
3579
3580        fs::write(
3581            &journal_path,
3582            serde_json::to_vec(&journal).expect("serialize journal"),
3583        )
3584        .expect("write journal");
3585
3586        let err = run([
3587            OsString::from("apply-report"),
3588            OsString::from("--journal"),
3589            OsString::from(journal_path.as_os_str()),
3590            OsString::from("--out"),
3591            OsString::from(out_path.as_os_str()),
3592            OsString::from("--require-no-attention"),
3593        ])
3594        .expect_err("attention report should fail requirement");
3595
3596        let report: RestoreApplyJournalReport =
3597            serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3598                .expect("decode apply report");
3599
3600        fs::remove_dir_all(root).expect("remove temp root");
3601        assert!(report.attention_required);
3602        assert_eq!(report.pending_operations, 1);
3603        assert!(matches!(
3604            err,
3605            RestoreCommandError::RestoreApplyReportNeedsAttention {
3606                outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3607                ..
3608            }
3609        ));
3610    }
3611
3612    // Ensure apply-status can fail closed after writing status for incomplete work.
3613    #[test]
3614    fn run_restore_apply_status_require_complete_writes_status_then_fails() {
3615        let root = temp_dir("canic-cli-restore-apply-status-incomplete");
3616        fs::create_dir_all(&root).expect("create temp root");
3617        let journal_path = root.join("restore-apply-journal.json");
3618        let out_path = root.join("restore-apply-status.json");
3619        let journal = ready_apply_journal();
3620
3621        fs::write(
3622            &journal_path,
3623            serde_json::to_vec(&journal).expect("serialize journal"),
3624        )
3625        .expect("write journal");
3626
3627        let err = run([
3628            OsString::from("apply-status"),
3629            OsString::from("--journal"),
3630            OsString::from(journal_path.as_os_str()),
3631            OsString::from("--out"),
3632            OsString::from(out_path.as_os_str()),
3633            OsString::from("--require-complete"),
3634        ])
3635        .expect_err("incomplete journal should fail requirement");
3636
3637        assert!(out_path.exists());
3638        let status: RestoreApplyJournalStatus =
3639            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3640                .expect("decode apply status");
3641
3642        fs::remove_dir_all(root).expect("remove temp root");
3643        assert!(!status.complete);
3644        assert_eq!(status.completed_operations, 0);
3645        assert_eq!(status.operation_count, 8);
3646        assert!(matches!(
3647            err,
3648            RestoreCommandError::RestoreApplyIncomplete {
3649                completed_operations: 0,
3650                operation_count: 8,
3651                ..
3652            }
3653        ));
3654    }
3655
3656    // Ensure apply-status can fail closed after writing status for failed work.
3657    #[test]
3658    fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
3659        let root = temp_dir("canic-cli-restore-apply-status-failed");
3660        fs::create_dir_all(&root).expect("create temp root");
3661        let journal_path = root.join("restore-apply-journal.json");
3662        let out_path = root.join("restore-apply-status.json");
3663        let mut journal = ready_apply_journal();
3664        journal
3665            .mark_operation_failed(0, "dfx-load-failed".to_string())
3666            .expect("mark failed operation");
3667
3668        fs::write(
3669            &journal_path,
3670            serde_json::to_vec(&journal).expect("serialize journal"),
3671        )
3672        .expect("write journal");
3673
3674        let err = run([
3675            OsString::from("apply-status"),
3676            OsString::from("--journal"),
3677            OsString::from(journal_path.as_os_str()),
3678            OsString::from("--out"),
3679            OsString::from(out_path.as_os_str()),
3680            OsString::from("--require-no-failed"),
3681        ])
3682        .expect_err("failed operation should fail requirement");
3683
3684        assert!(out_path.exists());
3685        let status: RestoreApplyJournalStatus =
3686            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3687                .expect("decode apply status");
3688
3689        fs::remove_dir_all(root).expect("remove temp root");
3690        assert_eq!(status.failed_operations, 1);
3691        assert!(matches!(
3692            err,
3693            RestoreCommandError::RestoreApplyFailed {
3694                failed_operations: 1,
3695                ..
3696            }
3697        ));
3698    }
3699
3700    // Ensure apply-status accepts a complete journal when required.
3701    #[test]
3702    fn run_restore_apply_status_require_complete_accepts_complete_journal() {
3703        let root = temp_dir("canic-cli-restore-apply-status-complete");
3704        fs::create_dir_all(&root).expect("create temp root");
3705        let journal_path = root.join("restore-apply-journal.json");
3706        let out_path = root.join("restore-apply-status.json");
3707        let mut journal = ready_apply_journal();
3708        for sequence in 0..journal.operation_count {
3709            journal
3710                .mark_operation_completed(sequence)
3711                .expect("complete operation");
3712        }
3713
3714        fs::write(
3715            &journal_path,
3716            serde_json::to_vec(&journal).expect("serialize journal"),
3717        )
3718        .expect("write journal");
3719
3720        run([
3721            OsString::from("apply-status"),
3722            OsString::from("--journal"),
3723            OsString::from(journal_path.as_os_str()),
3724            OsString::from("--out"),
3725            OsString::from(out_path.as_os_str()),
3726            OsString::from("--require-complete"),
3727        ])
3728        .expect("complete journal should pass requirement");
3729
3730        let status: RestoreApplyJournalStatus =
3731            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3732                .expect("decode apply status");
3733
3734        fs::remove_dir_all(root).expect("remove temp root");
3735        assert!(status.complete);
3736        assert_eq!(status.completed_operations, 8);
3737        assert_eq!(status.operation_count, 8);
3738    }
3739
3740    // Ensure apply-next writes the full next ready operation row for runners.
3741    #[test]
3742    fn run_restore_apply_next_writes_next_ready_operation() {
3743        let root = temp_dir("canic-cli-restore-apply-next");
3744        fs::create_dir_all(&root).expect("create temp root");
3745        let journal_path = root.join("restore-apply-journal.json");
3746        let out_path = root.join("restore-apply-next.json");
3747        let mut journal = ready_apply_journal();
3748        journal
3749            .mark_operation_completed(0)
3750            .expect("mark first operation complete");
3751
3752        fs::write(
3753            &journal_path,
3754            serde_json::to_vec(&journal).expect("serialize journal"),
3755        )
3756        .expect("write journal");
3757
3758        run([
3759            OsString::from("apply-next"),
3760            OsString::from("--journal"),
3761            OsString::from(journal_path.as_os_str()),
3762            OsString::from("--out"),
3763            OsString::from(out_path.as_os_str()),
3764        ])
3765        .expect("write apply next");
3766
3767        let next: RestoreApplyNextOperation =
3768            serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
3769                .expect("decode next operation");
3770        let operation = next.operation.expect("operation should be available");
3771
3772        fs::remove_dir_all(root).expect("remove temp root");
3773        assert!(next.ready);
3774        assert!(next.operation_available);
3775        assert_eq!(operation.sequence, 1);
3776        assert_eq!(
3777            operation.operation,
3778            canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
3779        );
3780    }
3781
3782    // Ensure apply-command writes a no-execute command preview for the next operation.
3783    #[test]
3784    fn run_restore_apply_command_writes_next_command_preview() {
3785        let root = temp_dir("canic-cli-restore-apply-command");
3786        fs::create_dir_all(&root).expect("create temp root");
3787        let journal_path = root.join("restore-apply-journal.json");
3788        let out_path = root.join("restore-apply-command.json");
3789        let journal = ready_apply_journal();
3790
3791        fs::write(
3792            &journal_path,
3793            serde_json::to_vec(&journal).expect("serialize journal"),
3794        )
3795        .expect("write journal");
3796
3797        run([
3798            OsString::from("apply-command"),
3799            OsString::from("--journal"),
3800            OsString::from(journal_path.as_os_str()),
3801            OsString::from("--dfx"),
3802            OsString::from("/tmp/dfx"),
3803            OsString::from("--network"),
3804            OsString::from("local"),
3805            OsString::from("--out"),
3806            OsString::from(out_path.as_os_str()),
3807        ])
3808        .expect("write command preview");
3809
3810        let preview: RestoreApplyCommandPreview =
3811            serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
3812                .expect("decode command preview");
3813        let command = preview.command.expect("command should be available");
3814
3815        fs::remove_dir_all(root).expect("remove temp root");
3816        assert!(preview.ready);
3817        assert!(preview.command_available);
3818        assert_eq!(command.program, "/tmp/dfx");
3819        assert_eq!(
3820            command.args,
3821            vec![
3822                "canister".to_string(),
3823                "--network".to_string(),
3824                "local".to_string(),
3825                "snapshot".to_string(),
3826                "upload".to_string(),
3827                "--dir".to_string(),
3828                "artifacts/root".to_string(),
3829                ROOT.to_string(),
3830            ]
3831        );
3832        assert!(command.mutates);
3833    }
3834
3835    // Ensure apply-command can fail closed after writing a command preview.
3836    #[test]
3837    fn run_restore_apply_command_require_command_writes_preview_then_fails() {
3838        let root = temp_dir("canic-cli-restore-apply-command-require");
3839        fs::create_dir_all(&root).expect("create temp root");
3840        let journal_path = root.join("restore-apply-journal.json");
3841        let out_path = root.join("restore-apply-command.json");
3842        let mut journal = ready_apply_journal();
3843
3844        for sequence in 0..journal.operation_count {
3845            journal
3846                .mark_operation_completed(sequence)
3847                .expect("mark operation completed");
3848        }
3849
3850        fs::write(
3851            &journal_path,
3852            serde_json::to_vec(&journal).expect("serialize journal"),
3853        )
3854        .expect("write journal");
3855
3856        let err = run([
3857            OsString::from("apply-command"),
3858            OsString::from("--journal"),
3859            OsString::from(journal_path.as_os_str()),
3860            OsString::from("--out"),
3861            OsString::from(out_path.as_os_str()),
3862            OsString::from("--require-command"),
3863        ])
3864        .expect_err("missing command should fail");
3865
3866        let preview: RestoreApplyCommandPreview =
3867            serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
3868                .expect("decode command preview");
3869
3870        fs::remove_dir_all(root).expect("remove temp root");
3871        assert!(preview.complete);
3872        assert!(!preview.operation_available);
3873        assert!(!preview.command_available);
3874        assert!(matches!(
3875            err,
3876            RestoreCommandError::RestoreApplyCommandUnavailable {
3877                operation_available: false,
3878                complete: true,
3879                ..
3880            }
3881        ));
3882    }
3883
3884    // Ensure apply-claim marks the next operation pending before runner execution.
3885    #[test]
3886    fn run_restore_apply_claim_marks_next_operation_pending() {
3887        let root = temp_dir("canic-cli-restore-apply-claim");
3888        fs::create_dir_all(&root).expect("create temp root");
3889        let journal_path = root.join("restore-apply-journal.json");
3890        let claimed_path = root.join("restore-apply-journal.claimed.json");
3891        let journal = ready_apply_journal();
3892
3893        fs::write(
3894            &journal_path,
3895            serde_json::to_vec(&journal).expect("serialize journal"),
3896        )
3897        .expect("write journal");
3898
3899        run([
3900            OsString::from("apply-claim"),
3901            OsString::from("--journal"),
3902            OsString::from(journal_path.as_os_str()),
3903            OsString::from("--sequence"),
3904            OsString::from("0"),
3905            OsString::from("--updated-at"),
3906            OsString::from("2026-05-04T12:00:00Z"),
3907            OsString::from("--out"),
3908            OsString::from(claimed_path.as_os_str()),
3909        ])
3910        .expect("claim operation");
3911
3912        let claimed: RestoreApplyJournal =
3913            serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
3914                .expect("decode claimed journal");
3915        let status = claimed.status();
3916        let next = claimed.next_operation();
3917
3918        fs::remove_dir_all(root).expect("remove temp root");
3919        assert_eq!(claimed.pending_operations, 1);
3920        assert_eq!(claimed.ready_operations, 7);
3921        assert_eq!(
3922            claimed.operations[0].state,
3923            RestoreApplyOperationState::Pending
3924        );
3925        assert_eq!(
3926            claimed.operations[0].state_updated_at.as_deref(),
3927            Some("2026-05-04T12:00:00Z")
3928        );
3929        assert_eq!(status.next_transition_sequence, Some(0));
3930        assert_eq!(
3931            status.next_transition_state,
3932            Some(RestoreApplyOperationState::Pending)
3933        );
3934        assert_eq!(
3935            status.next_transition_updated_at.as_deref(),
3936            Some("2026-05-04T12:00:00Z")
3937        );
3938        assert_eq!(
3939            next.operation.expect("next operation").state,
3940            RestoreApplyOperationState::Pending
3941        );
3942    }
3943
3944    // Ensure apply-claim can reject a stale command preview sequence.
3945    #[test]
3946    fn run_restore_apply_claim_rejects_sequence_mismatch() {
3947        let root = temp_dir("canic-cli-restore-apply-claim-sequence");
3948        fs::create_dir_all(&root).expect("create temp root");
3949        let journal_path = root.join("restore-apply-journal.json");
3950        let claimed_path = root.join("restore-apply-journal.claimed.json");
3951        let journal = ready_apply_journal();
3952
3953        fs::write(
3954            &journal_path,
3955            serde_json::to_vec(&journal).expect("serialize journal"),
3956        )
3957        .expect("write journal");
3958
3959        let err = run([
3960            OsString::from("apply-claim"),
3961            OsString::from("--journal"),
3962            OsString::from(journal_path.as_os_str()),
3963            OsString::from("--sequence"),
3964            OsString::from("1"),
3965            OsString::from("--out"),
3966            OsString::from(claimed_path.as_os_str()),
3967        ])
3968        .expect_err("stale sequence should fail claim");
3969
3970        assert!(!claimed_path.exists());
3971        fs::remove_dir_all(root).expect("remove temp root");
3972        assert!(matches!(
3973            err,
3974            RestoreCommandError::RestoreApplyClaimSequenceMismatch {
3975                expected: 1,
3976                actual: Some(0),
3977            }
3978        ));
3979    }
3980
3981    // Ensure apply-unclaim releases the current pending operation back to ready.
3982    #[test]
3983    fn run_restore_apply_unclaim_marks_pending_operation_ready() {
3984        let root = temp_dir("canic-cli-restore-apply-unclaim");
3985        fs::create_dir_all(&root).expect("create temp root");
3986        let journal_path = root.join("restore-apply-journal.json");
3987        let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
3988        let mut journal = ready_apply_journal();
3989        journal
3990            .mark_next_operation_pending()
3991            .expect("claim operation");
3992
3993        fs::write(
3994            &journal_path,
3995            serde_json::to_vec(&journal).expect("serialize journal"),
3996        )
3997        .expect("write journal");
3998
3999        run([
4000            OsString::from("apply-unclaim"),
4001            OsString::from("--journal"),
4002            OsString::from(journal_path.as_os_str()),
4003            OsString::from("--sequence"),
4004            OsString::from("0"),
4005            OsString::from("--updated-at"),
4006            OsString::from("2026-05-04T12:01:00Z"),
4007            OsString::from("--out"),
4008            OsString::from(unclaimed_path.as_os_str()),
4009        ])
4010        .expect("unclaim operation");
4011
4012        let unclaimed: RestoreApplyJournal =
4013            serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
4014                .expect("decode unclaimed journal");
4015        let status = unclaimed.status();
4016
4017        fs::remove_dir_all(root).expect("remove temp root");
4018        assert_eq!(unclaimed.pending_operations, 0);
4019        assert_eq!(unclaimed.ready_operations, 8);
4020        assert_eq!(
4021            unclaimed.operations[0].state,
4022            RestoreApplyOperationState::Ready
4023        );
4024        assert_eq!(
4025            unclaimed.operations[0].state_updated_at.as_deref(),
4026            Some("2026-05-04T12:01:00Z")
4027        );
4028        assert_eq!(status.next_ready_sequence, Some(0));
4029        assert_eq!(
4030            status.next_transition_state,
4031            Some(RestoreApplyOperationState::Ready)
4032        );
4033        assert_eq!(
4034            status.next_transition_updated_at.as_deref(),
4035            Some("2026-05-04T12:01:00Z")
4036        );
4037    }
4038
4039    // Ensure apply-unclaim can reject a stale pending operation sequence.
4040    #[test]
4041    fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
4042        let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
4043        fs::create_dir_all(&root).expect("create temp root");
4044        let journal_path = root.join("restore-apply-journal.json");
4045        let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
4046        let mut journal = ready_apply_journal();
4047        journal
4048            .mark_next_operation_pending()
4049            .expect("claim operation");
4050
4051        fs::write(
4052            &journal_path,
4053            serde_json::to_vec(&journal).expect("serialize journal"),
4054        )
4055        .expect("write journal");
4056
4057        let err = run([
4058            OsString::from("apply-unclaim"),
4059            OsString::from("--journal"),
4060            OsString::from(journal_path.as_os_str()),
4061            OsString::from("--sequence"),
4062            OsString::from("1"),
4063            OsString::from("--out"),
4064            OsString::from(unclaimed_path.as_os_str()),
4065        ])
4066        .expect_err("stale sequence should fail unclaim");
4067
4068        assert!(!unclaimed_path.exists());
4069        fs::remove_dir_all(root).expect("remove temp root");
4070        assert!(matches!(
4071            err,
4072            RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
4073                expected: 1,
4074                actual: Some(0),
4075            }
4076        ));
4077    }
4078
4079    // Ensure apply-mark can advance one journal operation and keep counts consistent.
4080    #[test]
4081    fn run_restore_apply_mark_completes_operation() {
4082        let root = temp_dir("canic-cli-restore-apply-mark-complete");
4083        fs::create_dir_all(&root).expect("create temp root");
4084        let journal_path = root.join("restore-apply-journal.json");
4085        let updated_path = root.join("restore-apply-journal.updated.json");
4086        let journal = ready_apply_journal();
4087
4088        fs::write(
4089            &journal_path,
4090            serde_json::to_vec(&journal).expect("serialize journal"),
4091        )
4092        .expect("write journal");
4093
4094        run([
4095            OsString::from("apply-mark"),
4096            OsString::from("--journal"),
4097            OsString::from(journal_path.as_os_str()),
4098            OsString::from("--sequence"),
4099            OsString::from("0"),
4100            OsString::from("--state"),
4101            OsString::from("completed"),
4102            OsString::from("--updated-at"),
4103            OsString::from("2026-05-04T12:02:00Z"),
4104            OsString::from("--out"),
4105            OsString::from(updated_path.as_os_str()),
4106        ])
4107        .expect("mark operation completed");
4108
4109        let updated: RestoreApplyJournal =
4110            serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
4111                .expect("decode updated journal");
4112        let status = updated.status();
4113
4114        fs::remove_dir_all(root).expect("remove temp root");
4115        assert_eq!(updated.completed_operations, 1);
4116        assert_eq!(updated.ready_operations, 7);
4117        assert_eq!(
4118            updated.operations[0].state_updated_at.as_deref(),
4119            Some("2026-05-04T12:02:00Z")
4120        );
4121        assert_eq!(status.next_ready_sequence, Some(1));
4122    }
4123
4124    // Ensure apply-mark can require an operation claim before completion.
4125    #[test]
4126    fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
4127        let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
4128        fs::create_dir_all(&root).expect("create temp root");
4129        let journal_path = root.join("restore-apply-journal.json");
4130        let updated_path = root.join("restore-apply-journal.updated.json");
4131        let journal = ready_apply_journal();
4132
4133        fs::write(
4134            &journal_path,
4135            serde_json::to_vec(&journal).expect("serialize journal"),
4136        )
4137        .expect("write journal");
4138
4139        let err = run([
4140            OsString::from("apply-mark"),
4141            OsString::from("--journal"),
4142            OsString::from(journal_path.as_os_str()),
4143            OsString::from("--sequence"),
4144            OsString::from("0"),
4145            OsString::from("--state"),
4146            OsString::from("completed"),
4147            OsString::from("--out"),
4148            OsString::from(updated_path.as_os_str()),
4149            OsString::from("--require-pending"),
4150        ])
4151        .expect_err("ready operation should fail pending requirement");
4152
4153        assert!(!updated_path.exists());
4154        fs::remove_dir_all(root).expect("remove temp root");
4155        assert!(matches!(
4156            err,
4157            RestoreCommandError::RestoreApplyMarkRequiresPending {
4158                sequence: 0,
4159                state: RestoreApplyOperationState::Ready,
4160            }
4161        ));
4162    }
4163
4164    // Ensure apply-mark refuses to skip earlier ready operations.
4165    #[test]
4166    fn run_restore_apply_mark_rejects_out_of_order_operation() {
4167        let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
4168        fs::create_dir_all(&root).expect("create temp root");
4169        let journal_path = root.join("restore-apply-journal.json");
4170        let updated_path = root.join("restore-apply-journal.updated.json");
4171        let journal = ready_apply_journal();
4172
4173        fs::write(
4174            &journal_path,
4175            serde_json::to_vec(&journal).expect("serialize journal"),
4176        )
4177        .expect("write journal");
4178
4179        let err = run([
4180            OsString::from("apply-mark"),
4181            OsString::from("--journal"),
4182            OsString::from(journal_path.as_os_str()),
4183            OsString::from("--sequence"),
4184            OsString::from("1"),
4185            OsString::from("--state"),
4186            OsString::from("completed"),
4187            OsString::from("--out"),
4188            OsString::from(updated_path.as_os_str()),
4189        ])
4190        .expect_err("out-of-order operation should fail");
4191
4192        assert!(!updated_path.exists());
4193        fs::remove_dir_all(root).expect("remove temp root");
4194        assert!(matches!(
4195            err,
4196            RestoreCommandError::RestoreApplyJournal(
4197                RestoreApplyJournalError::OutOfOrderOperationTransition {
4198                    requested: 1,
4199                    next: 0
4200                }
4201            )
4202        ));
4203    }
4204
4205    // Ensure apply-mark requires failure reasons for failed operation state.
4206    #[test]
4207    fn run_restore_apply_mark_failed_requires_reason() {
4208        let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
4209        fs::create_dir_all(&root).expect("create temp root");
4210        let journal_path = root.join("restore-apply-journal.json");
4211        let journal = ready_apply_journal();
4212
4213        fs::write(
4214            &journal_path,
4215            serde_json::to_vec(&journal).expect("serialize journal"),
4216        )
4217        .expect("write journal");
4218
4219        let err = run([
4220            OsString::from("apply-mark"),
4221            OsString::from("--journal"),
4222            OsString::from(journal_path.as_os_str()),
4223            OsString::from("--sequence"),
4224            OsString::from("0"),
4225            OsString::from("--state"),
4226            OsString::from("failed"),
4227        ])
4228        .expect_err("failed state should require reason");
4229
4230        fs::remove_dir_all(root).expect("remove temp root");
4231        assert!(matches!(
4232            err,
4233            RestoreCommandError::RestoreApplyJournal(
4234                RestoreApplyJournalError::FailureReasonRequired(0)
4235            )
4236        ));
4237    }
4238
4239    // Ensure restore apply dry-run rejects status files from another plan.
4240    #[test]
4241    fn run_restore_apply_dry_run_rejects_mismatched_status() {
4242        let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
4243        fs::create_dir_all(&root).expect("create temp root");
4244        let plan_path = root.join("restore-plan.json");
4245        let status_path = root.join("restore-status.json");
4246        let out_path = root.join("restore-apply-dry-run.json");
4247        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4248        let mut status = RestoreStatus::from_plan(&plan);
4249        status.backup_id = "other-backup".to_string();
4250
4251        fs::write(
4252            &plan_path,
4253            serde_json::to_vec(&plan).expect("serialize plan"),
4254        )
4255        .expect("write plan");
4256        fs::write(
4257            &status_path,
4258            serde_json::to_vec(&status).expect("serialize status"),
4259        )
4260        .expect("write status");
4261
4262        let err = run([
4263            OsString::from("apply"),
4264            OsString::from("--plan"),
4265            OsString::from(plan_path.as_os_str()),
4266            OsString::from("--status"),
4267            OsString::from(status_path.as_os_str()),
4268            OsString::from("--dry-run"),
4269            OsString::from("--out"),
4270            OsString::from(out_path.as_os_str()),
4271        ])
4272        .expect_err("mismatched status should fail");
4273
4274        assert!(!out_path.exists());
4275        fs::remove_dir_all(root).expect("remove temp root");
4276        assert!(matches!(
4277            err,
4278            RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
4279                field: "backup_id",
4280                ..
4281            })
4282        ));
4283    }
4284
4285    // Build one manually ready apply journal for runner-focused CLI tests.
4286    fn ready_apply_journal() -> RestoreApplyJournal {
4287        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4288        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4289        let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4290
4291        journal.ready = true;
4292        journal.blocked_reasons = Vec::new();
4293        for operation in &mut journal.operations {
4294            operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
4295            operation.blocking_reasons = Vec::new();
4296        }
4297        journal.blocked_operations = 0;
4298        journal.ready_operations = journal.operation_count;
4299        journal.validate().expect("journal should validate");
4300        journal
4301    }
4302
4303    // Build one valid manifest for restore planning tests.
4304    fn valid_manifest() -> FleetBackupManifest {
4305        FleetBackupManifest {
4306            manifest_version: 1,
4307            backup_id: "backup-test".to_string(),
4308            created_at: "2026-05-03T00:00:00Z".to_string(),
4309            tool: ToolMetadata {
4310                name: "canic".to_string(),
4311                version: "0.30.1".to_string(),
4312            },
4313            source: SourceMetadata {
4314                environment: "local".to_string(),
4315                root_canister: ROOT.to_string(),
4316            },
4317            consistency: ConsistencySection {
4318                mode: ConsistencyMode::CrashConsistent,
4319                backup_units: vec![BackupUnit {
4320                    unit_id: "fleet".to_string(),
4321                    kind: BackupUnitKind::SubtreeRooted,
4322                    roles: vec!["root".to_string(), "app".to_string()],
4323                    consistency_reason: None,
4324                    dependency_closure: Vec::new(),
4325                    topology_validation: "subtree-closed".to_string(),
4326                    quiescence_strategy: None,
4327                }],
4328            },
4329            fleet: FleetSection {
4330                topology_hash_algorithm: "sha256".to_string(),
4331                topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
4332                discovery_topology_hash: HASH.to_string(),
4333                pre_snapshot_topology_hash: HASH.to_string(),
4334                topology_hash: HASH.to_string(),
4335                members: vec![
4336                    fleet_member("root", ROOT, None, IdentityMode::Fixed),
4337                    fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
4338                ],
4339            },
4340            verification: VerificationPlan::default(),
4341        }
4342    }
4343
4344    // Build one manifest whose restore readiness metadata is complete.
4345    fn restore_ready_manifest() -> FleetBackupManifest {
4346        let mut manifest = valid_manifest();
4347        for member in &mut manifest.fleet.members {
4348            member.source_snapshot.module_hash = Some(HASH.to_string());
4349            member.source_snapshot.wasm_hash = Some(HASH.to_string());
4350            member.source_snapshot.checksum = Some(HASH.to_string());
4351        }
4352        manifest
4353    }
4354
4355    // Build one valid manifest member.
4356    fn fleet_member(
4357        role: &str,
4358        canister_id: &str,
4359        parent_canister_id: Option<&str>,
4360        identity_mode: IdentityMode,
4361    ) -> FleetMember {
4362        FleetMember {
4363            role: role.to_string(),
4364            canister_id: canister_id.to_string(),
4365            parent_canister_id: parent_canister_id.map(str::to_string),
4366            subnet_canister_id: Some(ROOT.to_string()),
4367            controller_hint: None,
4368            identity_mode,
4369            restore_group: 1,
4370            verification_class: "basic".to_string(),
4371            verification_checks: vec![VerificationCheck {
4372                kind: "status".to_string(),
4373                method: None,
4374                roles: vec![role.to_string()],
4375            }],
4376            source_snapshot: SourceSnapshot {
4377                snapshot_id: format!("{role}-snapshot"),
4378                module_hash: None,
4379                wasm_hash: None,
4380                code_version: Some("v0.30.1".to_string()),
4381                artifact_path: format!("artifacts/{role}"),
4382                checksum_algorithm: "sha256".to_string(),
4383                checksum: None,
4384            },
4385        }
4386    }
4387
4388    // Write a canonical backup layout whose journal checksums match the artifacts.
4389    fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
4390        layout.write_manifest(manifest).expect("write manifest");
4391
4392        let artifacts = manifest
4393            .fleet
4394            .members
4395            .iter()
4396            .map(|member| {
4397                let bytes = format!("{} artifact", member.role);
4398                let artifact_path = root.join(&member.source_snapshot.artifact_path);
4399                if let Some(parent) = artifact_path.parent() {
4400                    fs::create_dir_all(parent).expect("create artifact parent");
4401                }
4402                fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4403                let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4404
4405                ArtifactJournalEntry {
4406                    canister_id: member.canister_id.clone(),
4407                    snapshot_id: member.source_snapshot.snapshot_id.clone(),
4408                    state: ArtifactState::Durable,
4409                    temp_path: None,
4410                    artifact_path: member.source_snapshot.artifact_path.clone(),
4411                    checksum_algorithm: checksum.algorithm,
4412                    checksum: Some(checksum.hash),
4413                    updated_at: "2026-05-03T00:00:00Z".to_string(),
4414                }
4415            })
4416            .collect();
4417
4418        layout
4419            .write_journal(&DownloadJournal {
4420                journal_version: 1,
4421                backup_id: manifest.backup_id.clone(),
4422                discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
4423                pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
4424                operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
4425                artifacts,
4426            })
4427            .expect("write journal");
4428    }
4429
4430    // Write artifact bytes and update the manifest checksums for apply validation.
4431    fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
4432        for member in &mut manifest.fleet.members {
4433            let bytes = format!("{} apply artifact", member.role);
4434            let artifact_path = root.join(&member.source_snapshot.artifact_path);
4435            if let Some(parent) = artifact_path.parent() {
4436                fs::create_dir_all(parent).expect("create artifact parent");
4437            }
4438            fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4439            let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4440            member.source_snapshot.checksum = Some(checksum.hash);
4441        }
4442    }
4443
4444    // Build a unique temporary directory.
4445    fn temp_dir(prefix: &str) -> PathBuf {
4446        let nanos = SystemTime::now()
4447            .duration_since(UNIX_EPOCH)
4448            .expect("system time after epoch")
4449            .as_nanos();
4450        std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
4451    }
4452}