Skip to main content

canic_cli/restore/
mod.rs

1use canic_backup::{
2    manifest::FleetBackupManifest,
3    persistence::{BackupLayout, PersistenceError},
4    restore::{
5        RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6        RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7        RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8        RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationState,
9        RestoreApplyReportOperation, RestoreApplyReportOutcome, RestoreApplyRunnerCommand,
10        RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner, RestoreStatus,
11    },
12};
13use serde::Serialize;
14use std::{
15    ffi::OsString,
16    fs,
17    io::{self, Write},
18    path::PathBuf,
19    process::Command,
20};
21use thiserror::Error as ThisError;
22
23///
24/// RestoreCommandError
25///
26
27#[derive(Debug, ThisError)]
28pub enum RestoreCommandError {
29    #[error("{0}")]
30    Usage(&'static str),
31
32    #[error("missing required option {0}")]
33    MissingOption(&'static str),
34
35    #[error("use either --manifest or --backup-dir, not both")]
36    ConflictingManifestSources,
37
38    #[error("--require-verified requires --backup-dir")]
39    RequireVerifiedNeedsBackupDir,
40
41    #[error("restore apply currently requires --dry-run")]
42    ApplyRequiresDryRun,
43
44    #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
45    RestoreRunRequiresMode,
46
47    #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
48    RestoreRunConflictingModes,
49
50    #[error("restore run command failed for operation {sequence}: status={status}")]
51    RestoreRunCommandFailed { sequence: usize, status: String },
52
53    #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
54    RestoreRunModeMismatch {
55        backup_id: String,
56        expected: String,
57        actual: String,
58    },
59
60    #[error(
61        "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
62    )]
63    RestoreRunStoppedReasonMismatch {
64        backup_id: String,
65        expected: String,
66        actual: String,
67    },
68
69    #[error(
70        "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
71    )]
72    RestoreRunNextActionMismatch {
73        backup_id: String,
74        expected: String,
75        actual: String,
76    },
77
78    #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
79    RestoreRunExecutedCountMismatch {
80        backup_id: String,
81        expected: usize,
82        actual: usize,
83    },
84
85    #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
86    RestoreNotReady {
87        backup_id: String,
88        reasons: Vec<String>,
89    },
90
91    #[error(
92        "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
93    )]
94    RestoreApplyPending {
95        backup_id: String,
96        pending_operations: usize,
97        next_transition_sequence: Option<usize>,
98    },
99
100    #[error(
101        "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
102    )]
103    RestoreApplyIncomplete {
104        backup_id: String,
105        completed_operations: usize,
106        operation_count: usize,
107    },
108
109    #[error(
110        "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
111    )]
112    RestoreApplyFailed {
113        backup_id: String,
114        failed_operations: usize,
115    },
116
117    #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
118    RestoreApplyNotReady {
119        backup_id: String,
120        reasons: Vec<String>,
121    },
122
123    #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
124    RestoreApplyReportNeedsAttention {
125        backup_id: String,
126        outcome: canic_backup::restore::RestoreApplyReportOutcome,
127    },
128
129    #[error(
130        "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
131    )]
132    RestoreApplyCommandUnavailable {
133        backup_id: String,
134        operation_available: bool,
135        complete: bool,
136        blocked_reasons: Vec<String>,
137    },
138
139    #[error(
140        "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
141    )]
142    RestoreApplyMarkRequiresPending {
143        sequence: usize,
144        state: RestoreApplyOperationState,
145    },
146
147    #[error(
148        "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
149    )]
150    RestoreApplyClaimSequenceMismatch {
151        expected: usize,
152        actual: Option<usize>,
153    },
154
155    #[error(
156        "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
157    )]
158    RestoreApplyUnclaimSequenceMismatch {
159        expected: usize,
160        actual: Option<usize>,
161    },
162
163    #[error("unknown option {0}")]
164    UnknownOption(String),
165
166    #[error("option {0} requires a value")]
167    MissingValue(&'static str),
168
169    #[error("option --sequence requires a non-negative integer value")]
170    InvalidSequence,
171
172    #[error("unsupported apply-mark state {0}; use completed or failed")]
173    InvalidApplyMarkState(String),
174
175    #[error(transparent)]
176    Io(#[from] std::io::Error),
177
178    #[error(transparent)]
179    Json(#[from] serde_json::Error),
180
181    #[error(transparent)]
182    Persistence(#[from] PersistenceError),
183
184    #[error(transparent)]
185    RestorePlan(#[from] RestorePlanError),
186
187    #[error(transparent)]
188    RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
189
190    #[error(transparent)]
191    RestoreApplyJournal(#[from] RestoreApplyJournalError),
192}
193
194///
195/// RestorePlanOptions
196///
197
198#[derive(Clone, Debug, Eq, PartialEq)]
199pub struct RestorePlanOptions {
200    pub manifest: Option<PathBuf>,
201    pub backup_dir: Option<PathBuf>,
202    pub mapping: Option<PathBuf>,
203    pub out: Option<PathBuf>,
204    pub require_verified: bool,
205    pub require_restore_ready: bool,
206}
207
208impl RestorePlanOptions {
209    /// Parse restore planning options from CLI arguments.
210    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
211    where
212        I: IntoIterator<Item = OsString>,
213    {
214        let mut manifest = None;
215        let mut backup_dir = None;
216        let mut mapping = None;
217        let mut out = None;
218        let mut require_verified = false;
219        let mut require_restore_ready = false;
220
221        let mut args = args.into_iter();
222        while let Some(arg) = args.next() {
223            let arg = arg
224                .into_string()
225                .map_err(|_| RestoreCommandError::Usage(usage()))?;
226            match arg.as_str() {
227                "--manifest" => {
228                    manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
229                }
230                "--backup-dir" => {
231                    backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
232                }
233                "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
234                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
235                "--require-verified" => require_verified = true,
236                "--require-restore-ready" => require_restore_ready = true,
237                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
238                _ => return Err(RestoreCommandError::UnknownOption(arg)),
239            }
240        }
241
242        if manifest.is_some() && backup_dir.is_some() {
243            return Err(RestoreCommandError::ConflictingManifestSources);
244        }
245
246        if manifest.is_none() && backup_dir.is_none() {
247            return Err(RestoreCommandError::MissingOption(
248                "--manifest or --backup-dir",
249            ));
250        }
251
252        if require_verified && backup_dir.is_none() {
253            return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
254        }
255
256        Ok(Self {
257            manifest,
258            backup_dir,
259            mapping,
260            out,
261            require_verified,
262            require_restore_ready,
263        })
264    }
265}
266
267///
268/// RestoreStatusOptions
269///
270
271#[derive(Clone, Debug, Eq, PartialEq)]
272pub struct RestoreStatusOptions {
273    pub plan: PathBuf,
274    pub out: Option<PathBuf>,
275}
276
277impl RestoreStatusOptions {
278    /// Parse restore status options from CLI arguments.
279    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
280    where
281        I: IntoIterator<Item = OsString>,
282    {
283        let mut plan = None;
284        let mut out = None;
285
286        let mut args = args.into_iter();
287        while let Some(arg) = args.next() {
288            let arg = arg
289                .into_string()
290                .map_err(|_| RestoreCommandError::Usage(usage()))?;
291            match arg.as_str() {
292                "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
293                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
294                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
295                _ => return Err(RestoreCommandError::UnknownOption(arg)),
296            }
297        }
298
299        Ok(Self {
300            plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
301            out,
302        })
303    }
304}
305
306///
307/// RestoreApplyOptions
308///
309
310#[derive(Clone, Debug, Eq, PartialEq)]
311pub struct RestoreApplyOptions {
312    pub plan: PathBuf,
313    pub status: Option<PathBuf>,
314    pub backup_dir: Option<PathBuf>,
315    pub out: Option<PathBuf>,
316    pub journal_out: Option<PathBuf>,
317    pub dry_run: bool,
318}
319
320impl RestoreApplyOptions {
321    /// Parse restore apply options from CLI arguments.
322    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
323    where
324        I: IntoIterator<Item = OsString>,
325    {
326        let mut plan = None;
327        let mut status = None;
328        let mut backup_dir = None;
329        let mut out = None;
330        let mut journal_out = None;
331        let mut dry_run = false;
332
333        let mut args = args.into_iter();
334        while let Some(arg) = args.next() {
335            let arg = arg
336                .into_string()
337                .map_err(|_| RestoreCommandError::Usage(usage()))?;
338            match arg.as_str() {
339                "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
340                "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
341                "--backup-dir" => {
342                    backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
343                }
344                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
345                "--journal-out" => {
346                    journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
347                }
348                "--dry-run" => dry_run = true,
349                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
350                _ => return Err(RestoreCommandError::UnknownOption(arg)),
351            }
352        }
353
354        if !dry_run {
355            return Err(RestoreCommandError::ApplyRequiresDryRun);
356        }
357
358        Ok(Self {
359            plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
360            status,
361            backup_dir,
362            out,
363            journal_out,
364            dry_run,
365        })
366    }
367}
368
369///
370/// RestoreApplyStatusOptions
371///
372
373#[derive(Clone, Debug, Eq, PartialEq)]
374#[expect(
375    clippy::struct_excessive_bools,
376    reason = "CLI status options mirror independent fail-closed guard flags"
377)]
378pub struct RestoreApplyStatusOptions {
379    pub journal: PathBuf,
380    pub require_ready: bool,
381    pub require_no_pending: bool,
382    pub require_no_failed: bool,
383    pub require_complete: bool,
384    pub out: Option<PathBuf>,
385}
386
387impl RestoreApplyStatusOptions {
388    /// Parse restore apply-status options from CLI arguments.
389    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
390    where
391        I: IntoIterator<Item = OsString>,
392    {
393        let mut journal = None;
394        let mut require_ready = false;
395        let mut require_no_pending = false;
396        let mut require_no_failed = false;
397        let mut require_complete = false;
398        let mut out = None;
399
400        let mut args = args.into_iter();
401        while let Some(arg) = args.next() {
402            let arg = arg
403                .into_string()
404                .map_err(|_| RestoreCommandError::Usage(usage()))?;
405            match arg.as_str() {
406                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
407                "--require-ready" => require_ready = true,
408                "--require-no-pending" => require_no_pending = true,
409                "--require-no-failed" => require_no_failed = true,
410                "--require-complete" => require_complete = true,
411                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
412                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
413                _ => return Err(RestoreCommandError::UnknownOption(arg)),
414            }
415        }
416
417        Ok(Self {
418            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
419            require_ready,
420            require_no_pending,
421            require_no_failed,
422            require_complete,
423            out,
424        })
425    }
426}
427
428///
429/// RestoreApplyReportOptions
430///
431
432#[derive(Clone, Debug, Eq, PartialEq)]
433pub struct RestoreApplyReportOptions {
434    pub journal: PathBuf,
435    pub require_no_attention: bool,
436    pub out: Option<PathBuf>,
437}
438
439impl RestoreApplyReportOptions {
440    /// Parse restore apply-report options from CLI arguments.
441    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
442    where
443        I: IntoIterator<Item = OsString>,
444    {
445        let mut journal = None;
446        let mut require_no_attention = false;
447        let mut out = None;
448
449        let mut args = args.into_iter();
450        while let Some(arg) = args.next() {
451            let arg = arg
452                .into_string()
453                .map_err(|_| RestoreCommandError::Usage(usage()))?;
454            match arg.as_str() {
455                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
456                "--require-no-attention" => require_no_attention = true,
457                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
458                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
459                _ => return Err(RestoreCommandError::UnknownOption(arg)),
460            }
461        }
462
463        Ok(Self {
464            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
465            require_no_attention,
466            out,
467        })
468    }
469}
470
471///
472/// RestoreRunOptions
473///
474
475#[derive(Clone, Debug, Eq, PartialEq)]
476#[expect(
477    clippy::struct_excessive_bools,
478    reason = "CLI runner options mirror independent mode and fail-closed guard flags"
479)]
480pub struct RestoreRunOptions {
481    pub journal: PathBuf,
482    pub dfx: String,
483    pub network: Option<String>,
484    pub out: Option<PathBuf>,
485    pub dry_run: bool,
486    pub execute: bool,
487    pub unclaim_pending: bool,
488    pub max_steps: Option<usize>,
489    pub require_complete: bool,
490    pub require_no_attention: bool,
491    pub require_run_mode: Option<String>,
492    pub require_stopped_reason: Option<String>,
493    pub require_next_action: Option<String>,
494    pub require_executed_count: Option<usize>,
495}
496
497impl RestoreRunOptions {
498    /// Parse restore run options from CLI arguments.
499    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
500    where
501        I: IntoIterator<Item = OsString>,
502    {
503        let mut journal = None;
504        let mut dfx = "dfx".to_string();
505        let mut network = None;
506        let mut out = None;
507        let mut dry_run = false;
508        let mut execute = false;
509        let mut unclaim_pending = false;
510        let mut max_steps = None;
511        let mut require_complete = false;
512        let mut require_no_attention = false;
513        let mut require_run_mode = None;
514        let mut require_stopped_reason = None;
515        let mut require_next_action = None;
516        let mut require_executed_count = None;
517
518        let mut args = args.into_iter();
519        while let Some(arg) = args.next() {
520            let arg = arg
521                .into_string()
522                .map_err(|_| RestoreCommandError::Usage(usage()))?;
523            match arg.as_str() {
524                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
525                "--dfx" => dfx = next_value(&mut args, "--dfx")?,
526                "--network" => network = Some(next_value(&mut args, "--network")?),
527                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
528                "--dry-run" => dry_run = true,
529                "--execute" => execute = true,
530                "--unclaim-pending" => unclaim_pending = true,
531                "--max-steps" => {
532                    max_steps = Some(parse_sequence(next_value(&mut args, "--max-steps")?)?);
533                }
534                "--require-complete" => require_complete = true,
535                "--require-no-attention" => require_no_attention = true,
536                "--require-run-mode" => {
537                    require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
538                }
539                "--require-stopped-reason" => {
540                    require_stopped_reason =
541                        Some(next_value(&mut args, "--require-stopped-reason")?);
542                }
543                "--require-next-action" => {
544                    require_next_action = Some(next_value(&mut args, "--require-next-action")?);
545                }
546                "--require-executed-count" => {
547                    require_executed_count = Some(parse_sequence(next_value(
548                        &mut args,
549                        "--require-executed-count",
550                    )?)?);
551                }
552                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
553                _ => return Err(RestoreCommandError::UnknownOption(arg)),
554            }
555        }
556
557        let mode_count = [dry_run, execute, unclaim_pending]
558            .into_iter()
559            .filter(|enabled| *enabled)
560            .count();
561        if mode_count > 1 {
562            return Err(RestoreCommandError::RestoreRunConflictingModes);
563        }
564
565        if mode_count == 0 {
566            return Err(RestoreCommandError::RestoreRunRequiresMode);
567        }
568
569        Ok(Self {
570            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
571            dfx,
572            network,
573            out,
574            dry_run,
575            execute,
576            unclaim_pending,
577            max_steps,
578            require_complete,
579            require_no_attention,
580            require_run_mode,
581            require_stopped_reason,
582            require_next_action,
583            require_executed_count,
584        })
585    }
586}
587
588///
589/// RestoreRunResult
590///
591
592struct RestoreRunResult {
593    response: RestoreRunResponse,
594    error: Option<RestoreCommandError>,
595}
596
597impl RestoreRunResult {
598    // Build a successful runner response with no deferred error.
599    const fn ok(response: RestoreRunResponse) -> Self {
600        Self {
601            response,
602            error: None,
603        }
604    }
605}
606
607const RESTORE_RUN_MODE_DRY_RUN: &str = "dry-run";
608const RESTORE_RUN_MODE_EXECUTE: &str = "execute";
609const RESTORE_RUN_MODE_UNCLAIM_PENDING: &str = "unclaim-pending";
610
611const RESTORE_RUN_STOPPED_BLOCKED: &str = "blocked";
612const RESTORE_RUN_STOPPED_COMMAND_FAILED: &str = "command-failed";
613const RESTORE_RUN_STOPPED_COMPLETE: &str = "complete";
614const RESTORE_RUN_STOPPED_MAX_STEPS: &str = "max-steps-reached";
615const RESTORE_RUN_STOPPED_PENDING: &str = "pending";
616const RESTORE_RUN_STOPPED_PREVIEW: &str = "preview";
617const RESTORE_RUN_STOPPED_READY: &str = "ready";
618const RESTORE_RUN_STOPPED_RECOVERED_PENDING: &str = "recovered-pending";
619
620const RESTORE_RUN_ACTION_DONE: &str = "done";
621const RESTORE_RUN_ACTION_FIX_BLOCKED: &str = "fix-blocked-journal";
622const RESTORE_RUN_ACTION_INSPECT_FAILED: &str = "inspect-failed-operation";
623const RESTORE_RUN_ACTION_RERUN: &str = "rerun";
624const RESTORE_RUN_ACTION_UNCLAIM_PENDING: &str = "unclaim-pending";
625
626const RESTORE_RUN_EXECUTED_COMPLETED: &str = "completed";
627const RESTORE_RUN_EXECUTED_FAILED: &str = "failed";
628const RESTORE_RUN_COMMAND_EXIT_PREFIX: &str = "runner-command-exit";
629const RESTORE_RUN_RESPONSE_VERSION: u16 = 1;
630
631///
632/// RestoreRunResponse
633///
634
635#[derive(Clone, Debug, Serialize)]
636#[expect(
637    clippy::struct_excessive_bools,
638    reason = "Runner response exposes stable JSON status flags for operators and CI"
639)]
640pub struct RestoreRunResponse {
641    run_version: u16,
642    backup_id: String,
643    run_mode: &'static str,
644    dry_run: bool,
645    execute: bool,
646    unclaim_pending: bool,
647    stopped_reason: &'static str,
648    next_action: &'static str,
649    #[serde(skip_serializing_if = "Option::is_none")]
650    max_steps_reached: Option<bool>,
651    #[serde(default, skip_serializing_if = "Vec::is_empty")]
652    executed_operations: Vec<RestoreRunExecutedOperation>,
653    #[serde(skip_serializing_if = "Option::is_none")]
654    executed_operation_count: Option<usize>,
655    #[serde(skip_serializing_if = "Option::is_none")]
656    recovered_operation: Option<RestoreApplyJournalOperation>,
657    ready: bool,
658    complete: bool,
659    attention_required: bool,
660    outcome: RestoreApplyReportOutcome,
661    operation_count: usize,
662    pending_operations: usize,
663    ready_operations: usize,
664    blocked_operations: usize,
665    completed_operations: usize,
666    failed_operations: usize,
667    blocked_reasons: Vec<String>,
668    next_transition: Option<RestoreApplyReportOperation>,
669    #[serde(skip_serializing_if = "Option::is_none")]
670    operation_available: Option<bool>,
671    #[serde(skip_serializing_if = "Option::is_none")]
672    command_available: Option<bool>,
673    #[serde(skip_serializing_if = "Option::is_none")]
674    command: Option<RestoreApplyRunnerCommand>,
675}
676
677impl RestoreRunResponse {
678    // Build the shared native runner response fields from an apply journal report.
679    fn from_report(
680        backup_id: String,
681        report: RestoreApplyJournalReport,
682        mode: RestoreRunResponseMode,
683    ) -> Self {
684        Self {
685            run_version: RESTORE_RUN_RESPONSE_VERSION,
686            backup_id,
687            run_mode: mode.run_mode,
688            dry_run: mode.dry_run,
689            execute: mode.execute,
690            unclaim_pending: mode.unclaim_pending,
691            stopped_reason: mode.stopped_reason,
692            next_action: mode.next_action,
693            max_steps_reached: None,
694            executed_operations: Vec::new(),
695            executed_operation_count: None,
696            recovered_operation: None,
697            ready: report.ready,
698            complete: report.complete,
699            attention_required: report.attention_required,
700            outcome: report.outcome,
701            operation_count: report.operation_count,
702            pending_operations: report.pending_operations,
703            ready_operations: report.ready_operations,
704            blocked_operations: report.blocked_operations,
705            completed_operations: report.completed_operations,
706            failed_operations: report.failed_operations,
707            blocked_reasons: report.blocked_reasons,
708            next_transition: report.next_transition,
709            operation_available: None,
710            command_available: None,
711            command: None,
712        }
713    }
714}
715
716///
717/// RestoreRunExecutedOperation
718///
719
720#[derive(Clone, Debug, Serialize)]
721struct RestoreRunExecutedOperation {
722    sequence: usize,
723    operation: RestoreApplyOperationKind,
724    target_canister: String,
725    command: RestoreApplyRunnerCommand,
726    status: String,
727    state: &'static str,
728}
729
730impl RestoreRunExecutedOperation {
731    // Build a completed executed-operation summary row from a runner operation.
732    fn completed(
733        operation: RestoreApplyJournalOperation,
734        command: RestoreApplyRunnerCommand,
735        status: String,
736    ) -> Self {
737        Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_COMPLETED)
738    }
739
740    // Build a failed executed-operation summary row from a runner operation.
741    fn failed(
742        operation: RestoreApplyJournalOperation,
743        command: RestoreApplyRunnerCommand,
744        status: String,
745    ) -> Self {
746        Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_FAILED)
747    }
748
749    // Map a journal operation into the compact runner execution row.
750    fn from_operation(
751        operation: RestoreApplyJournalOperation,
752        command: RestoreApplyRunnerCommand,
753        status: String,
754        state: &'static str,
755    ) -> Self {
756        Self {
757            sequence: operation.sequence,
758            operation: operation.operation,
759            target_canister: operation.target_canister,
760            command,
761            status,
762            state,
763        }
764    }
765}
766
767///
768/// RestoreRunResponseMode
769///
770
771struct RestoreRunResponseMode {
772    run_mode: &'static str,
773    dry_run: bool,
774    execute: bool,
775    unclaim_pending: bool,
776    stopped_reason: &'static str,
777    next_action: &'static str,
778}
779
780impl RestoreRunResponseMode {
781    // Build a response mode from the stable JSON mode flags and action labels.
782    const fn new(
783        run_mode: &'static str,
784        dry_run: bool,
785        execute: bool,
786        unclaim_pending: bool,
787        stopped_reason: &'static str,
788        next_action: &'static str,
789    ) -> Self {
790        Self {
791            run_mode,
792            dry_run,
793            execute,
794            unclaim_pending,
795            stopped_reason,
796            next_action,
797        }
798    }
799
800    // Build a dry-run response mode with a computed stop reason and action.
801    const fn dry_run(stopped_reason: &'static str, next_action: &'static str) -> Self {
802        Self::new(
803            RESTORE_RUN_MODE_DRY_RUN,
804            true,
805            false,
806            false,
807            stopped_reason,
808            next_action,
809        )
810    }
811
812    // Build an execute response mode with a computed stop reason and action.
813    const fn execute(stopped_reason: &'static str, next_action: &'static str) -> Self {
814        Self::new(
815            RESTORE_RUN_MODE_EXECUTE,
816            false,
817            true,
818            false,
819            stopped_reason,
820            next_action,
821        )
822    }
823
824    // Build the pending-operation recovery response mode.
825    const fn unclaim_pending(next_action: &'static str) -> Self {
826        Self::new(
827            RESTORE_RUN_MODE_UNCLAIM_PENDING,
828            false,
829            false,
830            true,
831            RESTORE_RUN_STOPPED_RECOVERED_PENDING,
832            next_action,
833        )
834    }
835}
836
837///
838/// RestoreApplyNextOptions
839///
840
841#[derive(Clone, Debug, Eq, PartialEq)]
842pub struct RestoreApplyNextOptions {
843    pub journal: PathBuf,
844    pub out: Option<PathBuf>,
845}
846
847impl RestoreApplyNextOptions {
848    /// Parse restore apply-next options from CLI arguments.
849    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
850    where
851        I: IntoIterator<Item = OsString>,
852    {
853        let mut journal = None;
854        let mut out = None;
855
856        let mut args = args.into_iter();
857        while let Some(arg) = args.next() {
858            let arg = arg
859                .into_string()
860                .map_err(|_| RestoreCommandError::Usage(usage()))?;
861            match arg.as_str() {
862                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
863                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
864                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
865                _ => return Err(RestoreCommandError::UnknownOption(arg)),
866            }
867        }
868
869        Ok(Self {
870            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
871            out,
872        })
873    }
874}
875
876///
877/// RestoreApplyCommandOptions
878///
879
880#[derive(Clone, Debug, Eq, PartialEq)]
881pub struct RestoreApplyCommandOptions {
882    pub journal: PathBuf,
883    pub dfx: String,
884    pub network: Option<String>,
885    pub out: Option<PathBuf>,
886    pub require_command: bool,
887}
888
889impl RestoreApplyCommandOptions {
890    /// Parse restore apply-command options from CLI arguments.
891    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
892    where
893        I: IntoIterator<Item = OsString>,
894    {
895        let mut journal = None;
896        let mut dfx = "dfx".to_string();
897        let mut network = None;
898        let mut out = None;
899        let mut require_command = false;
900
901        let mut args = args.into_iter();
902        while let Some(arg) = args.next() {
903            let arg = arg
904                .into_string()
905                .map_err(|_| RestoreCommandError::Usage(usage()))?;
906            match arg.as_str() {
907                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
908                "--dfx" => dfx = next_value(&mut args, "--dfx")?,
909                "--network" => network = Some(next_value(&mut args, "--network")?),
910                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
911                "--require-command" => require_command = true,
912                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
913                _ => return Err(RestoreCommandError::UnknownOption(arg)),
914            }
915        }
916
917        Ok(Self {
918            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
919            dfx,
920            network,
921            out,
922            require_command,
923        })
924    }
925}
926
927///
928/// RestoreApplyClaimOptions
929///
930
931#[derive(Clone, Debug, Eq, PartialEq)]
932pub struct RestoreApplyClaimOptions {
933    pub journal: PathBuf,
934    pub sequence: Option<usize>,
935    pub updated_at: Option<String>,
936    pub out: Option<PathBuf>,
937}
938
939impl RestoreApplyClaimOptions {
940    /// Parse restore apply-claim options from CLI arguments.
941    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
942    where
943        I: IntoIterator<Item = OsString>,
944    {
945        let mut journal = None;
946        let mut sequence = None;
947        let mut updated_at = None;
948        let mut out = None;
949
950        let mut args = args.into_iter();
951        while let Some(arg) = args.next() {
952            let arg = arg
953                .into_string()
954                .map_err(|_| RestoreCommandError::Usage(usage()))?;
955            match arg.as_str() {
956                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
957                "--sequence" => {
958                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
959                }
960                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
961                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
962                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
963                _ => return Err(RestoreCommandError::UnknownOption(arg)),
964            }
965        }
966
967        Ok(Self {
968            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
969            sequence,
970            updated_at,
971            out,
972        })
973    }
974}
975
976///
977/// RestoreApplyMarkOptions
978///
979
980#[derive(Clone, Debug, Eq, PartialEq)]
981pub struct RestoreApplyUnclaimOptions {
982    pub journal: PathBuf,
983    pub sequence: Option<usize>,
984    pub updated_at: Option<String>,
985    pub out: Option<PathBuf>,
986}
987
988impl RestoreApplyUnclaimOptions {
989    /// Parse restore apply-unclaim options from CLI arguments.
990    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
991    where
992        I: IntoIterator<Item = OsString>,
993    {
994        let mut journal = None;
995        let mut sequence = None;
996        let mut updated_at = None;
997        let mut out = None;
998
999        let mut args = args.into_iter();
1000        while let Some(arg) = args.next() {
1001            let arg = arg
1002                .into_string()
1003                .map_err(|_| RestoreCommandError::Usage(usage()))?;
1004            match arg.as_str() {
1005                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1006                "--sequence" => {
1007                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1008                }
1009                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1010                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1011                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1012                _ => return Err(RestoreCommandError::UnknownOption(arg)),
1013            }
1014        }
1015
1016        Ok(Self {
1017            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1018            sequence,
1019            updated_at,
1020            out,
1021        })
1022    }
1023}
1024
1025///
1026/// RestoreApplyMarkOptions
1027///
1028
1029#[derive(Clone, Debug, Eq, PartialEq)]
1030pub struct RestoreApplyMarkOptions {
1031    pub journal: PathBuf,
1032    pub sequence: usize,
1033    pub state: RestoreApplyMarkState,
1034    pub reason: Option<String>,
1035    pub updated_at: Option<String>,
1036    pub out: Option<PathBuf>,
1037    pub require_pending: bool,
1038}
1039
1040impl RestoreApplyMarkOptions {
1041    /// Parse restore apply-mark options from CLI arguments.
1042    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1043    where
1044        I: IntoIterator<Item = OsString>,
1045    {
1046        let mut journal = None;
1047        let mut sequence = None;
1048        let mut state = None;
1049        let mut reason = None;
1050        let mut updated_at = None;
1051        let mut out = None;
1052        let mut require_pending = false;
1053
1054        let mut args = args.into_iter();
1055        while let Some(arg) = args.next() {
1056            let arg = arg
1057                .into_string()
1058                .map_err(|_| RestoreCommandError::Usage(usage()))?;
1059            match arg.as_str() {
1060                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1061                "--sequence" => {
1062                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1063                }
1064                "--state" => {
1065                    state = Some(RestoreApplyMarkState::parse(next_value(
1066                        &mut args, "--state",
1067                    )?)?);
1068                }
1069                "--reason" => reason = Some(next_value(&mut args, "--reason")?),
1070                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1071                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1072                "--require-pending" => require_pending = true,
1073                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1074                _ => return Err(RestoreCommandError::UnknownOption(arg)),
1075            }
1076        }
1077
1078        Ok(Self {
1079            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1080            sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
1081            state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
1082            reason,
1083            updated_at,
1084            out,
1085            require_pending,
1086        })
1087    }
1088}
1089
1090///
1091/// RestoreApplyMarkState
1092///
1093
1094#[derive(Clone, Debug, Eq, PartialEq)]
1095pub enum RestoreApplyMarkState {
1096    Completed,
1097    Failed,
1098}
1099
1100impl RestoreApplyMarkState {
1101    // Parse the restricted operation states accepted by apply-mark.
1102    fn parse(value: String) -> Result<Self, RestoreCommandError> {
1103        match value.as_str() {
1104            "completed" => Ok(Self::Completed),
1105            "failed" => Ok(Self::Failed),
1106            _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
1107        }
1108    }
1109}
1110
1111/// Run a restore subcommand.
1112pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
1113where
1114    I: IntoIterator<Item = OsString>,
1115{
1116    let mut args = args.into_iter();
1117    let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
1118        return Err(RestoreCommandError::Usage(usage()));
1119    };
1120
1121    match command.as_str() {
1122        "plan" => {
1123            let options = RestorePlanOptions::parse(args)?;
1124            let plan = plan_restore(&options)?;
1125            write_plan(&options, &plan)?;
1126            enforce_restore_plan_requirements(&options, &plan)?;
1127            Ok(())
1128        }
1129        "status" => {
1130            let options = RestoreStatusOptions::parse(args)?;
1131            let status = restore_status(&options)?;
1132            write_status(&options, &status)?;
1133            Ok(())
1134        }
1135        "apply" => {
1136            let options = RestoreApplyOptions::parse(args)?;
1137            let dry_run = restore_apply_dry_run(&options)?;
1138            write_apply_dry_run(&options, &dry_run)?;
1139            write_apply_journal_if_requested(&options, &dry_run)?;
1140            Ok(())
1141        }
1142        "apply-status" => {
1143            let options = RestoreApplyStatusOptions::parse(args)?;
1144            let status = restore_apply_status(&options)?;
1145            write_apply_status(&options, &status)?;
1146            enforce_apply_status_requirements(&options, &status)?;
1147            Ok(())
1148        }
1149        "apply-report" => {
1150            let options = RestoreApplyReportOptions::parse(args)?;
1151            let report = restore_apply_report(&options)?;
1152            write_apply_report(&options, &report)?;
1153            enforce_apply_report_requirements(&options, &report)?;
1154            Ok(())
1155        }
1156        "run" => {
1157            let options = RestoreRunOptions::parse(args)?;
1158            let run = if options.execute {
1159                restore_run_execute_result(&options)?
1160            } else if options.unclaim_pending {
1161                RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1162            } else {
1163                RestoreRunResult::ok(restore_run_dry_run(&options)?)
1164            };
1165            write_restore_run(&options, &run.response)?;
1166            if let Some(error) = run.error {
1167                return Err(error);
1168            }
1169            enforce_restore_run_requirements(&options, &run.response)?;
1170            Ok(())
1171        }
1172        "apply-next" => {
1173            let options = RestoreApplyNextOptions::parse(args)?;
1174            let next = restore_apply_next(&options)?;
1175            write_apply_next(&options, &next)?;
1176            Ok(())
1177        }
1178        "apply-command" => {
1179            let options = RestoreApplyCommandOptions::parse(args)?;
1180            let preview = restore_apply_command(&options)?;
1181            write_apply_command(&options, &preview)?;
1182            enforce_apply_command_requirements(&options, &preview)?;
1183            Ok(())
1184        }
1185        "apply-claim" => {
1186            let options = RestoreApplyClaimOptions::parse(args)?;
1187            let journal = restore_apply_claim(&options)?;
1188            write_apply_claim(&options, &journal)?;
1189            Ok(())
1190        }
1191        "apply-unclaim" => {
1192            let options = RestoreApplyUnclaimOptions::parse(args)?;
1193            let journal = restore_apply_unclaim(&options)?;
1194            write_apply_unclaim(&options, &journal)?;
1195            Ok(())
1196        }
1197        "apply-mark" => {
1198            let options = RestoreApplyMarkOptions::parse(args)?;
1199            let journal = restore_apply_mark(&options)?;
1200            write_apply_mark(&options, &journal)?;
1201            Ok(())
1202        }
1203        "help" | "--help" | "-h" => Err(RestoreCommandError::Usage(usage())),
1204        _ => Err(RestoreCommandError::UnknownOption(command)),
1205    }
1206}
1207
1208/// Build a no-mutation restore plan from a manifest and optional mapping.
1209pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1210    verify_backup_layout_if_required(options)?;
1211
1212    let manifest = read_manifest_source(options)?;
1213    let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1214
1215    RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1216}
1217
1218/// Build the initial no-mutation restore status from a restore plan.
1219pub fn restore_status(
1220    options: &RestoreStatusOptions,
1221) -> Result<RestoreStatus, RestoreCommandError> {
1222    let plan = read_plan(&options.plan)?;
1223    Ok(RestoreStatus::from_plan(&plan))
1224}
1225
1226/// Build a no-mutation restore apply dry-run from a restore plan.
1227pub fn restore_apply_dry_run(
1228    options: &RestoreApplyOptions,
1229) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1230    let plan = read_plan(&options.plan)?;
1231    let status = options.status.as_ref().map(read_status).transpose()?;
1232    if let Some(backup_dir) = &options.backup_dir {
1233        return RestoreApplyDryRun::try_from_plan_with_artifacts(
1234            &plan,
1235            status.as_ref(),
1236            backup_dir,
1237        )
1238        .map_err(RestoreCommandError::from);
1239    }
1240
1241    RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1242}
1243
1244/// Build a compact restore apply status from a journal file.
1245pub fn restore_apply_status(
1246    options: &RestoreApplyStatusOptions,
1247) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1248    let journal = read_apply_journal(&options.journal)?;
1249    Ok(journal.status())
1250}
1251
1252/// Build an operator-oriented restore apply report from a journal file.
1253pub fn restore_apply_report(
1254    options: &RestoreApplyReportOptions,
1255) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1256    let journal = read_apply_journal(&options.journal)?;
1257    Ok(journal.report())
1258}
1259
1260/// Build a no-mutation native restore runner preview from a journal file.
1261pub fn restore_run_dry_run(
1262    options: &RestoreRunOptions,
1263) -> Result<RestoreRunResponse, RestoreCommandError> {
1264    let journal = read_apply_journal(&options.journal)?;
1265    let report = journal.report();
1266    let preview = journal.next_command_preview_with_config(&restore_run_command_config(options));
1267    let stopped_reason = restore_run_stopped_reason(&report, false, false);
1268    let next_action = restore_run_next_action(&report, false);
1269
1270    let mut response = RestoreRunResponse::from_report(
1271        journal.backup_id,
1272        report,
1273        RestoreRunResponseMode::dry_run(stopped_reason, next_action),
1274    );
1275    response.operation_available = Some(preview.operation_available);
1276    response.command_available = Some(preview.command_available);
1277    response.command = preview.command;
1278    Ok(response)
1279}
1280
1281/// Recover an interrupted restore runner by unclaiming the pending operation.
1282pub fn restore_run_unclaim_pending(
1283    options: &RestoreRunOptions,
1284) -> Result<RestoreRunResponse, RestoreCommandError> {
1285    let mut journal = read_apply_journal(&options.journal)?;
1286    let recovered_operation = journal
1287        .next_transition_operation()
1288        .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1289        .cloned()
1290        .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1291
1292    journal.mark_next_operation_ready_at(Some(timestamp_placeholder()))?;
1293    write_apply_journal_file(&options.journal, &journal)?;
1294
1295    let report = journal.report();
1296    let next_action = restore_run_next_action(&report, true);
1297    let mut response = RestoreRunResponse::from_report(
1298        journal.backup_id,
1299        report,
1300        RestoreRunResponseMode::unclaim_pending(next_action),
1301    );
1302    response.recovered_operation = Some(recovered_operation);
1303    Ok(response)
1304}
1305
1306/// Execute ready restore apply journal operations through generated runner commands.
1307pub fn restore_run_execute(
1308    options: &RestoreRunOptions,
1309) -> Result<RestoreRunResponse, RestoreCommandError> {
1310    let run = restore_run_execute_result(options)?;
1311    if let Some(error) = run.error {
1312        return Err(error);
1313    }
1314
1315    Ok(run.response)
1316}
1317
1318// Execute ready restore apply operations and retain any deferred runner error.
1319fn restore_run_execute_result(
1320    options: &RestoreRunOptions,
1321) -> Result<RestoreRunResult, RestoreCommandError> {
1322    let mut journal = read_apply_journal(&options.journal)?;
1323    let mut executed_operations = Vec::new();
1324    let config = restore_run_command_config(options);
1325
1326    loop {
1327        let report = journal.report();
1328        let max_steps_reached =
1329            restore_run_max_steps_reached(options, executed_operations.len(), &report);
1330        if report.complete || max_steps_reached {
1331            return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1332                &journal,
1333                executed_operations,
1334                max_steps_reached,
1335            )));
1336        }
1337
1338        enforce_restore_run_executable(&journal, &report)?;
1339        let preview = journal.next_command_preview_with_config(&config);
1340        enforce_restore_run_command_available(&preview)?;
1341
1342        let operation = preview
1343            .operation
1344            .clone()
1345            .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1346        let command = preview
1347            .command
1348            .clone()
1349            .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1350        let sequence = operation.sequence;
1351
1352        enforce_apply_claim_sequence(sequence, &journal)?;
1353        journal.mark_operation_pending_at(sequence, Some(timestamp_placeholder()))?;
1354        write_apply_journal_file(&options.journal, &journal)?;
1355
1356        let status = Command::new(&command.program)
1357            .args(&command.args)
1358            .status()?;
1359        let status_label = exit_status_label(status);
1360        if status.success() {
1361            journal.mark_operation_completed_at(sequence, Some(timestamp_placeholder()))?;
1362            write_apply_journal_file(&options.journal, &journal)?;
1363            executed_operations.push(RestoreRunExecutedOperation::completed(
1364                operation,
1365                command,
1366                status_label,
1367            ));
1368            continue;
1369        }
1370
1371        journal.mark_operation_failed_at(
1372            sequence,
1373            format!("{RESTORE_RUN_COMMAND_EXIT_PREFIX}-{status_label}"),
1374            Some(timestamp_placeholder()),
1375        )?;
1376        write_apply_journal_file(&options.journal, &journal)?;
1377        executed_operations.push(RestoreRunExecutedOperation::failed(
1378            operation,
1379            command,
1380            status_label.clone(),
1381        ));
1382        let response = restore_run_execute_summary(&journal, executed_operations, false);
1383        return Ok(RestoreRunResult {
1384            response,
1385            error: Some(RestoreCommandError::RestoreRunCommandFailed {
1386                sequence,
1387                status: status_label,
1388            }),
1389        });
1390    }
1391}
1392
1393// Build the shared runner command-preview configuration from CLI options.
1394fn restore_run_command_config(options: &RestoreRunOptions) -> RestoreApplyCommandConfig {
1395    restore_command_config(&options.dfx, options.network.as_deref())
1396}
1397
1398// Build the shared apply-command preview configuration from CLI options.
1399fn restore_apply_command_config(options: &RestoreApplyCommandOptions) -> RestoreApplyCommandConfig {
1400    restore_command_config(&options.dfx, options.network.as_deref())
1401}
1402
1403// Build command-preview configuration from common dfx/network inputs.
1404fn restore_command_config(program: &str, network: Option<&str>) -> RestoreApplyCommandConfig {
1405    RestoreApplyCommandConfig {
1406        program: program.to_string(),
1407        network: network.map(str::to_string),
1408    }
1409}
1410
1411// Check whether execute mode has reached its requested operation batch size.
1412fn restore_run_max_steps_reached(
1413    options: &RestoreRunOptions,
1414    executed_operation_count: usize,
1415    report: &RestoreApplyJournalReport,
1416) -> bool {
1417    options.max_steps == Some(executed_operation_count) && !report.complete
1418}
1419
1420// Build the final native runner execution summary.
1421fn restore_run_execute_summary(
1422    journal: &RestoreApplyJournal,
1423    executed_operations: Vec<RestoreRunExecutedOperation>,
1424    max_steps_reached: bool,
1425) -> RestoreRunResponse {
1426    let report = journal.report();
1427    let executed_operation_count = executed_operations.len();
1428    let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
1429    let next_action = restore_run_next_action(&report, false);
1430
1431    let mut response = RestoreRunResponse::from_report(
1432        journal.backup_id.clone(),
1433        report,
1434        RestoreRunResponseMode::execute(stopped_reason, next_action),
1435    );
1436    response.max_steps_reached = Some(max_steps_reached);
1437    response.executed_operation_count = Some(executed_operation_count);
1438    response.executed_operations = executed_operations;
1439    response
1440}
1441
1442// Classify why the native runner stopped for operator summaries.
1443const fn restore_run_stopped_reason(
1444    report: &RestoreApplyJournalReport,
1445    max_steps_reached: bool,
1446    executed: bool,
1447) -> &'static str {
1448    if report.complete {
1449        return RESTORE_RUN_STOPPED_COMPLETE;
1450    }
1451    if report.failed_operations > 0 {
1452        return RESTORE_RUN_STOPPED_COMMAND_FAILED;
1453    }
1454    if report.pending_operations > 0 {
1455        return RESTORE_RUN_STOPPED_PENDING;
1456    }
1457    if !report.ready || report.blocked_operations > 0 {
1458        return RESTORE_RUN_STOPPED_BLOCKED;
1459    }
1460    if max_steps_reached {
1461        return RESTORE_RUN_STOPPED_MAX_STEPS;
1462    }
1463    if executed {
1464        return RESTORE_RUN_STOPPED_READY;
1465    }
1466    RESTORE_RUN_STOPPED_PREVIEW
1467}
1468
1469// Recommend the next operator action for the native runner summary.
1470const fn restore_run_next_action(
1471    report: &RestoreApplyJournalReport,
1472    recovered_pending: bool,
1473) -> &'static str {
1474    if report.complete {
1475        return RESTORE_RUN_ACTION_DONE;
1476    }
1477    if report.failed_operations > 0 {
1478        return RESTORE_RUN_ACTION_INSPECT_FAILED;
1479    }
1480    if report.pending_operations > 0 {
1481        return RESTORE_RUN_ACTION_UNCLAIM_PENDING;
1482    }
1483    if !report.ready || report.blocked_operations > 0 {
1484        return RESTORE_RUN_ACTION_FIX_BLOCKED;
1485    }
1486    if recovered_pending {
1487        return RESTORE_RUN_ACTION_RERUN;
1488    }
1489    RESTORE_RUN_ACTION_RERUN
1490}
1491
1492// Ensure the journal can be advanced by the native restore runner.
1493fn enforce_restore_run_executable(
1494    journal: &RestoreApplyJournal,
1495    report: &RestoreApplyJournalReport,
1496) -> Result<(), RestoreCommandError> {
1497    if report.pending_operations > 0 {
1498        return Err(RestoreCommandError::RestoreApplyPending {
1499            backup_id: report.backup_id.clone(),
1500            pending_operations: report.pending_operations,
1501            next_transition_sequence: report
1502                .next_transition
1503                .as_ref()
1504                .map(|operation| operation.sequence),
1505        });
1506    }
1507
1508    if report.failed_operations > 0 {
1509        return Err(RestoreCommandError::RestoreApplyFailed {
1510            backup_id: report.backup_id.clone(),
1511            failed_operations: report.failed_operations,
1512        });
1513    }
1514
1515    if report.ready {
1516        return Ok(());
1517    }
1518
1519    Err(RestoreCommandError::RestoreApplyNotReady {
1520        backup_id: journal.backup_id.clone(),
1521        reasons: report.blocked_reasons.clone(),
1522    })
1523}
1524
1525// Convert an unavailable native runner command into the shared fail-closed error.
1526fn enforce_restore_run_command_available(
1527    preview: &RestoreApplyCommandPreview,
1528) -> Result<(), RestoreCommandError> {
1529    if preview.command_available {
1530        return Ok(());
1531    }
1532
1533    Err(restore_command_unavailable_error(preview))
1534}
1535
1536// Build a shared command-unavailable error from a preview.
1537fn restore_command_unavailable_error(preview: &RestoreApplyCommandPreview) -> RestoreCommandError {
1538    RestoreCommandError::RestoreApplyCommandUnavailable {
1539        backup_id: preview.backup_id.clone(),
1540        operation_available: preview.operation_available,
1541        complete: preview.complete,
1542        blocked_reasons: preview.blocked_reasons.clone(),
1543    }
1544}
1545
1546// Render process exit status without relying on platform-specific internals.
1547fn exit_status_label(status: std::process::ExitStatus) -> String {
1548    status
1549        .code()
1550        .map_or_else(|| "signal".to_string(), |code| code.to_string())
1551}
1552
1553// Enforce caller-requested native runner requirements after output is emitted.
1554fn enforce_restore_run_requirements(
1555    options: &RestoreRunOptions,
1556    run: &RestoreRunResponse,
1557) -> Result<(), RestoreCommandError> {
1558    if options.require_complete && !run.complete {
1559        return Err(RestoreCommandError::RestoreApplyIncomplete {
1560            backup_id: run.backup_id.clone(),
1561            completed_operations: run.completed_operations,
1562            operation_count: run.operation_count,
1563        });
1564    }
1565
1566    if options.require_no_attention && run.attention_required {
1567        return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1568            backup_id: run.backup_id.clone(),
1569            outcome: run.outcome.clone(),
1570        });
1571    }
1572
1573    if let Some(expected) = &options.require_run_mode
1574        && run.run_mode != expected
1575    {
1576        return Err(RestoreCommandError::RestoreRunModeMismatch {
1577            backup_id: run.backup_id.clone(),
1578            expected: expected.clone(),
1579            actual: run.run_mode.to_string(),
1580        });
1581    }
1582
1583    if let Some(expected) = &options.require_stopped_reason
1584        && run.stopped_reason != expected
1585    {
1586        return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
1587            backup_id: run.backup_id.clone(),
1588            expected: expected.clone(),
1589            actual: run.stopped_reason.to_string(),
1590        });
1591    }
1592
1593    if let Some(expected) = &options.require_next_action
1594        && run.next_action != expected
1595    {
1596        return Err(RestoreCommandError::RestoreRunNextActionMismatch {
1597            backup_id: run.backup_id.clone(),
1598            expected: expected.clone(),
1599            actual: run.next_action.to_string(),
1600        });
1601    }
1602
1603    if let Some(expected) = options.require_executed_count {
1604        let actual = run.executed_operation_count.unwrap_or(0);
1605        if actual != expected {
1606            return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
1607                backup_id: run.backup_id.clone(),
1608                expected,
1609                actual,
1610            });
1611        }
1612    }
1613
1614    Ok(())
1615}
1616
1617// Enforce caller-requested apply report requirements after report output is emitted.
1618fn enforce_apply_report_requirements(
1619    options: &RestoreApplyReportOptions,
1620    report: &RestoreApplyJournalReport,
1621) -> Result<(), RestoreCommandError> {
1622    if !options.require_no_attention || !report.attention_required {
1623        return Ok(());
1624    }
1625
1626    Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1627        backup_id: report.backup_id.clone(),
1628        outcome: report.outcome.clone(),
1629    })
1630}
1631
1632// Enforce caller-requested apply journal requirements after status is emitted.
1633fn enforce_apply_status_requirements(
1634    options: &RestoreApplyStatusOptions,
1635    status: &RestoreApplyJournalStatus,
1636) -> Result<(), RestoreCommandError> {
1637    if options.require_ready && !status.ready {
1638        return Err(RestoreCommandError::RestoreApplyNotReady {
1639            backup_id: status.backup_id.clone(),
1640            reasons: status.blocked_reasons.clone(),
1641        });
1642    }
1643
1644    if options.require_no_pending && status.pending_operations > 0 {
1645        return Err(RestoreCommandError::RestoreApplyPending {
1646            backup_id: status.backup_id.clone(),
1647            pending_operations: status.pending_operations,
1648            next_transition_sequence: status.next_transition_sequence,
1649        });
1650    }
1651
1652    if options.require_no_failed && status.failed_operations > 0 {
1653        return Err(RestoreCommandError::RestoreApplyFailed {
1654            backup_id: status.backup_id.clone(),
1655            failed_operations: status.failed_operations,
1656        });
1657    }
1658
1659    if options.require_complete && !status.complete {
1660        return Err(RestoreCommandError::RestoreApplyIncomplete {
1661            backup_id: status.backup_id.clone(),
1662            completed_operations: status.completed_operations,
1663            operation_count: status.operation_count,
1664        });
1665    }
1666
1667    Ok(())
1668}
1669
1670/// Build the next restore apply operation response from a journal file.
1671pub fn restore_apply_next(
1672    options: &RestoreApplyNextOptions,
1673) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
1674    let journal = read_apply_journal(&options.journal)?;
1675    Ok(journal.next_operation())
1676}
1677
1678/// Build the next restore apply command preview from a journal file.
1679pub fn restore_apply_command(
1680    options: &RestoreApplyCommandOptions,
1681) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
1682    let journal = read_apply_journal(&options.journal)?;
1683    Ok(journal.next_command_preview_with_config(&restore_apply_command_config(options)))
1684}
1685
1686// Enforce caller-requested command preview requirements after preview output is emitted.
1687fn enforce_apply_command_requirements(
1688    options: &RestoreApplyCommandOptions,
1689    preview: &RestoreApplyCommandPreview,
1690) -> Result<(), RestoreCommandError> {
1691    if !options.require_command || preview.command_available {
1692        return Ok(());
1693    }
1694
1695    Err(restore_command_unavailable_error(preview))
1696}
1697
1698/// Mark the next restore apply journal operation pending.
1699pub fn restore_apply_claim(
1700    options: &RestoreApplyClaimOptions,
1701) -> Result<RestoreApplyJournal, RestoreCommandError> {
1702    let mut journal = read_apply_journal(&options.journal)?;
1703    let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
1704
1705    if let Some(sequence) = options.sequence {
1706        enforce_apply_claim_sequence(sequence, &journal)?;
1707        journal.mark_operation_pending_at(sequence, updated_at)?;
1708        return Ok(journal);
1709    }
1710
1711    journal.mark_next_operation_pending_at(updated_at)?;
1712    Ok(journal)
1713}
1714
1715// Ensure a runner claim still matches the operation it previewed.
1716fn enforce_apply_claim_sequence(
1717    expected: usize,
1718    journal: &RestoreApplyJournal,
1719) -> Result<(), RestoreCommandError> {
1720    let actual = journal
1721        .next_transition_operation()
1722        .map(|operation| operation.sequence);
1723
1724    if actual == Some(expected) {
1725        return Ok(());
1726    }
1727
1728    Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
1729}
1730
1731/// Mark the current pending restore apply journal operation ready again.
1732pub fn restore_apply_unclaim(
1733    options: &RestoreApplyUnclaimOptions,
1734) -> Result<RestoreApplyJournal, RestoreCommandError> {
1735    let mut journal = read_apply_journal(&options.journal)?;
1736    if let Some(sequence) = options.sequence {
1737        enforce_apply_unclaim_sequence(sequence, &journal)?;
1738    }
1739
1740    journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
1741    Ok(journal)
1742}
1743
1744// Ensure a runner unclaim still matches the pending operation it recovered.
1745fn enforce_apply_unclaim_sequence(
1746    expected: usize,
1747    journal: &RestoreApplyJournal,
1748) -> Result<(), RestoreCommandError> {
1749    let actual = journal
1750        .next_transition_operation()
1751        .map(|operation| operation.sequence);
1752
1753    if actual == Some(expected) {
1754        return Ok(());
1755    }
1756
1757    Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
1758}
1759
1760/// Mark one restore apply journal operation completed or failed.
1761pub fn restore_apply_mark(
1762    options: &RestoreApplyMarkOptions,
1763) -> Result<RestoreApplyJournal, RestoreCommandError> {
1764    let mut journal = read_apply_journal(&options.journal)?;
1765    enforce_apply_mark_pending_requirement(options, &journal)?;
1766
1767    match options.state {
1768        RestoreApplyMarkState::Completed => {
1769            journal.mark_operation_completed_at(
1770                options.sequence,
1771                Some(state_updated_at(options.updated_at.as_ref())),
1772            )?;
1773        }
1774        RestoreApplyMarkState::Failed => {
1775            let reason =
1776                options
1777                    .reason
1778                    .clone()
1779                    .ok_or(RestoreApplyJournalError::FailureReasonRequired(
1780                        options.sequence,
1781                    ))?;
1782            journal.mark_operation_failed_at(
1783                options.sequence,
1784                reason,
1785                Some(state_updated_at(options.updated_at.as_ref())),
1786            )?;
1787        }
1788    }
1789
1790    Ok(journal)
1791}
1792
1793// Enforce that apply-mark only records an already claimed operation when requested.
1794fn enforce_apply_mark_pending_requirement(
1795    options: &RestoreApplyMarkOptions,
1796    journal: &RestoreApplyJournal,
1797) -> Result<(), RestoreCommandError> {
1798    if !options.require_pending {
1799        return Ok(());
1800    }
1801
1802    let state = journal
1803        .operations
1804        .iter()
1805        .find(|operation| operation.sequence == options.sequence)
1806        .map(|operation| operation.state.clone())
1807        .ok_or(RestoreApplyJournalError::OperationNotFound(
1808            options.sequence,
1809        ))?;
1810
1811    if state == RestoreApplyOperationState::Pending {
1812        return Ok(());
1813    }
1814
1815    Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
1816        sequence: options.sequence,
1817        state,
1818    })
1819}
1820
1821// Enforce caller-requested restore plan requirements after the plan is emitted.
1822fn enforce_restore_plan_requirements(
1823    options: &RestorePlanOptions,
1824    plan: &RestorePlan,
1825) -> Result<(), RestoreCommandError> {
1826    if !options.require_restore_ready || plan.readiness_summary.ready {
1827        return Ok(());
1828    }
1829
1830    Err(RestoreCommandError::RestoreNotReady {
1831        backup_id: plan.backup_id.clone(),
1832        reasons: plan.readiness_summary.reasons.clone(),
1833    })
1834}
1835
1836// Verify backup layout integrity before restore planning when requested.
1837fn verify_backup_layout_if_required(
1838    options: &RestorePlanOptions,
1839) -> Result<(), RestoreCommandError> {
1840    if !options.require_verified {
1841        return Ok(());
1842    }
1843
1844    let Some(dir) = &options.backup_dir else {
1845        return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
1846    };
1847
1848    BackupLayout::new(dir.clone()).verify_integrity()?;
1849    Ok(())
1850}
1851
1852// Read the manifest from a direct path or canonical backup layout.
1853fn read_manifest_source(
1854    options: &RestorePlanOptions,
1855) -> Result<FleetBackupManifest, RestoreCommandError> {
1856    if let Some(path) = &options.manifest {
1857        return read_manifest(path);
1858    }
1859
1860    let Some(dir) = &options.backup_dir else {
1861        return Err(RestoreCommandError::MissingOption(
1862            "--manifest or --backup-dir",
1863        ));
1864    };
1865
1866    BackupLayout::new(dir.clone())
1867        .read_manifest()
1868        .map_err(RestoreCommandError::from)
1869}
1870
1871// Read and decode a fleet backup manifest from disk.
1872fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
1873    let data = fs::read_to_string(path)?;
1874    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1875}
1876
1877// Read and decode an optional source-to-target restore mapping from disk.
1878fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
1879    let data = fs::read_to_string(path)?;
1880    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1881}
1882
1883// Read and decode a restore plan from disk.
1884fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
1885    let data = fs::read_to_string(path)?;
1886    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1887}
1888
1889// Read and decode a restore status from disk.
1890fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
1891    let data = fs::read_to_string(path)?;
1892    serde_json::from_str(&data).map_err(RestoreCommandError::from)
1893}
1894
1895// Read and decode a restore apply journal from disk.
1896fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
1897    let data = fs::read_to_string(path)?;
1898    let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
1899    journal.validate()?;
1900    Ok(journal)
1901}
1902
1903// Parse a restore apply journal operation sequence value.
1904fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
1905    value
1906        .parse::<usize>()
1907        .map_err(|_| RestoreCommandError::InvalidSequence)
1908}
1909
1910// Return the caller-supplied journal update marker or the current placeholder.
1911fn state_updated_at(updated_at: Option<&String>) -> String {
1912    updated_at.cloned().unwrap_or_else(timestamp_placeholder)
1913}
1914
1915// Return a placeholder timestamp until the CLI owns a clock abstraction.
1916fn timestamp_placeholder() -> String {
1917    "unknown".to_string()
1918}
1919
1920// Write the computed plan to stdout or a requested output file.
1921fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
1922    if let Some(path) = &options.out {
1923        let data = serde_json::to_vec_pretty(plan)?;
1924        fs::write(path, data)?;
1925        return Ok(());
1926    }
1927
1928    let stdout = io::stdout();
1929    let mut handle = stdout.lock();
1930    serde_json::to_writer_pretty(&mut handle, plan)?;
1931    writeln!(handle)?;
1932    Ok(())
1933}
1934
1935// Write the computed status to stdout or a requested output file.
1936fn write_status(
1937    options: &RestoreStatusOptions,
1938    status: &RestoreStatus,
1939) -> Result<(), RestoreCommandError> {
1940    if let Some(path) = &options.out {
1941        let data = serde_json::to_vec_pretty(status)?;
1942        fs::write(path, data)?;
1943        return Ok(());
1944    }
1945
1946    let stdout = io::stdout();
1947    let mut handle = stdout.lock();
1948    serde_json::to_writer_pretty(&mut handle, status)?;
1949    writeln!(handle)?;
1950    Ok(())
1951}
1952
1953// Write the computed apply dry-run to stdout or a requested output file.
1954fn write_apply_dry_run(
1955    options: &RestoreApplyOptions,
1956    dry_run: &RestoreApplyDryRun,
1957) -> Result<(), RestoreCommandError> {
1958    if let Some(path) = &options.out {
1959        let data = serde_json::to_vec_pretty(dry_run)?;
1960        fs::write(path, data)?;
1961        return Ok(());
1962    }
1963
1964    let stdout = io::stdout();
1965    let mut handle = stdout.lock();
1966    serde_json::to_writer_pretty(&mut handle, dry_run)?;
1967    writeln!(handle)?;
1968    Ok(())
1969}
1970
1971// Write the initial apply journal when the caller requests one.
1972fn write_apply_journal_if_requested(
1973    options: &RestoreApplyOptions,
1974    dry_run: &RestoreApplyDryRun,
1975) -> Result<(), RestoreCommandError> {
1976    let Some(path) = &options.journal_out else {
1977        return Ok(());
1978    };
1979
1980    let journal = RestoreApplyJournal::from_dry_run(dry_run);
1981    let data = serde_json::to_vec_pretty(&journal)?;
1982    fs::write(path, data)?;
1983    Ok(())
1984}
1985
1986// Write the computed apply journal status to stdout or a requested output file.
1987fn write_apply_status(
1988    options: &RestoreApplyStatusOptions,
1989    status: &RestoreApplyJournalStatus,
1990) -> Result<(), RestoreCommandError> {
1991    if let Some(path) = &options.out {
1992        let data = serde_json::to_vec_pretty(status)?;
1993        fs::write(path, data)?;
1994        return Ok(());
1995    }
1996
1997    let stdout = io::stdout();
1998    let mut handle = stdout.lock();
1999    serde_json::to_writer_pretty(&mut handle, status)?;
2000    writeln!(handle)?;
2001    Ok(())
2002}
2003
2004// Write the computed apply journal report to stdout or a requested output file.
2005fn write_apply_report(
2006    options: &RestoreApplyReportOptions,
2007    report: &RestoreApplyJournalReport,
2008) -> Result<(), RestoreCommandError> {
2009    if let Some(path) = &options.out {
2010        let data = serde_json::to_vec_pretty(report)?;
2011        fs::write(path, data)?;
2012        return Ok(());
2013    }
2014
2015    let stdout = io::stdout();
2016    let mut handle = stdout.lock();
2017    serde_json::to_writer_pretty(&mut handle, report)?;
2018    writeln!(handle)?;
2019    Ok(())
2020}
2021
2022// Write the restore runner response to stdout or a requested output file.
2023fn write_restore_run(
2024    options: &RestoreRunOptions,
2025    run: &RestoreRunResponse,
2026) -> Result<(), RestoreCommandError> {
2027    if let Some(path) = &options.out {
2028        let data = serde_json::to_vec_pretty(run)?;
2029        fs::write(path, data)?;
2030        return Ok(());
2031    }
2032
2033    let stdout = io::stdout();
2034    let mut handle = stdout.lock();
2035    serde_json::to_writer_pretty(&mut handle, run)?;
2036    writeln!(handle)?;
2037    Ok(())
2038}
2039
2040// Persist the restore apply journal to its canonical runner path.
2041fn write_apply_journal_file(
2042    path: &PathBuf,
2043    journal: &RestoreApplyJournal,
2044) -> Result<(), RestoreCommandError> {
2045    let data = serde_json::to_vec_pretty(journal)?;
2046    fs::write(path, data)?;
2047    Ok(())
2048}
2049
2050// Write the computed apply next-operation response to stdout or a requested output file.
2051fn write_apply_next(
2052    options: &RestoreApplyNextOptions,
2053    next: &RestoreApplyNextOperation,
2054) -> Result<(), RestoreCommandError> {
2055    if let Some(path) = &options.out {
2056        let data = serde_json::to_vec_pretty(next)?;
2057        fs::write(path, data)?;
2058        return Ok(());
2059    }
2060
2061    let stdout = io::stdout();
2062    let mut handle = stdout.lock();
2063    serde_json::to_writer_pretty(&mut handle, next)?;
2064    writeln!(handle)?;
2065    Ok(())
2066}
2067
2068// Write the computed apply command preview to stdout or a requested output file.
2069fn write_apply_command(
2070    options: &RestoreApplyCommandOptions,
2071    preview: &RestoreApplyCommandPreview,
2072) -> Result<(), RestoreCommandError> {
2073    if let Some(path) = &options.out {
2074        let data = serde_json::to_vec_pretty(preview)?;
2075        fs::write(path, data)?;
2076        return Ok(());
2077    }
2078
2079    let stdout = io::stdout();
2080    let mut handle = stdout.lock();
2081    serde_json::to_writer_pretty(&mut handle, preview)?;
2082    writeln!(handle)?;
2083    Ok(())
2084}
2085
2086// Write the claimed apply journal to stdout or a requested output file.
2087fn write_apply_claim(
2088    options: &RestoreApplyClaimOptions,
2089    journal: &RestoreApplyJournal,
2090) -> Result<(), RestoreCommandError> {
2091    if let Some(path) = &options.out {
2092        let data = serde_json::to_vec_pretty(journal)?;
2093        fs::write(path, data)?;
2094        return Ok(());
2095    }
2096
2097    let stdout = io::stdout();
2098    let mut handle = stdout.lock();
2099    serde_json::to_writer_pretty(&mut handle, journal)?;
2100    writeln!(handle)?;
2101    Ok(())
2102}
2103
2104// Write the unclaimed apply journal to stdout or a requested output file.
2105fn write_apply_unclaim(
2106    options: &RestoreApplyUnclaimOptions,
2107    journal: &RestoreApplyJournal,
2108) -> Result<(), RestoreCommandError> {
2109    if let Some(path) = &options.out {
2110        let data = serde_json::to_vec_pretty(journal)?;
2111        fs::write(path, data)?;
2112        return Ok(());
2113    }
2114
2115    let stdout = io::stdout();
2116    let mut handle = stdout.lock();
2117    serde_json::to_writer_pretty(&mut handle, journal)?;
2118    writeln!(handle)?;
2119    Ok(())
2120}
2121
2122// Write the updated apply journal to stdout or a requested output file.
2123fn write_apply_mark(
2124    options: &RestoreApplyMarkOptions,
2125    journal: &RestoreApplyJournal,
2126) -> Result<(), RestoreCommandError> {
2127    if let Some(path) = &options.out {
2128        let data = serde_json::to_vec_pretty(journal)?;
2129        fs::write(path, data)?;
2130        return Ok(());
2131    }
2132
2133    let stdout = io::stdout();
2134    let mut handle = stdout.lock();
2135    serde_json::to_writer_pretty(&mut handle, journal)?;
2136    writeln!(handle)?;
2137    Ok(())
2138}
2139
2140// Read the next required option value.
2141fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
2142where
2143    I: Iterator<Item = OsString>,
2144{
2145    args.next()
2146        .and_then(|value| value.into_string().ok())
2147        .ok_or(RestoreCommandError::MissingValue(option))
2148}
2149
2150// Return restore command usage text.
2151const fn usage() -> &'static str {
2152    "usage: canic restore plan (--manifest <file> | --backup-dir <dir>) [--mapping <file>] [--out <file>] [--require-verified] [--require-restore-ready]\n       canic restore status --plan <file> [--out <file>]\n       canic restore apply --plan <file> [--status <file>] [--backup-dir <dir>] --dry-run [--out <file>] [--journal-out <file>]\n       canic restore apply-status --journal <file> [--out <file>] [--require-ready] [--require-no-pending] [--require-no-failed] [--require-complete]\n       canic restore apply-report --journal <file> [--out <file>] [--require-no-attention]\n       canic restore run --journal <file> (--dry-run | --execute | --unclaim-pending) [--dfx <path>] [--network <name>] [--max-steps <n>] [--out <file>] [--require-complete] [--require-no-attention] [--require-run-mode <text>] [--require-stopped-reason <text>] [--require-next-action <text>] [--require-executed-count <n>]\n       canic restore apply-next --journal <file> [--out <file>]\n       canic restore apply-command --journal <file> [--dfx <path>] [--network <name>] [--out <file>] [--require-command]\n       canic restore apply-claim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n       canic restore apply-unclaim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n       canic restore apply-mark --journal <file> --sequence <n> --state completed|failed [--reason <text>] [--updated-at <text>] [--out <file>] [--require-pending]"
2153}
2154
2155#[cfg(test)]
2156mod tests {
2157    use super::*;
2158    use canic_backup::restore::RestoreApplyOperationState;
2159    use canic_backup::{
2160        artifacts::ArtifactChecksum,
2161        journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
2162        manifest::{
2163            BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
2164            FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
2165            VerificationCheck, VerificationPlan,
2166        },
2167    };
2168    use serde_json::json;
2169    use std::{
2170        path::Path,
2171        time::{SystemTime, UNIX_EPOCH},
2172    };
2173
2174    const ROOT: &str = "aaaaa-aa";
2175    const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2176    const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2177    const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2178
2179    // Ensure restore plan options parse the intended no-mutation command.
2180    #[test]
2181    fn parses_restore_plan_options() {
2182        let options = RestorePlanOptions::parse([
2183            OsString::from("--manifest"),
2184            OsString::from("manifest.json"),
2185            OsString::from("--mapping"),
2186            OsString::from("mapping.json"),
2187            OsString::from("--out"),
2188            OsString::from("plan.json"),
2189            OsString::from("--require-restore-ready"),
2190        ])
2191        .expect("parse options");
2192
2193        assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
2194        assert_eq!(options.backup_dir, None);
2195        assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
2196        assert_eq!(options.out, Some(PathBuf::from("plan.json")));
2197        assert!(!options.require_verified);
2198        assert!(options.require_restore_ready);
2199    }
2200
2201    // Ensure verified restore plan options parse with the canonical backup source.
2202    #[test]
2203    fn parses_verified_restore_plan_options() {
2204        let options = RestorePlanOptions::parse([
2205            OsString::from("--backup-dir"),
2206            OsString::from("backups/run"),
2207            OsString::from("--require-verified"),
2208        ])
2209        .expect("parse verified options");
2210
2211        assert_eq!(options.manifest, None);
2212        assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2213        assert_eq!(options.mapping, None);
2214        assert_eq!(options.out, None);
2215        assert!(options.require_verified);
2216        assert!(!options.require_restore_ready);
2217    }
2218
2219    // Ensure restore status options parse the intended no-mutation command.
2220    #[test]
2221    fn parses_restore_status_options() {
2222        let options = RestoreStatusOptions::parse([
2223            OsString::from("--plan"),
2224            OsString::from("restore-plan.json"),
2225            OsString::from("--out"),
2226            OsString::from("restore-status.json"),
2227        ])
2228        .expect("parse status options");
2229
2230        assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2231        assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
2232    }
2233
2234    // Ensure restore apply options require the explicit dry-run mode.
2235    #[test]
2236    fn parses_restore_apply_dry_run_options() {
2237        let options = RestoreApplyOptions::parse([
2238            OsString::from("--plan"),
2239            OsString::from("restore-plan.json"),
2240            OsString::from("--status"),
2241            OsString::from("restore-status.json"),
2242            OsString::from("--backup-dir"),
2243            OsString::from("backups/run"),
2244            OsString::from("--dry-run"),
2245            OsString::from("--out"),
2246            OsString::from("restore-apply-dry-run.json"),
2247            OsString::from("--journal-out"),
2248            OsString::from("restore-apply-journal.json"),
2249        ])
2250        .expect("parse apply options");
2251
2252        assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2253        assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
2254        assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2255        assert_eq!(
2256            options.out,
2257            Some(PathBuf::from("restore-apply-dry-run.json"))
2258        );
2259        assert_eq!(
2260            options.journal_out,
2261            Some(PathBuf::from("restore-apply-journal.json"))
2262        );
2263        assert!(options.dry_run);
2264    }
2265
2266    // Ensure restore apply-status options parse the intended journal command.
2267    #[test]
2268    fn parses_restore_apply_status_options() {
2269        let options = RestoreApplyStatusOptions::parse([
2270            OsString::from("--journal"),
2271            OsString::from("restore-apply-journal.json"),
2272            OsString::from("--out"),
2273            OsString::from("restore-apply-status.json"),
2274            OsString::from("--require-ready"),
2275            OsString::from("--require-no-pending"),
2276            OsString::from("--require-no-failed"),
2277            OsString::from("--require-complete"),
2278        ])
2279        .expect("parse apply-status options");
2280
2281        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2282        assert!(options.require_ready);
2283        assert!(options.require_no_pending);
2284        assert!(options.require_no_failed);
2285        assert!(options.require_complete);
2286        assert_eq!(
2287            options.out,
2288            Some(PathBuf::from("restore-apply-status.json"))
2289        );
2290    }
2291
2292    // Ensure restore apply-report options parse the intended journal command.
2293    #[test]
2294    fn parses_restore_apply_report_options() {
2295        let options = RestoreApplyReportOptions::parse([
2296            OsString::from("--journal"),
2297            OsString::from("restore-apply-journal.json"),
2298            OsString::from("--out"),
2299            OsString::from("restore-apply-report.json"),
2300            OsString::from("--require-no-attention"),
2301        ])
2302        .expect("parse apply-report options");
2303
2304        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2305        assert!(options.require_no_attention);
2306        assert_eq!(
2307            options.out,
2308            Some(PathBuf::from("restore-apply-report.json"))
2309        );
2310    }
2311
2312    // Ensure restore run options parse the native runner dry-run command.
2313    #[test]
2314    fn parses_restore_run_dry_run_options() {
2315        let options = RestoreRunOptions::parse([
2316            OsString::from("--journal"),
2317            OsString::from("restore-apply-journal.json"),
2318            OsString::from("--dry-run"),
2319            OsString::from("--dfx"),
2320            OsString::from("/tmp/dfx"),
2321            OsString::from("--network"),
2322            OsString::from("local"),
2323            OsString::from("--out"),
2324            OsString::from("restore-run-dry-run.json"),
2325            OsString::from("--max-steps"),
2326            OsString::from("1"),
2327            OsString::from("--require-complete"),
2328            OsString::from("--require-no-attention"),
2329            OsString::from("--require-run-mode"),
2330            OsString::from("dry-run"),
2331            OsString::from("--require-stopped-reason"),
2332            OsString::from("preview"),
2333            OsString::from("--require-next-action"),
2334            OsString::from("rerun"),
2335            OsString::from("--require-executed-count"),
2336            OsString::from("0"),
2337        ])
2338        .expect("parse restore run options");
2339
2340        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2341        assert_eq!(options.dfx, "/tmp/dfx");
2342        assert_eq!(options.network.as_deref(), Some("local"));
2343        assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
2344        assert!(options.dry_run);
2345        assert!(!options.execute);
2346        assert!(!options.unclaim_pending);
2347        assert_eq!(options.max_steps, Some(1));
2348        assert!(options.require_complete);
2349        assert!(options.require_no_attention);
2350        assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
2351        assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
2352        assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
2353        assert_eq!(options.require_executed_count, Some(0));
2354    }
2355
2356    // Ensure restore run options parse the native execute command.
2357    #[test]
2358    fn parses_restore_run_execute_options() {
2359        let options = RestoreRunOptions::parse([
2360            OsString::from("--journal"),
2361            OsString::from("restore-apply-journal.json"),
2362            OsString::from("--execute"),
2363            OsString::from("--dfx"),
2364            OsString::from("/bin/true"),
2365            OsString::from("--max-steps"),
2366            OsString::from("4"),
2367        ])
2368        .expect("parse restore run execute options");
2369
2370        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2371        assert_eq!(options.dfx, "/bin/true");
2372        assert_eq!(options.network, None);
2373        assert_eq!(options.out, None);
2374        assert!(!options.dry_run);
2375        assert!(options.execute);
2376        assert!(!options.unclaim_pending);
2377        assert_eq!(options.max_steps, Some(4));
2378        assert!(!options.require_complete);
2379        assert!(!options.require_no_attention);
2380        assert_eq!(options.require_run_mode, None);
2381        assert_eq!(options.require_stopped_reason, None);
2382        assert_eq!(options.require_next_action, None);
2383        assert_eq!(options.require_executed_count, None);
2384    }
2385
2386    // Ensure restore run options parse the native pending-operation recovery mode.
2387    #[test]
2388    fn parses_restore_run_unclaim_pending_options() {
2389        let options = RestoreRunOptions::parse([
2390            OsString::from("--journal"),
2391            OsString::from("restore-apply-journal.json"),
2392            OsString::from("--unclaim-pending"),
2393            OsString::from("--out"),
2394            OsString::from("restore-run.json"),
2395        ])
2396        .expect("parse restore run unclaim options");
2397
2398        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2399        assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
2400        assert!(!options.dry_run);
2401        assert!(!options.execute);
2402        assert!(options.unclaim_pending);
2403    }
2404
2405    // Ensure restore apply-next options parse the intended journal command.
2406    #[test]
2407    fn parses_restore_apply_next_options() {
2408        let options = RestoreApplyNextOptions::parse([
2409            OsString::from("--journal"),
2410            OsString::from("restore-apply-journal.json"),
2411            OsString::from("--out"),
2412            OsString::from("restore-apply-next.json"),
2413        ])
2414        .expect("parse apply-next options");
2415
2416        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2417        assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
2418    }
2419
2420    // Ensure restore apply-command options parse the intended preview command.
2421    #[test]
2422    fn parses_restore_apply_command_options() {
2423        let options = RestoreApplyCommandOptions::parse([
2424            OsString::from("--journal"),
2425            OsString::from("restore-apply-journal.json"),
2426            OsString::from("--dfx"),
2427            OsString::from("/tmp/dfx"),
2428            OsString::from("--network"),
2429            OsString::from("local"),
2430            OsString::from("--out"),
2431            OsString::from("restore-apply-command.json"),
2432            OsString::from("--require-command"),
2433        ])
2434        .expect("parse apply-command options");
2435
2436        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2437        assert_eq!(options.dfx, "/tmp/dfx");
2438        assert_eq!(options.network.as_deref(), Some("local"));
2439        assert!(options.require_command);
2440        assert_eq!(
2441            options.out,
2442            Some(PathBuf::from("restore-apply-command.json"))
2443        );
2444    }
2445
2446    // Ensure restore apply-claim options parse the intended journal command.
2447    #[test]
2448    fn parses_restore_apply_claim_options() {
2449        let options = RestoreApplyClaimOptions::parse([
2450            OsString::from("--journal"),
2451            OsString::from("restore-apply-journal.json"),
2452            OsString::from("--sequence"),
2453            OsString::from("0"),
2454            OsString::from("--updated-at"),
2455            OsString::from("2026-05-04T12:00:00Z"),
2456            OsString::from("--out"),
2457            OsString::from("restore-apply-journal.claimed.json"),
2458        ])
2459        .expect("parse apply-claim options");
2460
2461        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2462        assert_eq!(options.sequence, Some(0));
2463        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
2464        assert_eq!(
2465            options.out,
2466            Some(PathBuf::from("restore-apply-journal.claimed.json"))
2467        );
2468    }
2469
2470    // Ensure restore apply-unclaim options parse the intended journal command.
2471    #[test]
2472    fn parses_restore_apply_unclaim_options() {
2473        let options = RestoreApplyUnclaimOptions::parse([
2474            OsString::from("--journal"),
2475            OsString::from("restore-apply-journal.json"),
2476            OsString::from("--sequence"),
2477            OsString::from("0"),
2478            OsString::from("--updated-at"),
2479            OsString::from("2026-05-04T12:01:00Z"),
2480            OsString::from("--out"),
2481            OsString::from("restore-apply-journal.unclaimed.json"),
2482        ])
2483        .expect("parse apply-unclaim options");
2484
2485        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2486        assert_eq!(options.sequence, Some(0));
2487        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
2488        assert_eq!(
2489            options.out,
2490            Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
2491        );
2492    }
2493
2494    // Ensure restore apply-mark options parse the intended journal update command.
2495    #[test]
2496    fn parses_restore_apply_mark_options() {
2497        let options = RestoreApplyMarkOptions::parse([
2498            OsString::from("--journal"),
2499            OsString::from("restore-apply-journal.json"),
2500            OsString::from("--sequence"),
2501            OsString::from("4"),
2502            OsString::from("--state"),
2503            OsString::from("failed"),
2504            OsString::from("--reason"),
2505            OsString::from("dfx-load-failed"),
2506            OsString::from("--updated-at"),
2507            OsString::from("2026-05-04T12:02:00Z"),
2508            OsString::from("--out"),
2509            OsString::from("restore-apply-journal.updated.json"),
2510            OsString::from("--require-pending"),
2511        ])
2512        .expect("parse apply-mark options");
2513
2514        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2515        assert_eq!(options.sequence, 4);
2516        assert_eq!(options.state, RestoreApplyMarkState::Failed);
2517        assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
2518        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
2519        assert!(options.require_pending);
2520        assert_eq!(
2521            options.out,
2522            Some(PathBuf::from("restore-apply-journal.updated.json"))
2523        );
2524    }
2525
2526    // Ensure restore apply refuses non-dry-run execution while apply is scaffolded.
2527    #[test]
2528    fn restore_apply_requires_dry_run() {
2529        let err = RestoreApplyOptions::parse([
2530            OsString::from("--plan"),
2531            OsString::from("restore-plan.json"),
2532        ])
2533        .expect_err("apply without dry-run should fail");
2534
2535        assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
2536    }
2537
2538    // Ensure restore run refuses mutation while native execution is scaffolded.
2539    #[test]
2540    fn restore_run_requires_mode() {
2541        let err = RestoreRunOptions::parse([
2542            OsString::from("--journal"),
2543            OsString::from("restore-apply-journal.json"),
2544        ])
2545        .expect_err("restore run without dry-run should fail");
2546
2547        assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
2548    }
2549
2550    // Ensure restore run rejects ambiguous execution modes.
2551    #[test]
2552    fn restore_run_rejects_conflicting_modes() {
2553        let err = RestoreRunOptions::parse([
2554            OsString::from("--journal"),
2555            OsString::from("restore-apply-journal.json"),
2556            OsString::from("--dry-run"),
2557            OsString::from("--execute"),
2558            OsString::from("--unclaim-pending"),
2559        ])
2560        .expect_err("restore run should reject conflicting modes");
2561
2562        assert!(matches!(
2563            err,
2564            RestoreCommandError::RestoreRunConflictingModes
2565        ));
2566    }
2567
2568    // Ensure backup-dir restore planning reads the canonical layout manifest.
2569    #[test]
2570    fn plan_restore_reads_manifest_from_backup_dir() {
2571        let root = temp_dir("canic-cli-restore-plan-layout");
2572        let layout = BackupLayout::new(root.clone());
2573        layout
2574            .write_manifest(&valid_manifest())
2575            .expect("write manifest");
2576
2577        let options = RestorePlanOptions {
2578            manifest: None,
2579            backup_dir: Some(root.clone()),
2580            mapping: None,
2581            out: None,
2582            require_verified: false,
2583            require_restore_ready: false,
2584        };
2585
2586        let plan = plan_restore(&options).expect("plan restore");
2587
2588        fs::remove_dir_all(root).expect("remove temp root");
2589        assert_eq!(plan.backup_id, "backup-test");
2590        assert_eq!(plan.member_count, 2);
2591    }
2592
2593    // Ensure restore planning has exactly one manifest source.
2594    #[test]
2595    fn parse_rejects_conflicting_manifest_sources() {
2596        let err = RestorePlanOptions::parse([
2597            OsString::from("--manifest"),
2598            OsString::from("manifest.json"),
2599            OsString::from("--backup-dir"),
2600            OsString::from("backups/run"),
2601        ])
2602        .expect_err("conflicting sources should fail");
2603
2604        assert!(matches!(
2605            err,
2606            RestoreCommandError::ConflictingManifestSources
2607        ));
2608    }
2609
2610    // Ensure verified planning requires the canonical backup layout source.
2611    #[test]
2612    fn parse_rejects_require_verified_with_manifest_source() {
2613        let err = RestorePlanOptions::parse([
2614            OsString::from("--manifest"),
2615            OsString::from("manifest.json"),
2616            OsString::from("--require-verified"),
2617        ])
2618        .expect_err("verification should require a backup layout");
2619
2620        assert!(matches!(
2621            err,
2622            RestoreCommandError::RequireVerifiedNeedsBackupDir
2623        ));
2624    }
2625
2626    // Ensure restore planning can require manifest, journal, and artifact integrity.
2627    #[test]
2628    fn plan_restore_requires_verified_backup_layout() {
2629        let root = temp_dir("canic-cli-restore-plan-verified");
2630        let layout = BackupLayout::new(root.clone());
2631        let manifest = valid_manifest();
2632        write_verified_layout(&root, &layout, &manifest);
2633
2634        let options = RestorePlanOptions {
2635            manifest: None,
2636            backup_dir: Some(root.clone()),
2637            mapping: None,
2638            out: None,
2639            require_verified: true,
2640            require_restore_ready: false,
2641        };
2642
2643        let plan = plan_restore(&options).expect("plan verified restore");
2644
2645        fs::remove_dir_all(root).expect("remove temp root");
2646        assert_eq!(plan.backup_id, "backup-test");
2647        assert_eq!(plan.member_count, 2);
2648    }
2649
2650    // Ensure required verification fails before planning when the layout is incomplete.
2651    #[test]
2652    fn plan_restore_rejects_unverified_backup_layout() {
2653        let root = temp_dir("canic-cli-restore-plan-unverified");
2654        let layout = BackupLayout::new(root.clone());
2655        layout
2656            .write_manifest(&valid_manifest())
2657            .expect("write manifest");
2658
2659        let options = RestorePlanOptions {
2660            manifest: None,
2661            backup_dir: Some(root.clone()),
2662            mapping: None,
2663            out: None,
2664            require_verified: true,
2665            require_restore_ready: false,
2666        };
2667
2668        let err = plan_restore(&options).expect_err("missing journal should fail");
2669
2670        fs::remove_dir_all(root).expect("remove temp root");
2671        assert!(matches!(err, RestoreCommandError::Persistence(_)));
2672    }
2673
2674    // Ensure the CLI planning path validates manifests and applies mappings.
2675    #[test]
2676    fn plan_restore_reads_manifest_and_mapping() {
2677        let root = temp_dir("canic-cli-restore-plan");
2678        fs::create_dir_all(&root).expect("create temp root");
2679        let manifest_path = root.join("manifest.json");
2680        let mapping_path = root.join("mapping.json");
2681
2682        fs::write(
2683            &manifest_path,
2684            serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2685        )
2686        .expect("write manifest");
2687        fs::write(
2688            &mapping_path,
2689            json!({
2690                "members": [
2691                    {"source_canister": ROOT, "target_canister": ROOT},
2692                    {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
2693                ]
2694            })
2695            .to_string(),
2696        )
2697        .expect("write mapping");
2698
2699        let options = RestorePlanOptions {
2700            manifest: Some(manifest_path),
2701            backup_dir: None,
2702            mapping: Some(mapping_path),
2703            out: None,
2704            require_verified: false,
2705            require_restore_ready: false,
2706        };
2707
2708        let plan = plan_restore(&options).expect("plan restore");
2709
2710        fs::remove_dir_all(root).expect("remove temp root");
2711        let members = plan.ordered_members();
2712        assert_eq!(members.len(), 2);
2713        assert_eq!(members[0].source_canister, ROOT);
2714        assert_eq!(members[1].target_canister, MAPPED_CHILD);
2715    }
2716
2717    // Ensure restore-readiness gating happens after writing the plan artifact.
2718    #[test]
2719    fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
2720        let root = temp_dir("canic-cli-restore-plan-require-ready");
2721        fs::create_dir_all(&root).expect("create temp root");
2722        let manifest_path = root.join("manifest.json");
2723        let out_path = root.join("plan.json");
2724
2725        fs::write(
2726            &manifest_path,
2727            serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
2728        )
2729        .expect("write manifest");
2730
2731        let err = run([
2732            OsString::from("plan"),
2733            OsString::from("--manifest"),
2734            OsString::from(manifest_path.as_os_str()),
2735            OsString::from("--out"),
2736            OsString::from(out_path.as_os_str()),
2737            OsString::from("--require-restore-ready"),
2738        ])
2739        .expect_err("restore readiness should be enforced");
2740
2741        assert!(out_path.exists());
2742        let plan: RestorePlan =
2743            serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2744
2745        fs::remove_dir_all(root).expect("remove temp root");
2746        assert!(!plan.readiness_summary.ready);
2747        assert!(matches!(
2748            err,
2749            RestoreCommandError::RestoreNotReady {
2750                reasons,
2751                ..
2752            } if reasons == [
2753                "missing-module-hash",
2754                "missing-wasm-hash",
2755                "missing-snapshot-checksum"
2756            ]
2757        ));
2758    }
2759
2760    // Ensure restore-readiness gating accepts plans with complete provenance.
2761    #[test]
2762    fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
2763        let root = temp_dir("canic-cli-restore-plan-ready");
2764        fs::create_dir_all(&root).expect("create temp root");
2765        let manifest_path = root.join("manifest.json");
2766        let out_path = root.join("plan.json");
2767
2768        fs::write(
2769            &manifest_path,
2770            serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
2771        )
2772        .expect("write manifest");
2773
2774        run([
2775            OsString::from("plan"),
2776            OsString::from("--manifest"),
2777            OsString::from(manifest_path.as_os_str()),
2778            OsString::from("--out"),
2779            OsString::from(out_path.as_os_str()),
2780            OsString::from("--require-restore-ready"),
2781        ])
2782        .expect("restore-ready plan should pass");
2783
2784        let plan: RestorePlan =
2785            serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
2786
2787        fs::remove_dir_all(root).expect("remove temp root");
2788        assert!(plan.readiness_summary.ready);
2789        assert!(plan.readiness_summary.reasons.is_empty());
2790    }
2791
2792    // Ensure restore status writes the initial planned execution journal.
2793    #[test]
2794    fn run_restore_status_writes_planned_status() {
2795        let root = temp_dir("canic-cli-restore-status");
2796        fs::create_dir_all(&root).expect("create temp root");
2797        let plan_path = root.join("restore-plan.json");
2798        let out_path = root.join("restore-status.json");
2799        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2800
2801        fs::write(
2802            &plan_path,
2803            serde_json::to_vec(&plan).expect("serialize plan"),
2804        )
2805        .expect("write plan");
2806
2807        run([
2808            OsString::from("status"),
2809            OsString::from("--plan"),
2810            OsString::from(plan_path.as_os_str()),
2811            OsString::from("--out"),
2812            OsString::from(out_path.as_os_str()),
2813        ])
2814        .expect("write restore status");
2815
2816        let status: RestoreStatus =
2817            serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
2818                .expect("decode restore status");
2819        let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
2820
2821        fs::remove_dir_all(root).expect("remove temp root");
2822        assert_eq!(status.status_version, 1);
2823        assert_eq!(status.backup_id.as_str(), "backup-test");
2824        assert!(status.ready);
2825        assert!(status.readiness_reasons.is_empty());
2826        assert_eq!(status.member_count, 2);
2827        assert_eq!(status.phase_count, 1);
2828        assert_eq!(status.planned_snapshot_loads, 2);
2829        assert_eq!(status.planned_code_reinstalls, 2);
2830        assert_eq!(status.planned_verification_checks, 2);
2831        assert_eq!(status.phases[0].members[0].source_canister, ROOT);
2832        assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
2833    }
2834
2835    // Ensure restore apply dry-run writes ordered operations from plan and status.
2836    #[test]
2837    fn run_restore_apply_dry_run_writes_operations() {
2838        let root = temp_dir("canic-cli-restore-apply-dry-run");
2839        fs::create_dir_all(&root).expect("create temp root");
2840        let plan_path = root.join("restore-plan.json");
2841        let status_path = root.join("restore-status.json");
2842        let out_path = root.join("restore-apply-dry-run.json");
2843        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2844        let status = RestoreStatus::from_plan(&plan);
2845
2846        fs::write(
2847            &plan_path,
2848            serde_json::to_vec(&plan).expect("serialize plan"),
2849        )
2850        .expect("write plan");
2851        fs::write(
2852            &status_path,
2853            serde_json::to_vec(&status).expect("serialize status"),
2854        )
2855        .expect("write status");
2856
2857        run([
2858            OsString::from("apply"),
2859            OsString::from("--plan"),
2860            OsString::from(plan_path.as_os_str()),
2861            OsString::from("--status"),
2862            OsString::from(status_path.as_os_str()),
2863            OsString::from("--dry-run"),
2864            OsString::from("--out"),
2865            OsString::from(out_path.as_os_str()),
2866        ])
2867        .expect("write apply dry-run");
2868
2869        let dry_run: RestoreApplyDryRun =
2870            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2871                .expect("decode dry-run");
2872        let dry_run_json: serde_json::Value =
2873            serde_json::to_value(&dry_run).expect("encode dry-run");
2874
2875        fs::remove_dir_all(root).expect("remove temp root");
2876        assert_eq!(dry_run.dry_run_version, 1);
2877        assert_eq!(dry_run.backup_id.as_str(), "backup-test");
2878        assert!(dry_run.ready);
2879        assert!(dry_run.status_supplied);
2880        assert_eq!(dry_run.member_count, 2);
2881        assert_eq!(dry_run.phase_count, 1);
2882        assert_eq!(dry_run.rendered_operations, 8);
2883        assert_eq!(
2884            dry_run_json["phases"][0]["operations"][0]["operation"],
2885            "upload-snapshot"
2886        );
2887        assert_eq!(
2888            dry_run_json["phases"][0]["operations"][3]["operation"],
2889            "verify-member"
2890        );
2891        assert_eq!(
2892            dry_run_json["phases"][0]["operations"][3]["verification_kind"],
2893            "status"
2894        );
2895        assert_eq!(
2896            dry_run_json["phases"][0]["operations"][3]["verification_method"],
2897            serde_json::Value::Null
2898        );
2899    }
2900
2901    // Ensure restore apply dry-run can validate artifacts under a backup directory.
2902    #[test]
2903    fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
2904        let root = temp_dir("canic-cli-restore-apply-artifacts");
2905        fs::create_dir_all(&root).expect("create temp root");
2906        let plan_path = root.join("restore-plan.json");
2907        let out_path = root.join("restore-apply-dry-run.json");
2908        let journal_path = root.join("restore-apply-journal.json");
2909        let status_path = root.join("restore-apply-status.json");
2910        let mut manifest = restore_ready_manifest();
2911        write_manifest_artifacts(&root, &mut manifest);
2912        let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
2913
2914        fs::write(
2915            &plan_path,
2916            serde_json::to_vec(&plan).expect("serialize plan"),
2917        )
2918        .expect("write plan");
2919
2920        run([
2921            OsString::from("apply"),
2922            OsString::from("--plan"),
2923            OsString::from(plan_path.as_os_str()),
2924            OsString::from("--backup-dir"),
2925            OsString::from(root.as_os_str()),
2926            OsString::from("--dry-run"),
2927            OsString::from("--out"),
2928            OsString::from(out_path.as_os_str()),
2929            OsString::from("--journal-out"),
2930            OsString::from(journal_path.as_os_str()),
2931        ])
2932        .expect("write apply dry-run");
2933        run([
2934            OsString::from("apply-status"),
2935            OsString::from("--journal"),
2936            OsString::from(journal_path.as_os_str()),
2937            OsString::from("--out"),
2938            OsString::from(status_path.as_os_str()),
2939        ])
2940        .expect("write apply status");
2941
2942        let dry_run: RestoreApplyDryRun =
2943            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
2944                .expect("decode dry-run");
2945        let validation = dry_run
2946            .artifact_validation
2947            .expect("artifact validation should be present");
2948        let journal_json: serde_json::Value =
2949            serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
2950                .expect("decode journal");
2951        let status_json: serde_json::Value =
2952            serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
2953                .expect("decode apply status");
2954
2955        fs::remove_dir_all(root).expect("remove temp root");
2956        assert_eq!(validation.checked_members, 2);
2957        assert!(validation.artifacts_present);
2958        assert!(validation.checksums_verified);
2959        assert_eq!(validation.members_with_expected_checksums, 2);
2960        assert_eq!(journal_json["ready"], true);
2961        assert_eq!(journal_json["operation_count"], 8);
2962        assert_eq!(journal_json["ready_operations"], 8);
2963        assert_eq!(journal_json["blocked_operations"], 0);
2964        assert_eq!(journal_json["operations"][0]["state"], "ready");
2965        assert_eq!(status_json["ready"], true);
2966        assert_eq!(status_json["operation_count"], 8);
2967        assert_eq!(status_json["next_ready_sequence"], 0);
2968        assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
2969    }
2970
2971    // Ensure apply-status rejects structurally inconsistent journals.
2972    #[test]
2973    fn run_restore_apply_status_rejects_invalid_journal() {
2974        let root = temp_dir("canic-cli-restore-apply-status-invalid");
2975        fs::create_dir_all(&root).expect("create temp root");
2976        let journal_path = root.join("restore-apply-journal.json");
2977        let out_path = root.join("restore-apply-status.json");
2978        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
2979        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
2980        let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
2981        journal.operation_count += 1;
2982
2983        fs::write(
2984            &journal_path,
2985            serde_json::to_vec(&journal).expect("serialize journal"),
2986        )
2987        .expect("write journal");
2988
2989        let err = run([
2990            OsString::from("apply-status"),
2991            OsString::from("--journal"),
2992            OsString::from(journal_path.as_os_str()),
2993            OsString::from("--out"),
2994            OsString::from(out_path.as_os_str()),
2995        ])
2996        .expect_err("invalid journal should fail");
2997
2998        assert!(!out_path.exists());
2999        fs::remove_dir_all(root).expect("remove temp root");
3000        assert!(matches!(
3001            err,
3002            RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
3003                field: "operation_count",
3004                ..
3005            })
3006        ));
3007    }
3008
3009    // Ensure apply-status can fail closed after writing status for pending work.
3010    #[test]
3011    fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
3012        let root = temp_dir("canic-cli-restore-apply-status-pending");
3013        fs::create_dir_all(&root).expect("create temp root");
3014        let journal_path = root.join("restore-apply-journal.json");
3015        let out_path = root.join("restore-apply-status.json");
3016        let mut journal = ready_apply_journal();
3017        journal
3018            .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
3019            .expect("claim operation");
3020
3021        fs::write(
3022            &journal_path,
3023            serde_json::to_vec(&journal).expect("serialize journal"),
3024        )
3025        .expect("write journal");
3026
3027        let err = run([
3028            OsString::from("apply-status"),
3029            OsString::from("--journal"),
3030            OsString::from(journal_path.as_os_str()),
3031            OsString::from("--out"),
3032            OsString::from(out_path.as_os_str()),
3033            OsString::from("--require-no-pending"),
3034        ])
3035        .expect_err("pending operation should fail requirement");
3036
3037        assert!(out_path.exists());
3038        let status: RestoreApplyJournalStatus =
3039            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3040                .expect("decode apply status");
3041
3042        fs::remove_dir_all(root).expect("remove temp root");
3043        assert_eq!(status.pending_operations, 1);
3044        assert_eq!(status.next_transition_sequence, Some(0));
3045        assert_eq!(
3046            status.next_transition_updated_at.as_deref(),
3047            Some("2026-05-04T12:00:00Z")
3048        );
3049        assert!(matches!(
3050            err,
3051            RestoreCommandError::RestoreApplyPending {
3052                pending_operations: 1,
3053                next_transition_sequence: Some(0),
3054                ..
3055            }
3056        ));
3057    }
3058
3059    // Ensure apply-status can fail closed after writing status for unready work.
3060    #[test]
3061    fn run_restore_apply_status_require_ready_writes_status_then_fails() {
3062        let root = temp_dir("canic-cli-restore-apply-status-ready");
3063        fs::create_dir_all(&root).expect("create temp root");
3064        let journal_path = root.join("restore-apply-journal.json");
3065        let out_path = root.join("restore-apply-status.json");
3066        let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
3067        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3068        let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3069
3070        fs::write(
3071            &journal_path,
3072            serde_json::to_vec(&journal).expect("serialize journal"),
3073        )
3074        .expect("write journal");
3075
3076        let err = run([
3077            OsString::from("apply-status"),
3078            OsString::from("--journal"),
3079            OsString::from(journal_path.as_os_str()),
3080            OsString::from("--out"),
3081            OsString::from(out_path.as_os_str()),
3082            OsString::from("--require-ready"),
3083        ])
3084        .expect_err("unready journal should fail requirement");
3085
3086        let status: RestoreApplyJournalStatus =
3087            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3088                .expect("decode apply status");
3089
3090        fs::remove_dir_all(root).expect("remove temp root");
3091        assert!(!status.ready);
3092        assert_eq!(status.blocked_operations, status.operation_count);
3093        assert!(
3094            status
3095                .blocked_reasons
3096                .contains(&"missing-snapshot-checksum".to_string())
3097        );
3098        assert!(matches!(
3099            err,
3100            RestoreCommandError::RestoreApplyNotReady { reasons, .. }
3101                if reasons.contains(&"missing-snapshot-checksum".to_string())
3102        ));
3103    }
3104
3105    // Ensure apply-report writes the operator-focused journal summary.
3106    #[test]
3107    fn run_restore_apply_report_writes_attention_summary() {
3108        let root = temp_dir("canic-cli-restore-apply-report");
3109        fs::create_dir_all(&root).expect("create temp root");
3110        let journal_path = root.join("restore-apply-journal.json");
3111        let out_path = root.join("restore-apply-report.json");
3112        let mut journal = ready_apply_journal();
3113        journal
3114            .mark_operation_failed_at(
3115                0,
3116                "dfx-upload-failed".to_string(),
3117                Some("2026-05-05T12:00:00Z".to_string()),
3118            )
3119            .expect("mark failed operation");
3120        journal
3121            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3122            .expect("mark pending operation");
3123
3124        fs::write(
3125            &journal_path,
3126            serde_json::to_vec(&journal).expect("serialize journal"),
3127        )
3128        .expect("write journal");
3129
3130        run([
3131            OsString::from("apply-report"),
3132            OsString::from("--journal"),
3133            OsString::from(journal_path.as_os_str()),
3134            OsString::from("--out"),
3135            OsString::from(out_path.as_os_str()),
3136        ])
3137        .expect("write apply report");
3138
3139        let report: RestoreApplyJournalReport =
3140            serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3141                .expect("decode apply report");
3142        let report_json: serde_json::Value =
3143            serde_json::to_value(&report).expect("encode apply report");
3144
3145        fs::remove_dir_all(root).expect("remove temp root");
3146        assert_eq!(report.backup_id, "backup-test");
3147        assert!(report.attention_required);
3148        assert_eq!(report.failed_operations, 1);
3149        assert_eq!(report.pending_operations, 1);
3150        assert_eq!(report.failed.len(), 1);
3151        assert_eq!(report.pending.len(), 1);
3152        assert_eq!(report.failed[0].sequence, 0);
3153        assert_eq!(report.pending[0].sequence, 1);
3154        assert_eq!(
3155            report.next_transition.as_ref().map(|op| op.sequence),
3156            Some(1)
3157        );
3158        assert_eq!(report_json["outcome"], "failed");
3159        assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
3160    }
3161
3162    // Ensure restore run writes a native no-mutation runner preview.
3163    #[test]
3164    fn run_restore_run_dry_run_writes_native_runner_preview() {
3165        let root = temp_dir("canic-cli-restore-run-dry-run");
3166        fs::create_dir_all(&root).expect("create temp root");
3167        let journal_path = root.join("restore-apply-journal.json");
3168        let out_path = root.join("restore-run-dry-run.json");
3169        let journal = ready_apply_journal();
3170
3171        fs::write(
3172            &journal_path,
3173            serde_json::to_vec(&journal).expect("serialize journal"),
3174        )
3175        .expect("write journal");
3176
3177        run([
3178            OsString::from("run"),
3179            OsString::from("--journal"),
3180            OsString::from(journal_path.as_os_str()),
3181            OsString::from("--dry-run"),
3182            OsString::from("--dfx"),
3183            OsString::from("/tmp/dfx"),
3184            OsString::from("--network"),
3185            OsString::from("local"),
3186            OsString::from("--out"),
3187            OsString::from(out_path.as_os_str()),
3188        ])
3189        .expect("write restore run dry-run");
3190
3191        let dry_run: serde_json::Value =
3192            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3193                .expect("decode dry-run");
3194
3195        fs::remove_dir_all(root).expect("remove temp root");
3196        assert_eq!(dry_run["run_version"], 1);
3197        assert_eq!(dry_run["backup_id"], "backup-test");
3198        assert_eq!(dry_run["run_mode"], "dry-run");
3199        assert_eq!(dry_run["dry_run"], true);
3200        assert_eq!(dry_run["ready"], true);
3201        assert_eq!(dry_run["complete"], false);
3202        assert_eq!(dry_run["attention_required"], false);
3203        assert_eq!(dry_run["stopped_reason"], "preview");
3204        assert_eq!(dry_run["next_action"], "rerun");
3205        assert_eq!(dry_run["operation_available"], true);
3206        assert_eq!(dry_run["command_available"], true);
3207        assert_eq!(dry_run["next_transition"]["sequence"], 0);
3208        assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
3209        assert_eq!(
3210            dry_run["command"]["args"],
3211            json!([
3212                "canister",
3213                "--network",
3214                "local",
3215                "snapshot",
3216                "upload",
3217                "--dir",
3218                "artifacts/root",
3219                ROOT
3220            ])
3221        );
3222        assert_eq!(dry_run["command"]["mutates"], true);
3223    }
3224
3225    // Ensure restore run can recover one interrupted pending operation.
3226    #[test]
3227    fn run_restore_run_unclaim_pending_marks_operation_ready() {
3228        let root = temp_dir("canic-cli-restore-run-unclaim-pending");
3229        fs::create_dir_all(&root).expect("create temp root");
3230        let journal_path = root.join("restore-apply-journal.json");
3231        let out_path = root.join("restore-run.json");
3232        let mut journal = ready_apply_journal();
3233        journal
3234            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3235            .expect("mark pending operation");
3236
3237        fs::write(
3238            &journal_path,
3239            serde_json::to_vec(&journal).expect("serialize journal"),
3240        )
3241        .expect("write journal");
3242
3243        run([
3244            OsString::from("run"),
3245            OsString::from("--journal"),
3246            OsString::from(journal_path.as_os_str()),
3247            OsString::from("--unclaim-pending"),
3248            OsString::from("--out"),
3249            OsString::from(out_path.as_os_str()),
3250        ])
3251        .expect("unclaim pending operation");
3252
3253        let run_summary: serde_json::Value =
3254            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3255                .expect("decode run summary");
3256        let updated: RestoreApplyJournal =
3257            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3258                .expect("decode updated journal");
3259
3260        fs::remove_dir_all(root).expect("remove temp root");
3261        assert_eq!(run_summary["run_mode"], "unclaim-pending");
3262        assert_eq!(run_summary["unclaim_pending"], true);
3263        assert_eq!(run_summary["stopped_reason"], "recovered-pending");
3264        assert_eq!(run_summary["next_action"], "rerun");
3265        assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
3266        assert_eq!(run_summary["recovered_operation"]["state"], "pending");
3267        assert_eq!(run_summary["pending_operations"], 0);
3268        assert_eq!(run_summary["ready_operations"], 8);
3269        assert_eq!(run_summary["attention_required"], false);
3270        assert_eq!(updated.pending_operations, 0);
3271        assert_eq!(updated.ready_operations, 8);
3272        assert_eq!(
3273            updated.operations[0].state,
3274            RestoreApplyOperationState::Ready
3275        );
3276    }
3277
3278    // Ensure restore run execute claims and completes one generated command.
3279    #[test]
3280    fn run_restore_run_execute_marks_completed_operation() {
3281        let root = temp_dir("canic-cli-restore-run-execute");
3282        fs::create_dir_all(&root).expect("create temp root");
3283        let journal_path = root.join("restore-apply-journal.json");
3284        let out_path = root.join("restore-run.json");
3285        let journal = ready_apply_journal();
3286
3287        fs::write(
3288            &journal_path,
3289            serde_json::to_vec(&journal).expect("serialize journal"),
3290        )
3291        .expect("write journal");
3292
3293        run([
3294            OsString::from("run"),
3295            OsString::from("--journal"),
3296            OsString::from(journal_path.as_os_str()),
3297            OsString::from("--execute"),
3298            OsString::from("--dfx"),
3299            OsString::from("/bin/true"),
3300            OsString::from("--max-steps"),
3301            OsString::from("1"),
3302            OsString::from("--out"),
3303            OsString::from(out_path.as_os_str()),
3304        ])
3305        .expect("execute one restore run step");
3306
3307        let run_summary: serde_json::Value =
3308            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3309                .expect("decode run summary");
3310        let updated: RestoreApplyJournal =
3311            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3312                .expect("decode updated journal");
3313
3314        fs::remove_dir_all(root).expect("remove temp root");
3315        assert_eq!(run_summary["run_mode"], "execute");
3316        assert_eq!(run_summary["execute"], true);
3317        assert_eq!(run_summary["dry_run"], false);
3318        assert_eq!(run_summary["max_steps_reached"], true);
3319        assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
3320        assert_eq!(run_summary["next_action"], "rerun");
3321        assert_eq!(run_summary["executed_operation_count"], 1);
3322        assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
3323        assert_eq!(
3324            run_summary["executed_operations"][0]["command"]["program"],
3325            "/bin/true"
3326        );
3327        assert_eq!(updated.completed_operations, 1);
3328        assert_eq!(updated.pending_operations, 0);
3329        assert_eq!(updated.failed_operations, 0);
3330        assert_eq!(
3331            updated.operations[0].state,
3332            RestoreApplyOperationState::Completed
3333        );
3334    }
3335
3336    // Ensure restore run can fail closed after writing an incomplete summary.
3337    #[test]
3338    fn run_restore_run_require_complete_writes_summary_then_fails() {
3339        let root = temp_dir("canic-cli-restore-run-require-complete");
3340        fs::create_dir_all(&root).expect("create temp root");
3341        let journal_path = root.join("restore-apply-journal.json");
3342        let out_path = root.join("restore-run.json");
3343        let journal = ready_apply_journal();
3344
3345        fs::write(
3346            &journal_path,
3347            serde_json::to_vec(&journal).expect("serialize journal"),
3348        )
3349        .expect("write journal");
3350
3351        let err = run([
3352            OsString::from("run"),
3353            OsString::from("--journal"),
3354            OsString::from(journal_path.as_os_str()),
3355            OsString::from("--execute"),
3356            OsString::from("--dfx"),
3357            OsString::from("/bin/true"),
3358            OsString::from("--max-steps"),
3359            OsString::from("1"),
3360            OsString::from("--out"),
3361            OsString::from(out_path.as_os_str()),
3362            OsString::from("--require-complete"),
3363        ])
3364        .expect_err("incomplete run should fail requirement");
3365
3366        let run_summary: serde_json::Value =
3367            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3368                .expect("decode run summary");
3369
3370        fs::remove_dir_all(root).expect("remove temp root");
3371        assert_eq!(run_summary["executed_operation_count"], 1);
3372        assert_eq!(run_summary["complete"], false);
3373        assert!(matches!(
3374            err,
3375            RestoreCommandError::RestoreApplyIncomplete {
3376                completed_operations: 1,
3377                operation_count: 8,
3378                ..
3379            }
3380        ));
3381    }
3382
3383    // Ensure restore run execute records failed command exits in the journal.
3384    #[test]
3385    fn run_restore_run_execute_marks_failed_operation() {
3386        let root = temp_dir("canic-cli-restore-run-execute-failed");
3387        fs::create_dir_all(&root).expect("create temp root");
3388        let journal_path = root.join("restore-apply-journal.json");
3389        let out_path = root.join("restore-run.json");
3390        let journal = ready_apply_journal();
3391
3392        fs::write(
3393            &journal_path,
3394            serde_json::to_vec(&journal).expect("serialize journal"),
3395        )
3396        .expect("write journal");
3397
3398        let err = run([
3399            OsString::from("run"),
3400            OsString::from("--journal"),
3401            OsString::from(journal_path.as_os_str()),
3402            OsString::from("--execute"),
3403            OsString::from("--dfx"),
3404            OsString::from("/bin/false"),
3405            OsString::from("--max-steps"),
3406            OsString::from("1"),
3407            OsString::from("--out"),
3408            OsString::from(out_path.as_os_str()),
3409        ])
3410        .expect_err("failing runner command should fail");
3411
3412        let run_summary: serde_json::Value =
3413            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3414                .expect("decode run summary");
3415        let updated: RestoreApplyJournal =
3416            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
3417                .expect("decode updated journal");
3418
3419        fs::remove_dir_all(root).expect("remove temp root");
3420        assert!(matches!(
3421            err,
3422            RestoreCommandError::RestoreRunCommandFailed {
3423                sequence: 0,
3424                status,
3425            } if status == "1"
3426        ));
3427        assert_eq!(updated.failed_operations, 1);
3428        assert_eq!(updated.pending_operations, 0);
3429        assert_eq!(
3430            updated.operations[0].state,
3431            RestoreApplyOperationState::Failed
3432        );
3433        assert_eq!(run_summary["execute"], true);
3434        assert_eq!(run_summary["attention_required"], true);
3435        assert_eq!(run_summary["outcome"], "failed");
3436        assert_eq!(run_summary["stopped_reason"], "command-failed");
3437        assert_eq!(run_summary["next_action"], "inspect-failed-operation");
3438        assert_eq!(run_summary["executed_operation_count"], 1);
3439        assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
3440        assert_eq!(run_summary["executed_operations"][0]["status"], "1");
3441        assert_eq!(
3442            updated.operations[0].blocking_reasons,
3443            vec!["runner-command-exit-1".to_string()]
3444        );
3445    }
3446
3447    // Ensure restore run can fail closed after writing an attention summary.
3448    #[test]
3449    fn run_restore_run_require_no_attention_writes_summary_then_fails() {
3450        let root = temp_dir("canic-cli-restore-run-require-attention");
3451        fs::create_dir_all(&root).expect("create temp root");
3452        let journal_path = root.join("restore-apply-journal.json");
3453        let out_path = root.join("restore-run.json");
3454        let mut journal = ready_apply_journal();
3455        journal
3456            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3457            .expect("mark pending operation");
3458
3459        fs::write(
3460            &journal_path,
3461            serde_json::to_vec(&journal).expect("serialize journal"),
3462        )
3463        .expect("write journal");
3464
3465        let err = run([
3466            OsString::from("run"),
3467            OsString::from("--journal"),
3468            OsString::from(journal_path.as_os_str()),
3469            OsString::from("--dry-run"),
3470            OsString::from("--out"),
3471            OsString::from(out_path.as_os_str()),
3472            OsString::from("--require-no-attention"),
3473        ])
3474        .expect_err("attention run should fail requirement");
3475
3476        let run_summary: serde_json::Value =
3477            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3478                .expect("decode run summary");
3479
3480        fs::remove_dir_all(root).expect("remove temp root");
3481        assert_eq!(run_summary["attention_required"], true);
3482        assert_eq!(run_summary["outcome"], "pending");
3483        assert_eq!(run_summary["stopped_reason"], "pending");
3484        assert_eq!(run_summary["next_action"], "unclaim-pending");
3485        assert!(matches!(
3486            err,
3487            RestoreCommandError::RestoreApplyReportNeedsAttention {
3488                outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3489                ..
3490            }
3491        ));
3492    }
3493
3494    // Ensure restore run can fail closed on an unexpected run mode.
3495    #[test]
3496    fn run_restore_run_require_run_mode_writes_summary_then_fails() {
3497        let root = temp_dir("canic-cli-restore-run-require-run-mode");
3498        fs::create_dir_all(&root).expect("create temp root");
3499        let journal_path = root.join("restore-apply-journal.json");
3500        let out_path = root.join("restore-run.json");
3501        let journal = ready_apply_journal();
3502
3503        fs::write(
3504            &journal_path,
3505            serde_json::to_vec(&journal).expect("serialize journal"),
3506        )
3507        .expect("write journal");
3508
3509        let err = run([
3510            OsString::from("run"),
3511            OsString::from("--journal"),
3512            OsString::from(journal_path.as_os_str()),
3513            OsString::from("--dry-run"),
3514            OsString::from("--out"),
3515            OsString::from(out_path.as_os_str()),
3516            OsString::from("--require-run-mode"),
3517            OsString::from("execute"),
3518        ])
3519        .expect_err("run mode mismatch should fail requirement");
3520
3521        let run_summary: serde_json::Value =
3522            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3523                .expect("decode run summary");
3524
3525        fs::remove_dir_all(root).expect("remove temp root");
3526        assert_eq!(run_summary["run_mode"], "dry-run");
3527        assert!(matches!(
3528            err,
3529            RestoreCommandError::RestoreRunModeMismatch {
3530                expected,
3531                actual,
3532                ..
3533            } if expected == "execute" && actual == "dry-run"
3534        ));
3535    }
3536
3537    // Ensure restore run can fail closed on an unexpected executed operation count.
3538    #[test]
3539    fn run_restore_run_require_executed_count_writes_summary_then_fails() {
3540        let root = temp_dir("canic-cli-restore-run-require-executed-count");
3541        fs::create_dir_all(&root).expect("create temp root");
3542        let journal_path = root.join("restore-apply-journal.json");
3543        let out_path = root.join("restore-run.json");
3544        let journal = ready_apply_journal();
3545
3546        fs::write(
3547            &journal_path,
3548            serde_json::to_vec(&journal).expect("serialize journal"),
3549        )
3550        .expect("write journal");
3551
3552        let err = run([
3553            OsString::from("run"),
3554            OsString::from("--journal"),
3555            OsString::from(journal_path.as_os_str()),
3556            OsString::from("--execute"),
3557            OsString::from("--dfx"),
3558            OsString::from("/bin/true"),
3559            OsString::from("--max-steps"),
3560            OsString::from("1"),
3561            OsString::from("--out"),
3562            OsString::from(out_path.as_os_str()),
3563            OsString::from("--require-executed-count"),
3564            OsString::from("2"),
3565        ])
3566        .expect_err("executed count mismatch should fail requirement");
3567
3568        let run_summary: serde_json::Value =
3569            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3570                .expect("decode run summary");
3571
3572        fs::remove_dir_all(root).expect("remove temp root");
3573        assert_eq!(run_summary["executed_operation_count"], 1);
3574        assert!(matches!(
3575            err,
3576            RestoreCommandError::RestoreRunExecutedCountMismatch {
3577                expected: 2,
3578                actual: 1,
3579                ..
3580            }
3581        ));
3582    }
3583
3584    // Ensure restore run can fail closed on an unexpected stopped reason.
3585    #[test]
3586    fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
3587        let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
3588        fs::create_dir_all(&root).expect("create temp root");
3589        let journal_path = root.join("restore-apply-journal.json");
3590        let out_path = root.join("restore-run.json");
3591        let journal = ready_apply_journal();
3592
3593        fs::write(
3594            &journal_path,
3595            serde_json::to_vec(&journal).expect("serialize journal"),
3596        )
3597        .expect("write journal");
3598
3599        let err = run([
3600            OsString::from("run"),
3601            OsString::from("--journal"),
3602            OsString::from(journal_path.as_os_str()),
3603            OsString::from("--dry-run"),
3604            OsString::from("--out"),
3605            OsString::from(out_path.as_os_str()),
3606            OsString::from("--require-stopped-reason"),
3607            OsString::from("complete"),
3608        ])
3609        .expect_err("stopped reason mismatch should fail requirement");
3610
3611        let run_summary: serde_json::Value =
3612            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3613                .expect("decode run summary");
3614
3615        fs::remove_dir_all(root).expect("remove temp root");
3616        assert_eq!(run_summary["stopped_reason"], "preview");
3617        assert!(matches!(
3618            err,
3619            RestoreCommandError::RestoreRunStoppedReasonMismatch {
3620                expected,
3621                actual,
3622                ..
3623            } if expected == "complete" && actual == "preview"
3624        ));
3625    }
3626
3627    // Ensure restore run can fail closed on an unexpected next action.
3628    #[test]
3629    fn run_restore_run_require_next_action_writes_summary_then_fails() {
3630        let root = temp_dir("canic-cli-restore-run-require-next-action");
3631        fs::create_dir_all(&root).expect("create temp root");
3632        let journal_path = root.join("restore-apply-journal.json");
3633        let out_path = root.join("restore-run.json");
3634        let journal = ready_apply_journal();
3635
3636        fs::write(
3637            &journal_path,
3638            serde_json::to_vec(&journal).expect("serialize journal"),
3639        )
3640        .expect("write journal");
3641
3642        let err = run([
3643            OsString::from("run"),
3644            OsString::from("--journal"),
3645            OsString::from(journal_path.as_os_str()),
3646            OsString::from("--dry-run"),
3647            OsString::from("--out"),
3648            OsString::from(out_path.as_os_str()),
3649            OsString::from("--require-next-action"),
3650            OsString::from("done"),
3651        ])
3652        .expect_err("next action mismatch should fail requirement");
3653
3654        let run_summary: serde_json::Value =
3655            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
3656                .expect("decode run summary");
3657
3658        fs::remove_dir_all(root).expect("remove temp root");
3659        assert_eq!(run_summary["next_action"], "rerun");
3660        assert!(matches!(
3661            err,
3662            RestoreCommandError::RestoreRunNextActionMismatch {
3663                expected,
3664                actual,
3665                ..
3666            } if expected == "done" && actual == "rerun"
3667        ));
3668    }
3669
3670    // Ensure apply-report can fail closed after writing an attention report.
3671    #[test]
3672    fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
3673        let root = temp_dir("canic-cli-restore-apply-report-attention");
3674        fs::create_dir_all(&root).expect("create temp root");
3675        let journal_path = root.join("restore-apply-journal.json");
3676        let out_path = root.join("restore-apply-report.json");
3677        let mut journal = ready_apply_journal();
3678        journal
3679            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3680            .expect("mark pending operation");
3681
3682        fs::write(
3683            &journal_path,
3684            serde_json::to_vec(&journal).expect("serialize journal"),
3685        )
3686        .expect("write journal");
3687
3688        let err = run([
3689            OsString::from("apply-report"),
3690            OsString::from("--journal"),
3691            OsString::from(journal_path.as_os_str()),
3692            OsString::from("--out"),
3693            OsString::from(out_path.as_os_str()),
3694            OsString::from("--require-no-attention"),
3695        ])
3696        .expect_err("attention report should fail requirement");
3697
3698        let report: RestoreApplyJournalReport =
3699            serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
3700                .expect("decode apply report");
3701
3702        fs::remove_dir_all(root).expect("remove temp root");
3703        assert!(report.attention_required);
3704        assert_eq!(report.pending_operations, 1);
3705        assert!(matches!(
3706            err,
3707            RestoreCommandError::RestoreApplyReportNeedsAttention {
3708                outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
3709                ..
3710            }
3711        ));
3712    }
3713
3714    // Ensure apply-status can fail closed after writing status for incomplete work.
3715    #[test]
3716    fn run_restore_apply_status_require_complete_writes_status_then_fails() {
3717        let root = temp_dir("canic-cli-restore-apply-status-incomplete");
3718        fs::create_dir_all(&root).expect("create temp root");
3719        let journal_path = root.join("restore-apply-journal.json");
3720        let out_path = root.join("restore-apply-status.json");
3721        let journal = ready_apply_journal();
3722
3723        fs::write(
3724            &journal_path,
3725            serde_json::to_vec(&journal).expect("serialize journal"),
3726        )
3727        .expect("write journal");
3728
3729        let err = run([
3730            OsString::from("apply-status"),
3731            OsString::from("--journal"),
3732            OsString::from(journal_path.as_os_str()),
3733            OsString::from("--out"),
3734            OsString::from(out_path.as_os_str()),
3735            OsString::from("--require-complete"),
3736        ])
3737        .expect_err("incomplete journal should fail requirement");
3738
3739        assert!(out_path.exists());
3740        let status: RestoreApplyJournalStatus =
3741            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3742                .expect("decode apply status");
3743
3744        fs::remove_dir_all(root).expect("remove temp root");
3745        assert!(!status.complete);
3746        assert_eq!(status.completed_operations, 0);
3747        assert_eq!(status.operation_count, 8);
3748        assert!(matches!(
3749            err,
3750            RestoreCommandError::RestoreApplyIncomplete {
3751                completed_operations: 0,
3752                operation_count: 8,
3753                ..
3754            }
3755        ));
3756    }
3757
3758    // Ensure apply-status can fail closed after writing status for failed work.
3759    #[test]
3760    fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
3761        let root = temp_dir("canic-cli-restore-apply-status-failed");
3762        fs::create_dir_all(&root).expect("create temp root");
3763        let journal_path = root.join("restore-apply-journal.json");
3764        let out_path = root.join("restore-apply-status.json");
3765        let mut journal = ready_apply_journal();
3766        journal
3767            .mark_operation_failed(0, "dfx-load-failed".to_string())
3768            .expect("mark failed operation");
3769
3770        fs::write(
3771            &journal_path,
3772            serde_json::to_vec(&journal).expect("serialize journal"),
3773        )
3774        .expect("write journal");
3775
3776        let err = run([
3777            OsString::from("apply-status"),
3778            OsString::from("--journal"),
3779            OsString::from(journal_path.as_os_str()),
3780            OsString::from("--out"),
3781            OsString::from(out_path.as_os_str()),
3782            OsString::from("--require-no-failed"),
3783        ])
3784        .expect_err("failed operation should fail requirement");
3785
3786        assert!(out_path.exists());
3787        let status: RestoreApplyJournalStatus =
3788            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3789                .expect("decode apply status");
3790
3791        fs::remove_dir_all(root).expect("remove temp root");
3792        assert_eq!(status.failed_operations, 1);
3793        assert!(matches!(
3794            err,
3795            RestoreCommandError::RestoreApplyFailed {
3796                failed_operations: 1,
3797                ..
3798            }
3799        ));
3800    }
3801
3802    // Ensure apply-status accepts a complete journal when required.
3803    #[test]
3804    fn run_restore_apply_status_require_complete_accepts_complete_journal() {
3805        let root = temp_dir("canic-cli-restore-apply-status-complete");
3806        fs::create_dir_all(&root).expect("create temp root");
3807        let journal_path = root.join("restore-apply-journal.json");
3808        let out_path = root.join("restore-apply-status.json");
3809        let mut journal = ready_apply_journal();
3810        for sequence in 0..journal.operation_count {
3811            journal
3812                .mark_operation_completed(sequence)
3813                .expect("complete operation");
3814        }
3815
3816        fs::write(
3817            &journal_path,
3818            serde_json::to_vec(&journal).expect("serialize journal"),
3819        )
3820        .expect("write journal");
3821
3822        run([
3823            OsString::from("apply-status"),
3824            OsString::from("--journal"),
3825            OsString::from(journal_path.as_os_str()),
3826            OsString::from("--out"),
3827            OsString::from(out_path.as_os_str()),
3828            OsString::from("--require-complete"),
3829        ])
3830        .expect("complete journal should pass requirement");
3831
3832        let status: RestoreApplyJournalStatus =
3833            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3834                .expect("decode apply status");
3835
3836        fs::remove_dir_all(root).expect("remove temp root");
3837        assert!(status.complete);
3838        assert_eq!(status.completed_operations, 8);
3839        assert_eq!(status.operation_count, 8);
3840    }
3841
3842    // Ensure apply-next writes the full next ready operation row for runners.
3843    #[test]
3844    fn run_restore_apply_next_writes_next_ready_operation() {
3845        let root = temp_dir("canic-cli-restore-apply-next");
3846        fs::create_dir_all(&root).expect("create temp root");
3847        let journal_path = root.join("restore-apply-journal.json");
3848        let out_path = root.join("restore-apply-next.json");
3849        let mut journal = ready_apply_journal();
3850        journal
3851            .mark_operation_completed(0)
3852            .expect("mark first operation complete");
3853
3854        fs::write(
3855            &journal_path,
3856            serde_json::to_vec(&journal).expect("serialize journal"),
3857        )
3858        .expect("write journal");
3859
3860        run([
3861            OsString::from("apply-next"),
3862            OsString::from("--journal"),
3863            OsString::from(journal_path.as_os_str()),
3864            OsString::from("--out"),
3865            OsString::from(out_path.as_os_str()),
3866        ])
3867        .expect("write apply next");
3868
3869        let next: RestoreApplyNextOperation =
3870            serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
3871                .expect("decode next operation");
3872        let operation = next.operation.expect("operation should be available");
3873
3874        fs::remove_dir_all(root).expect("remove temp root");
3875        assert!(next.ready);
3876        assert!(next.operation_available);
3877        assert_eq!(operation.sequence, 1);
3878        assert_eq!(
3879            operation.operation,
3880            canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
3881        );
3882    }
3883
3884    // Ensure apply-command writes a no-execute command preview for the next operation.
3885    #[test]
3886    fn run_restore_apply_command_writes_next_command_preview() {
3887        let root = temp_dir("canic-cli-restore-apply-command");
3888        fs::create_dir_all(&root).expect("create temp root");
3889        let journal_path = root.join("restore-apply-journal.json");
3890        let out_path = root.join("restore-apply-command.json");
3891        let journal = ready_apply_journal();
3892
3893        fs::write(
3894            &journal_path,
3895            serde_json::to_vec(&journal).expect("serialize journal"),
3896        )
3897        .expect("write journal");
3898
3899        run([
3900            OsString::from("apply-command"),
3901            OsString::from("--journal"),
3902            OsString::from(journal_path.as_os_str()),
3903            OsString::from("--dfx"),
3904            OsString::from("/tmp/dfx"),
3905            OsString::from("--network"),
3906            OsString::from("local"),
3907            OsString::from("--out"),
3908            OsString::from(out_path.as_os_str()),
3909        ])
3910        .expect("write command preview");
3911
3912        let preview: RestoreApplyCommandPreview =
3913            serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
3914                .expect("decode command preview");
3915        let command = preview.command.expect("command should be available");
3916
3917        fs::remove_dir_all(root).expect("remove temp root");
3918        assert!(preview.ready);
3919        assert!(preview.command_available);
3920        assert_eq!(command.program, "/tmp/dfx");
3921        assert_eq!(
3922            command.args,
3923            vec![
3924                "canister".to_string(),
3925                "--network".to_string(),
3926                "local".to_string(),
3927                "snapshot".to_string(),
3928                "upload".to_string(),
3929                "--dir".to_string(),
3930                "artifacts/root".to_string(),
3931                ROOT.to_string(),
3932            ]
3933        );
3934        assert!(command.mutates);
3935    }
3936
3937    // Ensure apply-command can fail closed after writing a command preview.
3938    #[test]
3939    fn run_restore_apply_command_require_command_writes_preview_then_fails() {
3940        let root = temp_dir("canic-cli-restore-apply-command-require");
3941        fs::create_dir_all(&root).expect("create temp root");
3942        let journal_path = root.join("restore-apply-journal.json");
3943        let out_path = root.join("restore-apply-command.json");
3944        let mut journal = ready_apply_journal();
3945
3946        for sequence in 0..journal.operation_count {
3947            journal
3948                .mark_operation_completed(sequence)
3949                .expect("mark operation completed");
3950        }
3951
3952        fs::write(
3953            &journal_path,
3954            serde_json::to_vec(&journal).expect("serialize journal"),
3955        )
3956        .expect("write journal");
3957
3958        let err = run([
3959            OsString::from("apply-command"),
3960            OsString::from("--journal"),
3961            OsString::from(journal_path.as_os_str()),
3962            OsString::from("--out"),
3963            OsString::from(out_path.as_os_str()),
3964            OsString::from("--require-command"),
3965        ])
3966        .expect_err("missing command should fail");
3967
3968        let preview: RestoreApplyCommandPreview =
3969            serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
3970                .expect("decode command preview");
3971
3972        fs::remove_dir_all(root).expect("remove temp root");
3973        assert!(preview.complete);
3974        assert!(!preview.operation_available);
3975        assert!(!preview.command_available);
3976        assert!(matches!(
3977            err,
3978            RestoreCommandError::RestoreApplyCommandUnavailable {
3979                operation_available: false,
3980                complete: true,
3981                ..
3982            }
3983        ));
3984    }
3985
3986    // Ensure apply-claim marks the next operation pending before runner execution.
3987    #[test]
3988    fn run_restore_apply_claim_marks_next_operation_pending() {
3989        let root = temp_dir("canic-cli-restore-apply-claim");
3990        fs::create_dir_all(&root).expect("create temp root");
3991        let journal_path = root.join("restore-apply-journal.json");
3992        let claimed_path = root.join("restore-apply-journal.claimed.json");
3993        let journal = ready_apply_journal();
3994
3995        fs::write(
3996            &journal_path,
3997            serde_json::to_vec(&journal).expect("serialize journal"),
3998        )
3999        .expect("write journal");
4000
4001        run([
4002            OsString::from("apply-claim"),
4003            OsString::from("--journal"),
4004            OsString::from(journal_path.as_os_str()),
4005            OsString::from("--sequence"),
4006            OsString::from("0"),
4007            OsString::from("--updated-at"),
4008            OsString::from("2026-05-04T12:00:00Z"),
4009            OsString::from("--out"),
4010            OsString::from(claimed_path.as_os_str()),
4011        ])
4012        .expect("claim operation");
4013
4014        let claimed: RestoreApplyJournal =
4015            serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
4016                .expect("decode claimed journal");
4017        let status = claimed.status();
4018        let next = claimed.next_operation();
4019
4020        fs::remove_dir_all(root).expect("remove temp root");
4021        assert_eq!(claimed.pending_operations, 1);
4022        assert_eq!(claimed.ready_operations, 7);
4023        assert_eq!(
4024            claimed.operations[0].state,
4025            RestoreApplyOperationState::Pending
4026        );
4027        assert_eq!(
4028            claimed.operations[0].state_updated_at.as_deref(),
4029            Some("2026-05-04T12:00:00Z")
4030        );
4031        assert_eq!(status.next_transition_sequence, Some(0));
4032        assert_eq!(
4033            status.next_transition_state,
4034            Some(RestoreApplyOperationState::Pending)
4035        );
4036        assert_eq!(
4037            status.next_transition_updated_at.as_deref(),
4038            Some("2026-05-04T12:00:00Z")
4039        );
4040        assert_eq!(
4041            next.operation.expect("next operation").state,
4042            RestoreApplyOperationState::Pending
4043        );
4044    }
4045
4046    // Ensure apply-claim can reject a stale command preview sequence.
4047    #[test]
4048    fn run_restore_apply_claim_rejects_sequence_mismatch() {
4049        let root = temp_dir("canic-cli-restore-apply-claim-sequence");
4050        fs::create_dir_all(&root).expect("create temp root");
4051        let journal_path = root.join("restore-apply-journal.json");
4052        let claimed_path = root.join("restore-apply-journal.claimed.json");
4053        let journal = ready_apply_journal();
4054
4055        fs::write(
4056            &journal_path,
4057            serde_json::to_vec(&journal).expect("serialize journal"),
4058        )
4059        .expect("write journal");
4060
4061        let err = run([
4062            OsString::from("apply-claim"),
4063            OsString::from("--journal"),
4064            OsString::from(journal_path.as_os_str()),
4065            OsString::from("--sequence"),
4066            OsString::from("1"),
4067            OsString::from("--out"),
4068            OsString::from(claimed_path.as_os_str()),
4069        ])
4070        .expect_err("stale sequence should fail claim");
4071
4072        assert!(!claimed_path.exists());
4073        fs::remove_dir_all(root).expect("remove temp root");
4074        assert!(matches!(
4075            err,
4076            RestoreCommandError::RestoreApplyClaimSequenceMismatch {
4077                expected: 1,
4078                actual: Some(0),
4079            }
4080        ));
4081    }
4082
4083    // Ensure apply-unclaim releases the current pending operation back to ready.
4084    #[test]
4085    fn run_restore_apply_unclaim_marks_pending_operation_ready() {
4086        let root = temp_dir("canic-cli-restore-apply-unclaim");
4087        fs::create_dir_all(&root).expect("create temp root");
4088        let journal_path = root.join("restore-apply-journal.json");
4089        let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
4090        let mut journal = ready_apply_journal();
4091        journal
4092            .mark_next_operation_pending()
4093            .expect("claim operation");
4094
4095        fs::write(
4096            &journal_path,
4097            serde_json::to_vec(&journal).expect("serialize journal"),
4098        )
4099        .expect("write journal");
4100
4101        run([
4102            OsString::from("apply-unclaim"),
4103            OsString::from("--journal"),
4104            OsString::from(journal_path.as_os_str()),
4105            OsString::from("--sequence"),
4106            OsString::from("0"),
4107            OsString::from("--updated-at"),
4108            OsString::from("2026-05-04T12:01:00Z"),
4109            OsString::from("--out"),
4110            OsString::from(unclaimed_path.as_os_str()),
4111        ])
4112        .expect("unclaim operation");
4113
4114        let unclaimed: RestoreApplyJournal =
4115            serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
4116                .expect("decode unclaimed journal");
4117        let status = unclaimed.status();
4118
4119        fs::remove_dir_all(root).expect("remove temp root");
4120        assert_eq!(unclaimed.pending_operations, 0);
4121        assert_eq!(unclaimed.ready_operations, 8);
4122        assert_eq!(
4123            unclaimed.operations[0].state,
4124            RestoreApplyOperationState::Ready
4125        );
4126        assert_eq!(
4127            unclaimed.operations[0].state_updated_at.as_deref(),
4128            Some("2026-05-04T12:01:00Z")
4129        );
4130        assert_eq!(status.next_ready_sequence, Some(0));
4131        assert_eq!(
4132            status.next_transition_state,
4133            Some(RestoreApplyOperationState::Ready)
4134        );
4135        assert_eq!(
4136            status.next_transition_updated_at.as_deref(),
4137            Some("2026-05-04T12:01:00Z")
4138        );
4139    }
4140
4141    // Ensure apply-unclaim can reject a stale pending operation sequence.
4142    #[test]
4143    fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
4144        let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
4145        fs::create_dir_all(&root).expect("create temp root");
4146        let journal_path = root.join("restore-apply-journal.json");
4147        let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
4148        let mut journal = ready_apply_journal();
4149        journal
4150            .mark_next_operation_pending()
4151            .expect("claim operation");
4152
4153        fs::write(
4154            &journal_path,
4155            serde_json::to_vec(&journal).expect("serialize journal"),
4156        )
4157        .expect("write journal");
4158
4159        let err = run([
4160            OsString::from("apply-unclaim"),
4161            OsString::from("--journal"),
4162            OsString::from(journal_path.as_os_str()),
4163            OsString::from("--sequence"),
4164            OsString::from("1"),
4165            OsString::from("--out"),
4166            OsString::from(unclaimed_path.as_os_str()),
4167        ])
4168        .expect_err("stale sequence should fail unclaim");
4169
4170        assert!(!unclaimed_path.exists());
4171        fs::remove_dir_all(root).expect("remove temp root");
4172        assert!(matches!(
4173            err,
4174            RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
4175                expected: 1,
4176                actual: Some(0),
4177            }
4178        ));
4179    }
4180
4181    // Ensure apply-mark can advance one journal operation and keep counts consistent.
4182    #[test]
4183    fn run_restore_apply_mark_completes_operation() {
4184        let root = temp_dir("canic-cli-restore-apply-mark-complete");
4185        fs::create_dir_all(&root).expect("create temp root");
4186        let journal_path = root.join("restore-apply-journal.json");
4187        let updated_path = root.join("restore-apply-journal.updated.json");
4188        let journal = ready_apply_journal();
4189
4190        fs::write(
4191            &journal_path,
4192            serde_json::to_vec(&journal).expect("serialize journal"),
4193        )
4194        .expect("write journal");
4195
4196        run([
4197            OsString::from("apply-mark"),
4198            OsString::from("--journal"),
4199            OsString::from(journal_path.as_os_str()),
4200            OsString::from("--sequence"),
4201            OsString::from("0"),
4202            OsString::from("--state"),
4203            OsString::from("completed"),
4204            OsString::from("--updated-at"),
4205            OsString::from("2026-05-04T12:02:00Z"),
4206            OsString::from("--out"),
4207            OsString::from(updated_path.as_os_str()),
4208        ])
4209        .expect("mark operation completed");
4210
4211        let updated: RestoreApplyJournal =
4212            serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
4213                .expect("decode updated journal");
4214        let status = updated.status();
4215
4216        fs::remove_dir_all(root).expect("remove temp root");
4217        assert_eq!(updated.completed_operations, 1);
4218        assert_eq!(updated.ready_operations, 7);
4219        assert_eq!(
4220            updated.operations[0].state_updated_at.as_deref(),
4221            Some("2026-05-04T12:02:00Z")
4222        );
4223        assert_eq!(status.next_ready_sequence, Some(1));
4224    }
4225
4226    // Ensure apply-mark can require an operation claim before completion.
4227    #[test]
4228    fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
4229        let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
4230        fs::create_dir_all(&root).expect("create temp root");
4231        let journal_path = root.join("restore-apply-journal.json");
4232        let updated_path = root.join("restore-apply-journal.updated.json");
4233        let journal = ready_apply_journal();
4234
4235        fs::write(
4236            &journal_path,
4237            serde_json::to_vec(&journal).expect("serialize journal"),
4238        )
4239        .expect("write journal");
4240
4241        let err = run([
4242            OsString::from("apply-mark"),
4243            OsString::from("--journal"),
4244            OsString::from(journal_path.as_os_str()),
4245            OsString::from("--sequence"),
4246            OsString::from("0"),
4247            OsString::from("--state"),
4248            OsString::from("completed"),
4249            OsString::from("--out"),
4250            OsString::from(updated_path.as_os_str()),
4251            OsString::from("--require-pending"),
4252        ])
4253        .expect_err("ready operation should fail pending requirement");
4254
4255        assert!(!updated_path.exists());
4256        fs::remove_dir_all(root).expect("remove temp root");
4257        assert!(matches!(
4258            err,
4259            RestoreCommandError::RestoreApplyMarkRequiresPending {
4260                sequence: 0,
4261                state: RestoreApplyOperationState::Ready,
4262            }
4263        ));
4264    }
4265
4266    // Ensure apply-mark refuses to skip earlier ready operations.
4267    #[test]
4268    fn run_restore_apply_mark_rejects_out_of_order_operation() {
4269        let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
4270        fs::create_dir_all(&root).expect("create temp root");
4271        let journal_path = root.join("restore-apply-journal.json");
4272        let updated_path = root.join("restore-apply-journal.updated.json");
4273        let journal = ready_apply_journal();
4274
4275        fs::write(
4276            &journal_path,
4277            serde_json::to_vec(&journal).expect("serialize journal"),
4278        )
4279        .expect("write journal");
4280
4281        let err = run([
4282            OsString::from("apply-mark"),
4283            OsString::from("--journal"),
4284            OsString::from(journal_path.as_os_str()),
4285            OsString::from("--sequence"),
4286            OsString::from("1"),
4287            OsString::from("--state"),
4288            OsString::from("completed"),
4289            OsString::from("--out"),
4290            OsString::from(updated_path.as_os_str()),
4291        ])
4292        .expect_err("out-of-order operation should fail");
4293
4294        assert!(!updated_path.exists());
4295        fs::remove_dir_all(root).expect("remove temp root");
4296        assert!(matches!(
4297            err,
4298            RestoreCommandError::RestoreApplyJournal(
4299                RestoreApplyJournalError::OutOfOrderOperationTransition {
4300                    requested: 1,
4301                    next: 0
4302                }
4303            )
4304        ));
4305    }
4306
4307    // Ensure apply-mark requires failure reasons for failed operation state.
4308    #[test]
4309    fn run_restore_apply_mark_failed_requires_reason() {
4310        let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
4311        fs::create_dir_all(&root).expect("create temp root");
4312        let journal_path = root.join("restore-apply-journal.json");
4313        let journal = ready_apply_journal();
4314
4315        fs::write(
4316            &journal_path,
4317            serde_json::to_vec(&journal).expect("serialize journal"),
4318        )
4319        .expect("write journal");
4320
4321        let err = run([
4322            OsString::from("apply-mark"),
4323            OsString::from("--journal"),
4324            OsString::from(journal_path.as_os_str()),
4325            OsString::from("--sequence"),
4326            OsString::from("0"),
4327            OsString::from("--state"),
4328            OsString::from("failed"),
4329        ])
4330        .expect_err("failed state should require reason");
4331
4332        fs::remove_dir_all(root).expect("remove temp root");
4333        assert!(matches!(
4334            err,
4335            RestoreCommandError::RestoreApplyJournal(
4336                RestoreApplyJournalError::FailureReasonRequired(0)
4337            )
4338        ));
4339    }
4340
4341    // Ensure restore apply dry-run rejects status files from another plan.
4342    #[test]
4343    fn run_restore_apply_dry_run_rejects_mismatched_status() {
4344        let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
4345        fs::create_dir_all(&root).expect("create temp root");
4346        let plan_path = root.join("restore-plan.json");
4347        let status_path = root.join("restore-status.json");
4348        let out_path = root.join("restore-apply-dry-run.json");
4349        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4350        let mut status = RestoreStatus::from_plan(&plan);
4351        status.backup_id = "other-backup".to_string();
4352
4353        fs::write(
4354            &plan_path,
4355            serde_json::to_vec(&plan).expect("serialize plan"),
4356        )
4357        .expect("write plan");
4358        fs::write(
4359            &status_path,
4360            serde_json::to_vec(&status).expect("serialize status"),
4361        )
4362        .expect("write status");
4363
4364        let err = run([
4365            OsString::from("apply"),
4366            OsString::from("--plan"),
4367            OsString::from(plan_path.as_os_str()),
4368            OsString::from("--status"),
4369            OsString::from(status_path.as_os_str()),
4370            OsString::from("--dry-run"),
4371            OsString::from("--out"),
4372            OsString::from(out_path.as_os_str()),
4373        ])
4374        .expect_err("mismatched status should fail");
4375
4376        assert!(!out_path.exists());
4377        fs::remove_dir_all(root).expect("remove temp root");
4378        assert!(matches!(
4379            err,
4380            RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
4381                field: "backup_id",
4382                ..
4383            })
4384        ));
4385    }
4386
4387    // Build one manually ready apply journal for runner-focused CLI tests.
4388    fn ready_apply_journal() -> RestoreApplyJournal {
4389        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
4390        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
4391        let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
4392
4393        journal.ready = true;
4394        journal.blocked_reasons = Vec::new();
4395        for operation in &mut journal.operations {
4396            operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
4397            operation.blocking_reasons = Vec::new();
4398        }
4399        journal.blocked_operations = 0;
4400        journal.ready_operations = journal.operation_count;
4401        journal.validate().expect("journal should validate");
4402        journal
4403    }
4404
4405    // Build one valid manifest for restore planning tests.
4406    fn valid_manifest() -> FleetBackupManifest {
4407        FleetBackupManifest {
4408            manifest_version: 1,
4409            backup_id: "backup-test".to_string(),
4410            created_at: "2026-05-03T00:00:00Z".to_string(),
4411            tool: ToolMetadata {
4412                name: "canic".to_string(),
4413                version: "0.30.1".to_string(),
4414            },
4415            source: SourceMetadata {
4416                environment: "local".to_string(),
4417                root_canister: ROOT.to_string(),
4418            },
4419            consistency: ConsistencySection {
4420                mode: ConsistencyMode::CrashConsistent,
4421                backup_units: vec![BackupUnit {
4422                    unit_id: "fleet".to_string(),
4423                    kind: BackupUnitKind::SubtreeRooted,
4424                    roles: vec!["root".to_string(), "app".to_string()],
4425                    consistency_reason: None,
4426                    dependency_closure: Vec::new(),
4427                    topology_validation: "subtree-closed".to_string(),
4428                    quiescence_strategy: None,
4429                }],
4430            },
4431            fleet: FleetSection {
4432                topology_hash_algorithm: "sha256".to_string(),
4433                topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
4434                discovery_topology_hash: HASH.to_string(),
4435                pre_snapshot_topology_hash: HASH.to_string(),
4436                topology_hash: HASH.to_string(),
4437                members: vec![
4438                    fleet_member("root", ROOT, None, IdentityMode::Fixed),
4439                    fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
4440                ],
4441            },
4442            verification: VerificationPlan::default(),
4443        }
4444    }
4445
4446    // Build one manifest whose restore readiness metadata is complete.
4447    fn restore_ready_manifest() -> FleetBackupManifest {
4448        let mut manifest = valid_manifest();
4449        for member in &mut manifest.fleet.members {
4450            member.source_snapshot.module_hash = Some(HASH.to_string());
4451            member.source_snapshot.wasm_hash = Some(HASH.to_string());
4452            member.source_snapshot.checksum = Some(HASH.to_string());
4453        }
4454        manifest
4455    }
4456
4457    // Build one valid manifest member.
4458    fn fleet_member(
4459        role: &str,
4460        canister_id: &str,
4461        parent_canister_id: Option<&str>,
4462        identity_mode: IdentityMode,
4463    ) -> FleetMember {
4464        FleetMember {
4465            role: role.to_string(),
4466            canister_id: canister_id.to_string(),
4467            parent_canister_id: parent_canister_id.map(str::to_string),
4468            subnet_canister_id: Some(ROOT.to_string()),
4469            controller_hint: None,
4470            identity_mode,
4471            restore_group: 1,
4472            verification_class: "basic".to_string(),
4473            verification_checks: vec![VerificationCheck {
4474                kind: "status".to_string(),
4475                method: None,
4476                roles: vec![role.to_string()],
4477            }],
4478            source_snapshot: SourceSnapshot {
4479                snapshot_id: format!("{role}-snapshot"),
4480                module_hash: None,
4481                wasm_hash: None,
4482                code_version: Some("v0.30.1".to_string()),
4483                artifact_path: format!("artifacts/{role}"),
4484                checksum_algorithm: "sha256".to_string(),
4485                checksum: None,
4486            },
4487        }
4488    }
4489
4490    // Write a canonical backup layout whose journal checksums match the artifacts.
4491    fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
4492        layout.write_manifest(manifest).expect("write manifest");
4493
4494        let artifacts = manifest
4495            .fleet
4496            .members
4497            .iter()
4498            .map(|member| {
4499                let bytes = format!("{} artifact", member.role);
4500                let artifact_path = root.join(&member.source_snapshot.artifact_path);
4501                if let Some(parent) = artifact_path.parent() {
4502                    fs::create_dir_all(parent).expect("create artifact parent");
4503                }
4504                fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4505                let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4506
4507                ArtifactJournalEntry {
4508                    canister_id: member.canister_id.clone(),
4509                    snapshot_id: member.source_snapshot.snapshot_id.clone(),
4510                    state: ArtifactState::Durable,
4511                    temp_path: None,
4512                    artifact_path: member.source_snapshot.artifact_path.clone(),
4513                    checksum_algorithm: checksum.algorithm,
4514                    checksum: Some(checksum.hash),
4515                    updated_at: "2026-05-03T00:00:00Z".to_string(),
4516                }
4517            })
4518            .collect();
4519
4520        layout
4521            .write_journal(&DownloadJournal {
4522                journal_version: 1,
4523                backup_id: manifest.backup_id.clone(),
4524                discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
4525                pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
4526                operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
4527                artifacts,
4528            })
4529            .expect("write journal");
4530    }
4531
4532    // Write artifact bytes and update the manifest checksums for apply validation.
4533    fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
4534        for member in &mut manifest.fleet.members {
4535            let bytes = format!("{} apply artifact", member.role);
4536            let artifact_path = root.join(&member.source_snapshot.artifact_path);
4537            if let Some(parent) = artifact_path.parent() {
4538                fs::create_dir_all(parent).expect("create artifact parent");
4539            }
4540            fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
4541            let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
4542            member.source_snapshot.checksum = Some(checksum.hash);
4543        }
4544    }
4545
4546    // Build a unique temporary directory.
4547    fn temp_dir(prefix: &str) -> PathBuf {
4548        let nanos = SystemTime::now()
4549            .duration_since(UNIX_EPOCH)
4550            .expect("system time after epoch")
4551            .as_nanos();
4552        std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
4553    }
4554}