Skip to main content

canic_cli/restore/
mod.rs

1use canic_backup::{
2    manifest::FleetBackupManifest,
3    persistence::{BackupLayout, PersistenceError},
4    restore::{
5        RestoreApplyCommandConfig, RestoreApplyCommandPreview, RestoreApplyDryRun,
6        RestoreApplyDryRunError, RestoreApplyJournal, RestoreApplyJournalError,
7        RestoreApplyJournalOperation, RestoreApplyJournalReport, RestoreApplyJournalStatus,
8        RestoreApplyNextOperation, RestoreApplyOperationKind, RestoreApplyOperationKindCounts,
9        RestoreApplyOperationState, RestoreApplyPendingSummary, RestoreApplyProgressSummary,
10        RestoreApplyReportOperation, RestoreApplyReportOutcome, RestoreApplyRunnerCommand,
11        RestoreMapping, RestorePlan, RestorePlanError, RestorePlanner, RestoreStatus,
12    },
13};
14use serde::Serialize;
15use std::{
16    ffi::OsString,
17    fs,
18    io::{self, Write},
19    path::PathBuf,
20    process::Command,
21};
22use thiserror::Error as ThisError;
23
24///
25/// RestoreCommandError
26///
27
28#[derive(Debug, ThisError)]
29pub enum RestoreCommandError {
30    #[error("{0}")]
31    Usage(&'static str),
32
33    #[error("missing required option {0}")]
34    MissingOption(&'static str),
35
36    #[error("use either --manifest or --backup-dir, not both")]
37    ConflictingManifestSources,
38
39    #[error("--require-verified requires --backup-dir")]
40    RequireVerifiedNeedsBackupDir,
41
42    #[error("restore apply currently requires --dry-run")]
43    ApplyRequiresDryRun,
44
45    #[error("restore run requires --dry-run, --execute, or --unclaim-pending")]
46    RestoreRunRequiresMode,
47
48    #[error("use only one restore run mode: --dry-run, --execute, or --unclaim-pending")]
49    RestoreRunConflictingModes,
50
51    #[error("restore run command failed for operation {sequence}: status={status}")]
52    RestoreRunCommandFailed { sequence: usize, status: String },
53
54    #[error("restore run for backup {backup_id} used run_mode={actual}, expected {expected}")]
55    RestoreRunModeMismatch {
56        backup_id: String,
57        expected: String,
58        actual: String,
59    },
60
61    #[error(
62        "restore run for backup {backup_id} stopped for {actual}, expected stopped_reason={expected}"
63    )]
64    RestoreRunStoppedReasonMismatch {
65        backup_id: String,
66        expected: String,
67        actual: String,
68    },
69
70    #[error(
71        "restore run for backup {backup_id} reported next_action={actual}, expected {expected}"
72    )]
73    RestoreRunNextActionMismatch {
74        backup_id: String,
75        expected: String,
76        actual: String,
77    },
78
79    #[error("restore run for backup {backup_id} executed {actual} operations, expected {expected}")]
80    RestoreRunExecutedCountMismatch {
81        backup_id: String,
82        expected: usize,
83        actual: usize,
84    },
85
86    #[error("restore run for backup {backup_id} wrote {actual} receipts, expected {expected}")]
87    RestoreRunReceiptCountMismatch {
88        backup_id: String,
89        expected: usize,
90        actual: usize,
91    },
92
93    #[error(
94        "restore run for backup {backup_id} wrote {actual} {receipt_kind} receipts, expected {expected}"
95    )]
96    RestoreRunReceiptKindCountMismatch {
97        backup_id: String,
98        receipt_kind: &'static str,
99        expected: usize,
100        actual: usize,
101    },
102
103    #[error("restore plan for backup {backup_id} is not restore-ready: reasons={reasons:?}")]
104    RestoreNotReady {
105        backup_id: String,
106        reasons: Vec<String>,
107    },
108
109    #[error(
110        "restore apply journal for backup {backup_id} has pending operations: pending={pending_operations}, next={next_transition_sequence:?}"
111    )]
112    RestoreApplyPending {
113        backup_id: String,
114        pending_operations: usize,
115        next_transition_sequence: Option<usize>,
116    },
117
118    #[error(
119        "restore apply journal for backup {backup_id} has stale or untracked pending work before {cutoff_updated_at}: pending_sequence={pending_sequence:?}, pending_updated_at={pending_updated_at:?}"
120    )]
121    RestoreApplyPendingStale {
122        backup_id: String,
123        cutoff_updated_at: String,
124        pending_sequence: Option<usize>,
125        pending_updated_at: Option<String>,
126    },
127
128    #[error(
129        "restore apply journal for backup {backup_id} is incomplete: completed={completed_operations}, total={operation_count}"
130    )]
131    RestoreApplyIncomplete {
132        backup_id: String,
133        completed_operations: usize,
134        operation_count: usize,
135    },
136
137    #[error(
138        "restore apply journal for backup {backup_id} has failed operations: failed={failed_operations}"
139    )]
140    RestoreApplyFailed {
141        backup_id: String,
142        failed_operations: usize,
143    },
144
145    #[error("restore apply journal for backup {backup_id} is not ready: reasons={reasons:?}")]
146    RestoreApplyNotReady {
147        backup_id: String,
148        reasons: Vec<String>,
149    },
150
151    #[error("restore apply report for backup {backup_id} requires attention: outcome={outcome:?}")]
152    RestoreApplyReportNeedsAttention {
153        backup_id: String,
154        outcome: canic_backup::restore::RestoreApplyReportOutcome,
155    },
156
157    #[error(
158        "restore apply progress for backup {backup_id} has unexpected {field}: expected={expected}, actual={actual}"
159    )]
160    RestoreApplyProgressMismatch {
161        backup_id: String,
162        field: &'static str,
163        expected: usize,
164        actual: usize,
165    },
166
167    #[error(
168        "restore apply journal for backup {backup_id} has no executable command: operation_available={operation_available}, complete={complete}, blocked_reasons={blocked_reasons:?}"
169    )]
170    RestoreApplyCommandUnavailable {
171        backup_id: String,
172        operation_available: bool,
173        complete: bool,
174        blocked_reasons: Vec<String>,
175    },
176
177    #[error(
178        "restore apply journal operation {sequence} must be pending before apply-mark: state={state:?}"
179    )]
180    RestoreApplyMarkRequiresPending {
181        sequence: usize,
182        state: RestoreApplyOperationState,
183    },
184
185    #[error(
186        "restore apply journal next operation changed before claim: expected={expected}, actual={actual:?}"
187    )]
188    RestoreApplyClaimSequenceMismatch {
189        expected: usize,
190        actual: Option<usize>,
191    },
192
193    #[error(
194        "restore apply journal pending operation changed before unclaim: expected={expected}, actual={actual:?}"
195    )]
196    RestoreApplyUnclaimSequenceMismatch {
197        expected: usize,
198        actual: Option<usize>,
199    },
200
201    #[error("unknown option {0}")]
202    UnknownOption(String),
203
204    #[error("option {0} requires a value")]
205    MissingValue(&'static str),
206
207    #[error("option --sequence requires a non-negative integer value")]
208    InvalidSequence,
209
210    #[error("unsupported apply-mark state {0}; use completed or failed")]
211    InvalidApplyMarkState(String),
212
213    #[error(transparent)]
214    Io(#[from] std::io::Error),
215
216    #[error(transparent)]
217    Json(#[from] serde_json::Error),
218
219    #[error(transparent)]
220    Persistence(#[from] PersistenceError),
221
222    #[error(transparent)]
223    RestorePlan(#[from] RestorePlanError),
224
225    #[error(transparent)]
226    RestoreApplyDryRun(#[from] RestoreApplyDryRunError),
227
228    #[error(transparent)]
229    RestoreApplyJournal(#[from] RestoreApplyJournalError),
230}
231
232///
233/// RestorePlanOptions
234///
235
236#[derive(Clone, Debug, Eq, PartialEq)]
237pub struct RestorePlanOptions {
238    pub manifest: Option<PathBuf>,
239    pub backup_dir: Option<PathBuf>,
240    pub mapping: Option<PathBuf>,
241    pub out: Option<PathBuf>,
242    pub require_verified: bool,
243    pub require_restore_ready: bool,
244}
245
246impl RestorePlanOptions {
247    /// Parse restore planning options from CLI arguments.
248    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
249    where
250        I: IntoIterator<Item = OsString>,
251    {
252        let mut manifest = None;
253        let mut backup_dir = None;
254        let mut mapping = None;
255        let mut out = None;
256        let mut require_verified = false;
257        let mut require_restore_ready = false;
258
259        let mut args = args.into_iter();
260        while let Some(arg) = args.next() {
261            let arg = arg
262                .into_string()
263                .map_err(|_| RestoreCommandError::Usage(usage()))?;
264            match arg.as_str() {
265                "--manifest" => {
266                    manifest = Some(PathBuf::from(next_value(&mut args, "--manifest")?));
267                }
268                "--backup-dir" => {
269                    backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
270                }
271                "--mapping" => mapping = Some(PathBuf::from(next_value(&mut args, "--mapping")?)),
272                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
273                "--require-verified" => require_verified = true,
274                "--require-restore-ready" => require_restore_ready = true,
275                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
276                _ => return Err(RestoreCommandError::UnknownOption(arg)),
277            }
278        }
279
280        if manifest.is_some() && backup_dir.is_some() {
281            return Err(RestoreCommandError::ConflictingManifestSources);
282        }
283
284        if manifest.is_none() && backup_dir.is_none() {
285            return Err(RestoreCommandError::MissingOption(
286                "--manifest or --backup-dir",
287            ));
288        }
289
290        if require_verified && backup_dir.is_none() {
291            return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
292        }
293
294        Ok(Self {
295            manifest,
296            backup_dir,
297            mapping,
298            out,
299            require_verified,
300            require_restore_ready,
301        })
302    }
303}
304
305///
306/// RestoreStatusOptions
307///
308
309#[derive(Clone, Debug, Eq, PartialEq)]
310pub struct RestoreStatusOptions {
311    pub plan: PathBuf,
312    pub out: Option<PathBuf>,
313}
314
315impl RestoreStatusOptions {
316    /// Parse restore status options from CLI arguments.
317    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
318    where
319        I: IntoIterator<Item = OsString>,
320    {
321        let mut plan = None;
322        let mut out = None;
323
324        let mut args = args.into_iter();
325        while let Some(arg) = args.next() {
326            let arg = arg
327                .into_string()
328                .map_err(|_| RestoreCommandError::Usage(usage()))?;
329            match arg.as_str() {
330                "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
331                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
332                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
333                _ => return Err(RestoreCommandError::UnknownOption(arg)),
334            }
335        }
336
337        Ok(Self {
338            plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
339            out,
340        })
341    }
342}
343
344///
345/// RestoreApplyOptions
346///
347
348#[derive(Clone, Debug, Eq, PartialEq)]
349pub struct RestoreApplyOptions {
350    pub plan: PathBuf,
351    pub status: Option<PathBuf>,
352    pub backup_dir: Option<PathBuf>,
353    pub out: Option<PathBuf>,
354    pub journal_out: Option<PathBuf>,
355    pub dry_run: bool,
356}
357
358impl RestoreApplyOptions {
359    /// Parse restore apply options from CLI arguments.
360    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
361    where
362        I: IntoIterator<Item = OsString>,
363    {
364        let mut plan = None;
365        let mut status = None;
366        let mut backup_dir = None;
367        let mut out = None;
368        let mut journal_out = None;
369        let mut dry_run = false;
370
371        let mut args = args.into_iter();
372        while let Some(arg) = args.next() {
373            let arg = arg
374                .into_string()
375                .map_err(|_| RestoreCommandError::Usage(usage()))?;
376            match arg.as_str() {
377                "--plan" => plan = Some(PathBuf::from(next_value(&mut args, "--plan")?)),
378                "--status" => status = Some(PathBuf::from(next_value(&mut args, "--status")?)),
379                "--backup-dir" => {
380                    backup_dir = Some(PathBuf::from(next_value(&mut args, "--backup-dir")?));
381                }
382                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
383                "--journal-out" => {
384                    journal_out = Some(PathBuf::from(next_value(&mut args, "--journal-out")?));
385                }
386                "--dry-run" => dry_run = true,
387                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
388                _ => return Err(RestoreCommandError::UnknownOption(arg)),
389            }
390        }
391
392        if !dry_run {
393            return Err(RestoreCommandError::ApplyRequiresDryRun);
394        }
395
396        Ok(Self {
397            plan: plan.ok_or(RestoreCommandError::MissingOption("--plan"))?,
398            status,
399            backup_dir,
400            out,
401            journal_out,
402            dry_run,
403        })
404    }
405}
406
407///
408/// RestoreApplyStatusOptions
409///
410
411#[derive(Clone, Debug, Eq, PartialEq)]
412#[expect(
413    clippy::struct_excessive_bools,
414    reason = "CLI status options mirror independent fail-closed guard flags"
415)]
416pub struct RestoreApplyStatusOptions {
417    pub journal: PathBuf,
418    pub require_ready: bool,
419    pub require_no_pending: bool,
420    pub require_no_failed: bool,
421    pub require_complete: bool,
422    pub require_remaining_count: Option<usize>,
423    pub require_attention_count: Option<usize>,
424    pub require_completion_basis_points: Option<usize>,
425    pub require_no_pending_before: Option<String>,
426    pub out: Option<PathBuf>,
427}
428
429impl RestoreApplyStatusOptions {
430    /// Parse restore apply-status options from CLI arguments.
431    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
432    where
433        I: IntoIterator<Item = OsString>,
434    {
435        let mut journal = None;
436        let mut require_ready = false;
437        let mut require_no_pending = false;
438        let mut require_no_failed = false;
439        let mut require_complete = false;
440        let mut require_remaining_count = None;
441        let mut require_attention_count = None;
442        let mut require_completion_basis_points = None;
443        let mut require_no_pending_before = None;
444        let mut out = None;
445
446        let mut args = args.into_iter();
447        while let Some(arg) = args.next() {
448            let arg = arg
449                .into_string()
450                .map_err(|_| RestoreCommandError::Usage(usage()))?;
451            if parse_progress_requirement_option(
452                arg.as_str(),
453                &mut args,
454                &mut require_remaining_count,
455                &mut require_attention_count,
456                &mut require_completion_basis_points,
457            )? {
458                continue;
459            }
460            if parse_pending_requirement_option(
461                arg.as_str(),
462                &mut args,
463                &mut require_no_pending_before,
464            )? {
465                continue;
466            }
467            match arg.as_str() {
468                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
469                "--require-ready" => require_ready = true,
470                "--require-no-pending" => require_no_pending = true,
471                "--require-no-failed" => require_no_failed = true,
472                "--require-complete" => require_complete = true,
473                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
474                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
475                _ => return Err(RestoreCommandError::UnknownOption(arg)),
476            }
477        }
478
479        Ok(Self {
480            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
481            require_ready,
482            require_no_pending,
483            require_no_failed,
484            require_complete,
485            require_remaining_count,
486            require_attention_count,
487            require_completion_basis_points,
488            require_no_pending_before,
489            out,
490        })
491    }
492}
493
494///
495/// RestoreApplyReportOptions
496///
497
498#[derive(Clone, Debug, Eq, PartialEq)]
499pub struct RestoreApplyReportOptions {
500    pub journal: PathBuf,
501    pub require_no_attention: bool,
502    pub require_remaining_count: Option<usize>,
503    pub require_attention_count: Option<usize>,
504    pub require_completion_basis_points: Option<usize>,
505    pub require_no_pending_before: Option<String>,
506    pub out: Option<PathBuf>,
507}
508
509impl RestoreApplyReportOptions {
510    /// Parse restore apply-report options from CLI arguments.
511    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
512    where
513        I: IntoIterator<Item = OsString>,
514    {
515        let mut journal = None;
516        let mut require_no_attention = false;
517        let mut require_remaining_count = None;
518        let mut require_attention_count = None;
519        let mut require_completion_basis_points = None;
520        let mut require_no_pending_before = None;
521        let mut out = None;
522
523        let mut args = args.into_iter();
524        while let Some(arg) = args.next() {
525            let arg = arg
526                .into_string()
527                .map_err(|_| RestoreCommandError::Usage(usage()))?;
528            if parse_progress_requirement_option(
529                arg.as_str(),
530                &mut args,
531                &mut require_remaining_count,
532                &mut require_attention_count,
533                &mut require_completion_basis_points,
534            )? {
535                continue;
536            }
537            if parse_pending_requirement_option(
538                arg.as_str(),
539                &mut args,
540                &mut require_no_pending_before,
541            )? {
542                continue;
543            }
544            match arg.as_str() {
545                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
546                "--require-no-attention" => require_no_attention = true,
547                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
548                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
549                _ => return Err(RestoreCommandError::UnknownOption(arg)),
550            }
551        }
552
553        Ok(Self {
554            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
555            require_no_attention,
556            require_remaining_count,
557            require_attention_count,
558            require_completion_basis_points,
559            require_no_pending_before,
560            out,
561        })
562    }
563}
564
565///
566/// RestoreRunOptions
567///
568
569#[derive(Clone, Debug, Eq, PartialEq)]
570#[expect(
571    clippy::struct_excessive_bools,
572    reason = "CLI runner options mirror independent mode and fail-closed guard flags"
573)]
574pub struct RestoreRunOptions {
575    pub journal: PathBuf,
576    pub dfx: String,
577    pub network: Option<String>,
578    pub out: Option<PathBuf>,
579    pub dry_run: bool,
580    pub execute: bool,
581    pub unclaim_pending: bool,
582    pub max_steps: Option<usize>,
583    pub require_complete: bool,
584    pub require_no_attention: bool,
585    pub require_run_mode: Option<String>,
586    pub require_stopped_reason: Option<String>,
587    pub require_next_action: Option<String>,
588    pub require_executed_count: Option<usize>,
589    pub require_receipt_count: Option<usize>,
590    pub require_completed_receipt_count: Option<usize>,
591    pub require_failed_receipt_count: Option<usize>,
592    pub require_recovered_receipt_count: Option<usize>,
593    pub require_remaining_count: Option<usize>,
594    pub require_attention_count: Option<usize>,
595    pub require_completion_basis_points: Option<usize>,
596    pub require_no_pending_before: Option<String>,
597}
598
599impl RestoreRunOptions {
600    /// Parse restore run options from CLI arguments.
601    #[expect(
602        clippy::too_many_lines,
603        reason = "Restore runner options intentionally parse a broad flat CLI surface"
604    )]
605    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
606    where
607        I: IntoIterator<Item = OsString>,
608    {
609        let mut journal = None;
610        let mut dfx = "dfx".to_string();
611        let mut network = None;
612        let mut out = None;
613        let mut dry_run = false;
614        let mut execute = false;
615        let mut unclaim_pending = false;
616        let mut max_steps = None;
617        let mut require_complete = false;
618        let mut require_no_attention = false;
619        let mut require_run_mode = None;
620        let mut require_stopped_reason = None;
621        let mut require_next_action = None;
622        let mut require_executed_count = None;
623        let mut require_receipt_count = None;
624        let mut require_completed_receipt_count = None;
625        let mut require_failed_receipt_count = None;
626        let mut require_recovered_receipt_count = None;
627        let mut require_remaining_count = None;
628        let mut require_attention_count = None;
629        let mut require_completion_basis_points = None;
630        let mut require_no_pending_before = None;
631
632        let mut args = args.into_iter();
633        while let Some(arg) = args.next() {
634            let arg = arg
635                .into_string()
636                .map_err(|_| RestoreCommandError::Usage(usage()))?;
637            if parse_progress_requirement_option(
638                arg.as_str(),
639                &mut args,
640                &mut require_remaining_count,
641                &mut require_attention_count,
642                &mut require_completion_basis_points,
643            )? {
644                continue;
645            }
646            if parse_pending_requirement_option(
647                arg.as_str(),
648                &mut args,
649                &mut require_no_pending_before,
650            )? {
651                continue;
652            }
653            if parse_run_count_requirement_option(
654                arg.as_str(),
655                &mut args,
656                &mut require_executed_count,
657                &mut require_receipt_count,
658            )? {
659                continue;
660            }
661            if parse_run_receipt_kind_requirement_option(
662                arg.as_str(),
663                &mut args,
664                &mut require_completed_receipt_count,
665                &mut require_failed_receipt_count,
666                &mut require_recovered_receipt_count,
667            )? {
668                continue;
669            }
670
671            match arg.as_str() {
672                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
673                "--dfx" => dfx = next_value(&mut args, "--dfx")?,
674                "--network" => network = Some(next_value(&mut args, "--network")?),
675                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
676                "--dry-run" => dry_run = true,
677                "--execute" => execute = true,
678                "--unclaim-pending" => unclaim_pending = true,
679                "--max-steps" => {
680                    max_steps = Some(parse_sequence(next_value(&mut args, "--max-steps")?)?);
681                }
682                "--require-complete" => require_complete = true,
683                "--require-no-attention" => require_no_attention = true,
684                "--require-run-mode" => {
685                    require_run_mode = Some(next_value(&mut args, "--require-run-mode")?);
686                }
687                "--require-stopped-reason" => {
688                    require_stopped_reason =
689                        Some(next_value(&mut args, "--require-stopped-reason")?);
690                }
691                "--require-next-action" => {
692                    require_next_action = Some(next_value(&mut args, "--require-next-action")?);
693                }
694                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
695                _ => return Err(RestoreCommandError::UnknownOption(arg)),
696            }
697        }
698
699        validate_restore_run_mode_selection(dry_run, execute, unclaim_pending)?;
700
701        Ok(Self {
702            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
703            dfx,
704            network,
705            out,
706            dry_run,
707            execute,
708            unclaim_pending,
709            max_steps,
710            require_complete,
711            require_no_attention,
712            require_run_mode,
713            require_stopped_reason,
714            require_next_action,
715            require_executed_count,
716            require_receipt_count,
717            require_completed_receipt_count,
718            require_failed_receipt_count,
719            require_recovered_receipt_count,
720            require_remaining_count,
721            require_attention_count,
722            require_completion_basis_points,
723            require_no_pending_before,
724        })
725    }
726}
727
728// Validate that restore run received exactly one execution mode.
729fn validate_restore_run_mode_selection(
730    dry_run: bool,
731    execute: bool,
732    unclaim_pending: bool,
733) -> Result<(), RestoreCommandError> {
734    let mode_count = [dry_run, execute, unclaim_pending]
735        .into_iter()
736        .filter(|enabled| *enabled)
737        .count();
738    if mode_count > 1 {
739        return Err(RestoreCommandError::RestoreRunConflictingModes);
740    }
741
742    if mode_count == 0 {
743        return Err(RestoreCommandError::RestoreRunRequiresMode);
744    }
745
746    Ok(())
747}
748
749///
750/// RestoreRunResult
751///
752
753struct RestoreRunResult {
754    response: RestoreRunResponse,
755    error: Option<RestoreCommandError>,
756}
757
758impl RestoreRunResult {
759    // Build a successful runner response with no deferred error.
760    const fn ok(response: RestoreRunResponse) -> Self {
761        Self {
762            response,
763            error: None,
764        }
765    }
766}
767
768const RESTORE_RUN_MODE_DRY_RUN: &str = "dry-run";
769const RESTORE_RUN_MODE_EXECUTE: &str = "execute";
770const RESTORE_RUN_MODE_UNCLAIM_PENDING: &str = "unclaim-pending";
771
772const RESTORE_RUN_STOPPED_BLOCKED: &str = "blocked";
773const RESTORE_RUN_STOPPED_COMMAND_FAILED: &str = "command-failed";
774const RESTORE_RUN_STOPPED_COMPLETE: &str = "complete";
775const RESTORE_RUN_STOPPED_MAX_STEPS: &str = "max-steps-reached";
776const RESTORE_RUN_STOPPED_PENDING: &str = "pending";
777const RESTORE_RUN_STOPPED_PREVIEW: &str = "preview";
778const RESTORE_RUN_STOPPED_READY: &str = "ready";
779const RESTORE_RUN_STOPPED_RECOVERED_PENDING: &str = "recovered-pending";
780
781const RESTORE_RUN_ACTION_DONE: &str = "done";
782const RESTORE_RUN_ACTION_FIX_BLOCKED: &str = "fix-blocked-journal";
783const RESTORE_RUN_ACTION_INSPECT_FAILED: &str = "inspect-failed-operation";
784const RESTORE_RUN_ACTION_RERUN: &str = "rerun";
785const RESTORE_RUN_ACTION_UNCLAIM_PENDING: &str = "unclaim-pending";
786
787const RESTORE_RUN_EXECUTED_COMPLETED: &str = "completed";
788const RESTORE_RUN_EXECUTED_FAILED: &str = "failed";
789const RESTORE_RUN_RECEIPT_COMPLETED: &str = "command-completed";
790const RESTORE_RUN_RECEIPT_FAILED: &str = "command-failed";
791const RESTORE_RUN_RECEIPT_RECOVERED_PENDING: &str = "pending-recovered";
792const RESTORE_RUN_RECEIPT_STATE_READY: &str = "ready";
793const RESTORE_RUN_COMMAND_EXIT_PREFIX: &str = "runner-command-exit";
794const RESTORE_RUN_RESPONSE_VERSION: u16 = 1;
795
796///
797/// RestoreRunResponse
798///
799
800#[derive(Clone, Debug, Serialize)]
801#[expect(
802    clippy::struct_excessive_bools,
803    reason = "Runner response exposes stable JSON status flags for operators and CI"
804)]
805pub struct RestoreRunResponse {
806    run_version: u16,
807    backup_id: String,
808    run_mode: &'static str,
809    dry_run: bool,
810    execute: bool,
811    unclaim_pending: bool,
812    stopped_reason: &'static str,
813    next_action: &'static str,
814    #[serde(skip_serializing_if = "Option::is_none")]
815    max_steps_reached: Option<bool>,
816    #[serde(default, skip_serializing_if = "Vec::is_empty")]
817    executed_operations: Vec<RestoreRunExecutedOperation>,
818    #[serde(default, skip_serializing_if = "Vec::is_empty")]
819    operation_receipts: Vec<RestoreRunOperationReceipt>,
820    #[serde(skip_serializing_if = "Option::is_none")]
821    operation_receipt_count: Option<usize>,
822    operation_receipt_summary: RestoreRunReceiptSummary,
823    #[serde(skip_serializing_if = "Option::is_none")]
824    executed_operation_count: Option<usize>,
825    #[serde(skip_serializing_if = "Option::is_none")]
826    recovered_operation: Option<RestoreApplyJournalOperation>,
827    ready: bool,
828    complete: bool,
829    attention_required: bool,
830    outcome: RestoreApplyReportOutcome,
831    operation_count: usize,
832    operation_counts: RestoreApplyOperationKindCounts,
833    operation_counts_supplied: bool,
834    progress: RestoreApplyProgressSummary,
835    pending_summary: RestoreApplyPendingSummary,
836    pending_operations: usize,
837    ready_operations: usize,
838    blocked_operations: usize,
839    completed_operations: usize,
840    failed_operations: usize,
841    blocked_reasons: Vec<String>,
842    next_transition: Option<RestoreApplyReportOperation>,
843    #[serde(skip_serializing_if = "Option::is_none")]
844    operation_available: Option<bool>,
845    #[serde(skip_serializing_if = "Option::is_none")]
846    command_available: Option<bool>,
847    #[serde(skip_serializing_if = "Option::is_none")]
848    command: Option<RestoreApplyRunnerCommand>,
849}
850
851impl RestoreRunResponse {
852    // Build the shared native runner response fields from an apply journal report.
853    fn from_report(
854        backup_id: String,
855        report: RestoreApplyJournalReport,
856        mode: RestoreRunResponseMode,
857    ) -> Self {
858        Self {
859            run_version: RESTORE_RUN_RESPONSE_VERSION,
860            backup_id,
861            run_mode: mode.run_mode,
862            dry_run: mode.dry_run,
863            execute: mode.execute,
864            unclaim_pending: mode.unclaim_pending,
865            stopped_reason: mode.stopped_reason,
866            next_action: mode.next_action,
867            max_steps_reached: None,
868            executed_operations: Vec::new(),
869            operation_receipts: Vec::new(),
870            operation_receipt_count: Some(0),
871            operation_receipt_summary: RestoreRunReceiptSummary::default(),
872            executed_operation_count: None,
873            recovered_operation: None,
874            ready: report.ready,
875            complete: report.complete,
876            attention_required: report.attention_required,
877            outcome: report.outcome,
878            operation_count: report.operation_count,
879            operation_counts: report.operation_counts,
880            operation_counts_supplied: report.operation_counts_supplied,
881            progress: report.progress,
882            pending_summary: report.pending_summary,
883            pending_operations: report.pending_operations,
884            ready_operations: report.ready_operations,
885            blocked_operations: report.blocked_operations,
886            completed_operations: report.completed_operations,
887            failed_operations: report.failed_operations,
888            blocked_reasons: report.blocked_reasons,
889            next_transition: report.next_transition,
890            operation_available: None,
891            command_available: None,
892            command: None,
893        }
894    }
895
896    // Replace the detailed receipt stream and refresh the compact counters.
897    fn set_operation_receipts(&mut self, receipts: Vec<RestoreRunOperationReceipt>) {
898        self.operation_receipt_summary = RestoreRunReceiptSummary::from_receipts(&receipts);
899        self.operation_receipt_count = Some(receipts.len());
900        self.operation_receipts = receipts;
901    }
902}
903
904///
905/// RestoreRunReceiptSummary
906///
907
908#[derive(Clone, Debug, Default, Serialize)]
909struct RestoreRunReceiptSummary {
910    total_receipts: usize,
911    command_completed: usize,
912    command_failed: usize,
913    pending_recovered: usize,
914}
915
916impl RestoreRunReceiptSummary {
917    // Count restore runner receipt classes for script-friendly summaries.
918    fn from_receipts(receipts: &[RestoreRunOperationReceipt]) -> Self {
919        let mut summary = Self {
920            total_receipts: receipts.len(),
921            ..Self::default()
922        };
923
924        for receipt in receipts {
925            match receipt.event {
926                RESTORE_RUN_RECEIPT_COMPLETED => summary.command_completed += 1,
927                RESTORE_RUN_RECEIPT_FAILED => summary.command_failed += 1,
928                RESTORE_RUN_RECEIPT_RECOVERED_PENDING => summary.pending_recovered += 1,
929                _ => {}
930            }
931        }
932
933        summary
934    }
935}
936
937///
938/// RestoreRunOperationReceipt
939///
940
941#[derive(Clone, Debug, Serialize)]
942struct RestoreRunOperationReceipt {
943    event: &'static str,
944    sequence: usize,
945    operation: RestoreApplyOperationKind,
946    target_canister: String,
947    state: &'static str,
948    #[serde(skip_serializing_if = "Option::is_none")]
949    updated_at: Option<String>,
950    #[serde(skip_serializing_if = "Option::is_none")]
951    command: Option<RestoreApplyRunnerCommand>,
952    #[serde(skip_serializing_if = "Option::is_none")]
953    status: Option<String>,
954}
955
956impl RestoreRunOperationReceipt {
957    // Build a receipt for a completed runner command.
958    fn completed(
959        operation: RestoreApplyJournalOperation,
960        command: RestoreApplyRunnerCommand,
961        status: String,
962        updated_at: Option<String>,
963    ) -> Self {
964        Self::from_operation(
965            RESTORE_RUN_RECEIPT_COMPLETED,
966            operation,
967            RESTORE_RUN_EXECUTED_COMPLETED,
968            updated_at,
969            Some(command),
970            Some(status),
971        )
972    }
973
974    // Build a receipt for a failed runner command.
975    fn failed(
976        operation: RestoreApplyJournalOperation,
977        command: RestoreApplyRunnerCommand,
978        status: String,
979        updated_at: Option<String>,
980    ) -> Self {
981        Self::from_operation(
982            RESTORE_RUN_RECEIPT_FAILED,
983            operation,
984            RESTORE_RUN_EXECUTED_FAILED,
985            updated_at,
986            Some(command),
987            Some(status),
988        )
989    }
990
991    // Build a receipt for a recovered pending operation.
992    fn recovered_pending(
993        operation: RestoreApplyJournalOperation,
994        updated_at: Option<String>,
995    ) -> Self {
996        Self::from_operation(
997            RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
998            operation,
999            RESTORE_RUN_RECEIPT_STATE_READY,
1000            updated_at,
1001            None,
1002            None,
1003        )
1004    }
1005
1006    // Map one operation event into a compact audit receipt.
1007    fn from_operation(
1008        event: &'static str,
1009        operation: RestoreApplyJournalOperation,
1010        state: &'static str,
1011        updated_at: Option<String>,
1012        command: Option<RestoreApplyRunnerCommand>,
1013        status: Option<String>,
1014    ) -> Self {
1015        Self {
1016            event,
1017            sequence: operation.sequence,
1018            operation: operation.operation,
1019            target_canister: operation.target_canister,
1020            state,
1021            updated_at,
1022            command,
1023            status,
1024        }
1025    }
1026}
1027
1028///
1029/// RestoreRunExecutedOperation
1030///
1031
1032#[derive(Clone, Debug, Serialize)]
1033struct RestoreRunExecutedOperation {
1034    sequence: usize,
1035    operation: RestoreApplyOperationKind,
1036    target_canister: String,
1037    command: RestoreApplyRunnerCommand,
1038    status: String,
1039    state: &'static str,
1040}
1041
1042impl RestoreRunExecutedOperation {
1043    // Build a completed executed-operation summary row from a runner operation.
1044    fn completed(
1045        operation: RestoreApplyJournalOperation,
1046        command: RestoreApplyRunnerCommand,
1047        status: String,
1048    ) -> Self {
1049        Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_COMPLETED)
1050    }
1051
1052    // Build a failed executed-operation summary row from a runner operation.
1053    fn failed(
1054        operation: RestoreApplyJournalOperation,
1055        command: RestoreApplyRunnerCommand,
1056        status: String,
1057    ) -> Self {
1058        Self::from_operation(operation, command, status, RESTORE_RUN_EXECUTED_FAILED)
1059    }
1060
1061    // Map a journal operation into the compact runner execution row.
1062    fn from_operation(
1063        operation: RestoreApplyJournalOperation,
1064        command: RestoreApplyRunnerCommand,
1065        status: String,
1066        state: &'static str,
1067    ) -> Self {
1068        Self {
1069            sequence: operation.sequence,
1070            operation: operation.operation,
1071            target_canister: operation.target_canister,
1072            command,
1073            status,
1074            state,
1075        }
1076    }
1077}
1078
1079///
1080/// RestoreRunResponseMode
1081///
1082
1083struct RestoreRunResponseMode {
1084    run_mode: &'static str,
1085    dry_run: bool,
1086    execute: bool,
1087    unclaim_pending: bool,
1088    stopped_reason: &'static str,
1089    next_action: &'static str,
1090}
1091
1092impl RestoreRunResponseMode {
1093    // Build a response mode from the stable JSON mode flags and action labels.
1094    const fn new(
1095        run_mode: &'static str,
1096        dry_run: bool,
1097        execute: bool,
1098        unclaim_pending: bool,
1099        stopped_reason: &'static str,
1100        next_action: &'static str,
1101    ) -> Self {
1102        Self {
1103            run_mode,
1104            dry_run,
1105            execute,
1106            unclaim_pending,
1107            stopped_reason,
1108            next_action,
1109        }
1110    }
1111
1112    // Build a dry-run response mode with a computed stop reason and action.
1113    const fn dry_run(stopped_reason: &'static str, next_action: &'static str) -> Self {
1114        Self::new(
1115            RESTORE_RUN_MODE_DRY_RUN,
1116            true,
1117            false,
1118            false,
1119            stopped_reason,
1120            next_action,
1121        )
1122    }
1123
1124    // Build an execute response mode with a computed stop reason and action.
1125    const fn execute(stopped_reason: &'static str, next_action: &'static str) -> Self {
1126        Self::new(
1127            RESTORE_RUN_MODE_EXECUTE,
1128            false,
1129            true,
1130            false,
1131            stopped_reason,
1132            next_action,
1133        )
1134    }
1135
1136    // Build the pending-operation recovery response mode.
1137    const fn unclaim_pending(next_action: &'static str) -> Self {
1138        Self::new(
1139            RESTORE_RUN_MODE_UNCLAIM_PENDING,
1140            false,
1141            false,
1142            true,
1143            RESTORE_RUN_STOPPED_RECOVERED_PENDING,
1144            next_action,
1145        )
1146    }
1147}
1148
1149///
1150/// RestoreApplyNextOptions
1151///
1152
1153#[derive(Clone, Debug, Eq, PartialEq)]
1154pub struct RestoreApplyNextOptions {
1155    pub journal: PathBuf,
1156    pub out: Option<PathBuf>,
1157}
1158
1159impl RestoreApplyNextOptions {
1160    /// Parse restore apply-next options from CLI arguments.
1161    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1162    where
1163        I: IntoIterator<Item = OsString>,
1164    {
1165        let mut journal = None;
1166        let mut out = None;
1167
1168        let mut args = args.into_iter();
1169        while let Some(arg) = args.next() {
1170            let arg = arg
1171                .into_string()
1172                .map_err(|_| RestoreCommandError::Usage(usage()))?;
1173            match arg.as_str() {
1174                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1175                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1176                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1177                _ => return Err(RestoreCommandError::UnknownOption(arg)),
1178            }
1179        }
1180
1181        Ok(Self {
1182            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1183            out,
1184        })
1185    }
1186}
1187
1188///
1189/// RestoreApplyCommandOptions
1190///
1191
1192#[derive(Clone, Debug, Eq, PartialEq)]
1193pub struct RestoreApplyCommandOptions {
1194    pub journal: PathBuf,
1195    pub dfx: String,
1196    pub network: Option<String>,
1197    pub out: Option<PathBuf>,
1198    pub require_command: bool,
1199}
1200
1201impl RestoreApplyCommandOptions {
1202    /// Parse restore apply-command options from CLI arguments.
1203    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1204    where
1205        I: IntoIterator<Item = OsString>,
1206    {
1207        let mut journal = None;
1208        let mut dfx = "dfx".to_string();
1209        let mut network = None;
1210        let mut out = None;
1211        let mut require_command = false;
1212
1213        let mut args = args.into_iter();
1214        while let Some(arg) = args.next() {
1215            let arg = arg
1216                .into_string()
1217                .map_err(|_| RestoreCommandError::Usage(usage()))?;
1218            match arg.as_str() {
1219                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1220                "--dfx" => dfx = next_value(&mut args, "--dfx")?,
1221                "--network" => network = Some(next_value(&mut args, "--network")?),
1222                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1223                "--require-command" => require_command = true,
1224                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1225                _ => return Err(RestoreCommandError::UnknownOption(arg)),
1226            }
1227        }
1228
1229        Ok(Self {
1230            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1231            dfx,
1232            network,
1233            out,
1234            require_command,
1235        })
1236    }
1237}
1238
1239///
1240/// RestoreApplyClaimOptions
1241///
1242
1243#[derive(Clone, Debug, Eq, PartialEq)]
1244pub struct RestoreApplyClaimOptions {
1245    pub journal: PathBuf,
1246    pub sequence: Option<usize>,
1247    pub updated_at: Option<String>,
1248    pub out: Option<PathBuf>,
1249}
1250
1251impl RestoreApplyClaimOptions {
1252    /// Parse restore apply-claim options from CLI arguments.
1253    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1254    where
1255        I: IntoIterator<Item = OsString>,
1256    {
1257        let mut journal = None;
1258        let mut sequence = None;
1259        let mut updated_at = None;
1260        let mut out = None;
1261
1262        let mut args = args.into_iter();
1263        while let Some(arg) = args.next() {
1264            let arg = arg
1265                .into_string()
1266                .map_err(|_| RestoreCommandError::Usage(usage()))?;
1267            match arg.as_str() {
1268                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1269                "--sequence" => {
1270                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1271                }
1272                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1273                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1274                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1275                _ => return Err(RestoreCommandError::UnknownOption(arg)),
1276            }
1277        }
1278
1279        Ok(Self {
1280            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1281            sequence,
1282            updated_at,
1283            out,
1284        })
1285    }
1286}
1287
1288///
1289/// RestoreApplyMarkOptions
1290///
1291
1292#[derive(Clone, Debug, Eq, PartialEq)]
1293pub struct RestoreApplyUnclaimOptions {
1294    pub journal: PathBuf,
1295    pub sequence: Option<usize>,
1296    pub updated_at: Option<String>,
1297    pub out: Option<PathBuf>,
1298}
1299
1300impl RestoreApplyUnclaimOptions {
1301    /// Parse restore apply-unclaim options from CLI arguments.
1302    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1303    where
1304        I: IntoIterator<Item = OsString>,
1305    {
1306        let mut journal = None;
1307        let mut sequence = None;
1308        let mut updated_at = None;
1309        let mut out = None;
1310
1311        let mut args = args.into_iter();
1312        while let Some(arg) = args.next() {
1313            let arg = arg
1314                .into_string()
1315                .map_err(|_| RestoreCommandError::Usage(usage()))?;
1316            match arg.as_str() {
1317                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1318                "--sequence" => {
1319                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1320                }
1321                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1322                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1323                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1324                _ => return Err(RestoreCommandError::UnknownOption(arg)),
1325            }
1326        }
1327
1328        Ok(Self {
1329            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1330            sequence,
1331            updated_at,
1332            out,
1333        })
1334    }
1335}
1336
1337///
1338/// RestoreApplyMarkOptions
1339///
1340
1341#[derive(Clone, Debug, Eq, PartialEq)]
1342pub struct RestoreApplyMarkOptions {
1343    pub journal: PathBuf,
1344    pub sequence: usize,
1345    pub state: RestoreApplyMarkState,
1346    pub reason: Option<String>,
1347    pub updated_at: Option<String>,
1348    pub out: Option<PathBuf>,
1349    pub require_pending: bool,
1350}
1351
1352impl RestoreApplyMarkOptions {
1353    /// Parse restore apply-mark options from CLI arguments.
1354    pub fn parse<I>(args: I) -> Result<Self, RestoreCommandError>
1355    where
1356        I: IntoIterator<Item = OsString>,
1357    {
1358        let mut journal = None;
1359        let mut sequence = None;
1360        let mut state = None;
1361        let mut reason = None;
1362        let mut updated_at = None;
1363        let mut out = None;
1364        let mut require_pending = false;
1365
1366        let mut args = args.into_iter();
1367        while let Some(arg) = args.next() {
1368            let arg = arg
1369                .into_string()
1370                .map_err(|_| RestoreCommandError::Usage(usage()))?;
1371            match arg.as_str() {
1372                "--journal" => journal = Some(PathBuf::from(next_value(&mut args, "--journal")?)),
1373                "--sequence" => {
1374                    sequence = Some(parse_sequence(next_value(&mut args, "--sequence")?)?);
1375                }
1376                "--state" => {
1377                    state = Some(RestoreApplyMarkState::parse(next_value(
1378                        &mut args, "--state",
1379                    )?)?);
1380                }
1381                "--reason" => reason = Some(next_value(&mut args, "--reason")?),
1382                "--updated-at" => updated_at = Some(next_value(&mut args, "--updated-at")?),
1383                "--out" => out = Some(PathBuf::from(next_value(&mut args, "--out")?)),
1384                "--require-pending" => require_pending = true,
1385                "--help" | "-h" => return Err(RestoreCommandError::Usage(usage())),
1386                _ => return Err(RestoreCommandError::UnknownOption(arg)),
1387            }
1388        }
1389
1390        Ok(Self {
1391            journal: journal.ok_or(RestoreCommandError::MissingOption("--journal"))?,
1392            sequence: sequence.ok_or(RestoreCommandError::MissingOption("--sequence"))?,
1393            state: state.ok_or(RestoreCommandError::MissingOption("--state"))?,
1394            reason,
1395            updated_at,
1396            out,
1397            require_pending,
1398        })
1399    }
1400}
1401
1402///
1403/// RestoreApplyMarkState
1404///
1405
1406#[derive(Clone, Debug, Eq, PartialEq)]
1407pub enum RestoreApplyMarkState {
1408    Completed,
1409    Failed,
1410}
1411
1412impl RestoreApplyMarkState {
1413    // Parse the restricted operation states accepted by apply-mark.
1414    fn parse(value: String) -> Result<Self, RestoreCommandError> {
1415        match value.as_str() {
1416            "completed" => Ok(Self::Completed),
1417            "failed" => Ok(Self::Failed),
1418            _ => Err(RestoreCommandError::InvalidApplyMarkState(value)),
1419        }
1420    }
1421}
1422
1423/// Run a restore subcommand.
1424pub fn run<I>(args: I) -> Result<(), RestoreCommandError>
1425where
1426    I: IntoIterator<Item = OsString>,
1427{
1428    let mut args = args.into_iter();
1429    let Some(command) = args.next().and_then(|arg| arg.into_string().ok()) else {
1430        return Err(RestoreCommandError::Usage(usage()));
1431    };
1432
1433    match command.as_str() {
1434        "plan" => {
1435            let options = RestorePlanOptions::parse(args)?;
1436            let plan = plan_restore(&options)?;
1437            write_plan(&options, &plan)?;
1438            enforce_restore_plan_requirements(&options, &plan)?;
1439            Ok(())
1440        }
1441        "status" => {
1442            let options = RestoreStatusOptions::parse(args)?;
1443            let status = restore_status(&options)?;
1444            write_status(&options, &status)?;
1445            Ok(())
1446        }
1447        "apply" => {
1448            let options = RestoreApplyOptions::parse(args)?;
1449            let dry_run = restore_apply_dry_run(&options)?;
1450            write_apply_dry_run(&options, &dry_run)?;
1451            write_apply_journal_if_requested(&options, &dry_run)?;
1452            Ok(())
1453        }
1454        "apply-status" => {
1455            let options = RestoreApplyStatusOptions::parse(args)?;
1456            let status = restore_apply_status(&options)?;
1457            write_apply_status(&options, &status)?;
1458            enforce_apply_status_requirements(&options, &status)?;
1459            Ok(())
1460        }
1461        "apply-report" => {
1462            let options = RestoreApplyReportOptions::parse(args)?;
1463            let report = restore_apply_report(&options)?;
1464            write_apply_report(&options, &report)?;
1465            enforce_apply_report_requirements(&options, &report)?;
1466            Ok(())
1467        }
1468        "run" => {
1469            let options = RestoreRunOptions::parse(args)?;
1470            let run = if options.execute {
1471                restore_run_execute_result(&options)?
1472            } else if options.unclaim_pending {
1473                RestoreRunResult::ok(restore_run_unclaim_pending(&options)?)
1474            } else {
1475                RestoreRunResult::ok(restore_run_dry_run(&options)?)
1476            };
1477            write_restore_run(&options, &run.response)?;
1478            if let Some(error) = run.error {
1479                return Err(error);
1480            }
1481            enforce_restore_run_requirements(&options, &run.response)?;
1482            Ok(())
1483        }
1484        "apply-next" => {
1485            let options = RestoreApplyNextOptions::parse(args)?;
1486            let next = restore_apply_next(&options)?;
1487            write_apply_next(&options, &next)?;
1488            Ok(())
1489        }
1490        "apply-command" => {
1491            let options = RestoreApplyCommandOptions::parse(args)?;
1492            let preview = restore_apply_command(&options)?;
1493            write_apply_command(&options, &preview)?;
1494            enforce_apply_command_requirements(&options, &preview)?;
1495            Ok(())
1496        }
1497        "apply-claim" => {
1498            let options = RestoreApplyClaimOptions::parse(args)?;
1499            let journal = restore_apply_claim(&options)?;
1500            write_apply_claim(&options, &journal)?;
1501            Ok(())
1502        }
1503        "apply-unclaim" => {
1504            let options = RestoreApplyUnclaimOptions::parse(args)?;
1505            let journal = restore_apply_unclaim(&options)?;
1506            write_apply_unclaim(&options, &journal)?;
1507            Ok(())
1508        }
1509        "apply-mark" => {
1510            let options = RestoreApplyMarkOptions::parse(args)?;
1511            let journal = restore_apply_mark(&options)?;
1512            write_apply_mark(&options, &journal)?;
1513            Ok(())
1514        }
1515        "help" | "--help" | "-h" => Err(RestoreCommandError::Usage(usage())),
1516        _ => Err(RestoreCommandError::UnknownOption(command)),
1517    }
1518}
1519
1520/// Build a no-mutation restore plan from a manifest and optional mapping.
1521pub fn plan_restore(options: &RestorePlanOptions) -> Result<RestorePlan, RestoreCommandError> {
1522    verify_backup_layout_if_required(options)?;
1523
1524    let manifest = read_manifest_source(options)?;
1525    let mapping = options.mapping.as_ref().map(read_mapping).transpose()?;
1526
1527    RestorePlanner::plan(&manifest, mapping.as_ref()).map_err(RestoreCommandError::from)
1528}
1529
1530/// Build the initial no-mutation restore status from a restore plan.
1531pub fn restore_status(
1532    options: &RestoreStatusOptions,
1533) -> Result<RestoreStatus, RestoreCommandError> {
1534    let plan = read_plan(&options.plan)?;
1535    Ok(RestoreStatus::from_plan(&plan))
1536}
1537
1538/// Build a no-mutation restore apply dry-run from a restore plan.
1539pub fn restore_apply_dry_run(
1540    options: &RestoreApplyOptions,
1541) -> Result<RestoreApplyDryRun, RestoreCommandError> {
1542    let plan = read_plan(&options.plan)?;
1543    let status = options.status.as_ref().map(read_status).transpose()?;
1544    if let Some(backup_dir) = &options.backup_dir {
1545        return RestoreApplyDryRun::try_from_plan_with_artifacts(
1546            &plan,
1547            status.as_ref(),
1548            backup_dir,
1549        )
1550        .map_err(RestoreCommandError::from);
1551    }
1552
1553    RestoreApplyDryRun::try_from_plan(&plan, status.as_ref()).map_err(RestoreCommandError::from)
1554}
1555
1556/// Build a compact restore apply status from a journal file.
1557pub fn restore_apply_status(
1558    options: &RestoreApplyStatusOptions,
1559) -> Result<RestoreApplyJournalStatus, RestoreCommandError> {
1560    let journal = read_apply_journal(&options.journal)?;
1561    Ok(journal.status())
1562}
1563
1564/// Build an operator-oriented restore apply report from a journal file.
1565pub fn restore_apply_report(
1566    options: &RestoreApplyReportOptions,
1567) -> Result<RestoreApplyJournalReport, RestoreCommandError> {
1568    let journal = read_apply_journal(&options.journal)?;
1569    Ok(journal.report())
1570}
1571
1572/// Build a no-mutation native restore runner preview from a journal file.
1573pub fn restore_run_dry_run(
1574    options: &RestoreRunOptions,
1575) -> Result<RestoreRunResponse, RestoreCommandError> {
1576    let journal = read_apply_journal(&options.journal)?;
1577    let report = journal.report();
1578    let preview = journal.next_command_preview_with_config(&restore_run_command_config(options));
1579    let stopped_reason = restore_run_stopped_reason(&report, false, false);
1580    let next_action = restore_run_next_action(&report, false);
1581
1582    let mut response = RestoreRunResponse::from_report(
1583        journal.backup_id,
1584        report,
1585        RestoreRunResponseMode::dry_run(stopped_reason, next_action),
1586    );
1587    response.operation_available = Some(preview.operation_available);
1588    response.command_available = Some(preview.command_available);
1589    response.command = preview.command;
1590    Ok(response)
1591}
1592
1593/// Recover an interrupted restore runner by unclaiming the pending operation.
1594pub fn restore_run_unclaim_pending(
1595    options: &RestoreRunOptions,
1596) -> Result<RestoreRunResponse, RestoreCommandError> {
1597    let mut journal = read_apply_journal(&options.journal)?;
1598    let recovered_operation = journal
1599        .next_transition_operation()
1600        .filter(|operation| operation.state == RestoreApplyOperationState::Pending)
1601        .cloned()
1602        .ok_or(RestoreApplyJournalError::NoPendingOperation)?;
1603
1604    let recovered_updated_at = timestamp_placeholder();
1605    journal.mark_next_operation_ready_at(Some(recovered_updated_at.clone()))?;
1606    write_apply_journal_file(&options.journal, &journal)?;
1607
1608    let report = journal.report();
1609    let next_action = restore_run_next_action(&report, true);
1610    let mut response = RestoreRunResponse::from_report(
1611        journal.backup_id,
1612        report,
1613        RestoreRunResponseMode::unclaim_pending(next_action),
1614    );
1615    response.set_operation_receipts(vec![RestoreRunOperationReceipt::recovered_pending(
1616        recovered_operation.clone(),
1617        Some(recovered_updated_at),
1618    )]);
1619    response.recovered_operation = Some(recovered_operation);
1620    Ok(response)
1621}
1622
1623/// Execute ready restore apply journal operations through generated runner commands.
1624pub fn restore_run_execute(
1625    options: &RestoreRunOptions,
1626) -> Result<RestoreRunResponse, RestoreCommandError> {
1627    let run = restore_run_execute_result(options)?;
1628    if let Some(error) = run.error {
1629        return Err(error);
1630    }
1631
1632    Ok(run.response)
1633}
1634
1635// Execute ready restore apply operations and retain any deferred runner error.
1636fn restore_run_execute_result(
1637    options: &RestoreRunOptions,
1638) -> Result<RestoreRunResult, RestoreCommandError> {
1639    let mut journal = read_apply_journal(&options.journal)?;
1640    let mut executed_operations = Vec::new();
1641    let mut operation_receipts = Vec::new();
1642    let config = restore_run_command_config(options);
1643
1644    loop {
1645        let report = journal.report();
1646        let max_steps_reached =
1647            restore_run_max_steps_reached(options, executed_operations.len(), &report);
1648        if report.complete || max_steps_reached {
1649            return Ok(RestoreRunResult::ok(restore_run_execute_summary(
1650                &journal,
1651                executed_operations,
1652                operation_receipts,
1653                max_steps_reached,
1654            )));
1655        }
1656
1657        enforce_restore_run_executable(&journal, &report)?;
1658        let preview = journal.next_command_preview_with_config(&config);
1659        enforce_restore_run_command_available(&preview)?;
1660
1661        let operation = preview
1662            .operation
1663            .clone()
1664            .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1665        let command = preview
1666            .command
1667            .clone()
1668            .ok_or_else(|| restore_command_unavailable_error(&preview))?;
1669        let sequence = operation.sequence;
1670
1671        enforce_apply_claim_sequence(sequence, &journal)?;
1672        journal.mark_operation_pending_at(sequence, Some(timestamp_placeholder()))?;
1673        write_apply_journal_file(&options.journal, &journal)?;
1674
1675        let status = Command::new(&command.program)
1676            .args(&command.args)
1677            .status()?;
1678        let status_label = exit_status_label(status);
1679        if status.success() {
1680            let completed_updated_at = timestamp_placeholder();
1681            journal.mark_operation_completed_at(sequence, Some(completed_updated_at.clone()))?;
1682            write_apply_journal_file(&options.journal, &journal)?;
1683            executed_operations.push(RestoreRunExecutedOperation::completed(
1684                operation.clone(),
1685                command.clone(),
1686                status_label.clone(),
1687            ));
1688            operation_receipts.push(RestoreRunOperationReceipt::completed(
1689                operation,
1690                command,
1691                status_label,
1692                Some(completed_updated_at),
1693            ));
1694            continue;
1695        }
1696
1697        let failed_updated_at = timestamp_placeholder();
1698        journal.mark_operation_failed_at(
1699            sequence,
1700            format!("{RESTORE_RUN_COMMAND_EXIT_PREFIX}-{status_label}"),
1701            Some(failed_updated_at.clone()),
1702        )?;
1703        write_apply_journal_file(&options.journal, &journal)?;
1704        executed_operations.push(RestoreRunExecutedOperation::failed(
1705            operation.clone(),
1706            command.clone(),
1707            status_label.clone(),
1708        ));
1709        operation_receipts.push(RestoreRunOperationReceipt::failed(
1710            operation,
1711            command,
1712            status_label.clone(),
1713            Some(failed_updated_at),
1714        ));
1715        let response =
1716            restore_run_execute_summary(&journal, executed_operations, operation_receipts, false);
1717        return Ok(RestoreRunResult {
1718            response,
1719            error: Some(RestoreCommandError::RestoreRunCommandFailed {
1720                sequence,
1721                status: status_label,
1722            }),
1723        });
1724    }
1725}
1726
1727// Build the shared runner command-preview configuration from CLI options.
1728fn restore_run_command_config(options: &RestoreRunOptions) -> RestoreApplyCommandConfig {
1729    restore_command_config(&options.dfx, options.network.as_deref())
1730}
1731
1732// Build the shared apply-command preview configuration from CLI options.
1733fn restore_apply_command_config(options: &RestoreApplyCommandOptions) -> RestoreApplyCommandConfig {
1734    restore_command_config(&options.dfx, options.network.as_deref())
1735}
1736
1737// Build command-preview configuration from common dfx/network inputs.
1738fn restore_command_config(program: &str, network: Option<&str>) -> RestoreApplyCommandConfig {
1739    RestoreApplyCommandConfig {
1740        program: program.to_string(),
1741        network: network.map(str::to_string),
1742    }
1743}
1744
1745// Check whether execute mode has reached its requested operation batch size.
1746fn restore_run_max_steps_reached(
1747    options: &RestoreRunOptions,
1748    executed_operation_count: usize,
1749    report: &RestoreApplyJournalReport,
1750) -> bool {
1751    options.max_steps == Some(executed_operation_count) && !report.complete
1752}
1753
1754// Build the final native runner execution summary.
1755fn restore_run_execute_summary(
1756    journal: &RestoreApplyJournal,
1757    executed_operations: Vec<RestoreRunExecutedOperation>,
1758    operation_receipts: Vec<RestoreRunOperationReceipt>,
1759    max_steps_reached: bool,
1760) -> RestoreRunResponse {
1761    let report = journal.report();
1762    let executed_operation_count = executed_operations.len();
1763    let stopped_reason = restore_run_stopped_reason(&report, max_steps_reached, true);
1764    let next_action = restore_run_next_action(&report, false);
1765
1766    let mut response = RestoreRunResponse::from_report(
1767        journal.backup_id.clone(),
1768        report,
1769        RestoreRunResponseMode::execute(stopped_reason, next_action),
1770    );
1771    response.max_steps_reached = Some(max_steps_reached);
1772    response.executed_operation_count = Some(executed_operation_count);
1773    response.executed_operations = executed_operations;
1774    response.set_operation_receipts(operation_receipts);
1775    response
1776}
1777
1778// Classify why the native runner stopped for operator summaries.
1779const fn restore_run_stopped_reason(
1780    report: &RestoreApplyJournalReport,
1781    max_steps_reached: bool,
1782    executed: bool,
1783) -> &'static str {
1784    if report.complete {
1785        return RESTORE_RUN_STOPPED_COMPLETE;
1786    }
1787    if report.failed_operations > 0 {
1788        return RESTORE_RUN_STOPPED_COMMAND_FAILED;
1789    }
1790    if report.pending_operations > 0 {
1791        return RESTORE_RUN_STOPPED_PENDING;
1792    }
1793    if !report.ready || report.blocked_operations > 0 {
1794        return RESTORE_RUN_STOPPED_BLOCKED;
1795    }
1796    if max_steps_reached {
1797        return RESTORE_RUN_STOPPED_MAX_STEPS;
1798    }
1799    if executed {
1800        return RESTORE_RUN_STOPPED_READY;
1801    }
1802    RESTORE_RUN_STOPPED_PREVIEW
1803}
1804
1805// Recommend the next operator action for the native runner summary.
1806const fn restore_run_next_action(
1807    report: &RestoreApplyJournalReport,
1808    recovered_pending: bool,
1809) -> &'static str {
1810    if report.complete {
1811        return RESTORE_RUN_ACTION_DONE;
1812    }
1813    if report.failed_operations > 0 {
1814        return RESTORE_RUN_ACTION_INSPECT_FAILED;
1815    }
1816    if report.pending_operations > 0 {
1817        return RESTORE_RUN_ACTION_UNCLAIM_PENDING;
1818    }
1819    if !report.ready || report.blocked_operations > 0 {
1820        return RESTORE_RUN_ACTION_FIX_BLOCKED;
1821    }
1822    if recovered_pending {
1823        return RESTORE_RUN_ACTION_RERUN;
1824    }
1825    RESTORE_RUN_ACTION_RERUN
1826}
1827
1828// Ensure the journal can be advanced by the native restore runner.
1829fn enforce_restore_run_executable(
1830    journal: &RestoreApplyJournal,
1831    report: &RestoreApplyJournalReport,
1832) -> Result<(), RestoreCommandError> {
1833    if report.pending_operations > 0 {
1834        return Err(RestoreCommandError::RestoreApplyPending {
1835            backup_id: report.backup_id.clone(),
1836            pending_operations: report.pending_operations,
1837            next_transition_sequence: report
1838                .next_transition
1839                .as_ref()
1840                .map(|operation| operation.sequence),
1841        });
1842    }
1843
1844    if report.failed_operations > 0 {
1845        return Err(RestoreCommandError::RestoreApplyFailed {
1846            backup_id: report.backup_id.clone(),
1847            failed_operations: report.failed_operations,
1848        });
1849    }
1850
1851    if report.ready {
1852        return Ok(());
1853    }
1854
1855    Err(RestoreCommandError::RestoreApplyNotReady {
1856        backup_id: journal.backup_id.clone(),
1857        reasons: report.blocked_reasons.clone(),
1858    })
1859}
1860
1861// Convert an unavailable native runner command into the shared fail-closed error.
1862fn enforce_restore_run_command_available(
1863    preview: &RestoreApplyCommandPreview,
1864) -> Result<(), RestoreCommandError> {
1865    if preview.command_available {
1866        return Ok(());
1867    }
1868
1869    Err(restore_command_unavailable_error(preview))
1870}
1871
1872// Build a shared command-unavailable error from a preview.
1873fn restore_command_unavailable_error(preview: &RestoreApplyCommandPreview) -> RestoreCommandError {
1874    RestoreCommandError::RestoreApplyCommandUnavailable {
1875        backup_id: preview.backup_id.clone(),
1876        operation_available: preview.operation_available,
1877        complete: preview.complete,
1878        blocked_reasons: preview.blocked_reasons.clone(),
1879    }
1880}
1881
1882// Render process exit status without relying on platform-specific internals.
1883fn exit_status_label(status: std::process::ExitStatus) -> String {
1884    status
1885        .code()
1886        .map_or_else(|| "signal".to_string(), |code| code.to_string())
1887}
1888
1889// Enforce caller-requested native runner requirements after output is emitted.
1890fn enforce_restore_run_requirements(
1891    options: &RestoreRunOptions,
1892    run: &RestoreRunResponse,
1893) -> Result<(), RestoreCommandError> {
1894    if options.require_complete && !run.complete {
1895        return Err(RestoreCommandError::RestoreApplyIncomplete {
1896            backup_id: run.backup_id.clone(),
1897            completed_operations: run.completed_operations,
1898            operation_count: run.operation_count,
1899        });
1900    }
1901
1902    if options.require_no_attention && run.attention_required {
1903        return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
1904            backup_id: run.backup_id.clone(),
1905            outcome: run.outcome.clone(),
1906        });
1907    }
1908
1909    if let Some(expected) = &options.require_run_mode
1910        && run.run_mode != expected
1911    {
1912        return Err(RestoreCommandError::RestoreRunModeMismatch {
1913            backup_id: run.backup_id.clone(),
1914            expected: expected.clone(),
1915            actual: run.run_mode.to_string(),
1916        });
1917    }
1918
1919    if let Some(expected) = &options.require_stopped_reason
1920        && run.stopped_reason != expected
1921    {
1922        return Err(RestoreCommandError::RestoreRunStoppedReasonMismatch {
1923            backup_id: run.backup_id.clone(),
1924            expected: expected.clone(),
1925            actual: run.stopped_reason.to_string(),
1926        });
1927    }
1928
1929    if let Some(expected) = &options.require_next_action
1930        && run.next_action != expected
1931    {
1932        return Err(RestoreCommandError::RestoreRunNextActionMismatch {
1933            backup_id: run.backup_id.clone(),
1934            expected: expected.clone(),
1935            actual: run.next_action.to_string(),
1936        });
1937    }
1938
1939    if let Some(expected) = options.require_executed_count {
1940        let actual = run.executed_operation_count.unwrap_or(0);
1941        if actual != expected {
1942            return Err(RestoreCommandError::RestoreRunExecutedCountMismatch {
1943                backup_id: run.backup_id.clone(),
1944                expected,
1945                actual,
1946            });
1947        }
1948    }
1949
1950    if let Some(expected) = options.require_receipt_count {
1951        let actual = run.operation_receipt_count.unwrap_or(0);
1952        if actual != expected {
1953            return Err(RestoreCommandError::RestoreRunReceiptCountMismatch {
1954                backup_id: run.backup_id.clone(),
1955                expected,
1956                actual,
1957            });
1958        }
1959    }
1960
1961    enforce_restore_run_receipt_kind_requirement(
1962        &run.backup_id,
1963        RESTORE_RUN_RECEIPT_COMPLETED,
1964        options.require_completed_receipt_count,
1965        run.operation_receipt_summary.command_completed,
1966    )?;
1967    enforce_restore_run_receipt_kind_requirement(
1968        &run.backup_id,
1969        RESTORE_RUN_RECEIPT_FAILED,
1970        options.require_failed_receipt_count,
1971        run.operation_receipt_summary.command_failed,
1972    )?;
1973    enforce_restore_run_receipt_kind_requirement(
1974        &run.backup_id,
1975        RESTORE_RUN_RECEIPT_RECOVERED_PENDING,
1976        options.require_recovered_receipt_count,
1977        run.operation_receipt_summary.pending_recovered,
1978    )?;
1979
1980    enforce_progress_requirements(
1981        &run.backup_id,
1982        &run.progress,
1983        options.require_remaining_count,
1984        options.require_attention_count,
1985        options.require_completion_basis_points,
1986    )?;
1987    enforce_pending_before_requirement(
1988        &run.backup_id,
1989        &run.pending_summary,
1990        options.require_no_pending_before.as_deref(),
1991    )?;
1992
1993    Ok(())
1994}
1995
1996// Fail when a runner receipt-kind count differs from the requested value.
1997fn enforce_restore_run_receipt_kind_requirement(
1998    backup_id: &str,
1999    receipt_kind: &'static str,
2000    expected: Option<usize>,
2001    actual: usize,
2002) -> Result<(), RestoreCommandError> {
2003    if let Some(expected) = expected
2004        && actual != expected
2005    {
2006        return Err(RestoreCommandError::RestoreRunReceiptKindCountMismatch {
2007            backup_id: backup_id.to_string(),
2008            receipt_kind,
2009            expected,
2010            actual,
2011        });
2012    }
2013
2014    Ok(())
2015}
2016
2017// Enforce caller-requested integer progress requirements after output is emitted.
2018fn enforce_progress_requirements(
2019    backup_id: &str,
2020    progress: &RestoreApplyProgressSummary,
2021    require_remaining_count: Option<usize>,
2022    require_attention_count: Option<usize>,
2023    require_completion_basis_points: Option<usize>,
2024) -> Result<(), RestoreCommandError> {
2025    if let Some(expected) = require_remaining_count
2026        && progress.remaining_operations != expected
2027    {
2028        return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2029            backup_id: backup_id.to_string(),
2030            field: "remaining_operations",
2031            expected,
2032            actual: progress.remaining_operations,
2033        });
2034    }
2035
2036    if let Some(expected) = require_attention_count
2037        && progress.attention_operations != expected
2038    {
2039        return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2040            backup_id: backup_id.to_string(),
2041            field: "attention_operations",
2042            expected,
2043            actual: progress.attention_operations,
2044        });
2045    }
2046
2047    if let Some(expected) = require_completion_basis_points
2048        && progress.completion_basis_points != expected
2049    {
2050        return Err(RestoreCommandError::RestoreApplyProgressMismatch {
2051            backup_id: backup_id.to_string(),
2052            field: "completion_basis_points",
2053            expected,
2054            actual: progress.completion_basis_points,
2055        });
2056    }
2057
2058    Ok(())
2059}
2060
2061// Enforce pending-work freshness using caller-supplied comparable update markers.
2062fn enforce_pending_before_requirement(
2063    backup_id: &str,
2064    pending: &RestoreApplyPendingSummary,
2065    require_no_pending_before: Option<&str>,
2066) -> Result<(), RestoreCommandError> {
2067    let Some(cutoff_updated_at) = require_no_pending_before else {
2068        return Ok(());
2069    };
2070
2071    if pending.pending_operations == 0 {
2072        return Ok(());
2073    }
2074
2075    if pending.pending_updated_at_known
2076        && pending
2077            .pending_updated_at
2078            .as_deref()
2079            .is_some_and(|updated_at| updated_at >= cutoff_updated_at)
2080    {
2081        return Ok(());
2082    }
2083
2084    Err(RestoreCommandError::RestoreApplyPendingStale {
2085        backup_id: backup_id.to_string(),
2086        cutoff_updated_at: cutoff_updated_at.to_string(),
2087        pending_sequence: pending.pending_sequence,
2088        pending_updated_at: pending.pending_updated_at.clone(),
2089    })
2090}
2091
2092// Enforce caller-requested apply report requirements after report output is emitted.
2093fn enforce_apply_report_requirements(
2094    options: &RestoreApplyReportOptions,
2095    report: &RestoreApplyJournalReport,
2096) -> Result<(), RestoreCommandError> {
2097    if options.require_no_attention && report.attention_required {
2098        return Err(RestoreCommandError::RestoreApplyReportNeedsAttention {
2099            backup_id: report.backup_id.clone(),
2100            outcome: report.outcome.clone(),
2101        });
2102    }
2103
2104    enforce_progress_requirements(
2105        &report.backup_id,
2106        &report.progress,
2107        options.require_remaining_count,
2108        options.require_attention_count,
2109        options.require_completion_basis_points,
2110    )?;
2111    enforce_pending_before_requirement(
2112        &report.backup_id,
2113        &report.pending_summary,
2114        options.require_no_pending_before.as_deref(),
2115    )
2116}
2117
2118// Enforce caller-requested apply journal requirements after status is emitted.
2119fn enforce_apply_status_requirements(
2120    options: &RestoreApplyStatusOptions,
2121    status: &RestoreApplyJournalStatus,
2122) -> Result<(), RestoreCommandError> {
2123    if options.require_ready && !status.ready {
2124        return Err(RestoreCommandError::RestoreApplyNotReady {
2125            backup_id: status.backup_id.clone(),
2126            reasons: status.blocked_reasons.clone(),
2127        });
2128    }
2129
2130    if options.require_no_pending && status.pending_operations > 0 {
2131        return Err(RestoreCommandError::RestoreApplyPending {
2132            backup_id: status.backup_id.clone(),
2133            pending_operations: status.pending_operations,
2134            next_transition_sequence: status.next_transition_sequence,
2135        });
2136    }
2137
2138    if options.require_no_failed && status.failed_operations > 0 {
2139        return Err(RestoreCommandError::RestoreApplyFailed {
2140            backup_id: status.backup_id.clone(),
2141            failed_operations: status.failed_operations,
2142        });
2143    }
2144
2145    if options.require_complete && !status.complete {
2146        return Err(RestoreCommandError::RestoreApplyIncomplete {
2147            backup_id: status.backup_id.clone(),
2148            completed_operations: status.completed_operations,
2149            operation_count: status.operation_count,
2150        });
2151    }
2152
2153    enforce_progress_requirements(
2154        &status.backup_id,
2155        &status.progress,
2156        options.require_remaining_count,
2157        options.require_attention_count,
2158        options.require_completion_basis_points,
2159    )?;
2160    enforce_pending_before_requirement(
2161        &status.backup_id,
2162        &status.pending_summary,
2163        options.require_no_pending_before.as_deref(),
2164    )?;
2165
2166    Ok(())
2167}
2168
2169/// Build the next restore apply operation response from a journal file.
2170pub fn restore_apply_next(
2171    options: &RestoreApplyNextOptions,
2172) -> Result<RestoreApplyNextOperation, RestoreCommandError> {
2173    let journal = read_apply_journal(&options.journal)?;
2174    Ok(journal.next_operation())
2175}
2176
2177/// Build the next restore apply command preview from a journal file.
2178pub fn restore_apply_command(
2179    options: &RestoreApplyCommandOptions,
2180) -> Result<RestoreApplyCommandPreview, RestoreCommandError> {
2181    let journal = read_apply_journal(&options.journal)?;
2182    Ok(journal.next_command_preview_with_config(&restore_apply_command_config(options)))
2183}
2184
2185// Enforce caller-requested command preview requirements after preview output is emitted.
2186fn enforce_apply_command_requirements(
2187    options: &RestoreApplyCommandOptions,
2188    preview: &RestoreApplyCommandPreview,
2189) -> Result<(), RestoreCommandError> {
2190    if !options.require_command || preview.command_available {
2191        return Ok(());
2192    }
2193
2194    Err(restore_command_unavailable_error(preview))
2195}
2196
2197/// Mark the next restore apply journal operation pending.
2198pub fn restore_apply_claim(
2199    options: &RestoreApplyClaimOptions,
2200) -> Result<RestoreApplyJournal, RestoreCommandError> {
2201    let mut journal = read_apply_journal(&options.journal)?;
2202    let updated_at = Some(state_updated_at(options.updated_at.as_ref()));
2203
2204    if let Some(sequence) = options.sequence {
2205        enforce_apply_claim_sequence(sequence, &journal)?;
2206        journal.mark_operation_pending_at(sequence, updated_at)?;
2207        return Ok(journal);
2208    }
2209
2210    journal.mark_next_operation_pending_at(updated_at)?;
2211    Ok(journal)
2212}
2213
2214// Ensure a runner claim still matches the operation it previewed.
2215fn enforce_apply_claim_sequence(
2216    expected: usize,
2217    journal: &RestoreApplyJournal,
2218) -> Result<(), RestoreCommandError> {
2219    let actual = journal
2220        .next_transition_operation()
2221        .map(|operation| operation.sequence);
2222
2223    if actual == Some(expected) {
2224        return Ok(());
2225    }
2226
2227    Err(RestoreCommandError::RestoreApplyClaimSequenceMismatch { expected, actual })
2228}
2229
2230/// Mark the current pending restore apply journal operation ready again.
2231pub fn restore_apply_unclaim(
2232    options: &RestoreApplyUnclaimOptions,
2233) -> Result<RestoreApplyJournal, RestoreCommandError> {
2234    let mut journal = read_apply_journal(&options.journal)?;
2235    if let Some(sequence) = options.sequence {
2236        enforce_apply_unclaim_sequence(sequence, &journal)?;
2237    }
2238
2239    journal.mark_next_operation_ready_at(Some(state_updated_at(options.updated_at.as_ref())))?;
2240    Ok(journal)
2241}
2242
2243// Ensure a runner unclaim still matches the pending operation it recovered.
2244fn enforce_apply_unclaim_sequence(
2245    expected: usize,
2246    journal: &RestoreApplyJournal,
2247) -> Result<(), RestoreCommandError> {
2248    let actual = journal
2249        .next_transition_operation()
2250        .map(|operation| operation.sequence);
2251
2252    if actual == Some(expected) {
2253        return Ok(());
2254    }
2255
2256    Err(RestoreCommandError::RestoreApplyUnclaimSequenceMismatch { expected, actual })
2257}
2258
2259/// Mark one restore apply journal operation completed or failed.
2260pub fn restore_apply_mark(
2261    options: &RestoreApplyMarkOptions,
2262) -> Result<RestoreApplyJournal, RestoreCommandError> {
2263    let mut journal = read_apply_journal(&options.journal)?;
2264    enforce_apply_mark_pending_requirement(options, &journal)?;
2265
2266    match options.state {
2267        RestoreApplyMarkState::Completed => {
2268            journal.mark_operation_completed_at(
2269                options.sequence,
2270                Some(state_updated_at(options.updated_at.as_ref())),
2271            )?;
2272        }
2273        RestoreApplyMarkState::Failed => {
2274            let reason =
2275                options
2276                    .reason
2277                    .clone()
2278                    .ok_or(RestoreApplyJournalError::FailureReasonRequired(
2279                        options.sequence,
2280                    ))?;
2281            journal.mark_operation_failed_at(
2282                options.sequence,
2283                reason,
2284                Some(state_updated_at(options.updated_at.as_ref())),
2285            )?;
2286        }
2287    }
2288
2289    Ok(journal)
2290}
2291
2292// Enforce that apply-mark only records an already claimed operation when requested.
2293fn enforce_apply_mark_pending_requirement(
2294    options: &RestoreApplyMarkOptions,
2295    journal: &RestoreApplyJournal,
2296) -> Result<(), RestoreCommandError> {
2297    if !options.require_pending {
2298        return Ok(());
2299    }
2300
2301    let state = journal
2302        .operations
2303        .iter()
2304        .find(|operation| operation.sequence == options.sequence)
2305        .map(|operation| operation.state.clone())
2306        .ok_or(RestoreApplyJournalError::OperationNotFound(
2307            options.sequence,
2308        ))?;
2309
2310    if state == RestoreApplyOperationState::Pending {
2311        return Ok(());
2312    }
2313
2314    Err(RestoreCommandError::RestoreApplyMarkRequiresPending {
2315        sequence: options.sequence,
2316        state,
2317    })
2318}
2319
2320// Enforce caller-requested restore plan requirements after the plan is emitted.
2321fn enforce_restore_plan_requirements(
2322    options: &RestorePlanOptions,
2323    plan: &RestorePlan,
2324) -> Result<(), RestoreCommandError> {
2325    if !options.require_restore_ready || plan.readiness_summary.ready {
2326        return Ok(());
2327    }
2328
2329    Err(RestoreCommandError::RestoreNotReady {
2330        backup_id: plan.backup_id.clone(),
2331        reasons: plan.readiness_summary.reasons.clone(),
2332    })
2333}
2334
2335// Verify backup layout integrity before restore planning when requested.
2336fn verify_backup_layout_if_required(
2337    options: &RestorePlanOptions,
2338) -> Result<(), RestoreCommandError> {
2339    if !options.require_verified {
2340        return Ok(());
2341    }
2342
2343    let Some(dir) = &options.backup_dir else {
2344        return Err(RestoreCommandError::RequireVerifiedNeedsBackupDir);
2345    };
2346
2347    BackupLayout::new(dir.clone()).verify_integrity()?;
2348    Ok(())
2349}
2350
2351// Read the manifest from a direct path or canonical backup layout.
2352fn read_manifest_source(
2353    options: &RestorePlanOptions,
2354) -> Result<FleetBackupManifest, RestoreCommandError> {
2355    if let Some(path) = &options.manifest {
2356        return read_manifest(path);
2357    }
2358
2359    let Some(dir) = &options.backup_dir else {
2360        return Err(RestoreCommandError::MissingOption(
2361            "--manifest or --backup-dir",
2362        ));
2363    };
2364
2365    BackupLayout::new(dir.clone())
2366        .read_manifest()
2367        .map_err(RestoreCommandError::from)
2368}
2369
2370// Read and decode a fleet backup manifest from disk.
2371fn read_manifest(path: &PathBuf) -> Result<FleetBackupManifest, RestoreCommandError> {
2372    let data = fs::read_to_string(path)?;
2373    serde_json::from_str(&data).map_err(RestoreCommandError::from)
2374}
2375
2376// Read and decode an optional source-to-target restore mapping from disk.
2377fn read_mapping(path: &PathBuf) -> Result<RestoreMapping, RestoreCommandError> {
2378    let data = fs::read_to_string(path)?;
2379    serde_json::from_str(&data).map_err(RestoreCommandError::from)
2380}
2381
2382// Read and decode a restore plan from disk.
2383fn read_plan(path: &PathBuf) -> Result<RestorePlan, RestoreCommandError> {
2384    let data = fs::read_to_string(path)?;
2385    serde_json::from_str(&data).map_err(RestoreCommandError::from)
2386}
2387
2388// Read and decode a restore status from disk.
2389fn read_status(path: &PathBuf) -> Result<RestoreStatus, RestoreCommandError> {
2390    let data = fs::read_to_string(path)?;
2391    serde_json::from_str(&data).map_err(RestoreCommandError::from)
2392}
2393
2394// Read and decode a restore apply journal from disk.
2395fn read_apply_journal(path: &PathBuf) -> Result<RestoreApplyJournal, RestoreCommandError> {
2396    let data = fs::read_to_string(path)?;
2397    let journal: RestoreApplyJournal = serde_json::from_str(&data)?;
2398    journal.validate()?;
2399    Ok(journal)
2400}
2401
2402// Parse shared restore apply progress requirement flags.
2403fn parse_progress_requirement_option<I>(
2404    arg: &str,
2405    args: &mut I,
2406    require_remaining_count: &mut Option<usize>,
2407    require_attention_count: &mut Option<usize>,
2408    require_completion_basis_points: &mut Option<usize>,
2409) -> Result<bool, RestoreCommandError>
2410where
2411    I: Iterator<Item = OsString>,
2412{
2413    match arg {
2414        "--require-remaining-count" => {
2415            *require_remaining_count = Some(parse_sequence(next_value(
2416                args,
2417                "--require-remaining-count",
2418            )?)?);
2419            Ok(true)
2420        }
2421        "--require-attention-count" => {
2422            *require_attention_count = Some(parse_sequence(next_value(
2423                args,
2424                "--require-attention-count",
2425            )?)?);
2426            Ok(true)
2427        }
2428        "--require-completion-basis-points" => {
2429            *require_completion_basis_points = Some(parse_sequence(next_value(
2430                args,
2431                "--require-completion-basis-points",
2432            )?)?);
2433            Ok(true)
2434        }
2435        _ => Ok(false),
2436    }
2437}
2438
2439// Parse shared restore apply pending freshness requirement flags.
2440fn parse_pending_requirement_option<I>(
2441    arg: &str,
2442    args: &mut I,
2443    require_no_pending_before: &mut Option<String>,
2444) -> Result<bool, RestoreCommandError>
2445where
2446    I: Iterator<Item = OsString>,
2447{
2448    match arg {
2449        "--require-no-pending-before" => {
2450            *require_no_pending_before = Some(next_value(args, "--require-no-pending-before")?);
2451            Ok(true)
2452        }
2453        _ => Ok(false),
2454    }
2455}
2456
2457// Parse restore-run count requirement flags.
2458fn parse_run_count_requirement_option<I>(
2459    arg: &str,
2460    args: &mut I,
2461    require_executed_count: &mut Option<usize>,
2462    require_receipt_count: &mut Option<usize>,
2463) -> Result<bool, RestoreCommandError>
2464where
2465    I: Iterator<Item = OsString>,
2466{
2467    match arg {
2468        "--require-executed-count" => {
2469            *require_executed_count = Some(parse_sequence(next_value(
2470                args,
2471                "--require-executed-count",
2472            )?)?);
2473            Ok(true)
2474        }
2475        "--require-receipt-count" => {
2476            *require_receipt_count = Some(parse_sequence(next_value(
2477                args,
2478                "--require-receipt-count",
2479            )?)?);
2480            Ok(true)
2481        }
2482        _ => Ok(false),
2483    }
2484}
2485
2486// Parse restore-run receipt-kind count requirement flags.
2487fn parse_run_receipt_kind_requirement_option<I>(
2488    arg: &str,
2489    args: &mut I,
2490    require_completed_receipt_count: &mut Option<usize>,
2491    require_failed_receipt_count: &mut Option<usize>,
2492    require_recovered_receipt_count: &mut Option<usize>,
2493) -> Result<bool, RestoreCommandError>
2494where
2495    I: Iterator<Item = OsString>,
2496{
2497    match arg {
2498        "--require-completed-receipt-count" => {
2499            *require_completed_receipt_count = Some(parse_sequence(next_value(
2500                args,
2501                "--require-completed-receipt-count",
2502            )?)?);
2503            Ok(true)
2504        }
2505        "--require-failed-receipt-count" => {
2506            *require_failed_receipt_count = Some(parse_sequence(next_value(
2507                args,
2508                "--require-failed-receipt-count",
2509            )?)?);
2510            Ok(true)
2511        }
2512        "--require-recovered-receipt-count" => {
2513            *require_recovered_receipt_count = Some(parse_sequence(next_value(
2514                args,
2515                "--require-recovered-receipt-count",
2516            )?)?);
2517            Ok(true)
2518        }
2519        _ => Ok(false),
2520    }
2521}
2522
2523// Parse a restore apply journal operation sequence value.
2524fn parse_sequence(value: String) -> Result<usize, RestoreCommandError> {
2525    value
2526        .parse::<usize>()
2527        .map_err(|_| RestoreCommandError::InvalidSequence)
2528}
2529
2530// Return the caller-supplied journal update marker or the current placeholder.
2531fn state_updated_at(updated_at: Option<&String>) -> String {
2532    updated_at.cloned().unwrap_or_else(timestamp_placeholder)
2533}
2534
2535// Return a placeholder timestamp until the CLI owns a clock abstraction.
2536fn timestamp_placeholder() -> String {
2537    "unknown".to_string()
2538}
2539
2540// Write the computed plan to stdout or a requested output file.
2541fn write_plan(options: &RestorePlanOptions, plan: &RestorePlan) -> Result<(), RestoreCommandError> {
2542    if let Some(path) = &options.out {
2543        let data = serde_json::to_vec_pretty(plan)?;
2544        fs::write(path, data)?;
2545        return Ok(());
2546    }
2547
2548    let stdout = io::stdout();
2549    let mut handle = stdout.lock();
2550    serde_json::to_writer_pretty(&mut handle, plan)?;
2551    writeln!(handle)?;
2552    Ok(())
2553}
2554
2555// Write the computed status to stdout or a requested output file.
2556fn write_status(
2557    options: &RestoreStatusOptions,
2558    status: &RestoreStatus,
2559) -> Result<(), RestoreCommandError> {
2560    if let Some(path) = &options.out {
2561        let data = serde_json::to_vec_pretty(status)?;
2562        fs::write(path, data)?;
2563        return Ok(());
2564    }
2565
2566    let stdout = io::stdout();
2567    let mut handle = stdout.lock();
2568    serde_json::to_writer_pretty(&mut handle, status)?;
2569    writeln!(handle)?;
2570    Ok(())
2571}
2572
2573// Write the computed apply dry-run to stdout or a requested output file.
2574fn write_apply_dry_run(
2575    options: &RestoreApplyOptions,
2576    dry_run: &RestoreApplyDryRun,
2577) -> Result<(), RestoreCommandError> {
2578    if let Some(path) = &options.out {
2579        let data = serde_json::to_vec_pretty(dry_run)?;
2580        fs::write(path, data)?;
2581        return Ok(());
2582    }
2583
2584    let stdout = io::stdout();
2585    let mut handle = stdout.lock();
2586    serde_json::to_writer_pretty(&mut handle, dry_run)?;
2587    writeln!(handle)?;
2588    Ok(())
2589}
2590
2591// Write the initial apply journal when the caller requests one.
2592fn write_apply_journal_if_requested(
2593    options: &RestoreApplyOptions,
2594    dry_run: &RestoreApplyDryRun,
2595) -> Result<(), RestoreCommandError> {
2596    let Some(path) = &options.journal_out else {
2597        return Ok(());
2598    };
2599
2600    let journal = RestoreApplyJournal::from_dry_run(dry_run);
2601    let data = serde_json::to_vec_pretty(&journal)?;
2602    fs::write(path, data)?;
2603    Ok(())
2604}
2605
2606// Write the computed apply journal status to stdout or a requested output file.
2607fn write_apply_status(
2608    options: &RestoreApplyStatusOptions,
2609    status: &RestoreApplyJournalStatus,
2610) -> Result<(), RestoreCommandError> {
2611    if let Some(path) = &options.out {
2612        let data = serde_json::to_vec_pretty(status)?;
2613        fs::write(path, data)?;
2614        return Ok(());
2615    }
2616
2617    let stdout = io::stdout();
2618    let mut handle = stdout.lock();
2619    serde_json::to_writer_pretty(&mut handle, status)?;
2620    writeln!(handle)?;
2621    Ok(())
2622}
2623
2624// Write the computed apply journal report to stdout or a requested output file.
2625fn write_apply_report(
2626    options: &RestoreApplyReportOptions,
2627    report: &RestoreApplyJournalReport,
2628) -> Result<(), RestoreCommandError> {
2629    if let Some(path) = &options.out {
2630        let data = serde_json::to_vec_pretty(report)?;
2631        fs::write(path, data)?;
2632        return Ok(());
2633    }
2634
2635    let stdout = io::stdout();
2636    let mut handle = stdout.lock();
2637    serde_json::to_writer_pretty(&mut handle, report)?;
2638    writeln!(handle)?;
2639    Ok(())
2640}
2641
2642// Write the restore runner response to stdout or a requested output file.
2643fn write_restore_run(
2644    options: &RestoreRunOptions,
2645    run: &RestoreRunResponse,
2646) -> Result<(), RestoreCommandError> {
2647    if let Some(path) = &options.out {
2648        let data = serde_json::to_vec_pretty(run)?;
2649        fs::write(path, data)?;
2650        return Ok(());
2651    }
2652
2653    let stdout = io::stdout();
2654    let mut handle = stdout.lock();
2655    serde_json::to_writer_pretty(&mut handle, run)?;
2656    writeln!(handle)?;
2657    Ok(())
2658}
2659
2660// Persist the restore apply journal to its canonical runner path.
2661fn write_apply_journal_file(
2662    path: &PathBuf,
2663    journal: &RestoreApplyJournal,
2664) -> Result<(), RestoreCommandError> {
2665    let data = serde_json::to_vec_pretty(journal)?;
2666    fs::write(path, data)?;
2667    Ok(())
2668}
2669
2670// Write the computed apply next-operation response to stdout or a requested output file.
2671fn write_apply_next(
2672    options: &RestoreApplyNextOptions,
2673    next: &RestoreApplyNextOperation,
2674) -> Result<(), RestoreCommandError> {
2675    if let Some(path) = &options.out {
2676        let data = serde_json::to_vec_pretty(next)?;
2677        fs::write(path, data)?;
2678        return Ok(());
2679    }
2680
2681    let stdout = io::stdout();
2682    let mut handle = stdout.lock();
2683    serde_json::to_writer_pretty(&mut handle, next)?;
2684    writeln!(handle)?;
2685    Ok(())
2686}
2687
2688// Write the computed apply command preview to stdout or a requested output file.
2689fn write_apply_command(
2690    options: &RestoreApplyCommandOptions,
2691    preview: &RestoreApplyCommandPreview,
2692) -> Result<(), RestoreCommandError> {
2693    if let Some(path) = &options.out {
2694        let data = serde_json::to_vec_pretty(preview)?;
2695        fs::write(path, data)?;
2696        return Ok(());
2697    }
2698
2699    let stdout = io::stdout();
2700    let mut handle = stdout.lock();
2701    serde_json::to_writer_pretty(&mut handle, preview)?;
2702    writeln!(handle)?;
2703    Ok(())
2704}
2705
2706// Write the claimed apply journal to stdout or a requested output file.
2707fn write_apply_claim(
2708    options: &RestoreApplyClaimOptions,
2709    journal: &RestoreApplyJournal,
2710) -> Result<(), RestoreCommandError> {
2711    if let Some(path) = &options.out {
2712        let data = serde_json::to_vec_pretty(journal)?;
2713        fs::write(path, data)?;
2714        return Ok(());
2715    }
2716
2717    let stdout = io::stdout();
2718    let mut handle = stdout.lock();
2719    serde_json::to_writer_pretty(&mut handle, journal)?;
2720    writeln!(handle)?;
2721    Ok(())
2722}
2723
2724// Write the unclaimed apply journal to stdout or a requested output file.
2725fn write_apply_unclaim(
2726    options: &RestoreApplyUnclaimOptions,
2727    journal: &RestoreApplyJournal,
2728) -> Result<(), RestoreCommandError> {
2729    if let Some(path) = &options.out {
2730        let data = serde_json::to_vec_pretty(journal)?;
2731        fs::write(path, data)?;
2732        return Ok(());
2733    }
2734
2735    let stdout = io::stdout();
2736    let mut handle = stdout.lock();
2737    serde_json::to_writer_pretty(&mut handle, journal)?;
2738    writeln!(handle)?;
2739    Ok(())
2740}
2741
2742// Write the updated apply journal to stdout or a requested output file.
2743fn write_apply_mark(
2744    options: &RestoreApplyMarkOptions,
2745    journal: &RestoreApplyJournal,
2746) -> Result<(), RestoreCommandError> {
2747    if let Some(path) = &options.out {
2748        let data = serde_json::to_vec_pretty(journal)?;
2749        fs::write(path, data)?;
2750        return Ok(());
2751    }
2752
2753    let stdout = io::stdout();
2754    let mut handle = stdout.lock();
2755    serde_json::to_writer_pretty(&mut handle, journal)?;
2756    writeln!(handle)?;
2757    Ok(())
2758}
2759
2760// Read the next required option value.
2761fn next_value<I>(args: &mut I, option: &'static str) -> Result<String, RestoreCommandError>
2762where
2763    I: Iterator<Item = OsString>,
2764{
2765    args.next()
2766        .and_then(|value| value.into_string().ok())
2767        .ok_or(RestoreCommandError::MissingValue(option))
2768}
2769
2770// Return restore command usage text.
2771const fn usage() -> &'static str {
2772    "usage: canic restore plan (--manifest <file> | --backup-dir <dir>) [--mapping <file>] [--out <file>] [--require-verified] [--require-restore-ready]\n       canic restore status --plan <file> [--out <file>]\n       canic restore apply --plan <file> [--status <file>] [--backup-dir <dir>] --dry-run [--out <file>] [--journal-out <file>]\n       canic restore apply-status --journal <file> [--out <file>] [--require-ready] [--require-no-pending] [--require-no-failed] [--require-complete] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n       canic restore apply-report --journal <file> [--out <file>] [--require-no-attention] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n       canic restore run --journal <file> (--dry-run | --execute | --unclaim-pending) [--dfx <path>] [--network <name>] [--max-steps <n>] [--out <file>] [--require-complete] [--require-no-attention] [--require-run-mode <text>] [--require-stopped-reason <text>] [--require-next-action <text>] [--require-executed-count <n>] [--require-receipt-count <n>] [--require-completed-receipt-count <n>] [--require-failed-receipt-count <n>] [--require-recovered-receipt-count <n>] [--require-remaining-count <n>] [--require-attention-count <n>] [--require-completion-basis-points <n>] [--require-no-pending-before <text>]\n       canic restore apply-next --journal <file> [--out <file>]\n       canic restore apply-command --journal <file> [--dfx <path>] [--network <name>] [--out <file>] [--require-command]\n       canic restore apply-claim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n       canic restore apply-unclaim --journal <file> [--sequence <n>] [--updated-at <text>] [--out <file>]\n       canic restore apply-mark --journal <file> --sequence <n> --state completed|failed [--reason <text>] [--updated-at <text>] [--out <file>] [--require-pending]"
2773}
2774
2775#[cfg(test)]
2776mod tests {
2777    use super::*;
2778    use canic_backup::restore::RestoreApplyOperationState;
2779    use canic_backup::{
2780        artifacts::ArtifactChecksum,
2781        journal::{ArtifactJournalEntry, ArtifactState, DownloadJournal},
2782        manifest::{
2783            BackupUnit, BackupUnitKind, ConsistencyMode, ConsistencySection, FleetMember,
2784            FleetSection, IdentityMode, SourceMetadata, SourceSnapshot, ToolMetadata,
2785            VerificationCheck, VerificationPlan,
2786        },
2787    };
2788    use serde_json::json;
2789    use std::{
2790        path::Path,
2791        time::{SystemTime, UNIX_EPOCH},
2792    };
2793
2794    const ROOT: &str = "aaaaa-aa";
2795    const CHILD: &str = "renrk-eyaaa-aaaaa-aaada-cai";
2796    const MAPPED_CHILD: &str = "rno2w-sqaaa-aaaaa-aaacq-cai";
2797    const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
2798
2799    ///
2800    /// RestoreCliFixture
2801    ///
2802
2803    struct RestoreCliFixture {
2804        root: PathBuf,
2805        journal_path: PathBuf,
2806        out_path: PathBuf,
2807    }
2808
2809    impl RestoreCliFixture {
2810        // Create a temp restore CLI fixture with canonical journal and output paths.
2811        fn new(prefix: &str, out_file: &str) -> Self {
2812            let root = temp_dir(prefix);
2813            fs::create_dir_all(&root).expect("create temp root");
2814
2815            Self {
2816                journal_path: root.join("restore-apply-journal.json"),
2817                out_path: root.join(out_file),
2818                root,
2819            }
2820        }
2821
2822        // Persist a restore apply journal at the fixture journal path.
2823        fn write_journal(&self, journal: &RestoreApplyJournal) {
2824            fs::write(
2825                &self.journal_path,
2826                serde_json::to_vec(journal).expect("serialize journal"),
2827            )
2828            .expect("write journal");
2829        }
2830
2831        // Run apply-status against the fixture journal and output paths.
2832        fn run_apply_status(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
2833            self.run_journal_command("apply-status", extra)
2834        }
2835
2836        // Run apply-report against the fixture journal and output paths.
2837        fn run_apply_report(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
2838            self.run_journal_command("apply-report", extra)
2839        }
2840
2841        // Run restore-run against the fixture journal and output paths.
2842        fn run_restore_run(&self, extra: &[&str]) -> Result<(), RestoreCommandError> {
2843            self.run_journal_command("run", extra)
2844        }
2845
2846        // Read the fixture output as a typed JSON value.
2847        fn read_out<T>(&self, label: &str) -> T
2848        where
2849            T: serde::de::DeserializeOwned,
2850        {
2851            serde_json::from_slice(&fs::read(&self.out_path).expect(label)).expect(label)
2852        }
2853
2854        // Build and run one journal-backed restore CLI command.
2855        fn run_journal_command(
2856            &self,
2857            command: &str,
2858            extra: &[&str],
2859        ) -> Result<(), RestoreCommandError> {
2860            let mut args = vec![
2861                OsString::from(command),
2862                OsString::from("--journal"),
2863                OsString::from(self.journal_path.as_os_str()),
2864                OsString::from("--out"),
2865                OsString::from(self.out_path.as_os_str()),
2866            ];
2867            args.extend(extra.iter().map(OsString::from));
2868            run(args)
2869        }
2870    }
2871
2872    impl Drop for RestoreCliFixture {
2873        // Remove the fixture directory after each test completes.
2874        fn drop(&mut self) {
2875            let _ = fs::remove_dir_all(&self.root);
2876        }
2877    }
2878
2879    // Ensure restore plan options parse the intended no-mutation command.
2880    #[test]
2881    fn parses_restore_plan_options() {
2882        let options = RestorePlanOptions::parse([
2883            OsString::from("--manifest"),
2884            OsString::from("manifest.json"),
2885            OsString::from("--mapping"),
2886            OsString::from("mapping.json"),
2887            OsString::from("--out"),
2888            OsString::from("plan.json"),
2889            OsString::from("--require-restore-ready"),
2890        ])
2891        .expect("parse options");
2892
2893        assert_eq!(options.manifest, Some(PathBuf::from("manifest.json")));
2894        assert_eq!(options.backup_dir, None);
2895        assert_eq!(options.mapping, Some(PathBuf::from("mapping.json")));
2896        assert_eq!(options.out, Some(PathBuf::from("plan.json")));
2897        assert!(!options.require_verified);
2898        assert!(options.require_restore_ready);
2899    }
2900
2901    // Ensure verified restore plan options parse with the canonical backup source.
2902    #[test]
2903    fn parses_verified_restore_plan_options() {
2904        let options = RestorePlanOptions::parse([
2905            OsString::from("--backup-dir"),
2906            OsString::from("backups/run"),
2907            OsString::from("--require-verified"),
2908        ])
2909        .expect("parse verified options");
2910
2911        assert_eq!(options.manifest, None);
2912        assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2913        assert_eq!(options.mapping, None);
2914        assert_eq!(options.out, None);
2915        assert!(options.require_verified);
2916        assert!(!options.require_restore_ready);
2917    }
2918
2919    // Ensure restore status options parse the intended no-mutation command.
2920    #[test]
2921    fn parses_restore_status_options() {
2922        let options = RestoreStatusOptions::parse([
2923            OsString::from("--plan"),
2924            OsString::from("restore-plan.json"),
2925            OsString::from("--out"),
2926            OsString::from("restore-status.json"),
2927        ])
2928        .expect("parse status options");
2929
2930        assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2931        assert_eq!(options.out, Some(PathBuf::from("restore-status.json")));
2932    }
2933
2934    // Ensure restore apply options require the explicit dry-run mode.
2935    #[test]
2936    fn parses_restore_apply_dry_run_options() {
2937        let options = RestoreApplyOptions::parse([
2938            OsString::from("--plan"),
2939            OsString::from("restore-plan.json"),
2940            OsString::from("--status"),
2941            OsString::from("restore-status.json"),
2942            OsString::from("--backup-dir"),
2943            OsString::from("backups/run"),
2944            OsString::from("--dry-run"),
2945            OsString::from("--out"),
2946            OsString::from("restore-apply-dry-run.json"),
2947            OsString::from("--journal-out"),
2948            OsString::from("restore-apply-journal.json"),
2949        ])
2950        .expect("parse apply options");
2951
2952        assert_eq!(options.plan, PathBuf::from("restore-plan.json"));
2953        assert_eq!(options.status, Some(PathBuf::from("restore-status.json")));
2954        assert_eq!(options.backup_dir, Some(PathBuf::from("backups/run")));
2955        assert_eq!(
2956            options.out,
2957            Some(PathBuf::from("restore-apply-dry-run.json"))
2958        );
2959        assert_eq!(
2960            options.journal_out,
2961            Some(PathBuf::from("restore-apply-journal.json"))
2962        );
2963        assert!(options.dry_run);
2964    }
2965
2966    // Ensure restore apply-status options parse the intended journal command.
2967    #[test]
2968    fn parses_restore_apply_status_options() {
2969        let options = RestoreApplyStatusOptions::parse([
2970            OsString::from("--journal"),
2971            OsString::from("restore-apply-journal.json"),
2972            OsString::from("--out"),
2973            OsString::from("restore-apply-status.json"),
2974            OsString::from("--require-ready"),
2975            OsString::from("--require-no-pending"),
2976            OsString::from("--require-no-failed"),
2977            OsString::from("--require-complete"),
2978            OsString::from("--require-remaining-count"),
2979            OsString::from("7"),
2980            OsString::from("--require-attention-count"),
2981            OsString::from("0"),
2982            OsString::from("--require-completion-basis-points"),
2983            OsString::from("1250"),
2984            OsString::from("--require-no-pending-before"),
2985            OsString::from("2026-05-05T12:00:00Z"),
2986        ])
2987        .expect("parse apply-status options");
2988
2989        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
2990        assert!(options.require_ready);
2991        assert!(options.require_no_pending);
2992        assert!(options.require_no_failed);
2993        assert!(options.require_complete);
2994        assert_eq!(options.require_remaining_count, Some(7));
2995        assert_eq!(options.require_attention_count, Some(0));
2996        assert_eq!(options.require_completion_basis_points, Some(1250));
2997        assert_eq!(
2998            options.require_no_pending_before.as_deref(),
2999            Some("2026-05-05T12:00:00Z")
3000        );
3001        assert_eq!(
3002            options.out,
3003            Some(PathBuf::from("restore-apply-status.json"))
3004        );
3005    }
3006
3007    // Ensure restore apply-report options parse the intended journal command.
3008    #[test]
3009    fn parses_restore_apply_report_options() {
3010        let options = RestoreApplyReportOptions::parse([
3011            OsString::from("--journal"),
3012            OsString::from("restore-apply-journal.json"),
3013            OsString::from("--out"),
3014            OsString::from("restore-apply-report.json"),
3015            OsString::from("--require-no-attention"),
3016            OsString::from("--require-remaining-count"),
3017            OsString::from("8"),
3018            OsString::from("--require-attention-count"),
3019            OsString::from("0"),
3020            OsString::from("--require-completion-basis-points"),
3021            OsString::from("0"),
3022            OsString::from("--require-no-pending-before"),
3023            OsString::from("2026-05-05T12:00:00Z"),
3024        ])
3025        .expect("parse apply-report options");
3026
3027        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3028        assert!(options.require_no_attention);
3029        assert_eq!(options.require_remaining_count, Some(8));
3030        assert_eq!(options.require_attention_count, Some(0));
3031        assert_eq!(options.require_completion_basis_points, Some(0));
3032        assert_eq!(
3033            options.require_no_pending_before.as_deref(),
3034            Some("2026-05-05T12:00:00Z")
3035        );
3036        assert_eq!(
3037            options.out,
3038            Some(PathBuf::from("restore-apply-report.json"))
3039        );
3040    }
3041
3042    // Ensure restore run options parse the native runner dry-run command.
3043    #[test]
3044    fn parses_restore_run_dry_run_options() {
3045        let options = RestoreRunOptions::parse([
3046            OsString::from("--journal"),
3047            OsString::from("restore-apply-journal.json"),
3048            OsString::from("--dry-run"),
3049            OsString::from("--dfx"),
3050            OsString::from("/tmp/dfx"),
3051            OsString::from("--network"),
3052            OsString::from("local"),
3053            OsString::from("--out"),
3054            OsString::from("restore-run-dry-run.json"),
3055            OsString::from("--max-steps"),
3056            OsString::from("1"),
3057            OsString::from("--require-complete"),
3058            OsString::from("--require-no-attention"),
3059            OsString::from("--require-run-mode"),
3060            OsString::from("dry-run"),
3061            OsString::from("--require-stopped-reason"),
3062            OsString::from("preview"),
3063            OsString::from("--require-next-action"),
3064            OsString::from("rerun"),
3065            OsString::from("--require-executed-count"),
3066            OsString::from("0"),
3067            OsString::from("--require-receipt-count"),
3068            OsString::from("0"),
3069            OsString::from("--require-completed-receipt-count"),
3070            OsString::from("0"),
3071            OsString::from("--require-failed-receipt-count"),
3072            OsString::from("0"),
3073            OsString::from("--require-recovered-receipt-count"),
3074            OsString::from("0"),
3075            OsString::from("--require-remaining-count"),
3076            OsString::from("8"),
3077            OsString::from("--require-attention-count"),
3078            OsString::from("0"),
3079            OsString::from("--require-completion-basis-points"),
3080            OsString::from("0"),
3081            OsString::from("--require-no-pending-before"),
3082            OsString::from("2026-05-05T12:00:00Z"),
3083        ])
3084        .expect("parse restore run options");
3085
3086        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3087        assert_eq!(options.dfx, "/tmp/dfx");
3088        assert_eq!(options.network.as_deref(), Some("local"));
3089        assert_eq!(options.out, Some(PathBuf::from("restore-run-dry-run.json")));
3090        assert!(options.dry_run);
3091        assert!(!options.execute);
3092        assert!(!options.unclaim_pending);
3093        assert_eq!(options.max_steps, Some(1));
3094        assert!(options.require_complete);
3095        assert!(options.require_no_attention);
3096        assert_eq!(options.require_run_mode.as_deref(), Some("dry-run"));
3097        assert_eq!(options.require_stopped_reason.as_deref(), Some("preview"));
3098        assert_eq!(options.require_next_action.as_deref(), Some("rerun"));
3099        assert_eq!(options.require_executed_count, Some(0));
3100        assert_eq!(options.require_receipt_count, Some(0));
3101        assert_eq!(options.require_completed_receipt_count, Some(0));
3102        assert_eq!(options.require_failed_receipt_count, Some(0));
3103        assert_eq!(options.require_recovered_receipt_count, Some(0));
3104        assert_eq!(options.require_remaining_count, Some(8));
3105        assert_eq!(options.require_attention_count, Some(0));
3106        assert_eq!(options.require_completion_basis_points, Some(0));
3107        assert_eq!(
3108            options.require_no_pending_before.as_deref(),
3109            Some("2026-05-05T12:00:00Z")
3110        );
3111    }
3112
3113    // Ensure restore run options parse the native execute command.
3114    #[test]
3115    fn parses_restore_run_execute_options() {
3116        let options = RestoreRunOptions::parse([
3117            OsString::from("--journal"),
3118            OsString::from("restore-apply-journal.json"),
3119            OsString::from("--execute"),
3120            OsString::from("--dfx"),
3121            OsString::from("/bin/true"),
3122            OsString::from("--max-steps"),
3123            OsString::from("4"),
3124        ])
3125        .expect("parse restore run execute options");
3126
3127        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3128        assert_eq!(options.dfx, "/bin/true");
3129        assert_eq!(options.network, None);
3130        assert_eq!(options.out, None);
3131        assert!(!options.dry_run);
3132        assert!(options.execute);
3133        assert!(!options.unclaim_pending);
3134        assert_eq!(options.max_steps, Some(4));
3135        assert!(!options.require_complete);
3136        assert!(!options.require_no_attention);
3137        assert_eq!(options.require_run_mode, None);
3138        assert_eq!(options.require_stopped_reason, None);
3139        assert_eq!(options.require_next_action, None);
3140        assert_eq!(options.require_executed_count, None);
3141        assert_eq!(options.require_receipt_count, None);
3142        assert_eq!(options.require_completed_receipt_count, None);
3143        assert_eq!(options.require_failed_receipt_count, None);
3144        assert_eq!(options.require_recovered_receipt_count, None);
3145    }
3146
3147    // Ensure restore run options parse the native pending-operation recovery mode.
3148    #[test]
3149    fn parses_restore_run_unclaim_pending_options() {
3150        let options = RestoreRunOptions::parse([
3151            OsString::from("--journal"),
3152            OsString::from("restore-apply-journal.json"),
3153            OsString::from("--unclaim-pending"),
3154            OsString::from("--out"),
3155            OsString::from("restore-run.json"),
3156        ])
3157        .expect("parse restore run unclaim options");
3158
3159        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3160        assert_eq!(options.out, Some(PathBuf::from("restore-run.json")));
3161        assert!(!options.dry_run);
3162        assert!(!options.execute);
3163        assert!(options.unclaim_pending);
3164    }
3165
3166    // Ensure restore apply-next options parse the intended journal command.
3167    #[test]
3168    fn parses_restore_apply_next_options() {
3169        let options = RestoreApplyNextOptions::parse([
3170            OsString::from("--journal"),
3171            OsString::from("restore-apply-journal.json"),
3172            OsString::from("--out"),
3173            OsString::from("restore-apply-next.json"),
3174        ])
3175        .expect("parse apply-next options");
3176
3177        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3178        assert_eq!(options.out, Some(PathBuf::from("restore-apply-next.json")));
3179    }
3180
3181    // Ensure restore apply-command options parse the intended preview command.
3182    #[test]
3183    fn parses_restore_apply_command_options() {
3184        let options = RestoreApplyCommandOptions::parse([
3185            OsString::from("--journal"),
3186            OsString::from("restore-apply-journal.json"),
3187            OsString::from("--dfx"),
3188            OsString::from("/tmp/dfx"),
3189            OsString::from("--network"),
3190            OsString::from("local"),
3191            OsString::from("--out"),
3192            OsString::from("restore-apply-command.json"),
3193            OsString::from("--require-command"),
3194        ])
3195        .expect("parse apply-command options");
3196
3197        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3198        assert_eq!(options.dfx, "/tmp/dfx");
3199        assert_eq!(options.network.as_deref(), Some("local"));
3200        assert!(options.require_command);
3201        assert_eq!(
3202            options.out,
3203            Some(PathBuf::from("restore-apply-command.json"))
3204        );
3205    }
3206
3207    // Ensure restore apply-claim options parse the intended journal command.
3208    #[test]
3209    fn parses_restore_apply_claim_options() {
3210        let options = RestoreApplyClaimOptions::parse([
3211            OsString::from("--journal"),
3212            OsString::from("restore-apply-journal.json"),
3213            OsString::from("--sequence"),
3214            OsString::from("0"),
3215            OsString::from("--updated-at"),
3216            OsString::from("2026-05-04T12:00:00Z"),
3217            OsString::from("--out"),
3218            OsString::from("restore-apply-journal.claimed.json"),
3219        ])
3220        .expect("parse apply-claim options");
3221
3222        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3223        assert_eq!(options.sequence, Some(0));
3224        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:00:00Z"));
3225        assert_eq!(
3226            options.out,
3227            Some(PathBuf::from("restore-apply-journal.claimed.json"))
3228        );
3229    }
3230
3231    // Ensure restore apply-unclaim options parse the intended journal command.
3232    #[test]
3233    fn parses_restore_apply_unclaim_options() {
3234        let options = RestoreApplyUnclaimOptions::parse([
3235            OsString::from("--journal"),
3236            OsString::from("restore-apply-journal.json"),
3237            OsString::from("--sequence"),
3238            OsString::from("0"),
3239            OsString::from("--updated-at"),
3240            OsString::from("2026-05-04T12:01:00Z"),
3241            OsString::from("--out"),
3242            OsString::from("restore-apply-journal.unclaimed.json"),
3243        ])
3244        .expect("parse apply-unclaim options");
3245
3246        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3247        assert_eq!(options.sequence, Some(0));
3248        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:01:00Z"));
3249        assert_eq!(
3250            options.out,
3251            Some(PathBuf::from("restore-apply-journal.unclaimed.json"))
3252        );
3253    }
3254
3255    // Ensure restore apply-mark options parse the intended journal update command.
3256    #[test]
3257    fn parses_restore_apply_mark_options() {
3258        let options = RestoreApplyMarkOptions::parse([
3259            OsString::from("--journal"),
3260            OsString::from("restore-apply-journal.json"),
3261            OsString::from("--sequence"),
3262            OsString::from("4"),
3263            OsString::from("--state"),
3264            OsString::from("failed"),
3265            OsString::from("--reason"),
3266            OsString::from("dfx-load-failed"),
3267            OsString::from("--updated-at"),
3268            OsString::from("2026-05-04T12:02:00Z"),
3269            OsString::from("--out"),
3270            OsString::from("restore-apply-journal.updated.json"),
3271            OsString::from("--require-pending"),
3272        ])
3273        .expect("parse apply-mark options");
3274
3275        assert_eq!(options.journal, PathBuf::from("restore-apply-journal.json"));
3276        assert_eq!(options.sequence, 4);
3277        assert_eq!(options.state, RestoreApplyMarkState::Failed);
3278        assert_eq!(options.reason.as_deref(), Some("dfx-load-failed"));
3279        assert_eq!(options.updated_at.as_deref(), Some("2026-05-04T12:02:00Z"));
3280        assert!(options.require_pending);
3281        assert_eq!(
3282            options.out,
3283            Some(PathBuf::from("restore-apply-journal.updated.json"))
3284        );
3285    }
3286
3287    // Ensure restore apply refuses non-dry-run execution while apply is scaffolded.
3288    #[test]
3289    fn restore_apply_requires_dry_run() {
3290        let err = RestoreApplyOptions::parse([
3291            OsString::from("--plan"),
3292            OsString::from("restore-plan.json"),
3293        ])
3294        .expect_err("apply without dry-run should fail");
3295
3296        assert!(matches!(err, RestoreCommandError::ApplyRequiresDryRun));
3297    }
3298
3299    // Ensure restore run refuses mutation while native execution is scaffolded.
3300    #[test]
3301    fn restore_run_requires_mode() {
3302        let err = RestoreRunOptions::parse([
3303            OsString::from("--journal"),
3304            OsString::from("restore-apply-journal.json"),
3305        ])
3306        .expect_err("restore run without dry-run should fail");
3307
3308        assert!(matches!(err, RestoreCommandError::RestoreRunRequiresMode));
3309    }
3310
3311    // Ensure restore run rejects ambiguous execution modes.
3312    #[test]
3313    fn restore_run_rejects_conflicting_modes() {
3314        let err = RestoreRunOptions::parse([
3315            OsString::from("--journal"),
3316            OsString::from("restore-apply-journal.json"),
3317            OsString::from("--dry-run"),
3318            OsString::from("--execute"),
3319            OsString::from("--unclaim-pending"),
3320        ])
3321        .expect_err("restore run should reject conflicting modes");
3322
3323        assert!(matches!(
3324            err,
3325            RestoreCommandError::RestoreRunConflictingModes
3326        ));
3327    }
3328
3329    // Ensure backup-dir restore planning reads the canonical layout manifest.
3330    #[test]
3331    fn plan_restore_reads_manifest_from_backup_dir() {
3332        let root = temp_dir("canic-cli-restore-plan-layout");
3333        let layout = BackupLayout::new(root.clone());
3334        layout
3335            .write_manifest(&valid_manifest())
3336            .expect("write manifest");
3337
3338        let options = RestorePlanOptions {
3339            manifest: None,
3340            backup_dir: Some(root.clone()),
3341            mapping: None,
3342            out: None,
3343            require_verified: false,
3344            require_restore_ready: false,
3345        };
3346
3347        let plan = plan_restore(&options).expect("plan restore");
3348
3349        fs::remove_dir_all(root).expect("remove temp root");
3350        assert_eq!(plan.backup_id, "backup-test");
3351        assert_eq!(plan.member_count, 2);
3352    }
3353
3354    // Ensure restore planning has exactly one manifest source.
3355    #[test]
3356    fn parse_rejects_conflicting_manifest_sources() {
3357        let err = RestorePlanOptions::parse([
3358            OsString::from("--manifest"),
3359            OsString::from("manifest.json"),
3360            OsString::from("--backup-dir"),
3361            OsString::from("backups/run"),
3362        ])
3363        .expect_err("conflicting sources should fail");
3364
3365        assert!(matches!(
3366            err,
3367            RestoreCommandError::ConflictingManifestSources
3368        ));
3369    }
3370
3371    // Ensure verified planning requires the canonical backup layout source.
3372    #[test]
3373    fn parse_rejects_require_verified_with_manifest_source() {
3374        let err = RestorePlanOptions::parse([
3375            OsString::from("--manifest"),
3376            OsString::from("manifest.json"),
3377            OsString::from("--require-verified"),
3378        ])
3379        .expect_err("verification should require a backup layout");
3380
3381        assert!(matches!(
3382            err,
3383            RestoreCommandError::RequireVerifiedNeedsBackupDir
3384        ));
3385    }
3386
3387    // Ensure restore planning can require manifest, journal, and artifact integrity.
3388    #[test]
3389    fn plan_restore_requires_verified_backup_layout() {
3390        let root = temp_dir("canic-cli-restore-plan-verified");
3391        let layout = BackupLayout::new(root.clone());
3392        let manifest = valid_manifest();
3393        write_verified_layout(&root, &layout, &manifest);
3394
3395        let options = RestorePlanOptions {
3396            manifest: None,
3397            backup_dir: Some(root.clone()),
3398            mapping: None,
3399            out: None,
3400            require_verified: true,
3401            require_restore_ready: false,
3402        };
3403
3404        let plan = plan_restore(&options).expect("plan verified restore");
3405
3406        fs::remove_dir_all(root).expect("remove temp root");
3407        assert_eq!(plan.backup_id, "backup-test");
3408        assert_eq!(plan.member_count, 2);
3409    }
3410
3411    // Ensure required verification fails before planning when the layout is incomplete.
3412    #[test]
3413    fn plan_restore_rejects_unverified_backup_layout() {
3414        let root = temp_dir("canic-cli-restore-plan-unverified");
3415        let layout = BackupLayout::new(root.clone());
3416        layout
3417            .write_manifest(&valid_manifest())
3418            .expect("write manifest");
3419
3420        let options = RestorePlanOptions {
3421            manifest: None,
3422            backup_dir: Some(root.clone()),
3423            mapping: None,
3424            out: None,
3425            require_verified: true,
3426            require_restore_ready: false,
3427        };
3428
3429        let err = plan_restore(&options).expect_err("missing journal should fail");
3430
3431        fs::remove_dir_all(root).expect("remove temp root");
3432        assert!(matches!(err, RestoreCommandError::Persistence(_)));
3433    }
3434
3435    // Ensure the CLI planning path validates manifests and applies mappings.
3436    #[test]
3437    fn plan_restore_reads_manifest_and_mapping() {
3438        let root = temp_dir("canic-cli-restore-plan");
3439        fs::create_dir_all(&root).expect("create temp root");
3440        let manifest_path = root.join("manifest.json");
3441        let mapping_path = root.join("mapping.json");
3442
3443        fs::write(
3444            &manifest_path,
3445            serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
3446        )
3447        .expect("write manifest");
3448        fs::write(
3449            &mapping_path,
3450            json!({
3451                "members": [
3452                    {"source_canister": ROOT, "target_canister": ROOT},
3453                    {"source_canister": CHILD, "target_canister": MAPPED_CHILD}
3454                ]
3455            })
3456            .to_string(),
3457        )
3458        .expect("write mapping");
3459
3460        let options = RestorePlanOptions {
3461            manifest: Some(manifest_path),
3462            backup_dir: None,
3463            mapping: Some(mapping_path),
3464            out: None,
3465            require_verified: false,
3466            require_restore_ready: false,
3467        };
3468
3469        let plan = plan_restore(&options).expect("plan restore");
3470
3471        fs::remove_dir_all(root).expect("remove temp root");
3472        let members = plan.ordered_members();
3473        assert_eq!(members.len(), 2);
3474        assert_eq!(members[0].source_canister, ROOT);
3475        assert_eq!(members[1].target_canister, MAPPED_CHILD);
3476    }
3477
3478    // Ensure restore-readiness gating happens after writing the plan artifact.
3479    #[test]
3480    fn run_restore_plan_require_restore_ready_writes_plan_then_fails() {
3481        let root = temp_dir("canic-cli-restore-plan-require-ready");
3482        fs::create_dir_all(&root).expect("create temp root");
3483        let manifest_path = root.join("manifest.json");
3484        let out_path = root.join("plan.json");
3485
3486        fs::write(
3487            &manifest_path,
3488            serde_json::to_vec(&valid_manifest()).expect("serialize manifest"),
3489        )
3490        .expect("write manifest");
3491
3492        let err = run([
3493            OsString::from("plan"),
3494            OsString::from("--manifest"),
3495            OsString::from(manifest_path.as_os_str()),
3496            OsString::from("--out"),
3497            OsString::from(out_path.as_os_str()),
3498            OsString::from("--require-restore-ready"),
3499        ])
3500        .expect_err("restore readiness should be enforced");
3501
3502        assert!(out_path.exists());
3503        let plan: RestorePlan =
3504            serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
3505
3506        fs::remove_dir_all(root).expect("remove temp root");
3507        assert!(!plan.readiness_summary.ready);
3508        assert!(matches!(
3509            err,
3510            RestoreCommandError::RestoreNotReady {
3511                reasons,
3512                ..
3513            } if reasons == [
3514                "missing-module-hash",
3515                "missing-wasm-hash",
3516                "missing-snapshot-checksum"
3517            ]
3518        ));
3519    }
3520
3521    // Ensure restore-readiness gating accepts plans with complete provenance.
3522    #[test]
3523    fn run_restore_plan_require_restore_ready_accepts_ready_plan() {
3524        let root = temp_dir("canic-cli-restore-plan-ready");
3525        fs::create_dir_all(&root).expect("create temp root");
3526        let manifest_path = root.join("manifest.json");
3527        let out_path = root.join("plan.json");
3528
3529        fs::write(
3530            &manifest_path,
3531            serde_json::to_vec(&restore_ready_manifest()).expect("serialize manifest"),
3532        )
3533        .expect("write manifest");
3534
3535        run([
3536            OsString::from("plan"),
3537            OsString::from("--manifest"),
3538            OsString::from(manifest_path.as_os_str()),
3539            OsString::from("--out"),
3540            OsString::from(out_path.as_os_str()),
3541            OsString::from("--require-restore-ready"),
3542        ])
3543        .expect("restore-ready plan should pass");
3544
3545        let plan: RestorePlan =
3546            serde_json::from_slice(&fs::read(&out_path).expect("read plan")).expect("decode plan");
3547
3548        fs::remove_dir_all(root).expect("remove temp root");
3549        assert!(plan.readiness_summary.ready);
3550        assert!(plan.readiness_summary.reasons.is_empty());
3551    }
3552
3553    // Ensure restore status writes the initial planned execution journal.
3554    #[test]
3555    fn run_restore_status_writes_planned_status() {
3556        let root = temp_dir("canic-cli-restore-status");
3557        fs::create_dir_all(&root).expect("create temp root");
3558        let plan_path = root.join("restore-plan.json");
3559        let out_path = root.join("restore-status.json");
3560        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3561
3562        fs::write(
3563            &plan_path,
3564            serde_json::to_vec(&plan).expect("serialize plan"),
3565        )
3566        .expect("write plan");
3567
3568        run([
3569            OsString::from("status"),
3570            OsString::from("--plan"),
3571            OsString::from(plan_path.as_os_str()),
3572            OsString::from("--out"),
3573            OsString::from(out_path.as_os_str()),
3574        ])
3575        .expect("write restore status");
3576
3577        let status: RestoreStatus =
3578            serde_json::from_slice(&fs::read(&out_path).expect("read restore status"))
3579                .expect("decode restore status");
3580        let status_json: serde_json::Value = serde_json::to_value(&status).expect("encode status");
3581
3582        fs::remove_dir_all(root).expect("remove temp root");
3583        assert_eq!(status.status_version, 1);
3584        assert_eq!(status.backup_id.as_str(), "backup-test");
3585        assert!(status.ready);
3586        assert!(status.readiness_reasons.is_empty());
3587        assert_eq!(status.member_count, 2);
3588        assert_eq!(status.phase_count, 1);
3589        assert_eq!(status.planned_snapshot_uploads, 2);
3590        assert_eq!(status.planned_snapshot_loads, 2);
3591        assert_eq!(status.planned_code_reinstalls, 2);
3592        assert_eq!(status.planned_verification_checks, 2);
3593        assert_eq!(status.planned_operations, 8);
3594        assert_eq!(status.phases[0].members[0].source_canister, ROOT);
3595        assert_eq!(status_json["phases"][0]["members"][0]["state"], "planned");
3596    }
3597
3598    // Ensure restore apply dry-run writes ordered operations from plan and status.
3599    #[test]
3600    fn run_restore_apply_dry_run_writes_operations() {
3601        let root = temp_dir("canic-cli-restore-apply-dry-run");
3602        fs::create_dir_all(&root).expect("create temp root");
3603        let plan_path = root.join("restore-plan.json");
3604        let status_path = root.join("restore-status.json");
3605        let out_path = root.join("restore-apply-dry-run.json");
3606        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3607        let status = RestoreStatus::from_plan(&plan);
3608
3609        fs::write(
3610            &plan_path,
3611            serde_json::to_vec(&plan).expect("serialize plan"),
3612        )
3613        .expect("write plan");
3614        fs::write(
3615            &status_path,
3616            serde_json::to_vec(&status).expect("serialize status"),
3617        )
3618        .expect("write status");
3619
3620        run([
3621            OsString::from("apply"),
3622            OsString::from("--plan"),
3623            OsString::from(plan_path.as_os_str()),
3624            OsString::from("--status"),
3625            OsString::from(status_path.as_os_str()),
3626            OsString::from("--dry-run"),
3627            OsString::from("--out"),
3628            OsString::from(out_path.as_os_str()),
3629        ])
3630        .expect("write apply dry-run");
3631
3632        let dry_run: RestoreApplyDryRun =
3633            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3634                .expect("decode dry-run");
3635        let dry_run_json: serde_json::Value =
3636            serde_json::to_value(&dry_run).expect("encode dry-run");
3637
3638        fs::remove_dir_all(root).expect("remove temp root");
3639        assert_eq!(dry_run.dry_run_version, 1);
3640        assert_eq!(dry_run.backup_id.as_str(), "backup-test");
3641        assert!(dry_run.ready);
3642        assert!(dry_run.status_supplied);
3643        assert_eq!(dry_run.member_count, 2);
3644        assert_eq!(dry_run.phase_count, 1);
3645        assert_eq!(dry_run.planned_snapshot_uploads, 2);
3646        assert_eq!(dry_run.planned_operations, 8);
3647        assert_eq!(dry_run.rendered_operations, 8);
3648        assert_eq!(dry_run_json["operation_counts"]["snapshot_uploads"], 2);
3649        assert_eq!(dry_run_json["operation_counts"]["snapshot_loads"], 2);
3650        assert_eq!(dry_run_json["operation_counts"]["code_reinstalls"], 2);
3651        assert_eq!(dry_run_json["operation_counts"]["member_verifications"], 2);
3652        assert_eq!(dry_run_json["operation_counts"]["fleet_verifications"], 0);
3653        assert_eq!(
3654            dry_run_json["operation_counts"]["verification_operations"],
3655            2
3656        );
3657        assert_eq!(
3658            dry_run_json["phases"][0]["operations"][0]["operation"],
3659            "upload-snapshot"
3660        );
3661        assert_eq!(
3662            dry_run_json["phases"][0]["operations"][3]["operation"],
3663            "verify-member"
3664        );
3665        assert_eq!(
3666            dry_run_json["phases"][0]["operations"][3]["verification_kind"],
3667            "status"
3668        );
3669        assert_eq!(
3670            dry_run_json["phases"][0]["operations"][3]["verification_method"],
3671            serde_json::Value::Null
3672        );
3673    }
3674
3675    // Ensure restore apply dry-run can validate artifacts under a backup directory.
3676    #[test]
3677    fn run_restore_apply_dry_run_validates_backup_dir_artifacts() {
3678        let root = temp_dir("canic-cli-restore-apply-artifacts");
3679        fs::create_dir_all(&root).expect("create temp root");
3680        let plan_path = root.join("restore-plan.json");
3681        let out_path = root.join("restore-apply-dry-run.json");
3682        let journal_path = root.join("restore-apply-journal.json");
3683        let status_path = root.join("restore-apply-status.json");
3684        let mut manifest = restore_ready_manifest();
3685        write_manifest_artifacts(&root, &mut manifest);
3686        let plan = RestorePlanner::plan(&manifest, None).expect("build plan");
3687
3688        fs::write(
3689            &plan_path,
3690            serde_json::to_vec(&plan).expect("serialize plan"),
3691        )
3692        .expect("write plan");
3693
3694        run([
3695            OsString::from("apply"),
3696            OsString::from("--plan"),
3697            OsString::from(plan_path.as_os_str()),
3698            OsString::from("--backup-dir"),
3699            OsString::from(root.as_os_str()),
3700            OsString::from("--dry-run"),
3701            OsString::from("--out"),
3702            OsString::from(out_path.as_os_str()),
3703            OsString::from("--journal-out"),
3704            OsString::from(journal_path.as_os_str()),
3705        ])
3706        .expect("write apply dry-run");
3707        run([
3708            OsString::from("apply-status"),
3709            OsString::from("--journal"),
3710            OsString::from(journal_path.as_os_str()),
3711            OsString::from("--out"),
3712            OsString::from(status_path.as_os_str()),
3713        ])
3714        .expect("write apply status");
3715
3716        let dry_run: RestoreApplyDryRun =
3717            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
3718                .expect("decode dry-run");
3719        let validation = dry_run
3720            .artifact_validation
3721            .expect("artifact validation should be present");
3722        let journal_json: serde_json::Value =
3723            serde_json::from_slice(&fs::read(&journal_path).expect("read journal"))
3724                .expect("decode journal");
3725        let status_json: serde_json::Value =
3726            serde_json::from_slice(&fs::read(&status_path).expect("read apply status"))
3727                .expect("decode apply status");
3728
3729        fs::remove_dir_all(root).expect("remove temp root");
3730        assert_eq!(validation.checked_members, 2);
3731        assert!(validation.artifacts_present);
3732        assert!(validation.checksums_verified);
3733        assert_eq!(validation.members_with_expected_checksums, 2);
3734        assert_eq!(journal_json["ready"], true);
3735        assert_eq!(journal_json["operation_count"], 8);
3736        assert_eq!(journal_json["operation_counts"]["snapshot_uploads"], 2);
3737        assert_eq!(journal_json["operation_counts"]["snapshot_loads"], 2);
3738        assert_eq!(journal_json["operation_counts"]["code_reinstalls"], 2);
3739        assert_eq!(journal_json["operation_counts"]["member_verifications"], 2);
3740        assert_eq!(journal_json["operation_counts"]["fleet_verifications"], 0);
3741        assert_eq!(
3742            journal_json["operation_counts"]["verification_operations"],
3743            2
3744        );
3745        assert_eq!(journal_json["ready_operations"], 8);
3746        assert_eq!(journal_json["blocked_operations"], 0);
3747        assert_eq!(journal_json["operations"][0]["state"], "ready");
3748        assert_eq!(status_json["ready"], true);
3749        assert_eq!(status_json["operation_count"], 8);
3750        assert_eq!(status_json["operation_counts"]["snapshot_uploads"], 2);
3751        assert_eq!(status_json["operation_counts"]["snapshot_loads"], 2);
3752        assert_eq!(status_json["operation_counts"]["code_reinstalls"], 2);
3753        assert_eq!(status_json["operation_counts"]["member_verifications"], 2);
3754        assert_eq!(status_json["operation_counts"]["fleet_verifications"], 0);
3755        assert_eq!(
3756            status_json["operation_counts"]["verification_operations"],
3757            2
3758        );
3759        assert_eq!(status_json["operation_counts_supplied"], true);
3760        assert_eq!(status_json["progress"]["operation_count"], 8);
3761        assert_eq!(status_json["progress"]["completed_operations"], 0);
3762        assert_eq!(status_json["progress"]["remaining_operations"], 8);
3763        assert_eq!(status_json["progress"]["transitionable_operations"], 8);
3764        assert_eq!(status_json["progress"]["attention_operations"], 0);
3765        assert_eq!(status_json["progress"]["completion_basis_points"], 0);
3766        assert_eq!(status_json["next_ready_sequence"], 0);
3767        assert_eq!(status_json["next_ready_operation"], "upload-snapshot");
3768    }
3769
3770    // Ensure apply-status rejects structurally inconsistent journals.
3771    #[test]
3772    fn run_restore_apply_status_rejects_invalid_journal() {
3773        let root = temp_dir("canic-cli-restore-apply-status-invalid");
3774        fs::create_dir_all(&root).expect("create temp root");
3775        let journal_path = root.join("restore-apply-journal.json");
3776        let out_path = root.join("restore-apply-status.json");
3777        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
3778        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3779        let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
3780        journal.operation_count += 1;
3781
3782        fs::write(
3783            &journal_path,
3784            serde_json::to_vec(&journal).expect("serialize journal"),
3785        )
3786        .expect("write journal");
3787
3788        let err = run([
3789            OsString::from("apply-status"),
3790            OsString::from("--journal"),
3791            OsString::from(journal_path.as_os_str()),
3792            OsString::from("--out"),
3793            OsString::from(out_path.as_os_str()),
3794        ])
3795        .expect_err("invalid journal should fail");
3796
3797        assert!(!out_path.exists());
3798        fs::remove_dir_all(root).expect("remove temp root");
3799        assert!(matches!(
3800            err,
3801            RestoreCommandError::RestoreApplyJournal(RestoreApplyJournalError::CountMismatch {
3802                field: "operation_count",
3803                ..
3804            })
3805        ));
3806    }
3807
3808    // Ensure apply-status can fail closed after writing status for pending work.
3809    #[test]
3810    fn run_restore_apply_status_require_no_pending_writes_status_then_fails() {
3811        let fixture = RestoreCliFixture::new(
3812            "canic-cli-restore-apply-status-pending",
3813            "restore-apply-status.json",
3814        );
3815        let mut journal = ready_apply_journal();
3816        journal
3817            .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
3818            .expect("claim operation");
3819        fixture.write_journal(&journal);
3820
3821        let err = fixture
3822            .run_apply_status(&["--require-no-pending"])
3823            .expect_err("pending operation should fail requirement");
3824
3825        assert!(fixture.out_path.exists());
3826        let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
3827
3828        assert_eq!(status.pending_operations, 1);
3829        assert_eq!(status.next_transition_sequence, Some(0));
3830        assert_eq!(status.pending_summary.pending_operations, 1);
3831        assert_eq!(status.pending_summary.pending_sequence, Some(0));
3832        assert_eq!(
3833            status.pending_summary.pending_updated_at.as_deref(),
3834            Some("2026-05-04T12:00:00Z")
3835        );
3836        assert!(status.pending_summary.pending_updated_at_known);
3837        assert_eq!(
3838            status.next_transition_updated_at.as_deref(),
3839            Some("2026-05-04T12:00:00Z")
3840        );
3841        assert!(matches!(
3842            err,
3843            RestoreCommandError::RestoreApplyPending {
3844                pending_operations: 1,
3845                next_transition_sequence: Some(0),
3846                ..
3847            }
3848        ));
3849    }
3850
3851    // Ensure apply-status can fail closed when pending work is older than a cutoff.
3852    #[test]
3853    fn run_restore_apply_status_require_no_pending_before_writes_status_then_fails() {
3854        let fixture = RestoreCliFixture::new(
3855            "canic-cli-restore-apply-status-stale-pending",
3856            "restore-apply-status.json",
3857        );
3858        let mut journal = ready_apply_journal();
3859        journal
3860            .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
3861            .expect("claim operation");
3862        fixture.write_journal(&journal);
3863
3864        let err = fixture
3865            .run_apply_status(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
3866            .expect_err("stale pending operation should fail requirement");
3867
3868        let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
3869
3870        assert_eq!(status.pending_summary.pending_sequence, Some(0));
3871        assert_eq!(
3872            status.pending_summary.pending_updated_at.as_deref(),
3873            Some("2026-05-04T12:00:00Z")
3874        );
3875        assert!(matches!(
3876            err,
3877            RestoreCommandError::RestoreApplyPendingStale {
3878                cutoff_updated_at,
3879                pending_sequence: Some(0),
3880                pending_updated_at,
3881                ..
3882            } if cutoff_updated_at == "2026-05-05T12:00:00Z"
3883                && pending_updated_at.as_deref() == Some("2026-05-04T12:00:00Z")
3884        ));
3885    }
3886
3887    // Ensure apply-status can fail closed on an unexpected progress summary.
3888    #[test]
3889    fn run_restore_apply_status_require_progress_writes_status_then_fails() {
3890        let fixture = RestoreCliFixture::new(
3891            "canic-cli-restore-apply-status-progress",
3892            "restore-apply-status.json",
3893        );
3894        let journal = ready_apply_journal();
3895        fixture.write_journal(&journal);
3896
3897        let err = fixture
3898            .run_apply_status(&[
3899                "--require-remaining-count",
3900                "7",
3901                "--require-attention-count",
3902                "0",
3903                "--require-completion-basis-points",
3904                "0",
3905            ])
3906            .expect_err("remaining progress mismatch should fail requirement");
3907
3908        let status: RestoreApplyJournalStatus = fixture.read_out("read apply status");
3909
3910        assert_eq!(status.progress.remaining_operations, 8);
3911        assert_eq!(status.progress.attention_operations, 0);
3912        assert_eq!(status.progress.completion_basis_points, 0);
3913        assert!(matches!(
3914            err,
3915            RestoreCommandError::RestoreApplyProgressMismatch {
3916                field: "remaining_operations",
3917                expected: 7,
3918                actual: 8,
3919                ..
3920            }
3921        ));
3922    }
3923
3924    // Ensure apply-status can fail closed after writing status for unready work.
3925    #[test]
3926    fn run_restore_apply_status_require_ready_writes_status_then_fails() {
3927        let root = temp_dir("canic-cli-restore-apply-status-ready");
3928        fs::create_dir_all(&root).expect("create temp root");
3929        let journal_path = root.join("restore-apply-journal.json");
3930        let out_path = root.join("restore-apply-status.json");
3931        let plan = RestorePlanner::plan(&valid_manifest(), None).expect("build plan");
3932        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
3933        let journal = RestoreApplyJournal::from_dry_run(&dry_run);
3934
3935        fs::write(
3936            &journal_path,
3937            serde_json::to_vec(&journal).expect("serialize journal"),
3938        )
3939        .expect("write journal");
3940
3941        let err = run([
3942            OsString::from("apply-status"),
3943            OsString::from("--journal"),
3944            OsString::from(journal_path.as_os_str()),
3945            OsString::from("--out"),
3946            OsString::from(out_path.as_os_str()),
3947            OsString::from("--require-ready"),
3948        ])
3949        .expect_err("unready journal should fail requirement");
3950
3951        let status: RestoreApplyJournalStatus =
3952            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
3953                .expect("decode apply status");
3954
3955        fs::remove_dir_all(root).expect("remove temp root");
3956        assert!(!status.ready);
3957        assert_eq!(status.blocked_operations, status.operation_count);
3958        assert!(
3959            status
3960                .blocked_reasons
3961                .contains(&"missing-snapshot-checksum".to_string())
3962        );
3963        assert!(matches!(
3964            err,
3965            RestoreCommandError::RestoreApplyNotReady { reasons, .. }
3966                if reasons.contains(&"missing-snapshot-checksum".to_string())
3967        ));
3968    }
3969
3970    // Ensure apply-report writes the operator-focused journal summary.
3971    #[test]
3972    fn run_restore_apply_report_writes_attention_summary() {
3973        let root = temp_dir("canic-cli-restore-apply-report");
3974        fs::create_dir_all(&root).expect("create temp root");
3975        let journal_path = root.join("restore-apply-journal.json");
3976        let out_path = root.join("restore-apply-report.json");
3977        let mut journal = ready_apply_journal();
3978        journal
3979            .mark_operation_failed_at(
3980                0,
3981                "dfx-upload-failed".to_string(),
3982                Some("2026-05-05T12:00:00Z".to_string()),
3983            )
3984            .expect("mark failed operation");
3985        journal
3986            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
3987            .expect("mark pending operation");
3988
3989        fs::write(
3990            &journal_path,
3991            serde_json::to_vec(&journal).expect("serialize journal"),
3992        )
3993        .expect("write journal");
3994
3995        run([
3996            OsString::from("apply-report"),
3997            OsString::from("--journal"),
3998            OsString::from(journal_path.as_os_str()),
3999            OsString::from("--out"),
4000            OsString::from(out_path.as_os_str()),
4001        ])
4002        .expect("write apply report");
4003
4004        let report: RestoreApplyJournalReport =
4005            serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
4006                .expect("decode apply report");
4007        let report_json: serde_json::Value =
4008            serde_json::to_value(&report).expect("encode apply report");
4009
4010        fs::remove_dir_all(root).expect("remove temp root");
4011        assert_eq!(report.backup_id, "backup-test");
4012        assert!(report.attention_required);
4013        assert_eq!(report.failed_operations, 1);
4014        assert_eq!(report.pending_operations, 1);
4015        assert_eq!(report.operation_counts.snapshot_uploads, 2);
4016        assert_eq!(report.operation_counts.snapshot_loads, 2);
4017        assert_eq!(report.operation_counts.code_reinstalls, 2);
4018        assert_eq!(report.operation_counts.member_verifications, 2);
4019        assert_eq!(report.operation_counts.fleet_verifications, 0);
4020        assert_eq!(report.operation_counts.verification_operations, 2);
4021        assert!(report.operation_counts_supplied);
4022        assert_eq!(report.progress.operation_count, 8);
4023        assert_eq!(report.progress.completed_operations, 0);
4024        assert_eq!(report.progress.remaining_operations, 8);
4025        assert_eq!(report.progress.transitionable_operations, 7);
4026        assert_eq!(report.progress.attention_operations, 2);
4027        assert_eq!(report.progress.completion_basis_points, 0);
4028        assert_eq!(report.pending_summary.pending_operations, 1);
4029        assert_eq!(report.pending_summary.pending_sequence, Some(1));
4030        assert_eq!(
4031            report.pending_summary.pending_updated_at.as_deref(),
4032            Some("2026-05-05T12:01:00Z")
4033        );
4034        assert!(report.pending_summary.pending_updated_at_known);
4035        assert_eq!(report.failed.len(), 1);
4036        assert_eq!(report.pending.len(), 1);
4037        assert_eq!(report.failed[0].sequence, 0);
4038        assert_eq!(report.pending[0].sequence, 1);
4039        assert_eq!(
4040            report.next_transition.as_ref().map(|op| op.sequence),
4041            Some(1)
4042        );
4043        assert_eq!(report_json["outcome"], "failed");
4044        assert_eq!(report_json["failed"][0]["reasons"][0], "dfx-upload-failed");
4045    }
4046
4047    // Ensure apply-report can fail closed on an unexpected progress summary.
4048    #[test]
4049    fn run_restore_apply_report_require_progress_writes_report_then_fails() {
4050        let fixture = RestoreCliFixture::new(
4051            "canic-cli-restore-apply-report-progress",
4052            "restore-apply-report.json",
4053        );
4054        let journal = ready_apply_journal();
4055        fixture.write_journal(&journal);
4056
4057        let err = fixture
4058            .run_apply_report(&[
4059                "--require-remaining-count",
4060                "8",
4061                "--require-attention-count",
4062                "1",
4063                "--require-completion-basis-points",
4064                "0",
4065            ])
4066            .expect_err("attention progress mismatch should fail requirement");
4067
4068        let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4069
4070        assert_eq!(report.progress.remaining_operations, 8);
4071        assert_eq!(report.progress.attention_operations, 0);
4072        assert_eq!(report.progress.completion_basis_points, 0);
4073        assert!(matches!(
4074            err,
4075            RestoreCommandError::RestoreApplyProgressMismatch {
4076                field: "attention_operations",
4077                expected: 1,
4078                actual: 0,
4079                ..
4080            }
4081        ));
4082    }
4083
4084    // Ensure apply-report can fail closed when pending work is older than a cutoff.
4085    #[test]
4086    fn run_restore_apply_report_require_no_pending_before_writes_report_then_fails() {
4087        let fixture = RestoreCliFixture::new(
4088            "canic-cli-restore-apply-report-stale-pending",
4089            "restore-apply-report.json",
4090        );
4091        let mut journal = ready_apply_journal();
4092        journal
4093            .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4094            .expect("mark pending operation");
4095        fixture.write_journal(&journal);
4096
4097        let err = fixture
4098            .run_apply_report(&["--require-no-pending-before", "2026-05-05T12:00:00Z"])
4099            .expect_err("stale pending report should fail requirement");
4100
4101        let report: RestoreApplyJournalReport = fixture.read_out("read apply report");
4102
4103        assert_eq!(report.pending_summary.pending_sequence, Some(0));
4104        assert!(matches!(
4105            err,
4106            RestoreCommandError::RestoreApplyPendingStale {
4107                pending_sequence: Some(0),
4108                ..
4109            }
4110        ));
4111    }
4112
4113    // Ensure restore run writes a native no-mutation runner preview.
4114    #[test]
4115    fn run_restore_run_dry_run_writes_native_runner_preview() {
4116        let root = temp_dir("canic-cli-restore-run-dry-run");
4117        fs::create_dir_all(&root).expect("create temp root");
4118        let journal_path = root.join("restore-apply-journal.json");
4119        let out_path = root.join("restore-run-dry-run.json");
4120        let journal = ready_apply_journal();
4121
4122        fs::write(
4123            &journal_path,
4124            serde_json::to_vec(&journal).expect("serialize journal"),
4125        )
4126        .expect("write journal");
4127
4128        run([
4129            OsString::from("run"),
4130            OsString::from("--journal"),
4131            OsString::from(journal_path.as_os_str()),
4132            OsString::from("--dry-run"),
4133            OsString::from("--dfx"),
4134            OsString::from("/tmp/dfx"),
4135            OsString::from("--network"),
4136            OsString::from("local"),
4137            OsString::from("--out"),
4138            OsString::from(out_path.as_os_str()),
4139        ])
4140        .expect("write restore run dry-run");
4141
4142        let dry_run: serde_json::Value =
4143            serde_json::from_slice(&fs::read(&out_path).expect("read dry-run"))
4144                .expect("decode dry-run");
4145
4146        fs::remove_dir_all(root).expect("remove temp root");
4147        assert_eq!(dry_run["run_version"], 1);
4148        assert_eq!(dry_run["backup_id"], "backup-test");
4149        assert_eq!(dry_run["run_mode"], "dry-run");
4150        assert_eq!(dry_run["dry_run"], true);
4151        assert_eq!(dry_run["ready"], true);
4152        assert_eq!(dry_run["complete"], false);
4153        assert_eq!(dry_run["attention_required"], false);
4154        assert_eq!(dry_run["operation_counts"]["snapshot_uploads"], 2);
4155        assert_eq!(dry_run["operation_counts"]["snapshot_loads"], 2);
4156        assert_eq!(dry_run["operation_counts"]["code_reinstalls"], 2);
4157        assert_eq!(dry_run["operation_counts"]["member_verifications"], 2);
4158        assert_eq!(dry_run["operation_counts"]["fleet_verifications"], 0);
4159        assert_eq!(dry_run["operation_counts"]["verification_operations"], 2);
4160        assert_eq!(dry_run["operation_counts_supplied"], true);
4161        assert_eq!(dry_run["progress"]["operation_count"], 8);
4162        assert_eq!(dry_run["progress"]["completed_operations"], 0);
4163        assert_eq!(dry_run["progress"]["remaining_operations"], 8);
4164        assert_eq!(dry_run["progress"]["transitionable_operations"], 8);
4165        assert_eq!(dry_run["progress"]["attention_operations"], 0);
4166        assert_eq!(dry_run["progress"]["completion_basis_points"], 0);
4167        assert_eq!(dry_run["pending_summary"]["pending_operations"], 0);
4168        assert_eq!(
4169            dry_run["pending_summary"]["pending_operation_available"],
4170            false
4171        );
4172        assert_eq!(dry_run["operation_receipt_count"], 0);
4173        assert_eq!(dry_run["operation_receipt_summary"]["total_receipts"], 0);
4174        assert_eq!(dry_run["operation_receipt_summary"]["command_completed"], 0);
4175        assert_eq!(dry_run["operation_receipt_summary"]["command_failed"], 0);
4176        assert_eq!(dry_run["operation_receipt_summary"]["pending_recovered"], 0);
4177        assert_eq!(dry_run["stopped_reason"], "preview");
4178        assert_eq!(dry_run["next_action"], "rerun");
4179        assert_eq!(dry_run["operation_available"], true);
4180        assert_eq!(dry_run["command_available"], true);
4181        assert_eq!(dry_run["next_transition"]["sequence"], 0);
4182        assert_eq!(dry_run["command"]["program"], "/tmp/dfx");
4183        assert_eq!(
4184            dry_run["command"]["args"],
4185            json!([
4186                "canister",
4187                "--network",
4188                "local",
4189                "snapshot",
4190                "upload",
4191                "--dir",
4192                "artifacts/root",
4193                ROOT
4194            ])
4195        );
4196        assert_eq!(dry_run["command"]["mutates"], true);
4197    }
4198
4199    // Ensure restore run can recover one interrupted pending operation.
4200    #[test]
4201    fn run_restore_run_unclaim_pending_marks_operation_ready() {
4202        let root = temp_dir("canic-cli-restore-run-unclaim-pending");
4203        fs::create_dir_all(&root).expect("create temp root");
4204        let journal_path = root.join("restore-apply-journal.json");
4205        let out_path = root.join("restore-run.json");
4206        let mut journal = ready_apply_journal();
4207        journal
4208            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4209            .expect("mark pending operation");
4210
4211        fs::write(
4212            &journal_path,
4213            serde_json::to_vec(&journal).expect("serialize journal"),
4214        )
4215        .expect("write journal");
4216
4217        run([
4218            OsString::from("run"),
4219            OsString::from("--journal"),
4220            OsString::from(journal_path.as_os_str()),
4221            OsString::from("--unclaim-pending"),
4222            OsString::from("--out"),
4223            OsString::from(out_path.as_os_str()),
4224        ])
4225        .expect("unclaim pending operation");
4226
4227        let run_summary: serde_json::Value =
4228            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4229                .expect("decode run summary");
4230        let updated: RestoreApplyJournal =
4231            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4232                .expect("decode updated journal");
4233
4234        fs::remove_dir_all(root).expect("remove temp root");
4235        assert_eq!(run_summary["run_mode"], "unclaim-pending");
4236        assert_eq!(run_summary["unclaim_pending"], true);
4237        assert_eq!(run_summary["stopped_reason"], "recovered-pending");
4238        assert_eq!(run_summary["next_action"], "rerun");
4239        assert_eq!(run_summary["recovered_operation"]["sequence"], 0);
4240        assert_eq!(run_summary["recovered_operation"]["state"], "pending");
4241        assert_eq!(run_summary["operation_receipt_count"], 1);
4242        assert_eq!(
4243            run_summary["operation_receipt_summary"]["total_receipts"],
4244            1
4245        );
4246        assert_eq!(
4247            run_summary["operation_receipt_summary"]["command_completed"],
4248            0
4249        );
4250        assert_eq!(
4251            run_summary["operation_receipt_summary"]["command_failed"],
4252            0
4253        );
4254        assert_eq!(
4255            run_summary["operation_receipt_summary"]["pending_recovered"],
4256            1
4257        );
4258        assert_eq!(
4259            run_summary["operation_receipts"][0]["event"],
4260            "pending-recovered"
4261        );
4262        assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
4263        assert_eq!(run_summary["operation_receipts"][0]["state"], "ready");
4264        assert_eq!(
4265            run_summary["operation_receipts"][0]["updated_at"],
4266            "unknown"
4267        );
4268        assert_eq!(run_summary["pending_operations"], 0);
4269        assert_eq!(run_summary["ready_operations"], 8);
4270        assert_eq!(run_summary["attention_required"], false);
4271        assert_eq!(updated.pending_operations, 0);
4272        assert_eq!(updated.ready_operations, 8);
4273        assert_eq!(
4274            updated.operations[0].state,
4275            RestoreApplyOperationState::Ready
4276        );
4277    }
4278
4279    // Ensure restore run execute claims and completes one generated command.
4280    #[test]
4281    fn run_restore_run_execute_marks_completed_operation() {
4282        let root = temp_dir("canic-cli-restore-run-execute");
4283        fs::create_dir_all(&root).expect("create temp root");
4284        let journal_path = root.join("restore-apply-journal.json");
4285        let out_path = root.join("restore-run.json");
4286        let journal = ready_apply_journal();
4287
4288        fs::write(
4289            &journal_path,
4290            serde_json::to_vec(&journal).expect("serialize journal"),
4291        )
4292        .expect("write journal");
4293
4294        run([
4295            OsString::from("run"),
4296            OsString::from("--journal"),
4297            OsString::from(journal_path.as_os_str()),
4298            OsString::from("--execute"),
4299            OsString::from("--dfx"),
4300            OsString::from("/bin/true"),
4301            OsString::from("--max-steps"),
4302            OsString::from("1"),
4303            OsString::from("--out"),
4304            OsString::from(out_path.as_os_str()),
4305        ])
4306        .expect("execute one restore run step");
4307
4308        let run_summary: serde_json::Value =
4309            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4310                .expect("decode run summary");
4311        let updated: RestoreApplyJournal =
4312            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4313                .expect("decode updated journal");
4314
4315        fs::remove_dir_all(root).expect("remove temp root");
4316        assert_eq!(run_summary["run_mode"], "execute");
4317        assert_eq!(run_summary["execute"], true);
4318        assert_eq!(run_summary["dry_run"], false);
4319        assert_eq!(run_summary["max_steps_reached"], true);
4320        assert_eq!(run_summary["stopped_reason"], "max-steps-reached");
4321        assert_eq!(run_summary["next_action"], "rerun");
4322        assert_eq!(run_summary["executed_operation_count"], 1);
4323        assert_eq!(run_summary["operation_receipt_count"], 1);
4324        assert_eq!(
4325            run_summary["operation_receipt_summary"]["total_receipts"],
4326            1
4327        );
4328        assert_eq!(
4329            run_summary["operation_receipt_summary"]["command_completed"],
4330            1
4331        );
4332        assert_eq!(
4333            run_summary["operation_receipt_summary"]["command_failed"],
4334            0
4335        );
4336        assert_eq!(
4337            run_summary["operation_receipt_summary"]["pending_recovered"],
4338            0
4339        );
4340        assert_eq!(run_summary["executed_operations"][0]["sequence"], 0);
4341        assert_eq!(
4342            run_summary["executed_operations"][0]["command"]["program"],
4343            "/bin/true"
4344        );
4345        assert_eq!(
4346            run_summary["operation_receipts"][0]["event"],
4347            "command-completed"
4348        );
4349        assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
4350        assert_eq!(run_summary["operation_receipts"][0]["state"], "completed");
4351        assert_eq!(
4352            run_summary["operation_receipts"][0]["command"]["program"],
4353            "/bin/true"
4354        );
4355        assert_eq!(run_summary["operation_receipts"][0]["status"], "0");
4356        assert_eq!(
4357            run_summary["operation_receipts"][0]["updated_at"],
4358            "unknown"
4359        );
4360        assert_eq!(updated.completed_operations, 1);
4361        assert_eq!(updated.pending_operations, 0);
4362        assert_eq!(updated.failed_operations, 0);
4363        assert_eq!(
4364            updated.operations[0].state,
4365            RestoreApplyOperationState::Completed
4366        );
4367    }
4368
4369    // Ensure restore run can fail closed after writing an incomplete summary.
4370    #[test]
4371    fn run_restore_run_require_complete_writes_summary_then_fails() {
4372        let root = temp_dir("canic-cli-restore-run-require-complete");
4373        fs::create_dir_all(&root).expect("create temp root");
4374        let journal_path = root.join("restore-apply-journal.json");
4375        let out_path = root.join("restore-run.json");
4376        let journal = ready_apply_journal();
4377
4378        fs::write(
4379            &journal_path,
4380            serde_json::to_vec(&journal).expect("serialize journal"),
4381        )
4382        .expect("write journal");
4383
4384        let err = run([
4385            OsString::from("run"),
4386            OsString::from("--journal"),
4387            OsString::from(journal_path.as_os_str()),
4388            OsString::from("--execute"),
4389            OsString::from("--dfx"),
4390            OsString::from("/bin/true"),
4391            OsString::from("--max-steps"),
4392            OsString::from("1"),
4393            OsString::from("--out"),
4394            OsString::from(out_path.as_os_str()),
4395            OsString::from("--require-complete"),
4396        ])
4397        .expect_err("incomplete run should fail requirement");
4398
4399        let run_summary: serde_json::Value =
4400            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4401                .expect("decode run summary");
4402
4403        fs::remove_dir_all(root).expect("remove temp root");
4404        assert_eq!(run_summary["executed_operation_count"], 1);
4405        assert_eq!(run_summary["complete"], false);
4406        assert!(matches!(
4407            err,
4408            RestoreCommandError::RestoreApplyIncomplete {
4409                completed_operations: 1,
4410                operation_count: 8,
4411                ..
4412            }
4413        ));
4414    }
4415
4416    // Ensure restore run execute records failed command exits in the journal.
4417    #[test]
4418    fn run_restore_run_execute_marks_failed_operation() {
4419        let root = temp_dir("canic-cli-restore-run-execute-failed");
4420        fs::create_dir_all(&root).expect("create temp root");
4421        let journal_path = root.join("restore-apply-journal.json");
4422        let out_path = root.join("restore-run.json");
4423        let journal = ready_apply_journal();
4424
4425        fs::write(
4426            &journal_path,
4427            serde_json::to_vec(&journal).expect("serialize journal"),
4428        )
4429        .expect("write journal");
4430
4431        let err = run([
4432            OsString::from("run"),
4433            OsString::from("--journal"),
4434            OsString::from(journal_path.as_os_str()),
4435            OsString::from("--execute"),
4436            OsString::from("--dfx"),
4437            OsString::from("/bin/false"),
4438            OsString::from("--max-steps"),
4439            OsString::from("1"),
4440            OsString::from("--out"),
4441            OsString::from(out_path.as_os_str()),
4442        ])
4443        .expect_err("failing runner command should fail");
4444
4445        let run_summary: serde_json::Value =
4446            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4447                .expect("decode run summary");
4448        let updated: RestoreApplyJournal =
4449            serde_json::from_slice(&fs::read(&journal_path).expect("read updated journal"))
4450                .expect("decode updated journal");
4451
4452        fs::remove_dir_all(root).expect("remove temp root");
4453        assert!(matches!(
4454            err,
4455            RestoreCommandError::RestoreRunCommandFailed {
4456                sequence: 0,
4457                status,
4458            } if status == "1"
4459        ));
4460        assert_eq!(updated.failed_operations, 1);
4461        assert_eq!(updated.pending_operations, 0);
4462        assert_eq!(
4463            updated.operations[0].state,
4464            RestoreApplyOperationState::Failed
4465        );
4466        assert_eq!(run_summary["execute"], true);
4467        assert_eq!(run_summary["attention_required"], true);
4468        assert_eq!(run_summary["outcome"], "failed");
4469        assert_eq!(run_summary["stopped_reason"], "command-failed");
4470        assert_eq!(run_summary["next_action"], "inspect-failed-operation");
4471        assert_eq!(run_summary["executed_operation_count"], 1);
4472        assert_eq!(run_summary["operation_receipt_count"], 1);
4473        assert_eq!(
4474            run_summary["operation_receipt_summary"]["total_receipts"],
4475            1
4476        );
4477        assert_eq!(
4478            run_summary["operation_receipt_summary"]["command_completed"],
4479            0
4480        );
4481        assert_eq!(
4482            run_summary["operation_receipt_summary"]["command_failed"],
4483            1
4484        );
4485        assert_eq!(
4486            run_summary["operation_receipt_summary"]["pending_recovered"],
4487            0
4488        );
4489        assert_eq!(run_summary["executed_operations"][0]["state"], "failed");
4490        assert_eq!(run_summary["executed_operations"][0]["status"], "1");
4491        assert_eq!(
4492            run_summary["operation_receipts"][0]["event"],
4493            "command-failed"
4494        );
4495        assert_eq!(run_summary["operation_receipts"][0]["sequence"], 0);
4496        assert_eq!(run_summary["operation_receipts"][0]["state"], "failed");
4497        assert_eq!(
4498            run_summary["operation_receipts"][0]["command"]["program"],
4499            "/bin/false"
4500        );
4501        assert_eq!(run_summary["operation_receipts"][0]["status"], "1");
4502        assert_eq!(
4503            run_summary["operation_receipts"][0]["updated_at"],
4504            "unknown"
4505        );
4506        assert_eq!(
4507            updated.operations[0].blocking_reasons,
4508            vec!["runner-command-exit-1".to_string()]
4509        );
4510    }
4511
4512    // Ensure restore run can fail closed after writing an attention summary.
4513    #[test]
4514    fn run_restore_run_require_no_attention_writes_summary_then_fails() {
4515        let fixture = RestoreCliFixture::new(
4516            "canic-cli-restore-run-require-attention",
4517            "restore-run.json",
4518        );
4519        let mut journal = ready_apply_journal();
4520        journal
4521            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4522            .expect("mark pending operation");
4523        fixture.write_journal(&journal);
4524
4525        let err = fixture
4526            .run_restore_run(&["--dry-run", "--require-no-attention"])
4527            .expect_err("attention run should fail requirement");
4528
4529        let run_summary: serde_json::Value = fixture.read_out("read run summary");
4530
4531        assert_eq!(run_summary["attention_required"], true);
4532        assert_eq!(run_summary["outcome"], "pending");
4533        assert_eq!(run_summary["stopped_reason"], "pending");
4534        assert_eq!(run_summary["next_action"], "unclaim-pending");
4535        assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
4536        assert_eq!(
4537            run_summary["pending_summary"]["pending_updated_at"],
4538            "2026-05-05T12:01:00Z"
4539        );
4540        assert!(matches!(
4541            err,
4542            RestoreCommandError::RestoreApplyReportNeedsAttention {
4543                outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
4544                ..
4545            }
4546        ));
4547    }
4548
4549    // Ensure restore run can fail closed when pending work is older than a cutoff.
4550    #[test]
4551    fn run_restore_run_require_no_pending_before_writes_summary_then_fails() {
4552        let fixture = RestoreCliFixture::new(
4553            "canic-cli-restore-run-require-stale-pending",
4554            "restore-run.json",
4555        );
4556        let mut journal = ready_apply_journal();
4557        journal
4558            .mark_next_operation_pending_at(Some("2026-05-04T12:00:00Z".to_string()))
4559            .expect("mark pending operation");
4560        fixture.write_journal(&journal);
4561
4562        let err = fixture
4563            .run_restore_run(&[
4564                "--dry-run",
4565                "--require-no-pending-before",
4566                "2026-05-05T12:00:00Z",
4567            ])
4568            .expect_err("stale pending run should fail requirement");
4569
4570        let run_summary: serde_json::Value = fixture.read_out("read run summary");
4571
4572        assert_eq!(run_summary["pending_summary"]["pending_sequence"], 0);
4573        assert!(matches!(
4574            err,
4575            RestoreCommandError::RestoreApplyPendingStale {
4576                pending_sequence: Some(0),
4577                ..
4578            }
4579        ));
4580    }
4581
4582    // Ensure restore run can fail closed on an unexpected run mode.
4583    #[test]
4584    fn run_restore_run_require_run_mode_writes_summary_then_fails() {
4585        let fixture =
4586            RestoreCliFixture::new("canic-cli-restore-run-require-run-mode", "restore-run.json");
4587        let journal = ready_apply_journal();
4588        fixture.write_journal(&journal);
4589
4590        let err = fixture
4591            .run_restore_run(&["--dry-run", "--require-run-mode", "execute"])
4592            .expect_err("run mode mismatch should fail requirement");
4593
4594        let run_summary: serde_json::Value = fixture.read_out("read run summary");
4595
4596        assert_eq!(run_summary["run_mode"], "dry-run");
4597        assert!(matches!(
4598            err,
4599            RestoreCommandError::RestoreRunModeMismatch {
4600                expected,
4601                actual,
4602                ..
4603            } if expected == "execute" && actual == "dry-run"
4604        ));
4605    }
4606
4607    // Ensure restore run can fail closed on an unexpected executed operation count.
4608    #[test]
4609    fn run_restore_run_require_executed_count_writes_summary_then_fails() {
4610        let root = temp_dir("canic-cli-restore-run-require-executed-count");
4611        fs::create_dir_all(&root).expect("create temp root");
4612        let journal_path = root.join("restore-apply-journal.json");
4613        let out_path = root.join("restore-run.json");
4614        let journal = ready_apply_journal();
4615
4616        fs::write(
4617            &journal_path,
4618            serde_json::to_vec(&journal).expect("serialize journal"),
4619        )
4620        .expect("write journal");
4621
4622        let err = run([
4623            OsString::from("run"),
4624            OsString::from("--journal"),
4625            OsString::from(journal_path.as_os_str()),
4626            OsString::from("--execute"),
4627            OsString::from("--dfx"),
4628            OsString::from("/bin/true"),
4629            OsString::from("--max-steps"),
4630            OsString::from("1"),
4631            OsString::from("--out"),
4632            OsString::from(out_path.as_os_str()),
4633            OsString::from("--require-executed-count"),
4634            OsString::from("2"),
4635        ])
4636        .expect_err("executed count mismatch should fail requirement");
4637
4638        let run_summary: serde_json::Value =
4639            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4640                .expect("decode run summary");
4641
4642        fs::remove_dir_all(root).expect("remove temp root");
4643        assert_eq!(run_summary["executed_operation_count"], 1);
4644        assert!(matches!(
4645            err,
4646            RestoreCommandError::RestoreRunExecutedCountMismatch {
4647                expected: 2,
4648                actual: 1,
4649                ..
4650            }
4651        ));
4652    }
4653
4654    // Ensure restore run can fail closed on an unexpected operation receipt count.
4655    #[test]
4656    fn run_restore_run_require_receipt_count_writes_summary_then_fails() {
4657        let fixture = RestoreCliFixture::new(
4658            "canic-cli-restore-run-require-receipt-count",
4659            "restore-run.json",
4660        );
4661        let journal = ready_apply_journal();
4662        fixture.write_journal(&journal);
4663
4664        let err = fixture
4665            .run_restore_run(&[
4666                "--execute",
4667                "--dfx",
4668                "/bin/true",
4669                "--max-steps",
4670                "1",
4671                "--require-receipt-count",
4672                "2",
4673            ])
4674            .expect_err("receipt count mismatch should fail requirement");
4675
4676        let run_summary: serde_json::Value = fixture.read_out("read run summary");
4677
4678        assert_eq!(run_summary["operation_receipt_count"], 1);
4679        assert_eq!(
4680            run_summary["operation_receipt_summary"]["total_receipts"],
4681            1
4682        );
4683        assert!(matches!(
4684            err,
4685            RestoreCommandError::RestoreRunReceiptCountMismatch {
4686                expected: 2,
4687                actual: 1,
4688                ..
4689            }
4690        ));
4691    }
4692
4693    // Ensure restore run can fail closed on an unexpected receipt-kind count.
4694    #[test]
4695    fn run_restore_run_require_receipt_kind_count_writes_summary_then_fails() {
4696        let fixture = RestoreCliFixture::new(
4697            "canic-cli-restore-run-require-receipt-kind-count",
4698            "restore-run.json",
4699        );
4700        let journal = ready_apply_journal();
4701        fixture.write_journal(&journal);
4702
4703        let err = fixture
4704            .run_restore_run(&[
4705                "--execute",
4706                "--dfx",
4707                "/bin/true",
4708                "--max-steps",
4709                "1",
4710                "--require-failed-receipt-count",
4711                "1",
4712            ])
4713            .expect_err("receipt kind count mismatch should fail requirement");
4714
4715        let run_summary: serde_json::Value = fixture.read_out("read run summary");
4716
4717        assert_eq!(
4718            run_summary["operation_receipt_summary"]["command_failed"],
4719            0
4720        );
4721        assert_eq!(
4722            run_summary["operation_receipt_summary"]["command_completed"],
4723            1
4724        );
4725        assert!(matches!(
4726            err,
4727            RestoreCommandError::RestoreRunReceiptKindCountMismatch {
4728                receipt_kind: "command-failed",
4729                expected: 1,
4730                actual: 0,
4731                ..
4732            }
4733        ));
4734    }
4735
4736    // Ensure restore run can fail closed on an unexpected progress summary.
4737    #[test]
4738    fn run_restore_run_require_progress_writes_summary_then_fails() {
4739        let root = temp_dir("canic-cli-restore-run-require-progress");
4740        fs::create_dir_all(&root).expect("create temp root");
4741        let journal_path = root.join("restore-apply-journal.json");
4742        let out_path = root.join("restore-run.json");
4743        let journal = ready_apply_journal();
4744
4745        fs::write(
4746            &journal_path,
4747            serde_json::to_vec(&journal).expect("serialize journal"),
4748        )
4749        .expect("write journal");
4750
4751        let err = run([
4752            OsString::from("run"),
4753            OsString::from("--journal"),
4754            OsString::from(journal_path.as_os_str()),
4755            OsString::from("--execute"),
4756            OsString::from("--dfx"),
4757            OsString::from("/bin/true"),
4758            OsString::from("--max-steps"),
4759            OsString::from("1"),
4760            OsString::from("--out"),
4761            OsString::from(out_path.as_os_str()),
4762            OsString::from("--require-remaining-count"),
4763            OsString::from("7"),
4764            OsString::from("--require-attention-count"),
4765            OsString::from("0"),
4766            OsString::from("--require-completion-basis-points"),
4767            OsString::from("0"),
4768        ])
4769        .expect_err("completion progress mismatch should fail requirement");
4770
4771        let run_summary: serde_json::Value =
4772            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4773                .expect("decode run summary");
4774
4775        fs::remove_dir_all(root).expect("remove temp root");
4776        assert_eq!(run_summary["progress"]["remaining_operations"], 7);
4777        assert_eq!(run_summary["progress"]["attention_operations"], 0);
4778        assert_eq!(run_summary["progress"]["completion_basis_points"], 1250);
4779        assert!(matches!(
4780            err,
4781            RestoreCommandError::RestoreApplyProgressMismatch {
4782                field: "completion_basis_points",
4783                expected: 0,
4784                actual: 1250,
4785                ..
4786            }
4787        ));
4788    }
4789
4790    // Ensure restore run can fail closed on an unexpected stopped reason.
4791    #[test]
4792    fn run_restore_run_require_stopped_reason_writes_summary_then_fails() {
4793        let root = temp_dir("canic-cli-restore-run-require-stopped-reason");
4794        fs::create_dir_all(&root).expect("create temp root");
4795        let journal_path = root.join("restore-apply-journal.json");
4796        let out_path = root.join("restore-run.json");
4797        let journal = ready_apply_journal();
4798
4799        fs::write(
4800            &journal_path,
4801            serde_json::to_vec(&journal).expect("serialize journal"),
4802        )
4803        .expect("write journal");
4804
4805        let err = run([
4806            OsString::from("run"),
4807            OsString::from("--journal"),
4808            OsString::from(journal_path.as_os_str()),
4809            OsString::from("--dry-run"),
4810            OsString::from("--out"),
4811            OsString::from(out_path.as_os_str()),
4812            OsString::from("--require-stopped-reason"),
4813            OsString::from("complete"),
4814        ])
4815        .expect_err("stopped reason mismatch should fail requirement");
4816
4817        let run_summary: serde_json::Value =
4818            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4819                .expect("decode run summary");
4820
4821        fs::remove_dir_all(root).expect("remove temp root");
4822        assert_eq!(run_summary["stopped_reason"], "preview");
4823        assert!(matches!(
4824            err,
4825            RestoreCommandError::RestoreRunStoppedReasonMismatch {
4826                expected,
4827                actual,
4828                ..
4829            } if expected == "complete" && actual == "preview"
4830        ));
4831    }
4832
4833    // Ensure restore run can fail closed on an unexpected next action.
4834    #[test]
4835    fn run_restore_run_require_next_action_writes_summary_then_fails() {
4836        let root = temp_dir("canic-cli-restore-run-require-next-action");
4837        fs::create_dir_all(&root).expect("create temp root");
4838        let journal_path = root.join("restore-apply-journal.json");
4839        let out_path = root.join("restore-run.json");
4840        let journal = ready_apply_journal();
4841
4842        fs::write(
4843            &journal_path,
4844            serde_json::to_vec(&journal).expect("serialize journal"),
4845        )
4846        .expect("write journal");
4847
4848        let err = run([
4849            OsString::from("run"),
4850            OsString::from("--journal"),
4851            OsString::from(journal_path.as_os_str()),
4852            OsString::from("--dry-run"),
4853            OsString::from("--out"),
4854            OsString::from(out_path.as_os_str()),
4855            OsString::from("--require-next-action"),
4856            OsString::from("done"),
4857        ])
4858        .expect_err("next action mismatch should fail requirement");
4859
4860        let run_summary: serde_json::Value =
4861            serde_json::from_slice(&fs::read(&out_path).expect("read run summary"))
4862                .expect("decode run summary");
4863
4864        fs::remove_dir_all(root).expect("remove temp root");
4865        assert_eq!(run_summary["next_action"], "rerun");
4866        assert!(matches!(
4867            err,
4868            RestoreCommandError::RestoreRunNextActionMismatch {
4869                expected,
4870                actual,
4871                ..
4872            } if expected == "done" && actual == "rerun"
4873        ));
4874    }
4875
4876    // Ensure apply-report can fail closed after writing an attention report.
4877    #[test]
4878    fn run_restore_apply_report_require_no_attention_writes_report_then_fails() {
4879        let root = temp_dir("canic-cli-restore-apply-report-attention");
4880        fs::create_dir_all(&root).expect("create temp root");
4881        let journal_path = root.join("restore-apply-journal.json");
4882        let out_path = root.join("restore-apply-report.json");
4883        let mut journal = ready_apply_journal();
4884        journal
4885            .mark_next_operation_pending_at(Some("2026-05-05T12:01:00Z".to_string()))
4886            .expect("mark pending operation");
4887
4888        fs::write(
4889            &journal_path,
4890            serde_json::to_vec(&journal).expect("serialize journal"),
4891        )
4892        .expect("write journal");
4893
4894        let err = run([
4895            OsString::from("apply-report"),
4896            OsString::from("--journal"),
4897            OsString::from(journal_path.as_os_str()),
4898            OsString::from("--out"),
4899            OsString::from(out_path.as_os_str()),
4900            OsString::from("--require-no-attention"),
4901        ])
4902        .expect_err("attention report should fail requirement");
4903
4904        let report: RestoreApplyJournalReport =
4905            serde_json::from_slice(&fs::read(&out_path).expect("read apply report"))
4906                .expect("decode apply report");
4907
4908        fs::remove_dir_all(root).expect("remove temp root");
4909        assert!(report.attention_required);
4910        assert_eq!(report.pending_operations, 1);
4911        assert!(matches!(
4912            err,
4913            RestoreCommandError::RestoreApplyReportNeedsAttention {
4914                outcome: canic_backup::restore::RestoreApplyReportOutcome::Pending,
4915                ..
4916            }
4917        ));
4918    }
4919
4920    // Ensure apply-status can fail closed after writing status for incomplete work.
4921    #[test]
4922    fn run_restore_apply_status_require_complete_writes_status_then_fails() {
4923        let root = temp_dir("canic-cli-restore-apply-status-incomplete");
4924        fs::create_dir_all(&root).expect("create temp root");
4925        let journal_path = root.join("restore-apply-journal.json");
4926        let out_path = root.join("restore-apply-status.json");
4927        let journal = ready_apply_journal();
4928
4929        fs::write(
4930            &journal_path,
4931            serde_json::to_vec(&journal).expect("serialize journal"),
4932        )
4933        .expect("write journal");
4934
4935        let err = run([
4936            OsString::from("apply-status"),
4937            OsString::from("--journal"),
4938            OsString::from(journal_path.as_os_str()),
4939            OsString::from("--out"),
4940            OsString::from(out_path.as_os_str()),
4941            OsString::from("--require-complete"),
4942        ])
4943        .expect_err("incomplete journal should fail requirement");
4944
4945        assert!(out_path.exists());
4946        let status: RestoreApplyJournalStatus =
4947            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
4948                .expect("decode apply status");
4949
4950        fs::remove_dir_all(root).expect("remove temp root");
4951        assert!(!status.complete);
4952        assert_eq!(status.completed_operations, 0);
4953        assert_eq!(status.operation_count, 8);
4954        assert!(matches!(
4955            err,
4956            RestoreCommandError::RestoreApplyIncomplete {
4957                completed_operations: 0,
4958                operation_count: 8,
4959                ..
4960            }
4961        ));
4962    }
4963
4964    // Ensure apply-status can fail closed after writing status for failed work.
4965    #[test]
4966    fn run_restore_apply_status_require_no_failed_writes_status_then_fails() {
4967        let root = temp_dir("canic-cli-restore-apply-status-failed");
4968        fs::create_dir_all(&root).expect("create temp root");
4969        let journal_path = root.join("restore-apply-journal.json");
4970        let out_path = root.join("restore-apply-status.json");
4971        let mut journal = ready_apply_journal();
4972        journal
4973            .mark_operation_failed(0, "dfx-load-failed".to_string())
4974            .expect("mark failed operation");
4975
4976        fs::write(
4977            &journal_path,
4978            serde_json::to_vec(&journal).expect("serialize journal"),
4979        )
4980        .expect("write journal");
4981
4982        let err = run([
4983            OsString::from("apply-status"),
4984            OsString::from("--journal"),
4985            OsString::from(journal_path.as_os_str()),
4986            OsString::from("--out"),
4987            OsString::from(out_path.as_os_str()),
4988            OsString::from("--require-no-failed"),
4989        ])
4990        .expect_err("failed operation should fail requirement");
4991
4992        assert!(out_path.exists());
4993        let status: RestoreApplyJournalStatus =
4994            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
4995                .expect("decode apply status");
4996
4997        fs::remove_dir_all(root).expect("remove temp root");
4998        assert_eq!(status.failed_operations, 1);
4999        assert!(matches!(
5000            err,
5001            RestoreCommandError::RestoreApplyFailed {
5002                failed_operations: 1,
5003                ..
5004            }
5005        ));
5006    }
5007
5008    // Ensure apply-status accepts a complete journal when required.
5009    #[test]
5010    fn run_restore_apply_status_require_complete_accepts_complete_journal() {
5011        let root = temp_dir("canic-cli-restore-apply-status-complete");
5012        fs::create_dir_all(&root).expect("create temp root");
5013        let journal_path = root.join("restore-apply-journal.json");
5014        let out_path = root.join("restore-apply-status.json");
5015        let mut journal = ready_apply_journal();
5016        for sequence in 0..journal.operation_count {
5017            journal
5018                .mark_operation_completed(sequence)
5019                .expect("complete operation");
5020        }
5021
5022        fs::write(
5023            &journal_path,
5024            serde_json::to_vec(&journal).expect("serialize journal"),
5025        )
5026        .expect("write journal");
5027
5028        run([
5029            OsString::from("apply-status"),
5030            OsString::from("--journal"),
5031            OsString::from(journal_path.as_os_str()),
5032            OsString::from("--out"),
5033            OsString::from(out_path.as_os_str()),
5034            OsString::from("--require-complete"),
5035        ])
5036        .expect("complete journal should pass requirement");
5037
5038        let status: RestoreApplyJournalStatus =
5039            serde_json::from_slice(&fs::read(&out_path).expect("read apply status"))
5040                .expect("decode apply status");
5041
5042        fs::remove_dir_all(root).expect("remove temp root");
5043        assert!(status.complete);
5044        assert_eq!(status.completed_operations, 8);
5045        assert_eq!(status.operation_count, 8);
5046    }
5047
5048    // Ensure apply-next writes the full next ready operation row for runners.
5049    #[test]
5050    fn run_restore_apply_next_writes_next_ready_operation() {
5051        let root = temp_dir("canic-cli-restore-apply-next");
5052        fs::create_dir_all(&root).expect("create temp root");
5053        let journal_path = root.join("restore-apply-journal.json");
5054        let out_path = root.join("restore-apply-next.json");
5055        let mut journal = ready_apply_journal();
5056        journal
5057            .mark_operation_completed(0)
5058            .expect("mark first operation complete");
5059
5060        fs::write(
5061            &journal_path,
5062            serde_json::to_vec(&journal).expect("serialize journal"),
5063        )
5064        .expect("write journal");
5065
5066        run([
5067            OsString::from("apply-next"),
5068            OsString::from("--journal"),
5069            OsString::from(journal_path.as_os_str()),
5070            OsString::from("--out"),
5071            OsString::from(out_path.as_os_str()),
5072        ])
5073        .expect("write apply next");
5074
5075        let next: RestoreApplyNextOperation =
5076            serde_json::from_slice(&fs::read(&out_path).expect("read next operation"))
5077                .expect("decode next operation");
5078        let operation = next.operation.expect("operation should be available");
5079
5080        fs::remove_dir_all(root).expect("remove temp root");
5081        assert!(next.ready);
5082        assert!(next.operation_available);
5083        assert_eq!(operation.sequence, 1);
5084        assert_eq!(
5085            operation.operation,
5086            canic_backup::restore::RestoreApplyOperationKind::LoadSnapshot
5087        );
5088    }
5089
5090    // Ensure apply-command writes a no-execute command preview for the next operation.
5091    #[test]
5092    fn run_restore_apply_command_writes_next_command_preview() {
5093        let root = temp_dir("canic-cli-restore-apply-command");
5094        fs::create_dir_all(&root).expect("create temp root");
5095        let journal_path = root.join("restore-apply-journal.json");
5096        let out_path = root.join("restore-apply-command.json");
5097        let journal = ready_apply_journal();
5098
5099        fs::write(
5100            &journal_path,
5101            serde_json::to_vec(&journal).expect("serialize journal"),
5102        )
5103        .expect("write journal");
5104
5105        run([
5106            OsString::from("apply-command"),
5107            OsString::from("--journal"),
5108            OsString::from(journal_path.as_os_str()),
5109            OsString::from("--dfx"),
5110            OsString::from("/tmp/dfx"),
5111            OsString::from("--network"),
5112            OsString::from("local"),
5113            OsString::from("--out"),
5114            OsString::from(out_path.as_os_str()),
5115        ])
5116        .expect("write command preview");
5117
5118        let preview: RestoreApplyCommandPreview =
5119            serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
5120                .expect("decode command preview");
5121        let command = preview.command.expect("command should be available");
5122
5123        fs::remove_dir_all(root).expect("remove temp root");
5124        assert!(preview.ready);
5125        assert!(preview.command_available);
5126        assert_eq!(command.program, "/tmp/dfx");
5127        assert_eq!(
5128            command.args,
5129            vec![
5130                "canister".to_string(),
5131                "--network".to_string(),
5132                "local".to_string(),
5133                "snapshot".to_string(),
5134                "upload".to_string(),
5135                "--dir".to_string(),
5136                "artifacts/root".to_string(),
5137                ROOT.to_string(),
5138            ]
5139        );
5140        assert!(command.mutates);
5141    }
5142
5143    // Ensure apply-command can fail closed after writing a command preview.
5144    #[test]
5145    fn run_restore_apply_command_require_command_writes_preview_then_fails() {
5146        let root = temp_dir("canic-cli-restore-apply-command-require");
5147        fs::create_dir_all(&root).expect("create temp root");
5148        let journal_path = root.join("restore-apply-journal.json");
5149        let out_path = root.join("restore-apply-command.json");
5150        let mut journal = ready_apply_journal();
5151
5152        for sequence in 0..journal.operation_count {
5153            journal
5154                .mark_operation_completed(sequence)
5155                .expect("mark operation completed");
5156        }
5157
5158        fs::write(
5159            &journal_path,
5160            serde_json::to_vec(&journal).expect("serialize journal"),
5161        )
5162        .expect("write journal");
5163
5164        let err = run([
5165            OsString::from("apply-command"),
5166            OsString::from("--journal"),
5167            OsString::from(journal_path.as_os_str()),
5168            OsString::from("--out"),
5169            OsString::from(out_path.as_os_str()),
5170            OsString::from("--require-command"),
5171        ])
5172        .expect_err("missing command should fail");
5173
5174        let preview: RestoreApplyCommandPreview =
5175            serde_json::from_slice(&fs::read(&out_path).expect("read command preview"))
5176                .expect("decode command preview");
5177
5178        fs::remove_dir_all(root).expect("remove temp root");
5179        assert!(preview.complete);
5180        assert!(!preview.operation_available);
5181        assert!(!preview.command_available);
5182        assert!(matches!(
5183            err,
5184            RestoreCommandError::RestoreApplyCommandUnavailable {
5185                operation_available: false,
5186                complete: true,
5187                ..
5188            }
5189        ));
5190    }
5191
5192    // Ensure apply-claim marks the next operation pending before runner execution.
5193    #[test]
5194    fn run_restore_apply_claim_marks_next_operation_pending() {
5195        let root = temp_dir("canic-cli-restore-apply-claim");
5196        fs::create_dir_all(&root).expect("create temp root");
5197        let journal_path = root.join("restore-apply-journal.json");
5198        let claimed_path = root.join("restore-apply-journal.claimed.json");
5199        let journal = ready_apply_journal();
5200
5201        fs::write(
5202            &journal_path,
5203            serde_json::to_vec(&journal).expect("serialize journal"),
5204        )
5205        .expect("write journal");
5206
5207        run([
5208            OsString::from("apply-claim"),
5209            OsString::from("--journal"),
5210            OsString::from(journal_path.as_os_str()),
5211            OsString::from("--sequence"),
5212            OsString::from("0"),
5213            OsString::from("--updated-at"),
5214            OsString::from("2026-05-04T12:00:00Z"),
5215            OsString::from("--out"),
5216            OsString::from(claimed_path.as_os_str()),
5217        ])
5218        .expect("claim operation");
5219
5220        let claimed: RestoreApplyJournal =
5221            serde_json::from_slice(&fs::read(&claimed_path).expect("read claimed journal"))
5222                .expect("decode claimed journal");
5223        let status = claimed.status();
5224        let next = claimed.next_operation();
5225
5226        fs::remove_dir_all(root).expect("remove temp root");
5227        assert_eq!(claimed.pending_operations, 1);
5228        assert_eq!(claimed.ready_operations, 7);
5229        assert_eq!(
5230            claimed.operations[0].state,
5231            RestoreApplyOperationState::Pending
5232        );
5233        assert_eq!(
5234            claimed.operations[0].state_updated_at.as_deref(),
5235            Some("2026-05-04T12:00:00Z")
5236        );
5237        assert_eq!(status.next_transition_sequence, Some(0));
5238        assert_eq!(
5239            status.next_transition_state,
5240            Some(RestoreApplyOperationState::Pending)
5241        );
5242        assert_eq!(
5243            status.next_transition_updated_at.as_deref(),
5244            Some("2026-05-04T12:00:00Z")
5245        );
5246        assert_eq!(
5247            next.operation.expect("next operation").state,
5248            RestoreApplyOperationState::Pending
5249        );
5250    }
5251
5252    // Ensure apply-claim can reject a stale command preview sequence.
5253    #[test]
5254    fn run_restore_apply_claim_rejects_sequence_mismatch() {
5255        let root = temp_dir("canic-cli-restore-apply-claim-sequence");
5256        fs::create_dir_all(&root).expect("create temp root");
5257        let journal_path = root.join("restore-apply-journal.json");
5258        let claimed_path = root.join("restore-apply-journal.claimed.json");
5259        let journal = ready_apply_journal();
5260
5261        fs::write(
5262            &journal_path,
5263            serde_json::to_vec(&journal).expect("serialize journal"),
5264        )
5265        .expect("write journal");
5266
5267        let err = run([
5268            OsString::from("apply-claim"),
5269            OsString::from("--journal"),
5270            OsString::from(journal_path.as_os_str()),
5271            OsString::from("--sequence"),
5272            OsString::from("1"),
5273            OsString::from("--out"),
5274            OsString::from(claimed_path.as_os_str()),
5275        ])
5276        .expect_err("stale sequence should fail claim");
5277
5278        assert!(!claimed_path.exists());
5279        fs::remove_dir_all(root).expect("remove temp root");
5280        assert!(matches!(
5281            err,
5282            RestoreCommandError::RestoreApplyClaimSequenceMismatch {
5283                expected: 1,
5284                actual: Some(0),
5285            }
5286        ));
5287    }
5288
5289    // Ensure apply-unclaim releases the current pending operation back to ready.
5290    #[test]
5291    fn run_restore_apply_unclaim_marks_pending_operation_ready() {
5292        let root = temp_dir("canic-cli-restore-apply-unclaim");
5293        fs::create_dir_all(&root).expect("create temp root");
5294        let journal_path = root.join("restore-apply-journal.json");
5295        let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
5296        let mut journal = ready_apply_journal();
5297        journal
5298            .mark_next_operation_pending()
5299            .expect("claim operation");
5300
5301        fs::write(
5302            &journal_path,
5303            serde_json::to_vec(&journal).expect("serialize journal"),
5304        )
5305        .expect("write journal");
5306
5307        run([
5308            OsString::from("apply-unclaim"),
5309            OsString::from("--journal"),
5310            OsString::from(journal_path.as_os_str()),
5311            OsString::from("--sequence"),
5312            OsString::from("0"),
5313            OsString::from("--updated-at"),
5314            OsString::from("2026-05-04T12:01:00Z"),
5315            OsString::from("--out"),
5316            OsString::from(unclaimed_path.as_os_str()),
5317        ])
5318        .expect("unclaim operation");
5319
5320        let unclaimed: RestoreApplyJournal =
5321            serde_json::from_slice(&fs::read(&unclaimed_path).expect("read unclaimed journal"))
5322                .expect("decode unclaimed journal");
5323        let status = unclaimed.status();
5324
5325        fs::remove_dir_all(root).expect("remove temp root");
5326        assert_eq!(unclaimed.pending_operations, 0);
5327        assert_eq!(unclaimed.ready_operations, 8);
5328        assert_eq!(
5329            unclaimed.operations[0].state,
5330            RestoreApplyOperationState::Ready
5331        );
5332        assert_eq!(
5333            unclaimed.operations[0].state_updated_at.as_deref(),
5334            Some("2026-05-04T12:01:00Z")
5335        );
5336        assert_eq!(status.next_ready_sequence, Some(0));
5337        assert_eq!(
5338            status.next_transition_state,
5339            Some(RestoreApplyOperationState::Ready)
5340        );
5341        assert_eq!(
5342            status.next_transition_updated_at.as_deref(),
5343            Some("2026-05-04T12:01:00Z")
5344        );
5345    }
5346
5347    // Ensure apply-unclaim can reject a stale pending operation sequence.
5348    #[test]
5349    fn run_restore_apply_unclaim_rejects_sequence_mismatch() {
5350        let root = temp_dir("canic-cli-restore-apply-unclaim-sequence");
5351        fs::create_dir_all(&root).expect("create temp root");
5352        let journal_path = root.join("restore-apply-journal.json");
5353        let unclaimed_path = root.join("restore-apply-journal.unclaimed.json");
5354        let mut journal = ready_apply_journal();
5355        journal
5356            .mark_next_operation_pending()
5357            .expect("claim operation");
5358
5359        fs::write(
5360            &journal_path,
5361            serde_json::to_vec(&journal).expect("serialize journal"),
5362        )
5363        .expect("write journal");
5364
5365        let err = run([
5366            OsString::from("apply-unclaim"),
5367            OsString::from("--journal"),
5368            OsString::from(journal_path.as_os_str()),
5369            OsString::from("--sequence"),
5370            OsString::from("1"),
5371            OsString::from("--out"),
5372            OsString::from(unclaimed_path.as_os_str()),
5373        ])
5374        .expect_err("stale sequence should fail unclaim");
5375
5376        assert!(!unclaimed_path.exists());
5377        fs::remove_dir_all(root).expect("remove temp root");
5378        assert!(matches!(
5379            err,
5380            RestoreCommandError::RestoreApplyUnclaimSequenceMismatch {
5381                expected: 1,
5382                actual: Some(0),
5383            }
5384        ));
5385    }
5386
5387    // Ensure apply-mark can advance one journal operation and keep counts consistent.
5388    #[test]
5389    fn run_restore_apply_mark_completes_operation() {
5390        let root = temp_dir("canic-cli-restore-apply-mark-complete");
5391        fs::create_dir_all(&root).expect("create temp root");
5392        let journal_path = root.join("restore-apply-journal.json");
5393        let updated_path = root.join("restore-apply-journal.updated.json");
5394        let journal = ready_apply_journal();
5395
5396        fs::write(
5397            &journal_path,
5398            serde_json::to_vec(&journal).expect("serialize journal"),
5399        )
5400        .expect("write journal");
5401
5402        run([
5403            OsString::from("apply-mark"),
5404            OsString::from("--journal"),
5405            OsString::from(journal_path.as_os_str()),
5406            OsString::from("--sequence"),
5407            OsString::from("0"),
5408            OsString::from("--state"),
5409            OsString::from("completed"),
5410            OsString::from("--updated-at"),
5411            OsString::from("2026-05-04T12:02:00Z"),
5412            OsString::from("--out"),
5413            OsString::from(updated_path.as_os_str()),
5414        ])
5415        .expect("mark operation completed");
5416
5417        let updated: RestoreApplyJournal =
5418            serde_json::from_slice(&fs::read(&updated_path).expect("read updated journal"))
5419                .expect("decode updated journal");
5420        let status = updated.status();
5421
5422        fs::remove_dir_all(root).expect("remove temp root");
5423        assert_eq!(updated.completed_operations, 1);
5424        assert_eq!(updated.ready_operations, 7);
5425        assert_eq!(
5426            updated.operations[0].state_updated_at.as_deref(),
5427            Some("2026-05-04T12:02:00Z")
5428        );
5429        assert_eq!(status.next_ready_sequence, Some(1));
5430    }
5431
5432    // Ensure apply-mark can require an operation claim before completion.
5433    #[test]
5434    fn run_restore_apply_mark_require_pending_rejects_ready_operation() {
5435        let root = temp_dir("canic-cli-restore-apply-mark-require-pending");
5436        fs::create_dir_all(&root).expect("create temp root");
5437        let journal_path = root.join("restore-apply-journal.json");
5438        let updated_path = root.join("restore-apply-journal.updated.json");
5439        let journal = ready_apply_journal();
5440
5441        fs::write(
5442            &journal_path,
5443            serde_json::to_vec(&journal).expect("serialize journal"),
5444        )
5445        .expect("write journal");
5446
5447        let err = run([
5448            OsString::from("apply-mark"),
5449            OsString::from("--journal"),
5450            OsString::from(journal_path.as_os_str()),
5451            OsString::from("--sequence"),
5452            OsString::from("0"),
5453            OsString::from("--state"),
5454            OsString::from("completed"),
5455            OsString::from("--out"),
5456            OsString::from(updated_path.as_os_str()),
5457            OsString::from("--require-pending"),
5458        ])
5459        .expect_err("ready operation should fail pending requirement");
5460
5461        assert!(!updated_path.exists());
5462        fs::remove_dir_all(root).expect("remove temp root");
5463        assert!(matches!(
5464            err,
5465            RestoreCommandError::RestoreApplyMarkRequiresPending {
5466                sequence: 0,
5467                state: RestoreApplyOperationState::Ready,
5468            }
5469        ));
5470    }
5471
5472    // Ensure apply-mark refuses to skip earlier ready operations.
5473    #[test]
5474    fn run_restore_apply_mark_rejects_out_of_order_operation() {
5475        let root = temp_dir("canic-cli-restore-apply-mark-out-of-order");
5476        fs::create_dir_all(&root).expect("create temp root");
5477        let journal_path = root.join("restore-apply-journal.json");
5478        let updated_path = root.join("restore-apply-journal.updated.json");
5479        let journal = ready_apply_journal();
5480
5481        fs::write(
5482            &journal_path,
5483            serde_json::to_vec(&journal).expect("serialize journal"),
5484        )
5485        .expect("write journal");
5486
5487        let err = run([
5488            OsString::from("apply-mark"),
5489            OsString::from("--journal"),
5490            OsString::from(journal_path.as_os_str()),
5491            OsString::from("--sequence"),
5492            OsString::from("1"),
5493            OsString::from("--state"),
5494            OsString::from("completed"),
5495            OsString::from("--out"),
5496            OsString::from(updated_path.as_os_str()),
5497        ])
5498        .expect_err("out-of-order operation should fail");
5499
5500        assert!(!updated_path.exists());
5501        fs::remove_dir_all(root).expect("remove temp root");
5502        assert!(matches!(
5503            err,
5504            RestoreCommandError::RestoreApplyJournal(
5505                RestoreApplyJournalError::OutOfOrderOperationTransition {
5506                    requested: 1,
5507                    next: 0
5508                }
5509            )
5510        ));
5511    }
5512
5513    // Ensure apply-mark requires failure reasons for failed operation state.
5514    #[test]
5515    fn run_restore_apply_mark_failed_requires_reason() {
5516        let root = temp_dir("canic-cli-restore-apply-mark-failed-reason");
5517        fs::create_dir_all(&root).expect("create temp root");
5518        let journal_path = root.join("restore-apply-journal.json");
5519        let journal = ready_apply_journal();
5520
5521        fs::write(
5522            &journal_path,
5523            serde_json::to_vec(&journal).expect("serialize journal"),
5524        )
5525        .expect("write journal");
5526
5527        let err = run([
5528            OsString::from("apply-mark"),
5529            OsString::from("--journal"),
5530            OsString::from(journal_path.as_os_str()),
5531            OsString::from("--sequence"),
5532            OsString::from("0"),
5533            OsString::from("--state"),
5534            OsString::from("failed"),
5535        ])
5536        .expect_err("failed state should require reason");
5537
5538        fs::remove_dir_all(root).expect("remove temp root");
5539        assert!(matches!(
5540            err,
5541            RestoreCommandError::RestoreApplyJournal(
5542                RestoreApplyJournalError::FailureReasonRequired(0)
5543            )
5544        ));
5545    }
5546
5547    // Ensure restore apply dry-run rejects status files from another plan.
5548    #[test]
5549    fn run_restore_apply_dry_run_rejects_mismatched_status() {
5550        let root = temp_dir("canic-cli-restore-apply-dry-run-mismatch");
5551        fs::create_dir_all(&root).expect("create temp root");
5552        let plan_path = root.join("restore-plan.json");
5553        let status_path = root.join("restore-status.json");
5554        let out_path = root.join("restore-apply-dry-run.json");
5555        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
5556        let mut status = RestoreStatus::from_plan(&plan);
5557        status.backup_id = "other-backup".to_string();
5558
5559        fs::write(
5560            &plan_path,
5561            serde_json::to_vec(&plan).expect("serialize plan"),
5562        )
5563        .expect("write plan");
5564        fs::write(
5565            &status_path,
5566            serde_json::to_vec(&status).expect("serialize status"),
5567        )
5568        .expect("write status");
5569
5570        let err = run([
5571            OsString::from("apply"),
5572            OsString::from("--plan"),
5573            OsString::from(plan_path.as_os_str()),
5574            OsString::from("--status"),
5575            OsString::from(status_path.as_os_str()),
5576            OsString::from("--dry-run"),
5577            OsString::from("--out"),
5578            OsString::from(out_path.as_os_str()),
5579        ])
5580        .expect_err("mismatched status should fail");
5581
5582        assert!(!out_path.exists());
5583        fs::remove_dir_all(root).expect("remove temp root");
5584        assert!(matches!(
5585            err,
5586            RestoreCommandError::RestoreApplyDryRun(RestoreApplyDryRunError::StatusPlanMismatch {
5587                field: "backup_id",
5588                ..
5589            })
5590        ));
5591    }
5592
5593    // Build one manually ready apply journal for runner-focused CLI tests.
5594    fn ready_apply_journal() -> RestoreApplyJournal {
5595        let plan = RestorePlanner::plan(&restore_ready_manifest(), None).expect("build plan");
5596        let dry_run = RestoreApplyDryRun::try_from_plan(&plan, None).expect("dry-run");
5597        let mut journal = RestoreApplyJournal::from_dry_run(&dry_run);
5598
5599        journal.ready = true;
5600        journal.blocked_reasons = Vec::new();
5601        for operation in &mut journal.operations {
5602            operation.state = canic_backup::restore::RestoreApplyOperationState::Ready;
5603            operation.blocking_reasons = Vec::new();
5604        }
5605        journal.blocked_operations = 0;
5606        journal.ready_operations = journal.operation_count;
5607        journal.validate().expect("journal should validate");
5608        journal
5609    }
5610
5611    // Build one valid manifest for restore planning tests.
5612    fn valid_manifest() -> FleetBackupManifest {
5613        FleetBackupManifest {
5614            manifest_version: 1,
5615            backup_id: "backup-test".to_string(),
5616            created_at: "2026-05-03T00:00:00Z".to_string(),
5617            tool: ToolMetadata {
5618                name: "canic".to_string(),
5619                version: "0.30.1".to_string(),
5620            },
5621            source: SourceMetadata {
5622                environment: "local".to_string(),
5623                root_canister: ROOT.to_string(),
5624            },
5625            consistency: ConsistencySection {
5626                mode: ConsistencyMode::CrashConsistent,
5627                backup_units: vec![BackupUnit {
5628                    unit_id: "fleet".to_string(),
5629                    kind: BackupUnitKind::SubtreeRooted,
5630                    roles: vec!["root".to_string(), "app".to_string()],
5631                    consistency_reason: None,
5632                    dependency_closure: Vec::new(),
5633                    topology_validation: "subtree-closed".to_string(),
5634                    quiescence_strategy: None,
5635                }],
5636            },
5637            fleet: FleetSection {
5638                topology_hash_algorithm: "sha256".to_string(),
5639                topology_hash_input: "sorted(pid,parent_pid,role,module_hash)".to_string(),
5640                discovery_topology_hash: HASH.to_string(),
5641                pre_snapshot_topology_hash: HASH.to_string(),
5642                topology_hash: HASH.to_string(),
5643                members: vec![
5644                    fleet_member("root", ROOT, None, IdentityMode::Fixed),
5645                    fleet_member("app", CHILD, Some(ROOT), IdentityMode::Relocatable),
5646                ],
5647            },
5648            verification: VerificationPlan::default(),
5649        }
5650    }
5651
5652    // Build one manifest whose restore readiness metadata is complete.
5653    fn restore_ready_manifest() -> FleetBackupManifest {
5654        let mut manifest = valid_manifest();
5655        for member in &mut manifest.fleet.members {
5656            member.source_snapshot.module_hash = Some(HASH.to_string());
5657            member.source_snapshot.wasm_hash = Some(HASH.to_string());
5658            member.source_snapshot.checksum = Some(HASH.to_string());
5659        }
5660        manifest
5661    }
5662
5663    // Build one valid manifest member.
5664    fn fleet_member(
5665        role: &str,
5666        canister_id: &str,
5667        parent_canister_id: Option<&str>,
5668        identity_mode: IdentityMode,
5669    ) -> FleetMember {
5670        FleetMember {
5671            role: role.to_string(),
5672            canister_id: canister_id.to_string(),
5673            parent_canister_id: parent_canister_id.map(str::to_string),
5674            subnet_canister_id: Some(ROOT.to_string()),
5675            controller_hint: None,
5676            identity_mode,
5677            restore_group: 1,
5678            verification_class: "basic".to_string(),
5679            verification_checks: vec![VerificationCheck {
5680                kind: "status".to_string(),
5681                method: None,
5682                roles: vec![role.to_string()],
5683            }],
5684            source_snapshot: SourceSnapshot {
5685                snapshot_id: format!("{role}-snapshot"),
5686                module_hash: None,
5687                wasm_hash: None,
5688                code_version: Some("v0.30.1".to_string()),
5689                artifact_path: format!("artifacts/{role}"),
5690                checksum_algorithm: "sha256".to_string(),
5691                checksum: None,
5692            },
5693        }
5694    }
5695
5696    // Write a canonical backup layout whose journal checksums match the artifacts.
5697    fn write_verified_layout(root: &Path, layout: &BackupLayout, manifest: &FleetBackupManifest) {
5698        layout.write_manifest(manifest).expect("write manifest");
5699
5700        let artifacts = manifest
5701            .fleet
5702            .members
5703            .iter()
5704            .map(|member| {
5705                let bytes = format!("{} artifact", member.role);
5706                let artifact_path = root.join(&member.source_snapshot.artifact_path);
5707                if let Some(parent) = artifact_path.parent() {
5708                    fs::create_dir_all(parent).expect("create artifact parent");
5709                }
5710                fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
5711                let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
5712
5713                ArtifactJournalEntry {
5714                    canister_id: member.canister_id.clone(),
5715                    snapshot_id: member.source_snapshot.snapshot_id.clone(),
5716                    state: ArtifactState::Durable,
5717                    temp_path: None,
5718                    artifact_path: member.source_snapshot.artifact_path.clone(),
5719                    checksum_algorithm: checksum.algorithm,
5720                    checksum: Some(checksum.hash),
5721                    updated_at: "2026-05-03T00:00:00Z".to_string(),
5722                }
5723            })
5724            .collect();
5725
5726        layout
5727            .write_journal(&DownloadJournal {
5728                journal_version: 1,
5729                backup_id: manifest.backup_id.clone(),
5730                discovery_topology_hash: Some(manifest.fleet.discovery_topology_hash.clone()),
5731                pre_snapshot_topology_hash: Some(manifest.fleet.pre_snapshot_topology_hash.clone()),
5732                operation_metrics: canic_backup::journal::DownloadOperationMetrics::default(),
5733                artifacts,
5734            })
5735            .expect("write journal");
5736    }
5737
5738    // Write artifact bytes and update the manifest checksums for apply validation.
5739    fn write_manifest_artifacts(root: &Path, manifest: &mut FleetBackupManifest) {
5740        for member in &mut manifest.fleet.members {
5741            let bytes = format!("{} apply artifact", member.role);
5742            let artifact_path = root.join(&member.source_snapshot.artifact_path);
5743            if let Some(parent) = artifact_path.parent() {
5744                fs::create_dir_all(parent).expect("create artifact parent");
5745            }
5746            fs::write(&artifact_path, bytes.as_bytes()).expect("write artifact");
5747            let checksum = ArtifactChecksum::from_bytes(bytes.as_bytes());
5748            member.source_snapshot.checksum = Some(checksum.hash);
5749        }
5750    }
5751
5752    // Build a unique temporary directory.
5753    fn temp_dir(prefix: &str) -> PathBuf {
5754        let nanos = SystemTime::now()
5755            .duration_since(UNIX_EPOCH)
5756            .expect("system time after epoch")
5757            .as_nanos();
5758        std::env::temp_dir().join(format!("{prefix}-{}-{nanos}", std::process::id()))
5759    }
5760}