Skip to main content

aft/
checkpoint.rs

1use std::collections::{HashMap, HashSet};
2use std::fs;
3use std::io::{self, Write};
4use std::path::{Path, PathBuf};
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, LazyLock, Mutex};
7use std::time::Duration;
8
9use serde::{Deserialize, Serialize};
10
11use crate::backup::{hash_session, BackupStore, CapturedRegularFile};
12use crate::error::AftError;
13use crate::fs_lock;
14
15const CHECKPOINT_LOCK_TIMEOUT: Duration = Duration::from_secs(30);
16
17/// Named checkpoints are deliberately bounded per session so a busy session
18/// cannot grow an unbounded durable artifact store.
19const MAX_NAMED_CHECKPOINTS_PER_SESSION: usize = 20;
20/// Durable named checkpoints keep decisions long enough to survive ordinary
21/// work interruptions without becoming permanent storage.
22const NAMED_CHECKPOINT_RETENTION_DAYS: u64 = 14;
23const NAMED_CHECKPOINT_RETENTION_SECS: u64 = NAMED_CHECKPOINT_RETENTION_DAYS * 24 * 60 * 60;
24const CHECKPOINT_SCHEMA_VERSION: u32 = 1;
25const UNBOUND_HARNESS_SEGMENT: &str = "unbound";
26
27static CHECKPOINT_MAINTENANCE_KEYS: LazyLock<Mutex<HashSet<(PathBuf, String)>>> =
28    LazyLock::new(|| Mutex::new(HashSet::new()));
29
30/// User-visible explanation when no durable checkpoints were found after hydration.
31pub const CHECKPOINT_RESTART_NOTICE: &str =
32    "no durable checkpoints found on disk; in-memory checkpoints do not survive restarts";
33/// User-visible explanation when a checkpoint list was hydrated from disk.
34pub const CHECKPOINT_HYDRATED_NOTICE: &str =
35    "durable checkpoints are hydrated from disk and survive restarts";
36
37/// Describe the durable location for a successful checkpoint.
38pub fn checkpoint_durability(storage_path: &Path) -> String {
39    format!(
40        "durable on disk at {}; survives restarts",
41        storage_path.display()
42    )
43}
44
45/// Metadata about a checkpoint, returned by list/create/restore.
46#[derive(Debug, Clone)]
47pub struct CheckpointInfo {
48    pub name: String,
49    pub file_count: usize,
50    pub created_at: u64,
51    /// Durable checkpoint directory, when the store has a storage namespace.
52    pub storage_path: Option<PathBuf>,
53    /// Older checkpoint names evicted to keep the per-session retention cap.
54    pub evicted: Vec<String>,
55    /// Paths that could not be snapshotted (e.g. deleted since last edit),
56    /// paired with the OS-level error that stopped us from reading them.
57    /// Empty on successful round-trips. Populated only on `create()` — the
58    /// `list()` / `restore()` paths leave it empty.
59    pub skipped: Vec<(PathBuf, String)>,
60}
61
62/// A stored checkpoint: a snapshot of multiple file contents and metadata.
63#[derive(Debug, Clone)]
64struct Checkpoint {
65    name: String,
66    file_contents: HashMap<PathBuf, CheckpointFile>,
67    created_at: u64,
68    /// Nanosecond-resolution creation ordering prevents ties from making
69    /// retention nondeterministic when callers create several checkpoints in a second.
70    created_order: u64,
71}
72
73#[derive(Debug, Clone)]
74struct CheckpointFile {
75    /// Fresh in-memory checkpoints retain the platform metadata so restore keeps
76    /// its existing behavior. Disk hydration rebuilds from the portable mode.
77    metadata: Option<fs::Metadata>,
78    mode: Option<u32>,
79    kind: CheckpointFileKind,
80}
81
82#[derive(Debug, Serialize, Deserialize)]
83struct DiskCheckpointMeta {
84    schema_version: u32,
85    session_id: String,
86    name: String,
87    created_at: u64,
88    created_order: u64,
89    files: Vec<DiskCheckpointFileMeta>,
90}
91
92#[derive(Debug, Serialize, Deserialize)]
93struct DiskCheckpointFileMeta {
94    original_path: String,
95    blob: String,
96    kind: DiskCheckpointFileKind,
97    mode: Option<u32>,
98    target_is_dir: bool,
99}
100
101#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
102#[serde(rename_all = "snake_case")]
103enum DiskCheckpointFileKind {
104    Regular,
105    Symlink,
106}
107
108#[derive(Debug, Clone)]
109enum CheckpointFileKind {
110    Regular {
111        bytes: Arc<[u8]>,
112    },
113    Symlink {
114        target: PathBuf,
115        target_is_dir: bool,
116    },
117}
118
119impl CheckpointFile {
120    fn read(path: &Path) -> io::Result<Self> {
121        let metadata = fs::symlink_metadata(path)?;
122        let file_type = metadata.file_type();
123        if file_type.is_symlink() {
124            let target = fs::read_link(path)?;
125            let target_is_dir = fs::metadata(path)
126                .map(|target_metadata| target_metadata.is_dir())
127                .unwrap_or(false);
128            return Ok(Self {
129                mode: checkpoint_mode(&metadata),
130                metadata: Some(metadata),
131                kind: CheckpointFileKind::Symlink {
132                    target,
133                    target_is_dir,
134                },
135            });
136        }
137
138        if metadata.is_file() {
139            let capture = CapturedRegularFile::read(path)?.ok_or_else(|| {
140                io::Error::new(
141                    io::ErrorKind::InvalidInput,
142                    "file changed while being captured",
143                )
144            })?;
145            return Ok(Self::from_fresh_capture(capture));
146        }
147
148        Err(io::Error::new(
149            io::ErrorKind::InvalidInput,
150            "not a regular file or symlink",
151        ))
152    }
153
154    /// Build a checkpoint from bytes captured earlier in the command.
155    ///
156    /// Size and modification time are checked immediately before the bytes enter
157    /// the checkpoint. If either changed, the capture is refreshed from disk so
158    /// rollback and undo never preserve stale pre-edit content. This constructor
159    /// is only for regular files; symlinks continue through [`Self::read`].
160    fn from_captured(path: &Path, capture: &mut CapturedRegularFile) -> io::Result<Self> {
161        capture.refresh_if_stale(path)?;
162        let metadata = capture.metadata().clone();
163        Ok(Self {
164            mode: checkpoint_mode(&metadata),
165            metadata: Some(metadata),
166            kind: CheckpointFileKind::Regular {
167                bytes: capture.shared_bytes(),
168            },
169        })
170    }
171
172    fn from_fresh_capture(capture: CapturedRegularFile) -> Self {
173        let metadata = capture.metadata().clone();
174        Self {
175            mode: checkpoint_mode(&metadata),
176            metadata: Some(metadata),
177            kind: CheckpointFileKind::Regular {
178                bytes: capture.shared_bytes(),
179            },
180        }
181    }
182
183    fn read_optional(path: &Path) -> io::Result<Option<Self>> {
184        match Self::read(path) {
185            Ok(snapshot) => Ok(Some(snapshot)),
186            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
187            Err(error) => Err(error),
188        }
189    }
190
191    fn from_disk(meta: &DiskCheckpointFileMeta, bytes: Vec<u8>) -> Result<Self, String> {
192        let kind = match &meta.kind {
193            DiskCheckpointFileKind::Regular => CheckpointFileKind::Regular {
194                bytes: bytes.into(),
195            },
196            DiskCheckpointFileKind::Symlink => {
197                let target = String::from_utf8(bytes)
198                    .map(PathBuf::from)
199                    .map_err(|error| format!("checkpoint symlink target is not UTF-8: {error}"))?;
200                CheckpointFileKind::Symlink {
201                    target,
202                    target_is_dir: meta.target_is_dir,
203                }
204            }
205        };
206        Ok(Self {
207            metadata: None,
208            mode: meta.mode,
209            kind,
210        })
211    }
212}
213
214#[cfg(unix)]
215fn checkpoint_mode(metadata: &fs::Metadata) -> Option<u32> {
216    use std::os::unix::fs::PermissionsExt;
217    Some(metadata.permissions().mode())
218}
219
220#[cfg(not(unix))]
221fn checkpoint_mode(_metadata: &fs::Metadata) -> Option<u32> {
222    None
223}
224
225/// Workspace-wide, per-session checkpoint store.
226///
227/// Partitioned by session: two sessions sharing one bridge can both create
228/// checkpoints named `snap1` without collision, and restoring from one session
229/// does not leak the other's file set. The durable disk tree is authoritative;
230/// in-memory entries are rehydrated under the mutation lock before each read or
231/// change that depends on them.
232#[derive(Debug)]
233pub struct CheckpointStore {
234    /// session -> name -> checkpoint, derived from the durable disk tree.
235    checkpoints: HashMap<String, HashMap<String, Checkpoint>>,
236    lock_path: PathBuf,
237    lock_timeout: Duration,
238    storage_dir: Option<PathBuf>,
239    storage_harness: Option<String>,
240    blob_counter: AtomicU64,
241}
242
243/// Owns a checkpoint mutation lock and removes its project scope directory after
244/// the filesystem lock has released. The directory scopes only the transient
245/// lockfile; durable checkpoint bytes live under the harness namespace instead.
246struct CheckpointLockGuard {
247    guard: Option<fs_lock::LockGuard>,
248    scope_dir: Option<PathBuf>,
249}
250
251impl Drop for CheckpointLockGuard {
252    fn drop(&mut self) {
253        // LockGuard::drop must join the heartbeat before removing the lockfile.
254        // Drop it first, then make the best-effort directory cleanup so a new
255        // owner can keep the scope directory when it races this release.
256        if let Some(guard) = self.guard.take() {
257            drop(guard);
258        }
259        if let Some(scope_dir) = &self.scope_dir {
260            remove_empty_scope_dir(scope_dir);
261        }
262    }
263}
264
265impl CheckpointStore {
266    pub fn new() -> Self {
267        let project_root = std::env::current_dir().unwrap_or_else(|_| std::env::temp_dir());
268        let project_key = crate::path_identity::project_scope_key(&project_root);
269        let storage_dir = crate::bash_background::storage_dir(None);
270        let lock_path = storage_dir
271            .join("checkpoints")
272            .join(project_key)
273            .join("checkpoint.lock");
274        let mut store = Self::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
275        // Commands received before configure still need an honest durable home.
276        // Configure replaces this isolated namespace with the concrete harness.
277        store.storage_dir = Some(storage_dir);
278        store.storage_harness = Some(UNBOUND_HARNESS_SEGMENT.to_string());
279        store
280    }
281
282    /// Point this store's mutation lock at a private path. Tests use this for
283    /// isolation instead of mutating the process-global `AFT_CACHE_DIR` env
284    /// var, which races parallel lib tests that resolve storage paths.
285    #[cfg(test)]
286    pub(crate) fn set_lock_path_for_test(&mut self, lock_path: PathBuf) {
287        self.storage_dir = lock_path.parent().map(Path::to_path_buf);
288        self.storage_harness = Some("test".to_string());
289        self.lock_path = lock_path;
290    }
291
292    /// Select the harness-scoped durable namespace. Rebinding a different
293    /// namespace drops only the derived in-memory cache; disk remains authoritative.
294    pub fn set_storage_dir_for_harness(&mut self, dir: PathBuf, harness: crate::harness::Harness) {
295        let harness = harness.storage_segment();
296        if self.storage_dir.as_ref() == Some(&dir)
297            && self.storage_harness.as_deref() == Some(&harness)
298        {
299            return;
300        }
301        if self.storage_dir.as_ref() == Some(&dir)
302            && self.storage_harness.as_deref() == Some(UNBOUND_HARNESS_SEGMENT)
303        {
304            match self.acquire_mutation_lock() {
305                Ok(_lock) => migrate_unbound_checkpoint_namespace(&dir, &harness),
306                Err(error) => crate::slog_warn!(
307                    "could not migrate unbound durable checkpoints into {}: {}",
308                    harness,
309                    error
310                ),
311            }
312        }
313        self.storage_dir = Some(dir);
314        self.storage_harness = Some(harness);
315        self.checkpoints.clear();
316    }
317
318    fn with_lock_path(lock_path: PathBuf, lock_timeout: Duration) -> Self {
319        CheckpointStore {
320            checkpoints: HashMap::new(),
321            lock_path,
322            lock_timeout,
323            storage_dir: None,
324            storage_harness: None,
325            blob_counter: AtomicU64::new(0),
326        }
327    }
328
329    fn acquire_mutation_lock(&self) -> Result<CheckpointLockGuard, AftError> {
330        let scope_dir = self.lock_path.parent().map(Path::to_path_buf);
331        if let Some(parent) = scope_dir.as_deref() {
332            fs::create_dir_all(parent).map_err(|error| AftError::IoError {
333                path: parent.display().to_string(),
334                message: format!("failed to create checkpoint lock directory: {error}"),
335            })?;
336        }
337
338        let acquire_result = match fs_lock::try_acquire(&self.lock_path, self.lock_timeout) {
339            // A releasing peer removes the empty lock scope after its heartbeat
340            // exits. It can win the tiny interval after our create_dir_all and
341            // before lock creation, so recreate once and retry the acquisition.
342            Err(fs_lock::AcquireError::Io(error)) if error.kind() == io::ErrorKind::NotFound => {
343                if let Some(parent) = scope_dir.as_deref() {
344                    fs::create_dir_all(parent).map_err(|error| AftError::IoError {
345                        path: parent.display().to_string(),
346                        message: format!("failed to recreate checkpoint lock directory: {error}"),
347                    })?;
348                }
349                fs_lock::try_acquire(&self.lock_path, self.lock_timeout)
350            }
351            result => result,
352        };
353        let guard = acquire_result.map_err(|error| match error {
354            fs_lock::AcquireError::Timeout => AftError::IoError {
355                path: self.lock_path.display().to_string(),
356                message: "timed out acquiring checkpoint mutation lock".to_string(),
357            },
358            fs_lock::AcquireError::Io(error) => AftError::IoError {
359                path: self.lock_path.display().to_string(),
360                message: format!("failed to acquire checkpoint mutation lock: {error}"),
361            },
362        })?;
363
364        Ok(CheckpointLockGuard {
365            guard: Some(guard),
366            scope_dir,
367        })
368    }
369
370    /// Create a checkpoint by reading the given files, scoped to `session`.
371    ///
372    /// If `files` is empty, snapshots all tracked files for **that session**
373    /// from the BackupStore (other sessions' tracked files are not visible).
374    /// Overwrites any existing checkpoint with the same name in this session.
375    ///
376    /// Unreadable paths (e.g. deleted since their last edit) are skipped with
377    /// a warning instead of failing the whole checkpoint. The paths and their
378    /// errors are returned via `CheckpointInfo::skipped` so callers can
379    /// surface them. A checkpoint is only rejected outright when *every*
380    /// requested path fails — that case still returns a `FileNotFound`
381    /// error so callers can distinguish "partial success" from "nothing
382    /// snapshotted at all".
383    pub fn create(
384        &mut self,
385        session: &str,
386        name: &str,
387        files: Vec<PathBuf>,
388        backup_store: &BackupStore,
389    ) -> Result<CheckpointInfo, AftError> {
390        self.create_impl(session, name, files, backup_store, None)
391    }
392
393    pub(crate) fn create_from_captures(
394        &mut self,
395        session: &str,
396        name: &str,
397        files: Vec<PathBuf>,
398        backup_store: &BackupStore,
399        captures: &mut HashMap<PathBuf, CapturedRegularFile>,
400    ) -> Result<CheckpointInfo, AftError> {
401        self.create_impl(session, name, files, backup_store, Some(captures))
402    }
403
404    fn create_impl(
405        &mut self,
406        session: &str,
407        name: &str,
408        files: Vec<PathBuf>,
409        backup_store: &BackupStore,
410        mut captures: Option<&mut HashMap<PathBuf, CapturedRegularFile>>,
411    ) -> Result<CheckpointInfo, AftError> {
412        let _mutation_lock = self.acquire_mutation_lock()?;
413        validate_checkpoint_name(name)?;
414        self.run_process_maintenance_once_locked()?;
415        self.hydrate_session_locked(session)?;
416        let explicit_request = !files.is_empty();
417        let file_list = if files.is_empty() {
418            backup_store.tracked_files(session)
419        } else {
420            files
421        };
422
423        let mut file_contents = HashMap::new();
424        let mut skipped: Vec<(PathBuf, String)> = Vec::new();
425        for path in &file_list {
426            let seeded = captures
427                .as_deref_mut()
428                .and_then(|captures| captures.get_mut(path))
429                .map(|capture| CheckpointFile::from_captured(path, capture));
430            let snapshot = match seeded {
431                Some(Err(error)) if error.kind() == io::ErrorKind::InvalidInput => {
432                    if let Some(captures) = captures.as_deref_mut() {
433                        captures.remove(path);
434                    }
435                    CheckpointFile::read(path)
436                }
437                Some(result) => result,
438                None => CheckpointFile::read(path),
439            };
440            match snapshot {
441                Ok(snapshot) => {
442                    file_contents.insert(path.clone(), snapshot);
443                }
444                Err(e) => {
445                    crate::slog_warn!(
446                        "checkpoint {}: skipping unreadable file {}: {}",
447                        name,
448                        path.display(),
449                        e
450                    );
451                    skipped.push((path.clone(), e.to_string()));
452                }
453            }
454        }
455
456        // If the caller explicitly named a single file and it was unreadable,
457        // that's a real error — surface it rather than silently returning an
458        // empty checkpoint. For empty `files` (tracked-file fallback) with no
459        // readable files at all, the empty-file checkpoint is a legitimate
460        // "nothing to snapshot" outcome and we keep it.
461        if explicit_request && file_contents.is_empty() && !skipped.is_empty() {
462            let (path, err) = &skipped[0];
463            return Err(AftError::FileNotFound {
464                path: format!("{}: {}", path.display(), err),
465            });
466        }
467
468        let created_at = current_timestamp();
469        let created_order = current_timestamp_nanos()
470            .saturating_add(self.blob_counter.fetch_add(1, Ordering::Relaxed));
471        let file_count = file_contents.len();
472        let checkpoint = Checkpoint {
473            name: name.to_string(),
474            file_contents,
475            created_at,
476            created_order,
477        };
478        let storage_path = self.durable_checkpoint_dir(session, name);
479
480        self.persist_checkpoint_locked(session, &checkpoint)?;
481        self.checkpoints
482            .entry(session.to_string())
483            .or_default()
484            .insert(name.to_string(), checkpoint);
485
486        let evicted = self.evict_excess_checkpoints_locked(session)?;
487
488        if skipped.is_empty() {
489            crate::slog_info!("checkpoint created: {} ({} files)", name, file_count);
490        } else {
491            crate::slog_info!(
492                "checkpoint created: {} ({} files, {} skipped)",
493                name,
494                file_count,
495                skipped.len()
496            );
497        }
498
499        Ok(CheckpointInfo {
500            name: name.to_string(),
501            file_count,
502            created_at,
503            storage_path,
504            evicted,
505            skipped,
506        })
507    }
508
509    /// Restore a checkpoint by overwriting files with stored content.
510    pub fn restore(&mut self, session: &str, name: &str) -> Result<CheckpointInfo, AftError> {
511        let _mutation_lock = self.acquire_mutation_lock()?;
512        self.run_process_maintenance_once_locked()?;
513        self.hydrate_session_locked(session)?;
514        let storage_path = self.durable_checkpoint_dir(session, name);
515        let checkpoint = self.get(session, name)?;
516        let mut paths = checkpoint.file_contents.keys().cloned().collect::<Vec<_>>();
517        paths.sort();
518
519        restore_paths_atomically(checkpoint, &paths)?;
520        crate::slog_info!("checkpoint restored: {}", name);
521
522        Ok(CheckpointInfo {
523            name: checkpoint.name.clone(),
524            file_count: checkpoint.file_contents.len(),
525            created_at: checkpoint.created_at,
526            storage_path,
527            evicted: Vec::new(),
528            skipped: Vec::new(),
529        })
530    }
531
532    /// Restore a checkpoint using a caller-validated path list.
533    pub fn restore_validated(
534        &mut self,
535        session: &str,
536        name: &str,
537        validated_paths: &[PathBuf],
538    ) -> Result<CheckpointInfo, AftError> {
539        let _mutation_lock = self.acquire_mutation_lock()?;
540        self.run_process_maintenance_once_locked()?;
541        self.hydrate_session_locked(session)?;
542        let storage_path = self.durable_checkpoint_dir(session, name);
543        let checkpoint = self.get(session, name)?;
544
545        for path in validated_paths {
546            checkpoint
547                .file_contents
548                .get(path)
549                .ok_or_else(|| AftError::FileNotFound {
550                    path: path.display().to_string(),
551                })?;
552        }
553        restore_paths_atomically(checkpoint, validated_paths)?;
554        crate::slog_info!("checkpoint restored: {}", name);
555
556        Ok(CheckpointInfo {
557            name: checkpoint.name.clone(),
558            file_count: checkpoint.file_contents.len(),
559            created_at: checkpoint.created_at,
560            storage_path,
561            evicted: Vec::new(),
562            skipped: Vec::new(),
563        })
564    }
565
566    /// Return the file paths stored for a checkpoint.
567    pub fn file_paths(&mut self, session: &str, name: &str) -> Result<Vec<PathBuf>, AftError> {
568        let _mutation_lock = self.acquire_mutation_lock()?;
569        self.run_process_maintenance_once_locked()?;
570        self.hydrate_session_locked(session)?;
571        let checkpoint = self.get(session, name)?;
572        Ok(checkpoint.file_contents.keys().cloned().collect())
573    }
574
575    /// Return absolute file paths stored for a checkpoint without restoring it.
576    pub fn absolute_file_paths(
577        &mut self,
578        session: &str,
579        name: &str,
580    ) -> Result<Vec<PathBuf>, AftError> {
581        let mut paths: Vec<PathBuf> = self
582            .file_paths(session, name)?
583            .into_iter()
584            .map(absolute_checkpoint_path)
585            .collect();
586        paths.sort();
587        Ok(paths)
588    }
589
590    /// Delete a checkpoint from a session. Returns true when a checkpoint was removed.
591    pub fn delete(&mut self, session: &str, name: &str) -> bool {
592        let _mutation_lock = match self.acquire_mutation_lock() {
593            Ok(lock) => lock,
594            Err(error) => {
595                crate::slog_warn!("checkpoint delete lock failed for {}: {}", name, error);
596                return false;
597            }
598        };
599        if let Err(error) = self.run_process_maintenance_once_locked() {
600            crate::slog_warn!(
601                "checkpoint delete maintenance failed for {}: {}",
602                name,
603                error
604            );
605            return false;
606        }
607        if let Err(error) = self.hydrate_session_locked(session) {
608            crate::slog_warn!("checkpoint delete hydration failed for {}: {}", name, error);
609            return false;
610        }
611        if self
612            .checkpoints
613            .get(session)
614            .is_none_or(|checkpoints| !checkpoints.contains_key(name))
615        {
616            return false;
617        }
618        if let Err(error) = self.remove_checkpoint_from_disk_locked(session, name) {
619            crate::slog_warn!("checkpoint delete failed for {}: {}", name, error);
620            return false;
621        }
622        let Some(session_checkpoints) = self.checkpoints.get_mut(session) else {
623            return false;
624        };
625        let removed = session_checkpoints.remove(name).is_some();
626        if session_checkpoints.is_empty() {
627            self.checkpoints.remove(session);
628        }
629        removed
630    }
631
632    /// List all checkpoints for this session with metadata, hydrating from the
633    /// authoritative durable tree before returning.
634    pub fn list(&mut self, session: &str) -> Result<Vec<CheckpointInfo>, AftError> {
635        let _mutation_lock = self.acquire_mutation_lock()?;
636        self.run_process_maintenance_once_locked()?;
637        self.hydrate_session_locked(session)?;
638        let mut list = self
639            .checkpoints
640            .get(session)
641            .map(|checkpoints| {
642                checkpoints
643                    .values()
644                    .map(|checkpoint| CheckpointInfo {
645                        name: checkpoint.name.clone(),
646                        file_count: checkpoint.file_contents.len(),
647                        created_at: checkpoint.created_at,
648                        storage_path: self.durable_checkpoint_dir(session, &checkpoint.name),
649                        evicted: Vec::new(),
650                        skipped: Vec::new(),
651                    })
652                    .collect::<Vec<_>>()
653            })
654            .unwrap_or_default();
655        list.sort_by(|left, right| left.name.cmp(&right.name));
656        Ok(list)
657    }
658
659    /// Total checkpoint count across all sessions already hydrated in this process.
660    pub fn total_count(&self) -> usize {
661        self.checkpoints
662            .values()
663            .map(|checkpoints| checkpoints.len())
664            .sum()
665    }
666
667    /// Sweep checkpoints older than the fixed fourteen-day retention window.
668    /// The limit is intentionally not configurable: named checkpoints protect
669    /// irreplaceable decisions, while predictable retention keeps the store bounded.
670    pub fn cleanup(&mut self) {
671        let _mutation_lock = match self.acquire_mutation_lock() {
672            Ok(lock) => lock,
673            Err(error) => {
674                crate::slog_warn!("checkpoint cleanup lock failed: {}", error);
675                return;
676            }
677        };
678        if let Err(error) = self.cleanup_locked() {
679            crate::slog_warn!("checkpoint cleanup failed: {}", error);
680        }
681    }
682
683    fn get(&self, session: &str, name: &str) -> Result<&Checkpoint, AftError> {
684        self.checkpoints
685            .get(session)
686            .and_then(|checkpoints| checkpoints.get(name))
687            .ok_or_else(|| AftError::CheckpointNotFound {
688                name: name.to_string(),
689            })
690    }
691
692    fn durable_checkpoints_dir(&self) -> Option<PathBuf> {
693        self.storage_dir
694            .as_ref()
695            .zip(self.storage_harness.as_ref())
696            .map(|(storage_dir, harness)| storage_dir.join(harness).join("checkpoints"))
697    }
698
699    fn durable_session_dir(&self, session: &str) -> Option<PathBuf> {
700        self.durable_checkpoints_dir()
701            .map(|checkpoints_dir| checkpoints_dir.join(hash_session(session)))
702    }
703
704    fn durable_checkpoint_dir(&self, session: &str, name: &str) -> Option<PathBuf> {
705        self.durable_session_dir(session)
706            .map(|session_dir| session_dir.join(name))
707    }
708
709    fn run_process_maintenance_once_locked(&mut self) -> Result<(), AftError> {
710        let Some(storage_dir) = self.storage_dir.clone() else {
711            return Ok(());
712        };
713        let Some(harness) = self.storage_harness.clone() else {
714            return Ok(());
715        };
716        if !CHECKPOINT_MAINTENANCE_KEYS
717            .lock()
718            .unwrap()
719            .insert((storage_dir, harness))
720        {
721            return Ok(());
722        }
723        self.cleanup_locked()
724    }
725
726    fn cleanup_locked(&mut self) -> Result<(), AftError> {
727        let now = current_timestamp();
728        self.checkpoints.retain(|_, session_checkpoints| {
729            session_checkpoints.retain(|_, checkpoint| {
730                now.saturating_sub(checkpoint.created_at) < NAMED_CHECKPOINT_RETENTION_SECS
731            });
732            !session_checkpoints.is_empty()
733        });
734
735        if let Some(checkpoints_dir) = self.durable_checkpoints_dir() {
736            sweep_expired_durable_checkpoints(&checkpoints_dir, now);
737        }
738        if let Some(checkpoints_root) = self.lock_path.parent().and_then(Path::parent) {
739            // Fail-closed guard: the sweep root is DERIVED from lock_path depth, and a
740            // caller with a nonstandard (shallower) lock path would resolve this to an
741            // unrelated directory - in tests, the OS temp root itself, where removing
742            // "empty scope dirs" deletes other processes' freshly created temp dirs.
743            // Only a directory actually named `checkpoints` is a legitimate sweep root.
744            if checkpoints_root.file_name() == Some(std::ffi::OsStr::new("checkpoints")) {
745                sweep_empty_scope_dirs(checkpoints_root);
746            }
747        }
748        Ok(())
749    }
750
751    fn hydrate_session_locked(&mut self, session: &str) -> Result<(), AftError> {
752        let Some(session_dir) = self.durable_session_dir(session) else {
753            return Ok(());
754        };
755        if !session_dir.exists() {
756            self.checkpoints.remove(session);
757            return Ok(());
758        }
759
760        let entries = fs::read_dir(&session_dir).map_err(|error| AftError::IoError {
761            path: session_dir.display().to_string(),
762            message: format!("failed to read durable checkpoint session: {error}"),
763        })?;
764        let mut hydrated = HashMap::new();
765        for entry in entries {
766            let entry = entry.map_err(|error| AftError::IoError {
767                path: session_dir.display().to_string(),
768                message: format!("failed to read durable checkpoint entry: {error}"),
769            })?;
770            let checkpoint_dir = entry.path();
771            if !entry
772                .file_type()
773                .map_err(|error| AftError::IoError {
774                    path: checkpoint_dir.display().to_string(),
775                    message: format!("failed to inspect durable checkpoint entry: {error}"),
776                })?
777                .is_dir()
778            {
779                continue;
780            }
781            let name = entry.file_name().to_string_lossy().into_owned();
782            if !is_safe_checkpoint_name(&name) {
783                continue;
784            }
785            let meta_path = checkpoint_dir.join("meta.json");
786            if !meta_path.exists() {
787                continue;
788            }
789            let checkpoint = read_checkpoint_from_disk(&checkpoint_dir, session, &name)?;
790            hydrated.insert(name, checkpoint);
791        }
792        if hydrated.is_empty() {
793            self.checkpoints.remove(session);
794        } else {
795            self.checkpoints.insert(session.to_string(), hydrated);
796            self.evict_excess_checkpoints_locked(session)?;
797        }
798        Ok(())
799    }
800
801    fn persist_checkpoint_locked(
802        &self,
803        session: &str,
804        checkpoint: &Checkpoint,
805    ) -> Result<(), AftError> {
806        let Some(checkpoint_dir) = self.durable_checkpoint_dir(session, &checkpoint.name) else {
807            return Ok(());
808        };
809        fs::create_dir_all(&checkpoint_dir).map_err(|error| AftError::IoError {
810            path: checkpoint_dir.display().to_string(),
811            message: format!("failed to create durable checkpoint directory: {error}"),
812        })?;
813
814        let mut files = Vec::with_capacity(checkpoint.file_contents.len());
815        for (index, (path, file)) in checkpoint.file_contents.iter().enumerate() {
816            let blob = format!(
817                "file_{}_{}_{}.blob",
818                checkpoint.created_order,
819                index,
820                self.blob_counter.fetch_add(1, Ordering::Relaxed)
821            );
822            let bytes = checkpoint_file_bytes(file);
823            write_temp_fsync_rename(&checkpoint_dir, &blob, &bytes).map_err(|error| {
824                AftError::IoError {
825                    path: checkpoint_dir.join(&blob).display().to_string(),
826                    message: format!("failed to write durable checkpoint blob: {error}"),
827                }
828            })?;
829            files.push(DiskCheckpointFileMeta {
830                original_path: path.display().to_string(),
831                blob,
832                kind: match &file.kind {
833                    CheckpointFileKind::Regular { .. } => DiskCheckpointFileKind::Regular,
834                    CheckpointFileKind::Symlink { .. } => DiskCheckpointFileKind::Symlink,
835                },
836                mode: file.mode,
837                target_is_dir: matches!(
838                    &file.kind,
839                    CheckpointFileKind::Symlink {
840                        target_is_dir: true,
841                        ..
842                    }
843                ),
844            });
845        }
846        fsync_dir(&checkpoint_dir).map_err(|error| AftError::IoError {
847            path: checkpoint_dir.display().to_string(),
848            message: format!("failed to sync durable checkpoint blobs: {error}"),
849        })?;
850
851        let meta = DiskCheckpointMeta {
852            schema_version: CHECKPOINT_SCHEMA_VERSION,
853            session_id: session.to_string(),
854            name: checkpoint.name.clone(),
855            created_at: checkpoint.created_at,
856            created_order: checkpoint.created_order,
857            files,
858        };
859        let bytes = serde_json::to_vec_pretty(&meta).map_err(|error| AftError::IoError {
860            path: checkpoint_dir.join("meta.json").display().to_string(),
861            message: format!("failed to serialize durable checkpoint metadata: {error}"),
862        })?;
863        write_temp_fsync_rename(&checkpoint_dir, "meta.json", &bytes).map_err(|error| {
864            AftError::IoError {
865                path: checkpoint_dir.join("meta.json").display().to_string(),
866                message: format!("failed to write durable checkpoint metadata: {error}"),
867            }
868        })?;
869        fsync_dir(&checkpoint_dir).map_err(|error| AftError::IoError {
870            path: checkpoint_dir.display().to_string(),
871            message: format!("failed to sync durable checkpoint metadata: {error}"),
872        })?;
873        prune_unreferenced_checkpoint_blobs(&checkpoint_dir, &meta.files).map_err(|error| {
874            AftError::IoError {
875                path: checkpoint_dir.display().to_string(),
876                message: format!("failed to prune stale durable checkpoint blobs: {error}"),
877            }
878        })?;
879        Ok(())
880    }
881
882    fn remove_checkpoint_from_disk_locked(
883        &self,
884        session: &str,
885        name: &str,
886    ) -> Result<(), AftError> {
887        let Some(checkpoint_dir) = self.durable_checkpoint_dir(session, name) else {
888            return Ok(());
889        };
890        match fs::remove_dir_all(&checkpoint_dir) {
891            Ok(()) => {
892                if let Some(session_dir) = checkpoint_dir.parent() {
893                    let _ = fs::remove_dir(session_dir);
894                }
895                Ok(())
896            }
897            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
898            Err(error) => Err(AftError::IoError {
899                path: checkpoint_dir.display().to_string(),
900                message: format!("failed to remove durable checkpoint: {error}"),
901            }),
902        }
903    }
904
905    fn evict_excess_checkpoints_locked(&mut self, session: &str) -> Result<Vec<String>, AftError> {
906        let Some(checkpoints) = self.checkpoints.get(session) else {
907            return Ok(Vec::new());
908        };
909        let overflow = checkpoints
910            .len()
911            .saturating_sub(MAX_NAMED_CHECKPOINTS_PER_SESSION);
912        let mut checkpoints = checkpoints
913            .values()
914            .map(|checkpoint| (checkpoint.created_order, checkpoint.name.clone()))
915            .collect::<Vec<_>>();
916        checkpoints.sort();
917        let evicted = checkpoints
918            .into_iter()
919            .take(overflow)
920            .map(|(_, name)| name)
921            .collect::<Vec<_>>();
922
923        for name in &evicted {
924            self.remove_checkpoint_from_disk_locked(session, name)?;
925        }
926        if let Some(checkpoints) = self.checkpoints.get_mut(session) {
927            for name in &evicted {
928                checkpoints.remove(name);
929            }
930        }
931        Ok(evicted)
932    }
933
934    pub fn session_is_empty(&self, session: &str) -> bool {
935        self.checkpoints.get(session).is_none_or(HashMap::is_empty)
936    }
937}
938
939fn migrate_unbound_checkpoint_namespace(storage_dir: &Path, harness: &str) {
940    let source = storage_dir
941        .join(UNBOUND_HARNESS_SEGMENT)
942        .join("checkpoints");
943    if !source.exists() {
944        return;
945    }
946    let target = storage_dir.join(harness).join("checkpoints");
947    if !target.exists() {
948        if let Some(parent) = target.parent() {
949            if let Err(error) = fs::create_dir_all(parent) {
950                crate::slog_warn!(
951                    "failed to create durable checkpoint harness directory {}: {}",
952                    parent.display(),
953                    error
954                );
955                return;
956            }
957        }
958        if let Err(error) = fs::rename(&source, &target) {
959            crate::slog_warn!(
960                "failed to move unbound durable checkpoints into {}: {}",
961                target.display(),
962                error
963            );
964        }
965        return;
966    }
967
968    // A vanished mounted child can make ReadDir::drop panic after closedir
969    // returns ENXIO, aborting the daemon. Keep namespace migration on its root
970    // filesystem before opening session directories.
971    let Ok(boundary) = crate::walk_boundary::DeviceBoundary::for_root(&source) else {
972        crate::slog_warn!(
973            "cannot establish filesystem boundary for checkpoint migration {}",
974            source.display()
975        );
976        return;
977    };
978    let mut skipped_foreign_mounts = 0usize;
979    let Ok(session_entries) = fs::read_dir(&source) else {
980        return;
981    };
982    for session_entry in session_entries.flatten() {
983        let source_session = session_entry.path();
984        if !source_session.is_dir() {
985            continue;
986        }
987        if !boundary.should_descend(&source_session).unwrap_or(false) {
988            skipped_foreign_mounts += 1;
989            continue;
990        }
991        let target_session = target.join(session_entry.file_name());
992        if !target_session.exists() {
993            let _ = fs::rename(&source_session, &target_session);
994            continue;
995        }
996        let Ok(checkpoint_entries) = fs::read_dir(&source_session) else {
997            continue;
998        };
999        for checkpoint_entry in checkpoint_entries.flatten() {
1000            let source_checkpoint = checkpoint_entry.path();
1001            let target_checkpoint = target_session.join(checkpoint_entry.file_name());
1002            if !target_checkpoint.exists() {
1003                let _ = fs::rename(source_checkpoint, target_checkpoint);
1004            }
1005        }
1006        let _ = fs::remove_dir(&source_session);
1007    }
1008    let _ = fs::remove_dir(&source);
1009    if skipped_foreign_mounts > 0 {
1010        crate::slog_warn!(
1011            "checkpoint migration skipped {} foreign filesystem mount(s) below {}",
1012            skipped_foreign_mounts,
1013            source.display()
1014        );
1015    }
1016}
1017
1018fn validate_checkpoint_name(name: &str) -> Result<(), AftError> {
1019    if is_safe_checkpoint_name(name) {
1020        Ok(())
1021    } else {
1022        Err(AftError::InvalidRequest {
1023            message: "checkpoint name must be a single non-empty path component".to_string(),
1024        })
1025    }
1026}
1027
1028fn is_safe_checkpoint_name(name: &str) -> bool {
1029    matches!(
1030        Path::new(name).components().collect::<Vec<_>>().as_slice(),
1031        [std::path::Component::Normal(_)]
1032    ) && !name.chars().any(|character| {
1033        character.is_control()
1034            || matches!(character, '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|')
1035    })
1036}
1037
1038fn is_safe_blob_name(name: &str) -> bool {
1039    is_safe_checkpoint_name(name) && name.starts_with("file_") && name.ends_with(".blob")
1040}
1041
1042fn read_checkpoint_from_disk(
1043    checkpoint_dir: &Path,
1044    session: &str,
1045    expected_name: &str,
1046) -> Result<Checkpoint, AftError> {
1047    let meta_path = checkpoint_dir.join("meta.json");
1048    let bytes = fs::read(&meta_path).map_err(|error| AftError::IoError {
1049        path: meta_path.display().to_string(),
1050        message: format!("failed to read durable checkpoint metadata: {error}"),
1051    })?;
1052    let meta = serde_json::from_slice::<DiskCheckpointMeta>(&bytes).map_err(|error| {
1053        AftError::IoError {
1054            path: meta_path.display().to_string(),
1055            message: format!("failed to parse durable checkpoint metadata: {error}"),
1056        }
1057    })?;
1058    if meta.schema_version != CHECKPOINT_SCHEMA_VERSION {
1059        return Err(AftError::IoError {
1060            path: meta_path.display().to_string(),
1061            message: format!(
1062                "unsupported durable checkpoint metadata schema {}",
1063                meta.schema_version
1064            ),
1065        });
1066    }
1067    if meta.session_id != session
1068        || meta.name != expected_name
1069        || !is_safe_checkpoint_name(&meta.name)
1070    {
1071        return Err(AftError::IoError {
1072            path: meta_path.display().to_string(),
1073            message: "durable checkpoint metadata does not match its session or directory"
1074                .to_string(),
1075        });
1076    }
1077
1078    let mut file_contents = HashMap::with_capacity(meta.files.len());
1079    for file in &meta.files {
1080        if !is_safe_blob_name(&file.blob) {
1081            return Err(AftError::IoError {
1082                path: meta_path.display().to_string(),
1083                message: format!("invalid durable checkpoint blob name {}", file.blob),
1084            });
1085        }
1086        let blob_path = checkpoint_dir.join(&file.blob);
1087        let blob = fs::read(&blob_path).map_err(|error| AftError::IoError {
1088            path: blob_path.display().to_string(),
1089            message: format!("failed to read durable checkpoint blob: {error}"),
1090        })?;
1091        let path = PathBuf::from(&file.original_path);
1092        let checkpoint_file =
1093            CheckpointFile::from_disk(file, blob).map_err(|message| AftError::IoError {
1094                path: blob_path.display().to_string(),
1095                message,
1096            })?;
1097        if file_contents
1098            .insert(path.clone(), checkpoint_file)
1099            .is_some()
1100        {
1101            return Err(AftError::IoError {
1102                path: meta_path.display().to_string(),
1103                message: format!("duplicate durable checkpoint path {}", path.display()),
1104            });
1105        }
1106    }
1107
1108    Ok(Checkpoint {
1109        name: meta.name,
1110        file_contents,
1111        created_at: meta.created_at,
1112        created_order: meta.created_order,
1113    })
1114}
1115
1116fn checkpoint_file_bytes(file: &CheckpointFile) -> Vec<u8> {
1117    match &file.kind {
1118        CheckpointFileKind::Regular { bytes } => bytes.to_vec(),
1119        CheckpointFileKind::Symlink { target, .. } => {
1120            target.as_os_str().to_string_lossy().as_bytes().to_vec()
1121        }
1122    }
1123}
1124
1125fn write_temp_fsync_rename(dir: &Path, final_name: &str, bytes: &[u8]) -> io::Result<()> {
1126    let tmp_name = format!(
1127        ".{}.{}.{}.tmp",
1128        final_name,
1129        std::process::id(),
1130        current_timestamp_nanos()
1131    );
1132    let tmp_path = dir.join(tmp_name);
1133    let final_path = dir.join(final_name);
1134    {
1135        let mut file = fs::OpenOptions::new()
1136            .write(true)
1137            .create_new(true)
1138            .open(&tmp_path)?;
1139        file.write_all(bytes)?;
1140        file.sync_all()?;
1141    }
1142    fs::rename(tmp_path, final_path)
1143}
1144
1145#[cfg(unix)]
1146fn fsync_dir(path: &Path) -> io::Result<()> {
1147    fs::File::open(path)?.sync_all()
1148}
1149
1150#[cfg(not(unix))]
1151fn fsync_dir(_path: &Path) -> io::Result<()> {
1152    Ok(())
1153}
1154
1155fn prune_unreferenced_checkpoint_blobs(
1156    checkpoint_dir: &Path,
1157    files: &[DiskCheckpointFileMeta],
1158) -> io::Result<()> {
1159    let referenced = files
1160        .iter()
1161        .map(|file| file.blob.as_str())
1162        .collect::<HashSet<_>>();
1163    for entry in fs::read_dir(checkpoint_dir)? {
1164        let entry = entry?;
1165        let path = entry.path();
1166        if !path.is_file() {
1167            continue;
1168        }
1169        let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
1170            continue;
1171        };
1172        if (name.starts_with("file_") && name.ends_with(".blob") && !referenced.contains(name))
1173            || name.contains(".tmp.")
1174            || name.ends_with(".tmp")
1175        {
1176            let _ = fs::remove_file(path);
1177        }
1178    }
1179    Ok(())
1180}
1181
1182fn sweep_expired_durable_checkpoints(checkpoints_dir: &Path, now: u64) {
1183    // A vanished mounted child can make ReadDir::drop panic after closedir
1184    // returns ENXIO, aborting the daemon. This background sweep must not open
1185    // checkpoint directories on a different filesystem.
1186    let Ok(boundary) = crate::walk_boundary::DeviceBoundary::for_root(checkpoints_dir) else {
1187        crate::slog_warn!(
1188            "cannot establish filesystem boundary for checkpoint sweep {}",
1189            checkpoints_dir.display()
1190        );
1191        return;
1192    };
1193    let mut skipped_foreign_mounts = 0usize;
1194    let Ok(session_entries) = fs::read_dir(checkpoints_dir) else {
1195        return;
1196    };
1197    for session_entry in session_entries.flatten() {
1198        let session_dir = session_entry.path();
1199        if !session_dir.is_dir() {
1200            continue;
1201        }
1202        if !boundary.should_descend(&session_dir).unwrap_or(false) {
1203            skipped_foreign_mounts += 1;
1204            continue;
1205        }
1206        let Ok(checkpoint_entries) = fs::read_dir(&session_dir) else {
1207            continue;
1208        };
1209        for checkpoint_entry in checkpoint_entries.flatten() {
1210            let checkpoint_dir = checkpoint_entry.path();
1211            if !checkpoint_dir.is_dir() {
1212                continue;
1213            }
1214            if !boundary.should_descend(&checkpoint_dir).unwrap_or(false) {
1215                skipped_foreign_mounts += 1;
1216                continue;
1217            }
1218            let meta_path = checkpoint_dir.join("meta.json");
1219            let Ok(bytes) = fs::read(&meta_path) else {
1220                continue;
1221            };
1222            let Ok(meta) = serde_json::from_slice::<DiskCheckpointMeta>(&bytes) else {
1223                continue;
1224            };
1225            if now.saturating_sub(meta.created_at) < NAMED_CHECKPOINT_RETENTION_SECS {
1226                continue;
1227            }
1228            if let Err(error) = fs::remove_dir_all(&checkpoint_dir) {
1229                crate::slog_warn!(
1230                    "failed to remove expired durable checkpoint {}: {}",
1231                    checkpoint_dir.display(),
1232                    error
1233                );
1234            }
1235        }
1236        let _ = fs::remove_dir(&session_dir);
1237    }
1238    if skipped_foreign_mounts > 0 {
1239        crate::slog_warn!(
1240            "checkpoint sweep skipped {} foreign filesystem mount(s) below {}",
1241            skipped_foreign_mounts,
1242            checkpoints_dir.display()
1243        );
1244    }
1245}
1246
1247fn absolute_checkpoint_path(path: PathBuf) -> PathBuf {
1248    if path.is_absolute() {
1249        return normalize_checkpoint_path(&path);
1250    }
1251    let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
1252    normalize_checkpoint_path(&cwd.join(path))
1253}
1254
1255fn normalize_checkpoint_path(path: &Path) -> PathBuf {
1256    let mut normalized = PathBuf::new();
1257    for component in path.components() {
1258        match component {
1259            std::path::Component::CurDir => {}
1260            std::path::Component::ParentDir => {
1261                if !normalized.pop() {
1262                    normalized.push(component.as_os_str());
1263                }
1264            }
1265            other => normalized.push(other.as_os_str()),
1266        }
1267    }
1268    normalized
1269}
1270
1271fn restore_paths_atomically(checkpoint: &Checkpoint, paths: &[PathBuf]) -> Result<(), AftError> {
1272    let mut pre_restore_snapshot: HashMap<PathBuf, Option<CheckpointFile>> = HashMap::new();
1273    for path in paths {
1274        let current = CheckpointFile::read_optional(path).map_err(|error| AftError::IoError {
1275            path: path.display().to_string(),
1276            message: format!("failed to snapshot pre-restore file metadata: {error}"),
1277        })?;
1278        pre_restore_snapshot.insert(path.clone(), current);
1279    }
1280
1281    let mut restored_paths: Vec<PathBuf> = Vec::new();
1282    let mut created_dirs: Vec<PathBuf> = Vec::new();
1283    for path in paths {
1284        let snapshot =
1285            checkpoint
1286                .file_contents
1287                .get(path)
1288                .ok_or_else(|| AftError::FileNotFound {
1289                    path: path.display().to_string(),
1290                })?;
1291        if let Err(e) = write_restored_file(path, snapshot, &mut created_dirs) {
1292            let mut rollback_errors = Vec::new();
1293            if let Some(snapshot) = pre_restore_snapshot.get(path) {
1294                if let Err(rollback_error) = restore_snapshot_file(path, snapshot.as_ref()) {
1295                    rollback_errors.push(format!("{}: {}", path.display(), rollback_error));
1296                }
1297            }
1298            for restored_path in restored_paths.iter().rev() {
1299                if let Some(snapshot) = pre_restore_snapshot.get(restored_path) {
1300                    if let Err(rollback_error) =
1301                        restore_snapshot_file(restored_path, snapshot.as_ref())
1302                    {
1303                        rollback_errors.push(format!(
1304                            "{}: {}",
1305                            restored_path.display(),
1306                            rollback_error
1307                        ));
1308                    }
1309                }
1310            }
1311            let dirs_rollback_ok = rollback_created_dirs(&created_dirs);
1312            if rollback_errors.is_empty() && dirs_rollback_ok {
1313                return Err(e);
1314            }
1315            return Err(AftError::IoError {
1316                path: path.display().to_string(),
1317                message: format!(
1318                    "{}; restore_checkpoint rollback_succeeded: {}; rollback_errors: {}",
1319                    e,
1320                    rollback_errors.is_empty() && dirs_rollback_ok,
1321                    if rollback_errors.is_empty() {
1322                        "none".to_string()
1323                    } else {
1324                        rollback_errors.join("; ")
1325                    }
1326                ),
1327            });
1328        }
1329        restored_paths.push(path.clone());
1330    }
1331
1332    Ok(())
1333}
1334
1335fn restore_snapshot_file(path: &Path, snapshot: Option<&CheckpointFile>) -> Result<(), AftError> {
1336    match snapshot {
1337        Some(snapshot) => write_restored_file(path, snapshot, &mut Vec::new()),
1338        None => remove_file_if_exists(path).map_err(|error| AftError::IoError {
1339            path: path.display().to_string(),
1340            message: format!("failed to remove file during checkpoint restore rollback: {error}"),
1341        }),
1342    }
1343}
1344
1345fn write_restored_file(
1346    path: &Path,
1347    snapshot: &CheckpointFile,
1348    created_dirs: &mut Vec<PathBuf>,
1349) -> Result<(), AftError> {
1350    create_parent_dirs(path, created_dirs)?;
1351
1352    match &snapshot.kind {
1353        CheckpointFileKind::Regular { bytes } => {
1354            if path_is_symlink(path) {
1355                remove_file_if_exists(path).map_err(|error| AftError::IoError {
1356                    path: path.display().to_string(),
1357                    message: format!("failed to replace symlink with regular file: {error}"),
1358                })?;
1359            }
1360            fs::write(path, bytes).map_err(|error| AftError::IoError {
1361                path: path.display().to_string(),
1362                message: format!("failed to restore checkpoint file contents: {error}"),
1363            })?;
1364            restore_checkpoint_permissions(path, snapshot).map_err(|error| AftError::IoError {
1365                path: path.display().to_string(),
1366                message: format!("failed to restore checkpoint file permissions: {error}"),
1367            })
1368        }
1369        CheckpointFileKind::Symlink {
1370            target,
1371            target_is_dir,
1372        } => {
1373            remove_file_if_exists(path).map_err(|error| AftError::IoError {
1374                path: path.display().to_string(),
1375                message: format!("failed to replace file with checkpoint symlink: {error}"),
1376            })?;
1377            create_symlink(target, path, *target_is_dir).map_err(|error| AftError::IoError {
1378                path: path.display().to_string(),
1379                message: format!("failed to restore checkpoint symlink: {error}"),
1380            })
1381        }
1382    }
1383}
1384
1385fn restore_checkpoint_permissions(path: &Path, snapshot: &CheckpointFile) -> io::Result<()> {
1386    if let Some(metadata) = &snapshot.metadata {
1387        return fs::set_permissions(path, metadata.permissions());
1388    }
1389    restore_checkpoint_mode(path, snapshot.mode)
1390}
1391
1392#[cfg(unix)]
1393fn restore_checkpoint_mode(path: &Path, mode: Option<u32>) -> io::Result<()> {
1394    use std::os::unix::fs::PermissionsExt;
1395    if let Some(mode) = mode {
1396        fs::set_permissions(path, fs::Permissions::from_mode(mode))?;
1397    }
1398    Ok(())
1399}
1400
1401#[cfg(not(unix))]
1402fn restore_checkpoint_mode(_path: &Path, _mode: Option<u32>) -> io::Result<()> {
1403    Ok(())
1404}
1405
1406fn create_parent_dirs(path: &Path, created_dirs: &mut Vec<PathBuf>) -> Result<(), AftError> {
1407    if let Some(parent) = path.parent() {
1408        let missing_dirs = missing_parent_dirs(parent);
1409        fs::create_dir_all(parent).map_err(|error| AftError::IoError {
1410            path: parent.display().to_string(),
1411            message: format!("failed to create checkpoint restore parent directories: {error}"),
1412        })?;
1413        created_dirs.extend(missing_dirs);
1414    }
1415    Ok(())
1416}
1417
1418fn path_is_symlink(path: &Path) -> bool {
1419    fs::symlink_metadata(path)
1420        .map(|metadata| metadata.file_type().is_symlink())
1421        .unwrap_or(false)
1422}
1423
1424fn remove_file_if_exists(path: &Path) -> io::Result<()> {
1425    match fs::remove_file(path) {
1426        Ok(()) => Ok(()),
1427        Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
1428        Err(error) => Err(error),
1429    }
1430}
1431
1432#[cfg(unix)]
1433fn create_symlink(target: &Path, link: &Path, target_is_dir: bool) -> io::Result<()> {
1434    let _ = target_is_dir;
1435    std::os::unix::fs::symlink(target, link)
1436}
1437
1438#[cfg(windows)]
1439fn create_symlink(target: &Path, link: &Path, target_is_dir: bool) -> io::Result<()> {
1440    if target_is_dir {
1441        std::os::windows::fs::symlink_dir(target, link)
1442    } else {
1443        std::os::windows::fs::symlink_file(target, link)
1444    }
1445}
1446
1447#[cfg(not(any(unix, windows)))]
1448fn create_symlink(_target: &Path, _link: &Path, _target_is_dir: bool) -> io::Result<()> {
1449    Err(io::Error::new(
1450        io::ErrorKind::Unsupported,
1451        "checkpoint symlink restore is unsupported on this platform",
1452    ))
1453}
1454
1455fn missing_parent_dirs(parent: &Path) -> Vec<PathBuf> {
1456    let mut dirs = Vec::new();
1457    let mut current = Some(parent);
1458
1459    while let Some(dir) = current {
1460        if dir.as_os_str().is_empty() || dir.exists() {
1461            break;
1462        }
1463        dirs.push(dir.to_path_buf());
1464        current = dir.parent();
1465    }
1466
1467    dirs
1468}
1469
1470fn rollback_created_dirs(dirs: &[PathBuf]) -> bool {
1471    let mut dirs = dirs.to_vec();
1472    dirs.sort_by_key(|dir| std::cmp::Reverse(dir.components().count()));
1473    dirs.dedup();
1474
1475    let mut ok = true;
1476    for dir in dirs {
1477        match std::fs::remove_dir(&dir) {
1478            Ok(()) => {}
1479            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1480            Err(_) => ok = false,
1481        }
1482    }
1483    ok
1484}
1485
1486/// Remove one project scope directory without ever deleting its contents.
1487/// Another process may acquire the lock or create a file between inspection and
1488/// removal, so every failure is intentionally ignored.
1489fn remove_empty_scope_dir(scope_dir: &Path) {
1490    let _ = fs::remove_dir(scope_dir);
1491}
1492
1493/// Sweep only the direct children of the checkpoints root. Scope directories
1494/// contain lockfiles, not durable checkpoint data, so an empty one is safe to
1495/// remove while a non-empty one is left untouched by `remove_dir`.
1496fn sweep_empty_scope_dirs(checkpoints_root: &Path) {
1497    let entries = match fs::read_dir(checkpoints_root) {
1498        Ok(entries) => entries,
1499        Err(_) => return,
1500    };
1501
1502    for entry in entries.flatten() {
1503        let Ok(file_type) = entry.file_type() else {
1504            continue;
1505        };
1506        if file_type.is_dir() {
1507            remove_empty_scope_dir(&entry.path());
1508        }
1509    }
1510}
1511
1512fn current_timestamp() -> u64 {
1513    std::time::SystemTime::now()
1514        .duration_since(std::time::UNIX_EPOCH)
1515        .unwrap_or_default()
1516        .as_secs()
1517}
1518
1519fn current_timestamp_nanos() -> u64 {
1520    u64::try_from(
1521        std::time::SystemTime::now()
1522            .duration_since(std::time::UNIX_EPOCH)
1523            .unwrap_or_default()
1524            .as_nanos(),
1525    )
1526    .unwrap_or(u64::MAX)
1527}
1528
1529#[cfg(test)]
1530mod tests {
1531    use super::*;
1532    use crate::protocol::DEFAULT_SESSION_ID;
1533    use std::fs;
1534
1535    fn temp_file(name: &str, content: &str) -> (PathBuf, tempfile::TempDir) {
1536        let dir = tempfile::Builder::new()
1537            .prefix("aft_checkpoint_tests_")
1538            .tempdir()
1539            .expect("create checkpoint temp dir");
1540        let path = dir.path().join(name);
1541        fs::write(&path, content).unwrap();
1542        (path, dir)
1543    }
1544
1545    fn fresh_checkpoint_store(storage: &Path) -> CheckpointStore {
1546        let lock_path = storage
1547            .join("checkpoints")
1548            .join("test-project")
1549            .join("checkpoint.lock");
1550        let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1551        store.set_storage_dir_for_harness(storage.to_path_buf(), crate::harness::Harness::Opencode);
1552        store
1553    }
1554
1555    fn checkpoint_store() -> (CheckpointStore, tempfile::TempDir) {
1556        let dir = tempfile::tempdir().unwrap();
1557        (fresh_checkpoint_store(dir.path()), dir)
1558    }
1559
1560    fn checkpoint_file(content: &str) -> CheckpointFile {
1561        let file = tempfile::NamedTempFile::new().unwrap();
1562        fs::write(file.path(), content).unwrap();
1563        CheckpointFile::read(file.path()).unwrap()
1564    }
1565
1566    #[test]
1567    fn create_and_restore_round_trip() {
1568        let (path1, _dir1) = temp_file("cp_rt1.txt", "hello");
1569        let (path2, _dir2) = temp_file("cp_rt2.txt", "world");
1570
1571        let backup_store = BackupStore::new();
1572        let (mut store, _store_dir) = checkpoint_store();
1573
1574        let info = store
1575            .create(
1576                DEFAULT_SESSION_ID,
1577                "snap1",
1578                vec![path1.clone(), path2.clone()],
1579                &backup_store,
1580            )
1581            .unwrap();
1582        assert_eq!(info.name, "snap1");
1583        assert_eq!(info.file_count, 2);
1584
1585        // Modify files
1586        fs::write(&path1, "changed1").unwrap();
1587        fs::write(&path2, "changed2").unwrap();
1588
1589        // Restore
1590        let info = store.restore(DEFAULT_SESSION_ID, "snap1").unwrap();
1591        assert_eq!(info.file_count, 2);
1592        assert_eq!(fs::read_to_string(&path1).unwrap(), "hello");
1593        assert_eq!(fs::read_to_string(&path2).unwrap(), "world");
1594    }
1595
1596    #[cfg(unix)]
1597    #[test]
1598    fn durable_checkpoint_hydrates_after_restart_with_bytes_and_mode() {
1599        use std::os::unix::fs::PermissionsExt;
1600
1601        let files = tempfile::tempdir().unwrap();
1602        let path = files.path().join("durable-mode.bin");
1603        let original = b"draft decision\n\0byte exact\n";
1604        fs::write(&path, original).unwrap();
1605        let mut mode = fs::metadata(&path).unwrap().permissions();
1606        mode.set_mode(0o600);
1607        fs::set_permissions(&path, mode).unwrap();
1608
1609        let backup_store = BackupStore::new();
1610        let (mut first, storage) = checkpoint_store();
1611        let info = first
1612            .create(
1613                DEFAULT_SESSION_ID,
1614                "restart-mode",
1615                vec![path.clone()],
1616                &backup_store,
1617            )
1618            .unwrap();
1619        let durable_path = info.storage_path.expect("durable checkpoint path");
1620        assert!(durable_path.join("meta.json").is_file());
1621        assert!(
1622            fs::read_dir(&durable_path)
1623                .unwrap()
1624                .flatten()
1625                .any(|entry| entry.path().extension().is_some_and(|ext| ext == "blob")),
1626            "checkpoint must persist one or more file blobs"
1627        );
1628
1629        fs::write(&path, b"mutated\n").unwrap();
1630        let mut changed_mode = fs::metadata(&path).unwrap().permissions();
1631        changed_mode.set_mode(0o644);
1632        fs::set_permissions(&path, changed_mode).unwrap();
1633        drop(first);
1634
1635        let mut restarted = fresh_checkpoint_store(storage.path());
1636        let listed = restarted.list(DEFAULT_SESSION_ID).unwrap();
1637        assert_eq!(
1638            listed.len(),
1639            1,
1640            "fresh store must hydrate durable checkpoint"
1641        );
1642        assert_eq!(listed[0].name, "restart-mode");
1643        restarted
1644            .restore(DEFAULT_SESSION_ID, "restart-mode")
1645            .unwrap();
1646
1647        assert_eq!(fs::read(&path).unwrap(), original);
1648        assert_eq!(
1649            fs::metadata(&path).unwrap().permissions().mode() & 0o777,
1650            0o600
1651        );
1652    }
1653
1654    #[cfg(unix)]
1655    #[test]
1656    fn durable_checkpoint_hydrates_symlink_without_following_target() {
1657        let files = tempfile::tempdir().unwrap();
1658        let target = files.path().join("target.txt");
1659        let link = files.path().join("link.txt");
1660        fs::write(&target, "target content").unwrap();
1661        std::os::unix::fs::symlink(&target, &link).unwrap();
1662
1663        let backup_store = BackupStore::new();
1664        let (mut first, storage) = checkpoint_store();
1665        first
1666            .create(
1667                DEFAULT_SESSION_ID,
1668                "restart-symlink",
1669                vec![link.clone()],
1670                &backup_store,
1671            )
1672            .unwrap();
1673        fs::remove_file(&link).unwrap();
1674        fs::write(&link, "plain replacement").unwrap();
1675        drop(first);
1676
1677        let mut restarted = fresh_checkpoint_store(storage.path());
1678        restarted
1679            .restore(DEFAULT_SESSION_ID, "restart-symlink")
1680            .unwrap();
1681        assert!(fs::symlink_metadata(&link)
1682            .unwrap()
1683            .file_type()
1684            .is_symlink());
1685        assert_eq!(fs::read_link(&link).unwrap(), target);
1686        assert_eq!(fs::read_to_string(&target).unwrap(), "target content");
1687    }
1688
1689    #[test]
1690    fn checkpoint_retention_evicts_oldest_name_from_memory_and_disk() {
1691        let (path, _files) = temp_file("retention.txt", "version-0");
1692        let backup_store = BackupStore::new();
1693        let (mut store, storage) = checkpoint_store();
1694
1695        for index in 0..=MAX_NAMED_CHECKPOINTS_PER_SESSION {
1696            fs::write(&path, format!("version-{index}")).unwrap();
1697            let info = store
1698                .create(
1699                    DEFAULT_SESSION_ID,
1700                    &format!("checkpoint-{index:02}"),
1701                    vec![path.clone()],
1702                    &backup_store,
1703                )
1704                .unwrap();
1705            if index == MAX_NAMED_CHECKPOINTS_PER_SESSION {
1706                assert_eq!(info.evicted, vec!["checkpoint-00"]);
1707            } else {
1708                assert!(info.evicted.is_empty());
1709            }
1710        }
1711
1712        let listed = store.list(DEFAULT_SESSION_ID).unwrap();
1713        assert_eq!(listed.len(), MAX_NAMED_CHECKPOINTS_PER_SESSION);
1714        assert!(listed.iter().all(|info| info.name != "checkpoint-00"));
1715        let old_dir = storage
1716            .path()
1717            .join("opencode")
1718            .join("checkpoints")
1719            .join(hash_session(DEFAULT_SESSION_ID))
1720            .join("checkpoint-00");
1721        assert!(!old_dir.exists(), "evicted checkpoint must leave disk too");
1722    }
1723
1724    #[test]
1725    fn hydration_finishes_retention_after_interrupted_create() {
1726        let (path, _files) = temp_file("interrupted-retention.txt", "checkpoint content");
1727        let backup_store = BackupStore::new();
1728        let (mut first, storage) = checkpoint_store();
1729
1730        for index in 0..MAX_NAMED_CHECKPOINTS_PER_SESSION {
1731            first
1732                .create(
1733                    DEFAULT_SESSION_ID,
1734                    &format!("checkpoint-{index:02}"),
1735                    vec![path.clone()],
1736                    &backup_store,
1737                )
1738                .unwrap();
1739        }
1740
1741        // The create path persists the new checkpoint before evicting older
1742        // checkpoints. Write only the durable checkpoint here to simulate the
1743        // process exiting after persistence but before retention eviction.
1744        let checkpoint = Checkpoint {
1745            name: "checkpoint-20".to_string(),
1746            file_contents: HashMap::from([(path, checkpoint_file("newest"))]),
1747            created_at: current_timestamp(),
1748            created_order: u64::MAX,
1749        };
1750        {
1751            let _lock = first.acquire_mutation_lock().unwrap();
1752            first
1753                .persist_checkpoint_locked(DEFAULT_SESSION_ID, &checkpoint)
1754                .unwrap();
1755        }
1756        drop(first);
1757
1758        let session_dir = storage
1759            .path()
1760            .join("opencode")
1761            .join("checkpoints")
1762            .join(hash_session(DEFAULT_SESSION_ID));
1763        assert_eq!(fs::read_dir(&session_dir).unwrap().count(), 21);
1764
1765        let mut restarted = fresh_checkpoint_store(storage.path());
1766        let listed = restarted.list(DEFAULT_SESSION_ID).unwrap();
1767        assert_eq!(listed.len(), MAX_NAMED_CHECKPOINTS_PER_SESSION);
1768        assert!(listed.iter().all(|info| info.name != "checkpoint-00"));
1769        assert!(
1770            !session_dir.join("checkpoint-00").exists(),
1771            "hydration must finish interrupted retention on disk"
1772        );
1773    }
1774
1775    #[test]
1776    fn cleanup_refuses_to_sweep_scope_dirs_outside_a_checkpoints_root() {
1777        // Regression: edit_match tests set lock_path = <TempDir>/checkpoint.lock, so
1778        // lock_path.parent().parent() is the OS TEMP ROOT. Before the named-root guard,
1779        // cleanup_locked swept empty sibling directories there and deleted other tests'
1780        // freshly created TempDirs (observed live: read.rs fixture writes failing with
1781        // NotFound under the parallel suite). Mutation control: drop the file_name()
1782        // guard in cleanup_locked and this test fails.
1783        let temp_root = tempfile::tempdir().expect("temp root");
1784        // lock_path.parent().parent() == temp_root, which is NOT named `checkpoints`,
1785        // so the guard must refuse the sweep and the empty sibling must survive.
1786        let victim = temp_root.path().join("innocent-empty-sibling");
1787        fs::create_dir(&victim).expect("victim dir");
1788        let scope_dir = temp_root.path().join("scope");
1789        fs::create_dir(&scope_dir).expect("scope dir");
1790        let lock_path = scope_dir.join("checkpoint.lock");
1791        let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1792        store.cleanup_locked().expect("cleanup");
1793        assert!(
1794            victim.exists(),
1795            "cleanup must not sweep empty dirs outside a `checkpoints` root"
1796        );
1797    }
1798
1799    #[test]
1800    fn cleanup_sweeps_durable_checkpoints_older_than_fourteen_days() {
1801        let (path, _files) = temp_file("durable-gc.txt", "original");
1802        let backup_store = BackupStore::new();
1803        let (mut store, _storage) = checkpoint_store();
1804        let info = store
1805            .create(
1806                DEFAULT_SESSION_ID,
1807                "expired-durable",
1808                vec![path],
1809                &backup_store,
1810            )
1811            .unwrap();
1812        let durable_path = info.storage_path.unwrap();
1813        let meta_path = durable_path.join("meta.json");
1814        let mut meta: DiskCheckpointMeta =
1815            serde_json::from_slice(&fs::read(&meta_path).unwrap()).unwrap();
1816        meta.created_at = current_timestamp()
1817            .saturating_sub(NAMED_CHECKPOINT_RETENTION_SECS)
1818            .saturating_sub(1);
1819        fs::write(&meta_path, serde_json::to_vec_pretty(&meta).unwrap()).unwrap();
1820
1821        store.cleanup();
1822        assert!(
1823            !durable_path.exists(),
1824            "fourteen-day cleanup must remove the durable checkpoint directory"
1825        );
1826    }
1827
1828    #[test]
1829    fn durable_hydration_fails_when_a_referenced_blob_is_missing() {
1830        let (path, _files) = temp_file("hydration-control.txt", "original");
1831        let backup_store = BackupStore::new();
1832        let (mut first, storage) = checkpoint_store();
1833        let info = first
1834            .create(
1835                DEFAULT_SESSION_ID,
1836                "hydration-control",
1837                vec![path],
1838                &backup_store,
1839            )
1840            .unwrap();
1841        let durable_path = info.storage_path.unwrap();
1842        let meta: DiskCheckpointMeta =
1843            serde_json::from_slice(&fs::read(durable_path.join("meta.json")).unwrap()).unwrap();
1844        fs::remove_file(durable_path.join(&meta.files[0].blob)).unwrap();
1845        drop(first);
1846
1847        let mut restarted = fresh_checkpoint_store(storage.path());
1848        let error = restarted.list(DEFAULT_SESSION_ID).unwrap_err();
1849        match error {
1850            AftError::IoError { message, .. } => {
1851                assert!(message.contains("failed to read durable checkpoint blob"));
1852            }
1853            other => panic!("expected durable hydration I/O error, got {other:?}"),
1854        }
1855    }
1856
1857    #[test]
1858    fn overwrite_existing_name() {
1859        let (path, _dir) = temp_file("cp_overwrite.txt", "v1");
1860        let backup_store = BackupStore::new();
1861        let (mut store, _store_dir) = checkpoint_store();
1862
1863        store
1864            .create(DEFAULT_SESSION_ID, "dup", vec![path.clone()], &backup_store)
1865            .unwrap();
1866        fs::write(&path, "v2").unwrap();
1867        store
1868            .create(DEFAULT_SESSION_ID, "dup", vec![path.clone()], &backup_store)
1869            .unwrap();
1870
1871        // Restore should give v2 (the overwritten checkpoint)
1872        fs::write(&path, "v3").unwrap();
1873        store.restore(DEFAULT_SESSION_ID, "dup").unwrap();
1874        assert_eq!(fs::read_to_string(&path).unwrap(), "v2");
1875    }
1876
1877    #[test]
1878    fn list_returns_metadata_scoped_to_session() {
1879        let (path, _dir) = temp_file("cp_list.txt", "data");
1880        let backup_store = BackupStore::new();
1881        let (mut store, _store_dir) = checkpoint_store();
1882
1883        store
1884            .create(DEFAULT_SESSION_ID, "a", vec![path.clone()], &backup_store)
1885            .unwrap();
1886        store
1887            .create(DEFAULT_SESSION_ID, "b", vec![path.clone()], &backup_store)
1888            .unwrap();
1889        store
1890            .create("other_session", "c", vec![path.clone()], &backup_store)
1891            .unwrap();
1892
1893        let default_list = store.list(DEFAULT_SESSION_ID).unwrap();
1894        assert_eq!(default_list.len(), 2);
1895        let names: Vec<&str> = default_list.iter().map(|i| i.name.as_str()).collect();
1896        assert!(names.contains(&"a"));
1897        assert!(names.contains(&"b"));
1898
1899        let other_list = store.list("other_session").unwrap();
1900        assert_eq!(other_list.len(), 1);
1901        assert_eq!(other_list[0].name, "c");
1902    }
1903
1904    #[test]
1905    fn sessions_isolate_checkpoint_names() {
1906        // Same checkpoint name in two sessions does not collide on restore.
1907        let (path_a, _dir_a) = temp_file("cp_isolated_a.txt", "a-original");
1908        let (path_b, _dir_b) = temp_file("cp_isolated_b.txt", "b-original");
1909        let backup_store = BackupStore::new();
1910        let (mut store, _store_dir) = checkpoint_store();
1911
1912        // Both sessions create a checkpoint with the same name but different files.
1913        store
1914            .create("session_a", "snap", vec![path_a.clone()], &backup_store)
1915            .unwrap();
1916        store
1917            .create("session_b", "snap", vec![path_b.clone()], &backup_store)
1918            .unwrap();
1919
1920        fs::write(&path_a, "a-modified").unwrap();
1921        fs::write(&path_b, "b-modified").unwrap();
1922
1923        // Restoring session A's "snap" only touches path_a.
1924        store.restore("session_a", "snap").unwrap();
1925        assert_eq!(fs::read_to_string(&path_a).unwrap(), "a-original");
1926        assert_eq!(fs::read_to_string(&path_b).unwrap(), "b-modified");
1927
1928        // Restoring session B's "snap" only touches path_b.
1929        fs::write(&path_a, "a-modified").unwrap();
1930        store.restore("session_b", "snap").unwrap();
1931        assert_eq!(fs::read_to_string(&path_a).unwrap(), "a-modified");
1932        assert_eq!(fs::read_to_string(&path_b).unwrap(), "b-original");
1933    }
1934
1935    #[test]
1936    fn checkpoint_lock_scope_is_removed_after_release() {
1937        let dir = tempfile::tempdir().unwrap();
1938        let scope_dir = dir.path().join("checkpoints").join("project-scope");
1939        let lock_path = scope_dir.join("checkpoint.lock");
1940        let path = dir.path().join("checkpoint.txt");
1941        fs::write(&path, "data").unwrap();
1942        let backup_store = BackupStore::new();
1943        let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1944
1945        store
1946            .create(DEFAULT_SESSION_ID, "released", vec![path], &backup_store)
1947            .unwrap();
1948
1949        assert!(!scope_dir.exists(), "released lock scope should be removed");
1950    }
1951
1952    #[test]
1953    fn cleanup_removes_expired_across_sessions() {
1954        let (path, _dir) = temp_file("cp_cleanup.txt", "data");
1955        let backup_store = BackupStore::new();
1956        let (mut store, _store_dir) = checkpoint_store();
1957
1958        store
1959            .create(
1960                DEFAULT_SESSION_ID,
1961                "recent",
1962                vec![path.clone()],
1963                &backup_store,
1964            )
1965            .unwrap();
1966
1967        // Manually insert an expired checkpoint in another session.
1968        store
1969            .checkpoints
1970            .entry("other".to_string())
1971            .or_default()
1972            .insert(
1973                "old".to_string(),
1974                Checkpoint {
1975                    name: "old".to_string(),
1976                    file_contents: HashMap::new(),
1977                    created_at: 1000, // far in the past
1978                    created_order: 1000,
1979                },
1980            );
1981
1982        assert_eq!(store.total_count(), 2);
1983        store.cleanup();
1984        assert_eq!(store.total_count(), 1);
1985        assert_eq!(store.list(DEFAULT_SESSION_ID).unwrap()[0].name, "recent");
1986        assert!(store.list("other").unwrap().is_empty());
1987    }
1988
1989    #[test]
1990    fn cleanup_sweeps_empty_scope_dirs_but_keeps_live_lock_scope() {
1991        let dir = tempfile::tempdir().unwrap();
1992        let checkpoints_root = dir.path().join("checkpoints");
1993        let empty_a = checkpoints_root.join("empty-a");
1994        let empty_b = checkpoints_root.join("empty-b");
1995        let live_scope = checkpoints_root.join("live-scope");
1996        fs::create_dir_all(&empty_a).unwrap();
1997        fs::create_dir_all(&empty_b).unwrap();
1998        fs::create_dir_all(&live_scope).unwrap();
1999        fs::write(live_scope.join("checkpoint.lock"), "live lock").unwrap();
2000
2001        let lock_path = checkpoints_root
2002            .join("current-scope")
2003            .join("checkpoint.lock");
2004        let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
2005        store.cleanup();
2006
2007        assert!(!empty_a.exists());
2008        assert!(!empty_b.exists());
2009        assert!(live_scope.is_dir());
2010        assert!(live_scope.join("checkpoint.lock").is_file());
2011    }
2012
2013    #[test]
2014    fn cleanup_ignores_non_empty_scope_dir_removal_failure() {
2015        let dir = tempfile::tempdir().unwrap();
2016        let checkpoints_root = dir.path().join("checkpoints");
2017        let scope_dir = checkpoints_root.join("racing-scope");
2018        fs::create_dir_all(&scope_dir).unwrap();
2019        // Model the post-race state where a concurrent lock acquisition adds
2020        // this file after the root readdir but before remove_dir.
2021        fs::write(scope_dir.join("checkpoint.lock"), "lock appeared").unwrap();
2022
2023        let lock_path = checkpoints_root
2024            .join("current-scope")
2025            .join("checkpoint.lock");
2026        let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
2027        store.cleanup();
2028
2029        assert!(scope_dir.is_dir());
2030        assert!(scope_dir.join("checkpoint.lock").is_file());
2031    }
2032
2033    #[test]
2034    fn restore_nonexistent_returns_error() {
2035        let (mut store, _store_dir) = checkpoint_store();
2036        let result = store.restore(DEFAULT_SESSION_ID, "nope");
2037        assert!(result.is_err());
2038        match result.unwrap_err() {
2039            AftError::CheckpointNotFound { name } => {
2040                assert_eq!(name, "nope");
2041            }
2042            other => panic!("expected CheckpointNotFound, got: {:?}", other),
2043        }
2044    }
2045
2046    #[test]
2047    fn restore_nonexistent_in_other_session_returns_error() {
2048        // A "snap" that exists in session A must NOT be visible from session B.
2049        let (path, _dir) = temp_file("cp_cross_session.txt", "data");
2050        let backup_store = BackupStore::new();
2051        let (mut store, _store_dir) = checkpoint_store();
2052        store
2053            .create("session_a", "only_a", vec![path], &backup_store)
2054            .unwrap();
2055        assert!(store.restore("session_b", "only_a").is_err());
2056    }
2057
2058    #[test]
2059    fn create_skips_missing_files_from_backup_tracked_set() {
2060        // Simulate the reported issue #15-follow-up: an agent deletes a
2061        // previously-edited file, then calls checkpoint with no explicit
2062        // file list. Before the fix, the stale backup-tracked entry caused
2063        // the whole checkpoint to fail on the missing path. Now the checkpoint
2064        // succeeds with the readable file and reports the skipped one.
2065        let (readable, _readable_dir) = temp_file("cp_skip_readable.txt", "still_here");
2066        let (deleted, _deleted_dir) = temp_file("cp_skip_deleted.txt", "about_to_vanish");
2067
2068        // Backup store canonicalizes keys, so the skipped path in the
2069        // checkpoint result is the canonical form, not the raw temp path.
2070        let deleted_canonical = fs::canonicalize(&deleted).unwrap();
2071
2072        let mut backup_store = BackupStore::new();
2073        backup_store
2074            .snapshot(DEFAULT_SESSION_ID, &readable, "auto")
2075            .unwrap();
2076        backup_store
2077            .snapshot(DEFAULT_SESSION_ID, &deleted, "auto")
2078            .unwrap();
2079
2080        fs::remove_file(&deleted).unwrap();
2081
2082        let (mut store, _store_dir) = checkpoint_store();
2083        let info = store
2084            .create(DEFAULT_SESSION_ID, "partial", vec![], &backup_store)
2085            .expect("checkpoint should succeed despite one missing file");
2086        assert_eq!(info.file_count, 1);
2087        assert_eq!(info.skipped.len(), 1);
2088        assert_eq!(info.skipped[0].0, deleted_canonical);
2089        assert!(!info.skipped[0].1.is_empty());
2090    }
2091
2092    #[test]
2093    fn create_with_explicit_single_missing_file_errors() {
2094        // When the caller names a single file explicitly and it can't be read,
2095        // fail loudly — an empty checkpoint isn't what the caller asked for.
2096        let dir = tempfile::tempdir().unwrap();
2097        let missing = dir.path().join("cp_explicit_missing_does_not_exist.txt");
2098
2099        let backup_store = BackupStore::new();
2100        let (mut store, _store_dir) = checkpoint_store();
2101        let result = store.create(
2102            DEFAULT_SESSION_ID,
2103            "explicit",
2104            vec![missing.clone()],
2105            &backup_store,
2106        );
2107
2108        assert!(result.is_err());
2109        match result.unwrap_err() {
2110            AftError::FileNotFound { path } => {
2111                assert!(path.contains(&missing.display().to_string()));
2112            }
2113            other => panic!("expected FileNotFound, got: {:?}", other),
2114        }
2115    }
2116
2117    #[test]
2118    fn create_with_explicit_mixed_files_keeps_readable_and_reports_skipped() {
2119        // Explicit file list with one readable + one missing: keep the
2120        // readable one in the checkpoint, report the missing one under
2121        // `skipped` instead of failing outright.
2122        let (good, _good_dir) = temp_file("cp_mixed_good.txt", "ok");
2123        let missing_dir = tempfile::tempdir().unwrap();
2124        let missing = missing_dir.path().join("cp_mixed_missing.txt");
2125
2126        let backup_store = BackupStore::new();
2127        let (mut store, _store_dir) = checkpoint_store();
2128        let info = store
2129            .create(
2130                DEFAULT_SESSION_ID,
2131                "mixed",
2132                vec![good.clone(), missing.clone()],
2133                &backup_store,
2134            )
2135            .expect("mixed checkpoint should succeed when any file is readable");
2136        assert_eq!(info.file_count, 1);
2137        assert_eq!(info.skipped.len(), 1);
2138        assert_eq!(info.skipped[0].0, missing);
2139    }
2140
2141    #[test]
2142    fn create_with_empty_files_uses_backup_tracked() {
2143        let (path, _dir) = temp_file("cp_tracked.txt", "tracked_content");
2144        let mut backup_store = BackupStore::new();
2145        backup_store
2146            .snapshot(DEFAULT_SESSION_ID, &path, "auto")
2147            .unwrap();
2148
2149        let (mut store, _store_dir) = checkpoint_store();
2150        let info = store
2151            .create(DEFAULT_SESSION_ID, "from_tracked", vec![], &backup_store)
2152            .unwrap();
2153        assert!(info.file_count >= 1);
2154
2155        // Modify and restore
2156        fs::write(&path, "modified").unwrap();
2157        store.restore(DEFAULT_SESSION_ID, "from_tracked").unwrap();
2158        assert_eq!(fs::read_to_string(&path).unwrap(), "tracked_content");
2159    }
2160
2161    #[test]
2162    fn restore_recreates_missing_parent_directories() {
2163        let dir = tempfile::tempdir().unwrap();
2164        let path = dir.path().join("nested").join("deeper").join("file.txt");
2165        fs::create_dir_all(path.parent().unwrap()).unwrap();
2166        fs::write(&path, "original nested content").unwrap();
2167
2168        let backup_store = BackupStore::new();
2169        let (mut store, _store_dir) = checkpoint_store();
2170        store
2171            .create(
2172                DEFAULT_SESSION_ID,
2173                "nested",
2174                vec![path.clone()],
2175                &backup_store,
2176            )
2177            .unwrap();
2178
2179        fs::remove_dir_all(dir.path().join("nested")).unwrap();
2180
2181        store.restore(DEFAULT_SESSION_ID, "nested").unwrap();
2182        assert_eq!(
2183            fs::read_to_string(&path).unwrap(),
2184            "original nested content"
2185        );
2186    }
2187
2188    #[cfg(unix)]
2189    #[test]
2190    fn checkpoint_restore_rolls_back_on_partial_failure() {
2191        use std::os::unix::fs::PermissionsExt;
2192
2193        let dir = tempfile::tempdir().unwrap();
2194        let path_a = dir.path().join("a.txt");
2195        let path_b = dir.path().join("b.txt");
2196        fs::write(&path_a, "checkpoint-a").unwrap();
2197        fs::write(&path_b, "checkpoint-b").unwrap();
2198
2199        let backup_store = BackupStore::new();
2200        let (mut store, _store_dir) = checkpoint_store();
2201        store
2202            .create(
2203                DEFAULT_SESSION_ID,
2204                "partial_failure",
2205                vec![path_a.clone(), path_b.clone()],
2206                &backup_store,
2207            )
2208            .unwrap();
2209
2210        fs::write(&path_a, "pre-restore-a").unwrap();
2211        fs::write(&path_b, "pre-restore-b").unwrap();
2212        let mut readonly = fs::metadata(&path_b).unwrap().permissions();
2213        readonly.set_mode(0o444);
2214        fs::set_permissions(&path_b, readonly).unwrap();
2215
2216        let result = store.restore(DEFAULT_SESSION_ID, "partial_failure");
2217        let mut writable = fs::metadata(&path_b).unwrap().permissions();
2218        writable.set_mode(0o644);
2219        fs::set_permissions(&path_b, writable).unwrap();
2220
2221        assert!(result.is_err(), "restore should surface write failure");
2222        assert_eq!(fs::read_to_string(&path_a).unwrap(), "pre-restore-a");
2223        assert_eq!(fs::read_to_string(&path_b).unwrap(), "pre-restore-b");
2224    }
2225
2226    #[test]
2227    fn checkpoint_create_and_restore_use_mutation_lock() {
2228        let dir = tempfile::tempdir().unwrap();
2229        let lock_path = dir.path().join("locks").join("checkpoint.lock");
2230        fs::create_dir_all(lock_path.parent().unwrap()).unwrap();
2231        let mut store =
2232            CheckpointStore::with_lock_path(lock_path.clone(), Duration::from_millis(50));
2233        let backup_store = BackupStore::new();
2234        let path = dir.path().join("locked.txt");
2235        fs::write(&path, "original").unwrap();
2236
2237        let held_lock =
2238            fs_lock::try_acquire(&lock_path, Duration::from_secs(1)).expect("hold checkpoint lock");
2239        let create_result = store.create(
2240            DEFAULT_SESSION_ID,
2241            "locked",
2242            vec![path.clone()],
2243            &backup_store,
2244        );
2245        assert!(matches!(create_result, Err(AftError::IoError { .. })));
2246        drop(held_lock);
2247
2248        store
2249            .create(
2250                DEFAULT_SESSION_ID,
2251                "locked",
2252                vec![path.clone()],
2253                &backup_store,
2254            )
2255            .unwrap();
2256        fs::write(&path, "changed").unwrap();
2257
2258        fs::create_dir_all(lock_path.parent().unwrap()).unwrap();
2259        let held_lock =
2260            fs_lock::try_acquire(&lock_path, Duration::from_secs(1)).expect("hold checkpoint lock");
2261        let restore_result = store.restore(DEFAULT_SESSION_ID, "locked");
2262        assert!(matches!(restore_result, Err(AftError::IoError { .. })));
2263        drop(held_lock);
2264
2265        store.restore(DEFAULT_SESSION_ID, "locked").unwrap();
2266        assert_eq!(fs::read_to_string(&path).unwrap(), "original");
2267    }
2268
2269    #[cfg(unix)]
2270    #[test]
2271    fn checkpoint_restore_preserves_regular_file_permissions() {
2272        use std::os::unix::fs::PermissionsExt;
2273
2274        let dir = tempfile::tempdir().unwrap();
2275        let path = dir.path().join("mode.txt");
2276        fs::write(&path, "original").unwrap();
2277        let mut original_permissions = fs::metadata(&path).unwrap().permissions();
2278        original_permissions.set_mode(0o600);
2279        fs::set_permissions(&path, original_permissions).unwrap();
2280
2281        let backup_store = BackupStore::new();
2282        let (mut store, _store_dir) = checkpoint_store();
2283        store
2284            .create(
2285                DEFAULT_SESSION_ID,
2286                "mode",
2287                vec![path.clone()],
2288                &backup_store,
2289            )
2290            .unwrap();
2291
2292        fs::write(&path, "changed").unwrap();
2293        let mut changed_permissions = fs::metadata(&path).unwrap().permissions();
2294        changed_permissions.set_mode(0o644);
2295        fs::set_permissions(&path, changed_permissions).unwrap();
2296
2297        store.restore(DEFAULT_SESSION_ID, "mode").unwrap();
2298
2299        assert_eq!(fs::read_to_string(&path).unwrap(), "original");
2300        let restored_mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
2301        assert_eq!(restored_mode, 0o600);
2302    }
2303
2304    #[cfg(unix)]
2305    #[test]
2306    fn checkpoint_restore_recreates_symlink() {
2307        let dir = tempfile::tempdir().unwrap();
2308        let target = dir.path().join("target.txt");
2309        let link = dir.path().join("link.txt");
2310        fs::write(&target, "target content").unwrap();
2311        std::os::unix::fs::symlink(&target, &link).unwrap();
2312
2313        let backup_store = BackupStore::new();
2314        let (mut store, _store_dir) = checkpoint_store();
2315        store
2316            .create(
2317                DEFAULT_SESSION_ID,
2318                "symlink",
2319                vec![link.clone()],
2320                &backup_store,
2321            )
2322            .unwrap();
2323
2324        fs::remove_file(&link).unwrap();
2325        fs::write(&link, "plain file").unwrap();
2326
2327        store.restore(DEFAULT_SESSION_ID, "symlink").unwrap();
2328
2329        assert!(fs::symlink_metadata(&link)
2330            .unwrap()
2331            .file_type()
2332            .is_symlink());
2333        assert_eq!(fs::read_link(&link).unwrap(), target);
2334        assert_eq!(fs::read_to_string(&link).unwrap(), "target content");
2335    }
2336
2337    #[test]
2338    fn captured_regular_file_is_shared_by_checkpoint_and_backup() {
2339        let (path, _dir) = temp_file("shared-capture.txt", "original bytes");
2340        crate::backup::reset_capture_read_count(&path);
2341        let mut capture = CapturedRegularFile::read(&path).unwrap().unwrap();
2342        assert_eq!(crate::backup::capture_read_count(&path), 1);
2343
2344        let checkpoint = CheckpointFile::from_captured(&path, &mut capture).unwrap();
2345        let mut backup = BackupStore::new();
2346        backup
2347            .snapshot_with_op_from_capture(
2348                DEFAULT_SESSION_ID,
2349                &path,
2350                "shared capture",
2351                Some("shared-op"),
2352                &capture,
2353            )
2354            .unwrap();
2355
2356        let history = backup.history(DEFAULT_SESSION_ID, &path);
2357        let CheckpointFileKind::Regular { bytes } = checkpoint.kind else {
2358            panic!("regular capture must create a regular checkpoint");
2359        };
2360        assert_eq!(bytes.as_ref(), b"original bytes");
2361        assert_eq!(history[0].content_bytes.as_ref(), b"original bytes");
2362        assert!(Arc::ptr_eq(&bytes, &history[0].content_bytes));
2363        assert_eq!(crate::backup::capture_read_count(&path), 1);
2364    }
2365
2366    #[test]
2367    fn stale_capture_refreshes_before_checkpoint_and_backup() {
2368        let (path, _dir) = temp_file("stale-capture.txt", "old");
2369        crate::backup::reset_capture_read_count(&path);
2370        let mut capture = CapturedRegularFile::read(&path).unwrap().unwrap();
2371        fs::write(&path, "fresh disk truth").unwrap();
2372
2373        let checkpoint = CheckpointFile::from_captured(&path, &mut capture).unwrap();
2374        let mut backup = BackupStore::new();
2375        backup
2376            .snapshot_with_op_from_capture(
2377                DEFAULT_SESSION_ID,
2378                &path,
2379                "freshened capture",
2380                Some("fresh-op"),
2381                &capture,
2382            )
2383            .unwrap();
2384
2385        let history = backup.history(DEFAULT_SESSION_ID, &path);
2386        let CheckpointFileKind::Regular { bytes } = checkpoint.kind else {
2387            panic!("regular capture must create a regular checkpoint");
2388        };
2389        assert_eq!(bytes.as_ref(), b"fresh disk truth");
2390        assert_eq!(history[0].content_bytes.as_ref(), b"fresh disk truth");
2391        assert!(Arc::ptr_eq(&bytes, &history[0].content_bytes));
2392        assert_eq!(crate::backup::capture_read_count(&path), 2);
2393    }
2394
2395    #[test]
2396    fn checkpoint_restore_failure_removes_created_parent_dirs() {
2397        let dir = tempfile::tempdir().unwrap();
2398        let missing_root = dir.path().join("created");
2399        let path_a = missing_root.join("nested").join("a.txt");
2400        let path_b = dir.path().join("blocking-dir");
2401        fs::create_dir(&path_b).unwrap();
2402
2403        let checkpoint = Checkpoint {
2404            name: "dir-cleanup".to_string(),
2405            file_contents: HashMap::from([
2406                (path_a.clone(), checkpoint_file("checkpoint-a")),
2407                (path_b.clone(), checkpoint_file("checkpoint-b")),
2408            ]),
2409            created_at: current_timestamp(),
2410            created_order: current_timestamp_nanos(),
2411        };
2412
2413        let result = restore_paths_atomically(&checkpoint, &[path_a.clone(), path_b.clone()]);
2414
2415        assert!(
2416            result.is_err(),
2417            "second restore write should fail on directory"
2418        );
2419        assert!(!path_a.exists(), "restored file should be rolled back");
2420        assert!(
2421            !missing_root.exists(),
2422            "new parent directories should be removed on rollback"
2423        );
2424        assert!(path_b.is_dir(), "pre-existing blocking directory remains");
2425    }
2426}