1use std::collections::{HashMap, HashSet};
2use std::fs;
3use std::io::{self, Write};
4use std::path::{Path, PathBuf};
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, LazyLock, Mutex};
7use std::time::Duration;
8
9use serde::{Deserialize, Serialize};
10
11use crate::backup::{hash_session, BackupStore, CapturedRegularFile};
12use crate::error::AftError;
13use crate::fs_lock;
14
15const CHECKPOINT_LOCK_TIMEOUT: Duration = Duration::from_secs(30);
16
17const MAX_NAMED_CHECKPOINTS_PER_SESSION: usize = 20;
20const NAMED_CHECKPOINT_RETENTION_DAYS: u64 = 14;
23const NAMED_CHECKPOINT_RETENTION_SECS: u64 = NAMED_CHECKPOINT_RETENTION_DAYS * 24 * 60 * 60;
24const CHECKPOINT_SCHEMA_VERSION: u32 = 1;
25const UNBOUND_HARNESS_SEGMENT: &str = "unbound";
26
27static CHECKPOINT_MAINTENANCE_KEYS: LazyLock<Mutex<HashSet<(PathBuf, String)>>> =
28 LazyLock::new(|| Mutex::new(HashSet::new()));
29
30pub const CHECKPOINT_RESTART_NOTICE: &str =
32 "no durable checkpoints found on disk; in-memory checkpoints do not survive restarts";
33pub const CHECKPOINT_HYDRATED_NOTICE: &str =
35 "durable checkpoints are hydrated from disk and survive restarts";
36
37pub fn checkpoint_durability(storage_path: &Path) -> String {
39 format!(
40 "durable on disk at {}; survives restarts",
41 storage_path.display()
42 )
43}
44
45#[derive(Debug, Clone)]
47pub struct CheckpointInfo {
48 pub name: String,
49 pub file_count: usize,
50 pub created_at: u64,
51 pub storage_path: Option<PathBuf>,
53 pub evicted: Vec<String>,
55 pub skipped: Vec<(PathBuf, String)>,
60}
61
62#[derive(Debug, Clone)]
64struct Checkpoint {
65 name: String,
66 file_contents: HashMap<PathBuf, CheckpointFile>,
67 created_at: u64,
68 created_order: u64,
71}
72
73#[derive(Debug, Clone)]
74struct CheckpointFile {
75 metadata: Option<fs::Metadata>,
78 mode: Option<u32>,
79 kind: CheckpointFileKind,
80}
81
82#[derive(Debug, Serialize, Deserialize)]
83struct DiskCheckpointMeta {
84 schema_version: u32,
85 session_id: String,
86 name: String,
87 created_at: u64,
88 created_order: u64,
89 files: Vec<DiskCheckpointFileMeta>,
90}
91
92#[derive(Debug, Serialize, Deserialize)]
93struct DiskCheckpointFileMeta {
94 original_path: String,
95 blob: String,
96 kind: DiskCheckpointFileKind,
97 mode: Option<u32>,
98 target_is_dir: bool,
99}
100
101#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
102#[serde(rename_all = "snake_case")]
103enum DiskCheckpointFileKind {
104 Regular,
105 Symlink,
106}
107
108#[derive(Debug, Clone)]
109enum CheckpointFileKind {
110 Regular {
111 bytes: Arc<[u8]>,
112 },
113 Symlink {
114 target: PathBuf,
115 target_is_dir: bool,
116 },
117}
118
119impl CheckpointFile {
120 fn read(path: &Path) -> io::Result<Self> {
121 let metadata = fs::symlink_metadata(path)?;
122 let file_type = metadata.file_type();
123 if file_type.is_symlink() {
124 let target = fs::read_link(path)?;
125 let target_is_dir = fs::metadata(path)
126 .map(|target_metadata| target_metadata.is_dir())
127 .unwrap_or(false);
128 return Ok(Self {
129 mode: checkpoint_mode(&metadata),
130 metadata: Some(metadata),
131 kind: CheckpointFileKind::Symlink {
132 target,
133 target_is_dir,
134 },
135 });
136 }
137
138 if metadata.is_file() {
139 let capture = CapturedRegularFile::read(path)?.ok_or_else(|| {
140 io::Error::new(
141 io::ErrorKind::InvalidInput,
142 "file changed while being captured",
143 )
144 })?;
145 return Ok(Self::from_fresh_capture(capture));
146 }
147
148 Err(io::Error::new(
149 io::ErrorKind::InvalidInput,
150 "not a regular file or symlink",
151 ))
152 }
153
154 fn from_captured(path: &Path, capture: &mut CapturedRegularFile) -> io::Result<Self> {
161 capture.refresh_if_stale(path)?;
162 let metadata = capture.metadata().clone();
163 Ok(Self {
164 mode: checkpoint_mode(&metadata),
165 metadata: Some(metadata),
166 kind: CheckpointFileKind::Regular {
167 bytes: capture.shared_bytes(),
168 },
169 })
170 }
171
172 fn from_fresh_capture(capture: CapturedRegularFile) -> Self {
173 let metadata = capture.metadata().clone();
174 Self {
175 mode: checkpoint_mode(&metadata),
176 metadata: Some(metadata),
177 kind: CheckpointFileKind::Regular {
178 bytes: capture.shared_bytes(),
179 },
180 }
181 }
182
183 fn read_optional(path: &Path) -> io::Result<Option<Self>> {
184 match Self::read(path) {
185 Ok(snapshot) => Ok(Some(snapshot)),
186 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
187 Err(error) => Err(error),
188 }
189 }
190
191 fn from_disk(meta: &DiskCheckpointFileMeta, bytes: Vec<u8>) -> Result<Self, String> {
192 let kind = match &meta.kind {
193 DiskCheckpointFileKind::Regular => CheckpointFileKind::Regular {
194 bytes: bytes.into(),
195 },
196 DiskCheckpointFileKind::Symlink => {
197 let target = String::from_utf8(bytes)
198 .map(PathBuf::from)
199 .map_err(|error| format!("checkpoint symlink target is not UTF-8: {error}"))?;
200 CheckpointFileKind::Symlink {
201 target,
202 target_is_dir: meta.target_is_dir,
203 }
204 }
205 };
206 Ok(Self {
207 metadata: None,
208 mode: meta.mode,
209 kind,
210 })
211 }
212}
213
214#[cfg(unix)]
215fn checkpoint_mode(metadata: &fs::Metadata) -> Option<u32> {
216 use std::os::unix::fs::PermissionsExt;
217 Some(metadata.permissions().mode())
218}
219
220#[cfg(not(unix))]
221fn checkpoint_mode(_metadata: &fs::Metadata) -> Option<u32> {
222 None
223}
224
225#[derive(Debug)]
233pub struct CheckpointStore {
234 checkpoints: HashMap<String, HashMap<String, Checkpoint>>,
236 lock_path: PathBuf,
237 lock_timeout: Duration,
238 storage_dir: Option<PathBuf>,
239 storage_harness: Option<String>,
240 blob_counter: AtomicU64,
241}
242
243struct CheckpointLockGuard {
247 guard: Option<fs_lock::LockGuard>,
248 scope_dir: Option<PathBuf>,
249}
250
251impl Drop for CheckpointLockGuard {
252 fn drop(&mut self) {
253 if let Some(guard) = self.guard.take() {
257 drop(guard);
258 }
259 if let Some(scope_dir) = &self.scope_dir {
260 remove_empty_scope_dir(scope_dir);
261 }
262 }
263}
264
265impl CheckpointStore {
266 pub fn new() -> Self {
267 let project_root = std::env::current_dir().unwrap_or_else(|_| std::env::temp_dir());
268 let project_key = crate::path_identity::project_scope_key(&project_root);
269 let storage_dir = crate::bash_background::storage_dir(None);
270 let lock_path = storage_dir
271 .join("checkpoints")
272 .join(project_key)
273 .join("checkpoint.lock");
274 let mut store = Self::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
275 store.storage_dir = Some(storage_dir);
278 store.storage_harness = Some(UNBOUND_HARNESS_SEGMENT.to_string());
279 store
280 }
281
282 #[cfg(test)]
286 pub(crate) fn set_lock_path_for_test(&mut self, lock_path: PathBuf) {
287 self.storage_dir = lock_path.parent().map(Path::to_path_buf);
288 self.storage_harness = Some("test".to_string());
289 self.lock_path = lock_path;
290 }
291
292 pub fn set_storage_dir_for_harness(&mut self, dir: PathBuf, harness: crate::harness::Harness) {
295 let harness = harness.storage_segment();
296 if self.storage_dir.as_ref() == Some(&dir)
297 && self.storage_harness.as_deref() == Some(&harness)
298 {
299 return;
300 }
301 if self.storage_dir.as_ref() == Some(&dir)
302 && self.storage_harness.as_deref() == Some(UNBOUND_HARNESS_SEGMENT)
303 {
304 match self.acquire_mutation_lock() {
305 Ok(_lock) => migrate_unbound_checkpoint_namespace(&dir, &harness),
306 Err(error) => crate::slog_warn!(
307 "could not migrate unbound durable checkpoints into {}: {}",
308 harness,
309 error
310 ),
311 }
312 }
313 self.storage_dir = Some(dir);
314 self.storage_harness = Some(harness);
315 self.checkpoints.clear();
316 }
317
318 fn with_lock_path(lock_path: PathBuf, lock_timeout: Duration) -> Self {
319 CheckpointStore {
320 checkpoints: HashMap::new(),
321 lock_path,
322 lock_timeout,
323 storage_dir: None,
324 storage_harness: None,
325 blob_counter: AtomicU64::new(0),
326 }
327 }
328
329 fn acquire_mutation_lock(&self) -> Result<CheckpointLockGuard, AftError> {
330 let scope_dir = self.lock_path.parent().map(Path::to_path_buf);
331 if let Some(parent) = scope_dir.as_deref() {
332 fs::create_dir_all(parent).map_err(|error| AftError::IoError {
333 path: parent.display().to_string(),
334 message: format!("failed to create checkpoint lock directory: {error}"),
335 })?;
336 }
337
338 let acquire_result = match fs_lock::try_acquire(&self.lock_path, self.lock_timeout) {
339 Err(fs_lock::AcquireError::Io(error)) if error.kind() == io::ErrorKind::NotFound => {
343 if let Some(parent) = scope_dir.as_deref() {
344 fs::create_dir_all(parent).map_err(|error| AftError::IoError {
345 path: parent.display().to_string(),
346 message: format!("failed to recreate checkpoint lock directory: {error}"),
347 })?;
348 }
349 fs_lock::try_acquire(&self.lock_path, self.lock_timeout)
350 }
351 result => result,
352 };
353 let guard = acquire_result.map_err(|error| match error {
354 fs_lock::AcquireError::Timeout => AftError::IoError {
355 path: self.lock_path.display().to_string(),
356 message: "timed out acquiring checkpoint mutation lock".to_string(),
357 },
358 fs_lock::AcquireError::Io(error) => AftError::IoError {
359 path: self.lock_path.display().to_string(),
360 message: format!("failed to acquire checkpoint mutation lock: {error}"),
361 },
362 })?;
363
364 Ok(CheckpointLockGuard {
365 guard: Some(guard),
366 scope_dir,
367 })
368 }
369
370 pub fn create(
384 &mut self,
385 session: &str,
386 name: &str,
387 files: Vec<PathBuf>,
388 backup_store: &BackupStore,
389 ) -> Result<CheckpointInfo, AftError> {
390 self.create_impl(session, name, files, backup_store, None)
391 }
392
393 pub(crate) fn create_from_captures(
394 &mut self,
395 session: &str,
396 name: &str,
397 files: Vec<PathBuf>,
398 backup_store: &BackupStore,
399 captures: &mut HashMap<PathBuf, CapturedRegularFile>,
400 ) -> Result<CheckpointInfo, AftError> {
401 self.create_impl(session, name, files, backup_store, Some(captures))
402 }
403
404 fn create_impl(
405 &mut self,
406 session: &str,
407 name: &str,
408 files: Vec<PathBuf>,
409 backup_store: &BackupStore,
410 mut captures: Option<&mut HashMap<PathBuf, CapturedRegularFile>>,
411 ) -> Result<CheckpointInfo, AftError> {
412 let _mutation_lock = self.acquire_mutation_lock()?;
413 validate_checkpoint_name(name)?;
414 self.run_process_maintenance_once_locked()?;
415 self.hydrate_session_locked(session)?;
416 let explicit_request = !files.is_empty();
417 let file_list = if files.is_empty() {
418 backup_store.tracked_files(session)
419 } else {
420 files
421 };
422
423 let mut file_contents = HashMap::new();
424 let mut skipped: Vec<(PathBuf, String)> = Vec::new();
425 for path in &file_list {
426 let seeded = captures
427 .as_deref_mut()
428 .and_then(|captures| captures.get_mut(path))
429 .map(|capture| CheckpointFile::from_captured(path, capture));
430 let snapshot = match seeded {
431 Some(Err(error)) if error.kind() == io::ErrorKind::InvalidInput => {
432 if let Some(captures) = captures.as_deref_mut() {
433 captures.remove(path);
434 }
435 CheckpointFile::read(path)
436 }
437 Some(result) => result,
438 None => CheckpointFile::read(path),
439 };
440 match snapshot {
441 Ok(snapshot) => {
442 file_contents.insert(path.clone(), snapshot);
443 }
444 Err(e) => {
445 crate::slog_warn!(
446 "checkpoint {}: skipping unreadable file {}: {}",
447 name,
448 path.display(),
449 e
450 );
451 skipped.push((path.clone(), e.to_string()));
452 }
453 }
454 }
455
456 if explicit_request && file_contents.is_empty() && !skipped.is_empty() {
462 let (path, err) = &skipped[0];
463 return Err(AftError::FileNotFound {
464 path: format!("{}: {}", path.display(), err),
465 });
466 }
467
468 let created_at = current_timestamp();
469 let created_order = current_timestamp_nanos()
470 .saturating_add(self.blob_counter.fetch_add(1, Ordering::Relaxed));
471 let file_count = file_contents.len();
472 let checkpoint = Checkpoint {
473 name: name.to_string(),
474 file_contents,
475 created_at,
476 created_order,
477 };
478 let storage_path = self.durable_checkpoint_dir(session, name);
479
480 self.persist_checkpoint_locked(session, &checkpoint)?;
481 self.checkpoints
482 .entry(session.to_string())
483 .or_default()
484 .insert(name.to_string(), checkpoint);
485
486 let evicted = self.evict_excess_checkpoints_locked(session)?;
487
488 if skipped.is_empty() {
489 crate::slog_info!("checkpoint created: {} ({} files)", name, file_count);
490 } else {
491 crate::slog_info!(
492 "checkpoint created: {} ({} files, {} skipped)",
493 name,
494 file_count,
495 skipped.len()
496 );
497 }
498
499 Ok(CheckpointInfo {
500 name: name.to_string(),
501 file_count,
502 created_at,
503 storage_path,
504 evicted,
505 skipped,
506 })
507 }
508
509 pub fn restore(&mut self, session: &str, name: &str) -> Result<CheckpointInfo, AftError> {
511 let _mutation_lock = self.acquire_mutation_lock()?;
512 self.run_process_maintenance_once_locked()?;
513 self.hydrate_session_locked(session)?;
514 let storage_path = self.durable_checkpoint_dir(session, name);
515 let checkpoint = self.get(session, name)?;
516 let mut paths = checkpoint.file_contents.keys().cloned().collect::<Vec<_>>();
517 paths.sort();
518
519 restore_paths_atomically(checkpoint, &paths)?;
520 crate::slog_info!("checkpoint restored: {}", name);
521
522 Ok(CheckpointInfo {
523 name: checkpoint.name.clone(),
524 file_count: checkpoint.file_contents.len(),
525 created_at: checkpoint.created_at,
526 storage_path,
527 evicted: Vec::new(),
528 skipped: Vec::new(),
529 })
530 }
531
532 pub fn restore_validated(
534 &mut self,
535 session: &str,
536 name: &str,
537 validated_paths: &[PathBuf],
538 ) -> Result<CheckpointInfo, AftError> {
539 let _mutation_lock = self.acquire_mutation_lock()?;
540 self.run_process_maintenance_once_locked()?;
541 self.hydrate_session_locked(session)?;
542 let storage_path = self.durable_checkpoint_dir(session, name);
543 let checkpoint = self.get(session, name)?;
544
545 for path in validated_paths {
546 checkpoint
547 .file_contents
548 .get(path)
549 .ok_or_else(|| AftError::FileNotFound {
550 path: path.display().to_string(),
551 })?;
552 }
553 restore_paths_atomically(checkpoint, validated_paths)?;
554 crate::slog_info!("checkpoint restored: {}", name);
555
556 Ok(CheckpointInfo {
557 name: checkpoint.name.clone(),
558 file_count: checkpoint.file_contents.len(),
559 created_at: checkpoint.created_at,
560 storage_path,
561 evicted: Vec::new(),
562 skipped: Vec::new(),
563 })
564 }
565
566 pub fn file_paths(&mut self, session: &str, name: &str) -> Result<Vec<PathBuf>, AftError> {
568 let _mutation_lock = self.acquire_mutation_lock()?;
569 self.run_process_maintenance_once_locked()?;
570 self.hydrate_session_locked(session)?;
571 let checkpoint = self.get(session, name)?;
572 Ok(checkpoint.file_contents.keys().cloned().collect())
573 }
574
575 pub fn absolute_file_paths(
577 &mut self,
578 session: &str,
579 name: &str,
580 ) -> Result<Vec<PathBuf>, AftError> {
581 let mut paths: Vec<PathBuf> = self
582 .file_paths(session, name)?
583 .into_iter()
584 .map(absolute_checkpoint_path)
585 .collect();
586 paths.sort();
587 Ok(paths)
588 }
589
590 pub fn delete(&mut self, session: &str, name: &str) -> bool {
592 let _mutation_lock = match self.acquire_mutation_lock() {
593 Ok(lock) => lock,
594 Err(error) => {
595 crate::slog_warn!("checkpoint delete lock failed for {}: {}", name, error);
596 return false;
597 }
598 };
599 if let Err(error) = self.run_process_maintenance_once_locked() {
600 crate::slog_warn!(
601 "checkpoint delete maintenance failed for {}: {}",
602 name,
603 error
604 );
605 return false;
606 }
607 if let Err(error) = self.hydrate_session_locked(session) {
608 crate::slog_warn!("checkpoint delete hydration failed for {}: {}", name, error);
609 return false;
610 }
611 if self
612 .checkpoints
613 .get(session)
614 .is_none_or(|checkpoints| !checkpoints.contains_key(name))
615 {
616 return false;
617 }
618 if let Err(error) = self.remove_checkpoint_from_disk_locked(session, name) {
619 crate::slog_warn!("checkpoint delete failed for {}: {}", name, error);
620 return false;
621 }
622 let Some(session_checkpoints) = self.checkpoints.get_mut(session) else {
623 return false;
624 };
625 let removed = session_checkpoints.remove(name).is_some();
626 if session_checkpoints.is_empty() {
627 self.checkpoints.remove(session);
628 }
629 removed
630 }
631
632 pub fn list(&mut self, session: &str) -> Result<Vec<CheckpointInfo>, AftError> {
635 let _mutation_lock = self.acquire_mutation_lock()?;
636 self.run_process_maintenance_once_locked()?;
637 self.hydrate_session_locked(session)?;
638 let mut list = self
639 .checkpoints
640 .get(session)
641 .map(|checkpoints| {
642 checkpoints
643 .values()
644 .map(|checkpoint| CheckpointInfo {
645 name: checkpoint.name.clone(),
646 file_count: checkpoint.file_contents.len(),
647 created_at: checkpoint.created_at,
648 storage_path: self.durable_checkpoint_dir(session, &checkpoint.name),
649 evicted: Vec::new(),
650 skipped: Vec::new(),
651 })
652 .collect::<Vec<_>>()
653 })
654 .unwrap_or_default();
655 list.sort_by(|left, right| left.name.cmp(&right.name));
656 Ok(list)
657 }
658
659 pub fn total_count(&self) -> usize {
661 self.checkpoints
662 .values()
663 .map(|checkpoints| checkpoints.len())
664 .sum()
665 }
666
667 pub fn cleanup(&mut self) {
671 let _mutation_lock = match self.acquire_mutation_lock() {
672 Ok(lock) => lock,
673 Err(error) => {
674 crate::slog_warn!("checkpoint cleanup lock failed: {}", error);
675 return;
676 }
677 };
678 if let Err(error) = self.cleanup_locked() {
679 crate::slog_warn!("checkpoint cleanup failed: {}", error);
680 }
681 }
682
683 fn get(&self, session: &str, name: &str) -> Result<&Checkpoint, AftError> {
684 self.checkpoints
685 .get(session)
686 .and_then(|checkpoints| checkpoints.get(name))
687 .ok_or_else(|| AftError::CheckpointNotFound {
688 name: name.to_string(),
689 })
690 }
691
692 fn durable_checkpoints_dir(&self) -> Option<PathBuf> {
693 self.storage_dir
694 .as_ref()
695 .zip(self.storage_harness.as_ref())
696 .map(|(storage_dir, harness)| storage_dir.join(harness).join("checkpoints"))
697 }
698
699 fn durable_session_dir(&self, session: &str) -> Option<PathBuf> {
700 self.durable_checkpoints_dir()
701 .map(|checkpoints_dir| checkpoints_dir.join(hash_session(session)))
702 }
703
704 fn durable_checkpoint_dir(&self, session: &str, name: &str) -> Option<PathBuf> {
705 self.durable_session_dir(session)
706 .map(|session_dir| session_dir.join(name))
707 }
708
709 fn run_process_maintenance_once_locked(&mut self) -> Result<(), AftError> {
710 let Some(storage_dir) = self.storage_dir.clone() else {
711 return Ok(());
712 };
713 let Some(harness) = self.storage_harness.clone() else {
714 return Ok(());
715 };
716 if !CHECKPOINT_MAINTENANCE_KEYS
717 .lock()
718 .unwrap()
719 .insert((storage_dir, harness))
720 {
721 return Ok(());
722 }
723 self.cleanup_locked()
724 }
725
726 fn cleanup_locked(&mut self) -> Result<(), AftError> {
727 let now = current_timestamp();
728 self.checkpoints.retain(|_, session_checkpoints| {
729 session_checkpoints.retain(|_, checkpoint| {
730 now.saturating_sub(checkpoint.created_at) < NAMED_CHECKPOINT_RETENTION_SECS
731 });
732 !session_checkpoints.is_empty()
733 });
734
735 if let Some(checkpoints_dir) = self.durable_checkpoints_dir() {
736 sweep_expired_durable_checkpoints(&checkpoints_dir, now);
737 }
738 if let Some(checkpoints_root) = self.lock_path.parent().and_then(Path::parent) {
739 if checkpoints_root.file_name() == Some(std::ffi::OsStr::new("checkpoints")) {
745 sweep_empty_scope_dirs(checkpoints_root);
746 }
747 }
748 Ok(())
749 }
750
751 fn hydrate_session_locked(&mut self, session: &str) -> Result<(), AftError> {
752 let Some(session_dir) = self.durable_session_dir(session) else {
753 return Ok(());
754 };
755 if !session_dir.exists() {
756 self.checkpoints.remove(session);
757 return Ok(());
758 }
759
760 let entries = fs::read_dir(&session_dir).map_err(|error| AftError::IoError {
761 path: session_dir.display().to_string(),
762 message: format!("failed to read durable checkpoint session: {error}"),
763 })?;
764 let mut hydrated = HashMap::new();
765 for entry in entries {
766 let entry = entry.map_err(|error| AftError::IoError {
767 path: session_dir.display().to_string(),
768 message: format!("failed to read durable checkpoint entry: {error}"),
769 })?;
770 let checkpoint_dir = entry.path();
771 if !entry
772 .file_type()
773 .map_err(|error| AftError::IoError {
774 path: checkpoint_dir.display().to_string(),
775 message: format!("failed to inspect durable checkpoint entry: {error}"),
776 })?
777 .is_dir()
778 {
779 continue;
780 }
781 let name = entry.file_name().to_string_lossy().into_owned();
782 if !is_safe_checkpoint_name(&name) {
783 continue;
784 }
785 let meta_path = checkpoint_dir.join("meta.json");
786 if !meta_path.exists() {
787 continue;
788 }
789 let checkpoint = read_checkpoint_from_disk(&checkpoint_dir, session, &name)?;
790 hydrated.insert(name, checkpoint);
791 }
792 if hydrated.is_empty() {
793 self.checkpoints.remove(session);
794 } else {
795 self.checkpoints.insert(session.to_string(), hydrated);
796 self.evict_excess_checkpoints_locked(session)?;
797 }
798 Ok(())
799 }
800
801 fn persist_checkpoint_locked(
802 &self,
803 session: &str,
804 checkpoint: &Checkpoint,
805 ) -> Result<(), AftError> {
806 let Some(checkpoint_dir) = self.durable_checkpoint_dir(session, &checkpoint.name) else {
807 return Ok(());
808 };
809 fs::create_dir_all(&checkpoint_dir).map_err(|error| AftError::IoError {
810 path: checkpoint_dir.display().to_string(),
811 message: format!("failed to create durable checkpoint directory: {error}"),
812 })?;
813
814 let mut files = Vec::with_capacity(checkpoint.file_contents.len());
815 for (index, (path, file)) in checkpoint.file_contents.iter().enumerate() {
816 let blob = format!(
817 "file_{}_{}_{}.blob",
818 checkpoint.created_order,
819 index,
820 self.blob_counter.fetch_add(1, Ordering::Relaxed)
821 );
822 let bytes = checkpoint_file_bytes(file);
823 write_temp_fsync_rename(&checkpoint_dir, &blob, &bytes).map_err(|error| {
824 AftError::IoError {
825 path: checkpoint_dir.join(&blob).display().to_string(),
826 message: format!("failed to write durable checkpoint blob: {error}"),
827 }
828 })?;
829 files.push(DiskCheckpointFileMeta {
830 original_path: path.display().to_string(),
831 blob,
832 kind: match &file.kind {
833 CheckpointFileKind::Regular { .. } => DiskCheckpointFileKind::Regular,
834 CheckpointFileKind::Symlink { .. } => DiskCheckpointFileKind::Symlink,
835 },
836 mode: file.mode,
837 target_is_dir: matches!(
838 &file.kind,
839 CheckpointFileKind::Symlink {
840 target_is_dir: true,
841 ..
842 }
843 ),
844 });
845 }
846 fsync_dir(&checkpoint_dir).map_err(|error| AftError::IoError {
847 path: checkpoint_dir.display().to_string(),
848 message: format!("failed to sync durable checkpoint blobs: {error}"),
849 })?;
850
851 let meta = DiskCheckpointMeta {
852 schema_version: CHECKPOINT_SCHEMA_VERSION,
853 session_id: session.to_string(),
854 name: checkpoint.name.clone(),
855 created_at: checkpoint.created_at,
856 created_order: checkpoint.created_order,
857 files,
858 };
859 let bytes = serde_json::to_vec_pretty(&meta).map_err(|error| AftError::IoError {
860 path: checkpoint_dir.join("meta.json").display().to_string(),
861 message: format!("failed to serialize durable checkpoint metadata: {error}"),
862 })?;
863 write_temp_fsync_rename(&checkpoint_dir, "meta.json", &bytes).map_err(|error| {
864 AftError::IoError {
865 path: checkpoint_dir.join("meta.json").display().to_string(),
866 message: format!("failed to write durable checkpoint metadata: {error}"),
867 }
868 })?;
869 fsync_dir(&checkpoint_dir).map_err(|error| AftError::IoError {
870 path: checkpoint_dir.display().to_string(),
871 message: format!("failed to sync durable checkpoint metadata: {error}"),
872 })?;
873 prune_unreferenced_checkpoint_blobs(&checkpoint_dir, &meta.files).map_err(|error| {
874 AftError::IoError {
875 path: checkpoint_dir.display().to_string(),
876 message: format!("failed to prune stale durable checkpoint blobs: {error}"),
877 }
878 })?;
879 Ok(())
880 }
881
882 fn remove_checkpoint_from_disk_locked(
883 &self,
884 session: &str,
885 name: &str,
886 ) -> Result<(), AftError> {
887 let Some(checkpoint_dir) = self.durable_checkpoint_dir(session, name) else {
888 return Ok(());
889 };
890 match fs::remove_dir_all(&checkpoint_dir) {
891 Ok(()) => {
892 if let Some(session_dir) = checkpoint_dir.parent() {
893 let _ = fs::remove_dir(session_dir);
894 }
895 Ok(())
896 }
897 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
898 Err(error) => Err(AftError::IoError {
899 path: checkpoint_dir.display().to_string(),
900 message: format!("failed to remove durable checkpoint: {error}"),
901 }),
902 }
903 }
904
905 fn evict_excess_checkpoints_locked(&mut self, session: &str) -> Result<Vec<String>, AftError> {
906 let Some(checkpoints) = self.checkpoints.get(session) else {
907 return Ok(Vec::new());
908 };
909 let overflow = checkpoints
910 .len()
911 .saturating_sub(MAX_NAMED_CHECKPOINTS_PER_SESSION);
912 let mut checkpoints = checkpoints
913 .values()
914 .map(|checkpoint| (checkpoint.created_order, checkpoint.name.clone()))
915 .collect::<Vec<_>>();
916 checkpoints.sort();
917 let evicted = checkpoints
918 .into_iter()
919 .take(overflow)
920 .map(|(_, name)| name)
921 .collect::<Vec<_>>();
922
923 for name in &evicted {
924 self.remove_checkpoint_from_disk_locked(session, name)?;
925 }
926 if let Some(checkpoints) = self.checkpoints.get_mut(session) {
927 for name in &evicted {
928 checkpoints.remove(name);
929 }
930 }
931 Ok(evicted)
932 }
933
934 pub fn session_is_empty(&self, session: &str) -> bool {
935 self.checkpoints.get(session).is_none_or(HashMap::is_empty)
936 }
937}
938
939fn migrate_unbound_checkpoint_namespace(storage_dir: &Path, harness: &str) {
940 let source = storage_dir
941 .join(UNBOUND_HARNESS_SEGMENT)
942 .join("checkpoints");
943 if !source.exists() {
944 return;
945 }
946 let target = storage_dir.join(harness).join("checkpoints");
947 if !target.exists() {
948 if let Some(parent) = target.parent() {
949 if let Err(error) = fs::create_dir_all(parent) {
950 crate::slog_warn!(
951 "failed to create durable checkpoint harness directory {}: {}",
952 parent.display(),
953 error
954 );
955 return;
956 }
957 }
958 if let Err(error) = fs::rename(&source, &target) {
959 crate::slog_warn!(
960 "failed to move unbound durable checkpoints into {}: {}",
961 target.display(),
962 error
963 );
964 }
965 return;
966 }
967
968 let Ok(boundary) = crate::walk_boundary::DeviceBoundary::for_root(&source) else {
972 crate::slog_warn!(
973 "cannot establish filesystem boundary for checkpoint migration {}",
974 source.display()
975 );
976 return;
977 };
978 let mut skipped_foreign_mounts = 0usize;
979 let Ok(session_entries) = fs::read_dir(&source) else {
980 return;
981 };
982 for session_entry in session_entries.flatten() {
983 let source_session = session_entry.path();
984 if !source_session.is_dir() {
985 continue;
986 }
987 if !boundary.should_descend(&source_session).unwrap_or(false) {
988 skipped_foreign_mounts += 1;
989 continue;
990 }
991 let target_session = target.join(session_entry.file_name());
992 if !target_session.exists() {
993 let _ = fs::rename(&source_session, &target_session);
994 continue;
995 }
996 let Ok(checkpoint_entries) = fs::read_dir(&source_session) else {
997 continue;
998 };
999 for checkpoint_entry in checkpoint_entries.flatten() {
1000 let source_checkpoint = checkpoint_entry.path();
1001 let target_checkpoint = target_session.join(checkpoint_entry.file_name());
1002 if !target_checkpoint.exists() {
1003 let _ = fs::rename(source_checkpoint, target_checkpoint);
1004 }
1005 }
1006 let _ = fs::remove_dir(&source_session);
1007 }
1008 let _ = fs::remove_dir(&source);
1009 if skipped_foreign_mounts > 0 {
1010 crate::slog_warn!(
1011 "checkpoint migration skipped {} foreign filesystem mount(s) below {}",
1012 skipped_foreign_mounts,
1013 source.display()
1014 );
1015 }
1016}
1017
1018fn validate_checkpoint_name(name: &str) -> Result<(), AftError> {
1019 if is_safe_checkpoint_name(name) {
1020 Ok(())
1021 } else {
1022 Err(AftError::InvalidRequest {
1023 message: "checkpoint name must be a single non-empty path component".to_string(),
1024 })
1025 }
1026}
1027
1028fn is_safe_checkpoint_name(name: &str) -> bool {
1029 matches!(
1030 Path::new(name).components().collect::<Vec<_>>().as_slice(),
1031 [std::path::Component::Normal(_)]
1032 ) && !name.chars().any(|character| {
1033 character.is_control()
1034 || matches!(character, '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|')
1035 })
1036}
1037
1038fn is_safe_blob_name(name: &str) -> bool {
1039 is_safe_checkpoint_name(name) && name.starts_with("file_") && name.ends_with(".blob")
1040}
1041
1042fn read_checkpoint_from_disk(
1043 checkpoint_dir: &Path,
1044 session: &str,
1045 expected_name: &str,
1046) -> Result<Checkpoint, AftError> {
1047 let meta_path = checkpoint_dir.join("meta.json");
1048 let bytes = fs::read(&meta_path).map_err(|error| AftError::IoError {
1049 path: meta_path.display().to_string(),
1050 message: format!("failed to read durable checkpoint metadata: {error}"),
1051 })?;
1052 let meta = serde_json::from_slice::<DiskCheckpointMeta>(&bytes).map_err(|error| {
1053 AftError::IoError {
1054 path: meta_path.display().to_string(),
1055 message: format!("failed to parse durable checkpoint metadata: {error}"),
1056 }
1057 })?;
1058 if meta.schema_version != CHECKPOINT_SCHEMA_VERSION {
1059 return Err(AftError::IoError {
1060 path: meta_path.display().to_string(),
1061 message: format!(
1062 "unsupported durable checkpoint metadata schema {}",
1063 meta.schema_version
1064 ),
1065 });
1066 }
1067 if meta.session_id != session
1068 || meta.name != expected_name
1069 || !is_safe_checkpoint_name(&meta.name)
1070 {
1071 return Err(AftError::IoError {
1072 path: meta_path.display().to_string(),
1073 message: "durable checkpoint metadata does not match its session or directory"
1074 .to_string(),
1075 });
1076 }
1077
1078 let mut file_contents = HashMap::with_capacity(meta.files.len());
1079 for file in &meta.files {
1080 if !is_safe_blob_name(&file.blob) {
1081 return Err(AftError::IoError {
1082 path: meta_path.display().to_string(),
1083 message: format!("invalid durable checkpoint blob name {}", file.blob),
1084 });
1085 }
1086 let blob_path = checkpoint_dir.join(&file.blob);
1087 let blob = fs::read(&blob_path).map_err(|error| AftError::IoError {
1088 path: blob_path.display().to_string(),
1089 message: format!("failed to read durable checkpoint blob: {error}"),
1090 })?;
1091 let path = PathBuf::from(&file.original_path);
1092 let checkpoint_file =
1093 CheckpointFile::from_disk(file, blob).map_err(|message| AftError::IoError {
1094 path: blob_path.display().to_string(),
1095 message,
1096 })?;
1097 if file_contents
1098 .insert(path.clone(), checkpoint_file)
1099 .is_some()
1100 {
1101 return Err(AftError::IoError {
1102 path: meta_path.display().to_string(),
1103 message: format!("duplicate durable checkpoint path {}", path.display()),
1104 });
1105 }
1106 }
1107
1108 Ok(Checkpoint {
1109 name: meta.name,
1110 file_contents,
1111 created_at: meta.created_at,
1112 created_order: meta.created_order,
1113 })
1114}
1115
1116fn checkpoint_file_bytes(file: &CheckpointFile) -> Vec<u8> {
1117 match &file.kind {
1118 CheckpointFileKind::Regular { bytes } => bytes.to_vec(),
1119 CheckpointFileKind::Symlink { target, .. } => {
1120 target.as_os_str().to_string_lossy().as_bytes().to_vec()
1121 }
1122 }
1123}
1124
1125fn write_temp_fsync_rename(dir: &Path, final_name: &str, bytes: &[u8]) -> io::Result<()> {
1126 let tmp_name = format!(
1127 ".{}.{}.{}.tmp",
1128 final_name,
1129 std::process::id(),
1130 current_timestamp_nanos()
1131 );
1132 let tmp_path = dir.join(tmp_name);
1133 let final_path = dir.join(final_name);
1134 {
1135 let mut file = fs::OpenOptions::new()
1136 .write(true)
1137 .create_new(true)
1138 .open(&tmp_path)?;
1139 file.write_all(bytes)?;
1140 file.sync_all()?;
1141 }
1142 fs::rename(tmp_path, final_path)
1143}
1144
1145#[cfg(unix)]
1146fn fsync_dir(path: &Path) -> io::Result<()> {
1147 fs::File::open(path)?.sync_all()
1148}
1149
1150#[cfg(not(unix))]
1151fn fsync_dir(_path: &Path) -> io::Result<()> {
1152 Ok(())
1153}
1154
1155fn prune_unreferenced_checkpoint_blobs(
1156 checkpoint_dir: &Path,
1157 files: &[DiskCheckpointFileMeta],
1158) -> io::Result<()> {
1159 let referenced = files
1160 .iter()
1161 .map(|file| file.blob.as_str())
1162 .collect::<HashSet<_>>();
1163 for entry in fs::read_dir(checkpoint_dir)? {
1164 let entry = entry?;
1165 let path = entry.path();
1166 if !path.is_file() {
1167 continue;
1168 }
1169 let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
1170 continue;
1171 };
1172 if (name.starts_with("file_") && name.ends_with(".blob") && !referenced.contains(name))
1173 || name.contains(".tmp.")
1174 || name.ends_with(".tmp")
1175 {
1176 let _ = fs::remove_file(path);
1177 }
1178 }
1179 Ok(())
1180}
1181
1182fn sweep_expired_durable_checkpoints(checkpoints_dir: &Path, now: u64) {
1183 let Ok(boundary) = crate::walk_boundary::DeviceBoundary::for_root(checkpoints_dir) else {
1187 crate::slog_warn!(
1188 "cannot establish filesystem boundary for checkpoint sweep {}",
1189 checkpoints_dir.display()
1190 );
1191 return;
1192 };
1193 let mut skipped_foreign_mounts = 0usize;
1194 let Ok(session_entries) = fs::read_dir(checkpoints_dir) else {
1195 return;
1196 };
1197 for session_entry in session_entries.flatten() {
1198 let session_dir = session_entry.path();
1199 if !session_dir.is_dir() {
1200 continue;
1201 }
1202 if !boundary.should_descend(&session_dir).unwrap_or(false) {
1203 skipped_foreign_mounts += 1;
1204 continue;
1205 }
1206 let Ok(checkpoint_entries) = fs::read_dir(&session_dir) else {
1207 continue;
1208 };
1209 for checkpoint_entry in checkpoint_entries.flatten() {
1210 let checkpoint_dir = checkpoint_entry.path();
1211 if !checkpoint_dir.is_dir() {
1212 continue;
1213 }
1214 if !boundary.should_descend(&checkpoint_dir).unwrap_or(false) {
1215 skipped_foreign_mounts += 1;
1216 continue;
1217 }
1218 let meta_path = checkpoint_dir.join("meta.json");
1219 let Ok(bytes) = fs::read(&meta_path) else {
1220 continue;
1221 };
1222 let Ok(meta) = serde_json::from_slice::<DiskCheckpointMeta>(&bytes) else {
1223 continue;
1224 };
1225 if now.saturating_sub(meta.created_at) < NAMED_CHECKPOINT_RETENTION_SECS {
1226 continue;
1227 }
1228 if let Err(error) = fs::remove_dir_all(&checkpoint_dir) {
1229 crate::slog_warn!(
1230 "failed to remove expired durable checkpoint {}: {}",
1231 checkpoint_dir.display(),
1232 error
1233 );
1234 }
1235 }
1236 let _ = fs::remove_dir(&session_dir);
1237 }
1238 if skipped_foreign_mounts > 0 {
1239 crate::slog_warn!(
1240 "checkpoint sweep skipped {} foreign filesystem mount(s) below {}",
1241 skipped_foreign_mounts,
1242 checkpoints_dir.display()
1243 );
1244 }
1245}
1246
1247fn absolute_checkpoint_path(path: PathBuf) -> PathBuf {
1248 if path.is_absolute() {
1249 return normalize_checkpoint_path(&path);
1250 }
1251 let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
1252 normalize_checkpoint_path(&cwd.join(path))
1253}
1254
1255fn normalize_checkpoint_path(path: &Path) -> PathBuf {
1256 let mut normalized = PathBuf::new();
1257 for component in path.components() {
1258 match component {
1259 std::path::Component::CurDir => {}
1260 std::path::Component::ParentDir => {
1261 if !normalized.pop() {
1262 normalized.push(component.as_os_str());
1263 }
1264 }
1265 other => normalized.push(other.as_os_str()),
1266 }
1267 }
1268 normalized
1269}
1270
1271fn restore_paths_atomically(checkpoint: &Checkpoint, paths: &[PathBuf]) -> Result<(), AftError> {
1272 let mut pre_restore_snapshot: HashMap<PathBuf, Option<CheckpointFile>> = HashMap::new();
1273 for path in paths {
1274 let current = CheckpointFile::read_optional(path).map_err(|error| AftError::IoError {
1275 path: path.display().to_string(),
1276 message: format!("failed to snapshot pre-restore file metadata: {error}"),
1277 })?;
1278 pre_restore_snapshot.insert(path.clone(), current);
1279 }
1280
1281 let mut restored_paths: Vec<PathBuf> = Vec::new();
1282 let mut created_dirs: Vec<PathBuf> = Vec::new();
1283 for path in paths {
1284 let snapshot =
1285 checkpoint
1286 .file_contents
1287 .get(path)
1288 .ok_or_else(|| AftError::FileNotFound {
1289 path: path.display().to_string(),
1290 })?;
1291 if let Err(e) = write_restored_file(path, snapshot, &mut created_dirs) {
1292 let mut rollback_errors = Vec::new();
1293 if let Some(snapshot) = pre_restore_snapshot.get(path) {
1294 if let Err(rollback_error) = restore_snapshot_file(path, snapshot.as_ref()) {
1295 rollback_errors.push(format!("{}: {}", path.display(), rollback_error));
1296 }
1297 }
1298 for restored_path in restored_paths.iter().rev() {
1299 if let Some(snapshot) = pre_restore_snapshot.get(restored_path) {
1300 if let Err(rollback_error) =
1301 restore_snapshot_file(restored_path, snapshot.as_ref())
1302 {
1303 rollback_errors.push(format!(
1304 "{}: {}",
1305 restored_path.display(),
1306 rollback_error
1307 ));
1308 }
1309 }
1310 }
1311 let dirs_rollback_ok = rollback_created_dirs(&created_dirs);
1312 if rollback_errors.is_empty() && dirs_rollback_ok {
1313 return Err(e);
1314 }
1315 return Err(AftError::IoError {
1316 path: path.display().to_string(),
1317 message: format!(
1318 "{}; restore_checkpoint rollback_succeeded: {}; rollback_errors: {}",
1319 e,
1320 rollback_errors.is_empty() && dirs_rollback_ok,
1321 if rollback_errors.is_empty() {
1322 "none".to_string()
1323 } else {
1324 rollback_errors.join("; ")
1325 }
1326 ),
1327 });
1328 }
1329 restored_paths.push(path.clone());
1330 }
1331
1332 Ok(())
1333}
1334
1335fn restore_snapshot_file(path: &Path, snapshot: Option<&CheckpointFile>) -> Result<(), AftError> {
1336 match snapshot {
1337 Some(snapshot) => write_restored_file(path, snapshot, &mut Vec::new()),
1338 None => remove_file_if_exists(path).map_err(|error| AftError::IoError {
1339 path: path.display().to_string(),
1340 message: format!("failed to remove file during checkpoint restore rollback: {error}"),
1341 }),
1342 }
1343}
1344
1345fn write_restored_file(
1346 path: &Path,
1347 snapshot: &CheckpointFile,
1348 created_dirs: &mut Vec<PathBuf>,
1349) -> Result<(), AftError> {
1350 create_parent_dirs(path, created_dirs)?;
1351
1352 match &snapshot.kind {
1353 CheckpointFileKind::Regular { bytes } => {
1354 if path_is_symlink(path) {
1355 remove_file_if_exists(path).map_err(|error| AftError::IoError {
1356 path: path.display().to_string(),
1357 message: format!("failed to replace symlink with regular file: {error}"),
1358 })?;
1359 }
1360 fs::write(path, bytes).map_err(|error| AftError::IoError {
1361 path: path.display().to_string(),
1362 message: format!("failed to restore checkpoint file contents: {error}"),
1363 })?;
1364 restore_checkpoint_permissions(path, snapshot).map_err(|error| AftError::IoError {
1365 path: path.display().to_string(),
1366 message: format!("failed to restore checkpoint file permissions: {error}"),
1367 })
1368 }
1369 CheckpointFileKind::Symlink {
1370 target,
1371 target_is_dir,
1372 } => {
1373 remove_file_if_exists(path).map_err(|error| AftError::IoError {
1374 path: path.display().to_string(),
1375 message: format!("failed to replace file with checkpoint symlink: {error}"),
1376 })?;
1377 create_symlink(target, path, *target_is_dir).map_err(|error| AftError::IoError {
1378 path: path.display().to_string(),
1379 message: format!("failed to restore checkpoint symlink: {error}"),
1380 })
1381 }
1382 }
1383}
1384
1385fn restore_checkpoint_permissions(path: &Path, snapshot: &CheckpointFile) -> io::Result<()> {
1386 if let Some(metadata) = &snapshot.metadata {
1387 return fs::set_permissions(path, metadata.permissions());
1388 }
1389 restore_checkpoint_mode(path, snapshot.mode)
1390}
1391
1392#[cfg(unix)]
1393fn restore_checkpoint_mode(path: &Path, mode: Option<u32>) -> io::Result<()> {
1394 use std::os::unix::fs::PermissionsExt;
1395 if let Some(mode) = mode {
1396 fs::set_permissions(path, fs::Permissions::from_mode(mode))?;
1397 }
1398 Ok(())
1399}
1400
1401#[cfg(not(unix))]
1402fn restore_checkpoint_mode(_path: &Path, _mode: Option<u32>) -> io::Result<()> {
1403 Ok(())
1404}
1405
1406fn create_parent_dirs(path: &Path, created_dirs: &mut Vec<PathBuf>) -> Result<(), AftError> {
1407 if let Some(parent) = path.parent() {
1408 let missing_dirs = missing_parent_dirs(parent);
1409 fs::create_dir_all(parent).map_err(|error| AftError::IoError {
1410 path: parent.display().to_string(),
1411 message: format!("failed to create checkpoint restore parent directories: {error}"),
1412 })?;
1413 created_dirs.extend(missing_dirs);
1414 }
1415 Ok(())
1416}
1417
1418fn path_is_symlink(path: &Path) -> bool {
1419 fs::symlink_metadata(path)
1420 .map(|metadata| metadata.file_type().is_symlink())
1421 .unwrap_or(false)
1422}
1423
1424fn remove_file_if_exists(path: &Path) -> io::Result<()> {
1425 match fs::remove_file(path) {
1426 Ok(()) => Ok(()),
1427 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
1428 Err(error) => Err(error),
1429 }
1430}
1431
1432#[cfg(unix)]
1433fn create_symlink(target: &Path, link: &Path, target_is_dir: bool) -> io::Result<()> {
1434 let _ = target_is_dir;
1435 std::os::unix::fs::symlink(target, link)
1436}
1437
1438#[cfg(windows)]
1439fn create_symlink(target: &Path, link: &Path, target_is_dir: bool) -> io::Result<()> {
1440 if target_is_dir {
1441 std::os::windows::fs::symlink_dir(target, link)
1442 } else {
1443 std::os::windows::fs::symlink_file(target, link)
1444 }
1445}
1446
1447#[cfg(not(any(unix, windows)))]
1448fn create_symlink(_target: &Path, _link: &Path, _target_is_dir: bool) -> io::Result<()> {
1449 Err(io::Error::new(
1450 io::ErrorKind::Unsupported,
1451 "checkpoint symlink restore is unsupported on this platform",
1452 ))
1453}
1454
1455fn missing_parent_dirs(parent: &Path) -> Vec<PathBuf> {
1456 let mut dirs = Vec::new();
1457 let mut current = Some(parent);
1458
1459 while let Some(dir) = current {
1460 if dir.as_os_str().is_empty() || dir.exists() {
1461 break;
1462 }
1463 dirs.push(dir.to_path_buf());
1464 current = dir.parent();
1465 }
1466
1467 dirs
1468}
1469
1470fn rollback_created_dirs(dirs: &[PathBuf]) -> bool {
1471 let mut dirs = dirs.to_vec();
1472 dirs.sort_by_key(|dir| std::cmp::Reverse(dir.components().count()));
1473 dirs.dedup();
1474
1475 let mut ok = true;
1476 for dir in dirs {
1477 match std::fs::remove_dir(&dir) {
1478 Ok(()) => {}
1479 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1480 Err(_) => ok = false,
1481 }
1482 }
1483 ok
1484}
1485
1486fn remove_empty_scope_dir(scope_dir: &Path) {
1490 let _ = fs::remove_dir(scope_dir);
1491}
1492
1493fn sweep_empty_scope_dirs(checkpoints_root: &Path) {
1497 let entries = match fs::read_dir(checkpoints_root) {
1498 Ok(entries) => entries,
1499 Err(_) => return,
1500 };
1501
1502 for entry in entries.flatten() {
1503 let Ok(file_type) = entry.file_type() else {
1504 continue;
1505 };
1506 if file_type.is_dir() {
1507 remove_empty_scope_dir(&entry.path());
1508 }
1509 }
1510}
1511
1512fn current_timestamp() -> u64 {
1513 std::time::SystemTime::now()
1514 .duration_since(std::time::UNIX_EPOCH)
1515 .unwrap_or_default()
1516 .as_secs()
1517}
1518
1519fn current_timestamp_nanos() -> u64 {
1520 u64::try_from(
1521 std::time::SystemTime::now()
1522 .duration_since(std::time::UNIX_EPOCH)
1523 .unwrap_or_default()
1524 .as_nanos(),
1525 )
1526 .unwrap_or(u64::MAX)
1527}
1528
1529#[cfg(test)]
1530mod tests {
1531 use super::*;
1532 use crate::protocol::DEFAULT_SESSION_ID;
1533 use std::fs;
1534
1535 fn temp_file(name: &str, content: &str) -> (PathBuf, tempfile::TempDir) {
1536 let dir = tempfile::Builder::new()
1537 .prefix("aft_checkpoint_tests_")
1538 .tempdir()
1539 .expect("create checkpoint temp dir");
1540 let path = dir.path().join(name);
1541 fs::write(&path, content).unwrap();
1542 (path, dir)
1543 }
1544
1545 fn fresh_checkpoint_store(storage: &Path) -> CheckpointStore {
1546 let lock_path = storage
1547 .join("checkpoints")
1548 .join("test-project")
1549 .join("checkpoint.lock");
1550 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1551 store.set_storage_dir_for_harness(storage.to_path_buf(), crate::harness::Harness::Opencode);
1552 store
1553 }
1554
1555 fn checkpoint_store() -> (CheckpointStore, tempfile::TempDir) {
1556 let dir = tempfile::tempdir().unwrap();
1557 (fresh_checkpoint_store(dir.path()), dir)
1558 }
1559
1560 fn checkpoint_file(content: &str) -> CheckpointFile {
1561 let file = tempfile::NamedTempFile::new().unwrap();
1562 fs::write(file.path(), content).unwrap();
1563 CheckpointFile::read(file.path()).unwrap()
1564 }
1565
1566 #[test]
1567 fn create_and_restore_round_trip() {
1568 let (path1, _dir1) = temp_file("cp_rt1.txt", "hello");
1569 let (path2, _dir2) = temp_file("cp_rt2.txt", "world");
1570
1571 let backup_store = BackupStore::new();
1572 let (mut store, _store_dir) = checkpoint_store();
1573
1574 let info = store
1575 .create(
1576 DEFAULT_SESSION_ID,
1577 "snap1",
1578 vec![path1.clone(), path2.clone()],
1579 &backup_store,
1580 )
1581 .unwrap();
1582 assert_eq!(info.name, "snap1");
1583 assert_eq!(info.file_count, 2);
1584
1585 fs::write(&path1, "changed1").unwrap();
1587 fs::write(&path2, "changed2").unwrap();
1588
1589 let info = store.restore(DEFAULT_SESSION_ID, "snap1").unwrap();
1591 assert_eq!(info.file_count, 2);
1592 assert_eq!(fs::read_to_string(&path1).unwrap(), "hello");
1593 assert_eq!(fs::read_to_string(&path2).unwrap(), "world");
1594 }
1595
1596 #[cfg(unix)]
1597 #[test]
1598 fn durable_checkpoint_hydrates_after_restart_with_bytes_and_mode() {
1599 use std::os::unix::fs::PermissionsExt;
1600
1601 let files = tempfile::tempdir().unwrap();
1602 let path = files.path().join("durable-mode.bin");
1603 let original = b"draft decision\n\0byte exact\n";
1604 fs::write(&path, original).unwrap();
1605 let mut mode = fs::metadata(&path).unwrap().permissions();
1606 mode.set_mode(0o600);
1607 fs::set_permissions(&path, mode).unwrap();
1608
1609 let backup_store = BackupStore::new();
1610 let (mut first, storage) = checkpoint_store();
1611 let info = first
1612 .create(
1613 DEFAULT_SESSION_ID,
1614 "restart-mode",
1615 vec![path.clone()],
1616 &backup_store,
1617 )
1618 .unwrap();
1619 let durable_path = info.storage_path.expect("durable checkpoint path");
1620 assert!(durable_path.join("meta.json").is_file());
1621 assert!(
1622 fs::read_dir(&durable_path)
1623 .unwrap()
1624 .flatten()
1625 .any(|entry| entry.path().extension().is_some_and(|ext| ext == "blob")),
1626 "checkpoint must persist one or more file blobs"
1627 );
1628
1629 fs::write(&path, b"mutated\n").unwrap();
1630 let mut changed_mode = fs::metadata(&path).unwrap().permissions();
1631 changed_mode.set_mode(0o644);
1632 fs::set_permissions(&path, changed_mode).unwrap();
1633 drop(first);
1634
1635 let mut restarted = fresh_checkpoint_store(storage.path());
1636 let listed = restarted.list(DEFAULT_SESSION_ID).unwrap();
1637 assert_eq!(
1638 listed.len(),
1639 1,
1640 "fresh store must hydrate durable checkpoint"
1641 );
1642 assert_eq!(listed[0].name, "restart-mode");
1643 restarted
1644 .restore(DEFAULT_SESSION_ID, "restart-mode")
1645 .unwrap();
1646
1647 assert_eq!(fs::read(&path).unwrap(), original);
1648 assert_eq!(
1649 fs::metadata(&path).unwrap().permissions().mode() & 0o777,
1650 0o600
1651 );
1652 }
1653
1654 #[cfg(unix)]
1655 #[test]
1656 fn durable_checkpoint_hydrates_symlink_without_following_target() {
1657 let files = tempfile::tempdir().unwrap();
1658 let target = files.path().join("target.txt");
1659 let link = files.path().join("link.txt");
1660 fs::write(&target, "target content").unwrap();
1661 std::os::unix::fs::symlink(&target, &link).unwrap();
1662
1663 let backup_store = BackupStore::new();
1664 let (mut first, storage) = checkpoint_store();
1665 first
1666 .create(
1667 DEFAULT_SESSION_ID,
1668 "restart-symlink",
1669 vec![link.clone()],
1670 &backup_store,
1671 )
1672 .unwrap();
1673 fs::remove_file(&link).unwrap();
1674 fs::write(&link, "plain replacement").unwrap();
1675 drop(first);
1676
1677 let mut restarted = fresh_checkpoint_store(storage.path());
1678 restarted
1679 .restore(DEFAULT_SESSION_ID, "restart-symlink")
1680 .unwrap();
1681 assert!(fs::symlink_metadata(&link)
1682 .unwrap()
1683 .file_type()
1684 .is_symlink());
1685 assert_eq!(fs::read_link(&link).unwrap(), target);
1686 assert_eq!(fs::read_to_string(&target).unwrap(), "target content");
1687 }
1688
1689 #[test]
1690 fn checkpoint_retention_evicts_oldest_name_from_memory_and_disk() {
1691 let (path, _files) = temp_file("retention.txt", "version-0");
1692 let backup_store = BackupStore::new();
1693 let (mut store, storage) = checkpoint_store();
1694
1695 for index in 0..=MAX_NAMED_CHECKPOINTS_PER_SESSION {
1696 fs::write(&path, format!("version-{index}")).unwrap();
1697 let info = store
1698 .create(
1699 DEFAULT_SESSION_ID,
1700 &format!("checkpoint-{index:02}"),
1701 vec![path.clone()],
1702 &backup_store,
1703 )
1704 .unwrap();
1705 if index == MAX_NAMED_CHECKPOINTS_PER_SESSION {
1706 assert_eq!(info.evicted, vec!["checkpoint-00"]);
1707 } else {
1708 assert!(info.evicted.is_empty());
1709 }
1710 }
1711
1712 let listed = store.list(DEFAULT_SESSION_ID).unwrap();
1713 assert_eq!(listed.len(), MAX_NAMED_CHECKPOINTS_PER_SESSION);
1714 assert!(listed.iter().all(|info| info.name != "checkpoint-00"));
1715 let old_dir = storage
1716 .path()
1717 .join("opencode")
1718 .join("checkpoints")
1719 .join(hash_session(DEFAULT_SESSION_ID))
1720 .join("checkpoint-00");
1721 assert!(!old_dir.exists(), "evicted checkpoint must leave disk too");
1722 }
1723
1724 #[test]
1725 fn hydration_finishes_retention_after_interrupted_create() {
1726 let (path, _files) = temp_file("interrupted-retention.txt", "checkpoint content");
1727 let backup_store = BackupStore::new();
1728 let (mut first, storage) = checkpoint_store();
1729
1730 for index in 0..MAX_NAMED_CHECKPOINTS_PER_SESSION {
1731 first
1732 .create(
1733 DEFAULT_SESSION_ID,
1734 &format!("checkpoint-{index:02}"),
1735 vec![path.clone()],
1736 &backup_store,
1737 )
1738 .unwrap();
1739 }
1740
1741 let checkpoint = Checkpoint {
1745 name: "checkpoint-20".to_string(),
1746 file_contents: HashMap::from([(path, checkpoint_file("newest"))]),
1747 created_at: current_timestamp(),
1748 created_order: u64::MAX,
1749 };
1750 {
1751 let _lock = first.acquire_mutation_lock().unwrap();
1752 first
1753 .persist_checkpoint_locked(DEFAULT_SESSION_ID, &checkpoint)
1754 .unwrap();
1755 }
1756 drop(first);
1757
1758 let session_dir = storage
1759 .path()
1760 .join("opencode")
1761 .join("checkpoints")
1762 .join(hash_session(DEFAULT_SESSION_ID));
1763 assert_eq!(fs::read_dir(&session_dir).unwrap().count(), 21);
1764
1765 let mut restarted = fresh_checkpoint_store(storage.path());
1766 let listed = restarted.list(DEFAULT_SESSION_ID).unwrap();
1767 assert_eq!(listed.len(), MAX_NAMED_CHECKPOINTS_PER_SESSION);
1768 assert!(listed.iter().all(|info| info.name != "checkpoint-00"));
1769 assert!(
1770 !session_dir.join("checkpoint-00").exists(),
1771 "hydration must finish interrupted retention on disk"
1772 );
1773 }
1774
1775 #[test]
1776 fn cleanup_refuses_to_sweep_scope_dirs_outside_a_checkpoints_root() {
1777 let temp_root = tempfile::tempdir().expect("temp root");
1784 let victim = temp_root.path().join("innocent-empty-sibling");
1787 fs::create_dir(&victim).expect("victim dir");
1788 let scope_dir = temp_root.path().join("scope");
1789 fs::create_dir(&scope_dir).expect("scope dir");
1790 let lock_path = scope_dir.join("checkpoint.lock");
1791 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1792 store.cleanup_locked().expect("cleanup");
1793 assert!(
1794 victim.exists(),
1795 "cleanup must not sweep empty dirs outside a `checkpoints` root"
1796 );
1797 }
1798
1799 #[test]
1800 fn cleanup_sweeps_durable_checkpoints_older_than_fourteen_days() {
1801 let (path, _files) = temp_file("durable-gc.txt", "original");
1802 let backup_store = BackupStore::new();
1803 let (mut store, _storage) = checkpoint_store();
1804 let info = store
1805 .create(
1806 DEFAULT_SESSION_ID,
1807 "expired-durable",
1808 vec![path],
1809 &backup_store,
1810 )
1811 .unwrap();
1812 let durable_path = info.storage_path.unwrap();
1813 let meta_path = durable_path.join("meta.json");
1814 let mut meta: DiskCheckpointMeta =
1815 serde_json::from_slice(&fs::read(&meta_path).unwrap()).unwrap();
1816 meta.created_at = current_timestamp()
1817 .saturating_sub(NAMED_CHECKPOINT_RETENTION_SECS)
1818 .saturating_sub(1);
1819 fs::write(&meta_path, serde_json::to_vec_pretty(&meta).unwrap()).unwrap();
1820
1821 store.cleanup();
1822 assert!(
1823 !durable_path.exists(),
1824 "fourteen-day cleanup must remove the durable checkpoint directory"
1825 );
1826 }
1827
1828 #[test]
1829 fn durable_hydration_fails_when_a_referenced_blob_is_missing() {
1830 let (path, _files) = temp_file("hydration-control.txt", "original");
1831 let backup_store = BackupStore::new();
1832 let (mut first, storage) = checkpoint_store();
1833 let info = first
1834 .create(
1835 DEFAULT_SESSION_ID,
1836 "hydration-control",
1837 vec![path],
1838 &backup_store,
1839 )
1840 .unwrap();
1841 let durable_path = info.storage_path.unwrap();
1842 let meta: DiskCheckpointMeta =
1843 serde_json::from_slice(&fs::read(durable_path.join("meta.json")).unwrap()).unwrap();
1844 fs::remove_file(durable_path.join(&meta.files[0].blob)).unwrap();
1845 drop(first);
1846
1847 let mut restarted = fresh_checkpoint_store(storage.path());
1848 let error = restarted.list(DEFAULT_SESSION_ID).unwrap_err();
1849 match error {
1850 AftError::IoError { message, .. } => {
1851 assert!(message.contains("failed to read durable checkpoint blob"));
1852 }
1853 other => panic!("expected durable hydration I/O error, got {other:?}"),
1854 }
1855 }
1856
1857 #[test]
1858 fn overwrite_existing_name() {
1859 let (path, _dir) = temp_file("cp_overwrite.txt", "v1");
1860 let backup_store = BackupStore::new();
1861 let (mut store, _store_dir) = checkpoint_store();
1862
1863 store
1864 .create(DEFAULT_SESSION_ID, "dup", vec![path.clone()], &backup_store)
1865 .unwrap();
1866 fs::write(&path, "v2").unwrap();
1867 store
1868 .create(DEFAULT_SESSION_ID, "dup", vec![path.clone()], &backup_store)
1869 .unwrap();
1870
1871 fs::write(&path, "v3").unwrap();
1873 store.restore(DEFAULT_SESSION_ID, "dup").unwrap();
1874 assert_eq!(fs::read_to_string(&path).unwrap(), "v2");
1875 }
1876
1877 #[test]
1878 fn list_returns_metadata_scoped_to_session() {
1879 let (path, _dir) = temp_file("cp_list.txt", "data");
1880 let backup_store = BackupStore::new();
1881 let (mut store, _store_dir) = checkpoint_store();
1882
1883 store
1884 .create(DEFAULT_SESSION_ID, "a", vec![path.clone()], &backup_store)
1885 .unwrap();
1886 store
1887 .create(DEFAULT_SESSION_ID, "b", vec![path.clone()], &backup_store)
1888 .unwrap();
1889 store
1890 .create("other_session", "c", vec![path.clone()], &backup_store)
1891 .unwrap();
1892
1893 let default_list = store.list(DEFAULT_SESSION_ID).unwrap();
1894 assert_eq!(default_list.len(), 2);
1895 let names: Vec<&str> = default_list.iter().map(|i| i.name.as_str()).collect();
1896 assert!(names.contains(&"a"));
1897 assert!(names.contains(&"b"));
1898
1899 let other_list = store.list("other_session").unwrap();
1900 assert_eq!(other_list.len(), 1);
1901 assert_eq!(other_list[0].name, "c");
1902 }
1903
1904 #[test]
1905 fn sessions_isolate_checkpoint_names() {
1906 let (path_a, _dir_a) = temp_file("cp_isolated_a.txt", "a-original");
1908 let (path_b, _dir_b) = temp_file("cp_isolated_b.txt", "b-original");
1909 let backup_store = BackupStore::new();
1910 let (mut store, _store_dir) = checkpoint_store();
1911
1912 store
1914 .create("session_a", "snap", vec![path_a.clone()], &backup_store)
1915 .unwrap();
1916 store
1917 .create("session_b", "snap", vec![path_b.clone()], &backup_store)
1918 .unwrap();
1919
1920 fs::write(&path_a, "a-modified").unwrap();
1921 fs::write(&path_b, "b-modified").unwrap();
1922
1923 store.restore("session_a", "snap").unwrap();
1925 assert_eq!(fs::read_to_string(&path_a).unwrap(), "a-original");
1926 assert_eq!(fs::read_to_string(&path_b).unwrap(), "b-modified");
1927
1928 fs::write(&path_a, "a-modified").unwrap();
1930 store.restore("session_b", "snap").unwrap();
1931 assert_eq!(fs::read_to_string(&path_a).unwrap(), "a-modified");
1932 assert_eq!(fs::read_to_string(&path_b).unwrap(), "b-original");
1933 }
1934
1935 #[test]
1936 fn checkpoint_lock_scope_is_removed_after_release() {
1937 let dir = tempfile::tempdir().unwrap();
1938 let scope_dir = dir.path().join("checkpoints").join("project-scope");
1939 let lock_path = scope_dir.join("checkpoint.lock");
1940 let path = dir.path().join("checkpoint.txt");
1941 fs::write(&path, "data").unwrap();
1942 let backup_store = BackupStore::new();
1943 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1944
1945 store
1946 .create(DEFAULT_SESSION_ID, "released", vec![path], &backup_store)
1947 .unwrap();
1948
1949 assert!(!scope_dir.exists(), "released lock scope should be removed");
1950 }
1951
1952 #[test]
1953 fn cleanup_removes_expired_across_sessions() {
1954 let (path, _dir) = temp_file("cp_cleanup.txt", "data");
1955 let backup_store = BackupStore::new();
1956 let (mut store, _store_dir) = checkpoint_store();
1957
1958 store
1959 .create(
1960 DEFAULT_SESSION_ID,
1961 "recent",
1962 vec![path.clone()],
1963 &backup_store,
1964 )
1965 .unwrap();
1966
1967 store
1969 .checkpoints
1970 .entry("other".to_string())
1971 .or_default()
1972 .insert(
1973 "old".to_string(),
1974 Checkpoint {
1975 name: "old".to_string(),
1976 file_contents: HashMap::new(),
1977 created_at: 1000, created_order: 1000,
1979 },
1980 );
1981
1982 assert_eq!(store.total_count(), 2);
1983 store.cleanup();
1984 assert_eq!(store.total_count(), 1);
1985 assert_eq!(store.list(DEFAULT_SESSION_ID).unwrap()[0].name, "recent");
1986 assert!(store.list("other").unwrap().is_empty());
1987 }
1988
1989 #[test]
1990 fn cleanup_sweeps_empty_scope_dirs_but_keeps_live_lock_scope() {
1991 let dir = tempfile::tempdir().unwrap();
1992 let checkpoints_root = dir.path().join("checkpoints");
1993 let empty_a = checkpoints_root.join("empty-a");
1994 let empty_b = checkpoints_root.join("empty-b");
1995 let live_scope = checkpoints_root.join("live-scope");
1996 fs::create_dir_all(&empty_a).unwrap();
1997 fs::create_dir_all(&empty_b).unwrap();
1998 fs::create_dir_all(&live_scope).unwrap();
1999 fs::write(live_scope.join("checkpoint.lock"), "live lock").unwrap();
2000
2001 let lock_path = checkpoints_root
2002 .join("current-scope")
2003 .join("checkpoint.lock");
2004 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
2005 store.cleanup();
2006
2007 assert!(!empty_a.exists());
2008 assert!(!empty_b.exists());
2009 assert!(live_scope.is_dir());
2010 assert!(live_scope.join("checkpoint.lock").is_file());
2011 }
2012
2013 #[test]
2014 fn cleanup_ignores_non_empty_scope_dir_removal_failure() {
2015 let dir = tempfile::tempdir().unwrap();
2016 let checkpoints_root = dir.path().join("checkpoints");
2017 let scope_dir = checkpoints_root.join("racing-scope");
2018 fs::create_dir_all(&scope_dir).unwrap();
2019 fs::write(scope_dir.join("checkpoint.lock"), "lock appeared").unwrap();
2022
2023 let lock_path = checkpoints_root
2024 .join("current-scope")
2025 .join("checkpoint.lock");
2026 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
2027 store.cleanup();
2028
2029 assert!(scope_dir.is_dir());
2030 assert!(scope_dir.join("checkpoint.lock").is_file());
2031 }
2032
2033 #[test]
2034 fn restore_nonexistent_returns_error() {
2035 let (mut store, _store_dir) = checkpoint_store();
2036 let result = store.restore(DEFAULT_SESSION_ID, "nope");
2037 assert!(result.is_err());
2038 match result.unwrap_err() {
2039 AftError::CheckpointNotFound { name } => {
2040 assert_eq!(name, "nope");
2041 }
2042 other => panic!("expected CheckpointNotFound, got: {:?}", other),
2043 }
2044 }
2045
2046 #[test]
2047 fn restore_nonexistent_in_other_session_returns_error() {
2048 let (path, _dir) = temp_file("cp_cross_session.txt", "data");
2050 let backup_store = BackupStore::new();
2051 let (mut store, _store_dir) = checkpoint_store();
2052 store
2053 .create("session_a", "only_a", vec![path], &backup_store)
2054 .unwrap();
2055 assert!(store.restore("session_b", "only_a").is_err());
2056 }
2057
2058 #[test]
2059 fn create_skips_missing_files_from_backup_tracked_set() {
2060 let (readable, _readable_dir) = temp_file("cp_skip_readable.txt", "still_here");
2066 let (deleted, _deleted_dir) = temp_file("cp_skip_deleted.txt", "about_to_vanish");
2067
2068 let deleted_canonical = fs::canonicalize(&deleted).unwrap();
2071
2072 let mut backup_store = BackupStore::new();
2073 backup_store
2074 .snapshot(DEFAULT_SESSION_ID, &readable, "auto")
2075 .unwrap();
2076 backup_store
2077 .snapshot(DEFAULT_SESSION_ID, &deleted, "auto")
2078 .unwrap();
2079
2080 fs::remove_file(&deleted).unwrap();
2081
2082 let (mut store, _store_dir) = checkpoint_store();
2083 let info = store
2084 .create(DEFAULT_SESSION_ID, "partial", vec![], &backup_store)
2085 .expect("checkpoint should succeed despite one missing file");
2086 assert_eq!(info.file_count, 1);
2087 assert_eq!(info.skipped.len(), 1);
2088 assert_eq!(info.skipped[0].0, deleted_canonical);
2089 assert!(!info.skipped[0].1.is_empty());
2090 }
2091
2092 #[test]
2093 fn create_with_explicit_single_missing_file_errors() {
2094 let dir = tempfile::tempdir().unwrap();
2097 let missing = dir.path().join("cp_explicit_missing_does_not_exist.txt");
2098
2099 let backup_store = BackupStore::new();
2100 let (mut store, _store_dir) = checkpoint_store();
2101 let result = store.create(
2102 DEFAULT_SESSION_ID,
2103 "explicit",
2104 vec![missing.clone()],
2105 &backup_store,
2106 );
2107
2108 assert!(result.is_err());
2109 match result.unwrap_err() {
2110 AftError::FileNotFound { path } => {
2111 assert!(path.contains(&missing.display().to_string()));
2112 }
2113 other => panic!("expected FileNotFound, got: {:?}", other),
2114 }
2115 }
2116
2117 #[test]
2118 fn create_with_explicit_mixed_files_keeps_readable_and_reports_skipped() {
2119 let (good, _good_dir) = temp_file("cp_mixed_good.txt", "ok");
2123 let missing_dir = tempfile::tempdir().unwrap();
2124 let missing = missing_dir.path().join("cp_mixed_missing.txt");
2125
2126 let backup_store = BackupStore::new();
2127 let (mut store, _store_dir) = checkpoint_store();
2128 let info = store
2129 .create(
2130 DEFAULT_SESSION_ID,
2131 "mixed",
2132 vec![good.clone(), missing.clone()],
2133 &backup_store,
2134 )
2135 .expect("mixed checkpoint should succeed when any file is readable");
2136 assert_eq!(info.file_count, 1);
2137 assert_eq!(info.skipped.len(), 1);
2138 assert_eq!(info.skipped[0].0, missing);
2139 }
2140
2141 #[test]
2142 fn create_with_empty_files_uses_backup_tracked() {
2143 let (path, _dir) = temp_file("cp_tracked.txt", "tracked_content");
2144 let mut backup_store = BackupStore::new();
2145 backup_store
2146 .snapshot(DEFAULT_SESSION_ID, &path, "auto")
2147 .unwrap();
2148
2149 let (mut store, _store_dir) = checkpoint_store();
2150 let info = store
2151 .create(DEFAULT_SESSION_ID, "from_tracked", vec![], &backup_store)
2152 .unwrap();
2153 assert!(info.file_count >= 1);
2154
2155 fs::write(&path, "modified").unwrap();
2157 store.restore(DEFAULT_SESSION_ID, "from_tracked").unwrap();
2158 assert_eq!(fs::read_to_string(&path).unwrap(), "tracked_content");
2159 }
2160
2161 #[test]
2162 fn restore_recreates_missing_parent_directories() {
2163 let dir = tempfile::tempdir().unwrap();
2164 let path = dir.path().join("nested").join("deeper").join("file.txt");
2165 fs::create_dir_all(path.parent().unwrap()).unwrap();
2166 fs::write(&path, "original nested content").unwrap();
2167
2168 let backup_store = BackupStore::new();
2169 let (mut store, _store_dir) = checkpoint_store();
2170 store
2171 .create(
2172 DEFAULT_SESSION_ID,
2173 "nested",
2174 vec![path.clone()],
2175 &backup_store,
2176 )
2177 .unwrap();
2178
2179 fs::remove_dir_all(dir.path().join("nested")).unwrap();
2180
2181 store.restore(DEFAULT_SESSION_ID, "nested").unwrap();
2182 assert_eq!(
2183 fs::read_to_string(&path).unwrap(),
2184 "original nested content"
2185 );
2186 }
2187
2188 #[cfg(unix)]
2189 #[test]
2190 fn checkpoint_restore_rolls_back_on_partial_failure() {
2191 use std::os::unix::fs::PermissionsExt;
2192
2193 let dir = tempfile::tempdir().unwrap();
2194 let path_a = dir.path().join("a.txt");
2195 let path_b = dir.path().join("b.txt");
2196 fs::write(&path_a, "checkpoint-a").unwrap();
2197 fs::write(&path_b, "checkpoint-b").unwrap();
2198
2199 let backup_store = BackupStore::new();
2200 let (mut store, _store_dir) = checkpoint_store();
2201 store
2202 .create(
2203 DEFAULT_SESSION_ID,
2204 "partial_failure",
2205 vec![path_a.clone(), path_b.clone()],
2206 &backup_store,
2207 )
2208 .unwrap();
2209
2210 fs::write(&path_a, "pre-restore-a").unwrap();
2211 fs::write(&path_b, "pre-restore-b").unwrap();
2212 let mut readonly = fs::metadata(&path_b).unwrap().permissions();
2213 readonly.set_mode(0o444);
2214 fs::set_permissions(&path_b, readonly).unwrap();
2215
2216 let result = store.restore(DEFAULT_SESSION_ID, "partial_failure");
2217 let mut writable = fs::metadata(&path_b).unwrap().permissions();
2218 writable.set_mode(0o644);
2219 fs::set_permissions(&path_b, writable).unwrap();
2220
2221 assert!(result.is_err(), "restore should surface write failure");
2222 assert_eq!(fs::read_to_string(&path_a).unwrap(), "pre-restore-a");
2223 assert_eq!(fs::read_to_string(&path_b).unwrap(), "pre-restore-b");
2224 }
2225
2226 #[test]
2227 fn checkpoint_create_and_restore_use_mutation_lock() {
2228 let dir = tempfile::tempdir().unwrap();
2229 let lock_path = dir.path().join("locks").join("checkpoint.lock");
2230 fs::create_dir_all(lock_path.parent().unwrap()).unwrap();
2231 let mut store =
2232 CheckpointStore::with_lock_path(lock_path.clone(), Duration::from_millis(50));
2233 let backup_store = BackupStore::new();
2234 let path = dir.path().join("locked.txt");
2235 fs::write(&path, "original").unwrap();
2236
2237 let held_lock =
2238 fs_lock::try_acquire(&lock_path, Duration::from_secs(1)).expect("hold checkpoint lock");
2239 let create_result = store.create(
2240 DEFAULT_SESSION_ID,
2241 "locked",
2242 vec![path.clone()],
2243 &backup_store,
2244 );
2245 assert!(matches!(create_result, Err(AftError::IoError { .. })));
2246 drop(held_lock);
2247
2248 store
2249 .create(
2250 DEFAULT_SESSION_ID,
2251 "locked",
2252 vec![path.clone()],
2253 &backup_store,
2254 )
2255 .unwrap();
2256 fs::write(&path, "changed").unwrap();
2257
2258 fs::create_dir_all(lock_path.parent().unwrap()).unwrap();
2259 let held_lock =
2260 fs_lock::try_acquire(&lock_path, Duration::from_secs(1)).expect("hold checkpoint lock");
2261 let restore_result = store.restore(DEFAULT_SESSION_ID, "locked");
2262 assert!(matches!(restore_result, Err(AftError::IoError { .. })));
2263 drop(held_lock);
2264
2265 store.restore(DEFAULT_SESSION_ID, "locked").unwrap();
2266 assert_eq!(fs::read_to_string(&path).unwrap(), "original");
2267 }
2268
2269 #[cfg(unix)]
2270 #[test]
2271 fn checkpoint_restore_preserves_regular_file_permissions() {
2272 use std::os::unix::fs::PermissionsExt;
2273
2274 let dir = tempfile::tempdir().unwrap();
2275 let path = dir.path().join("mode.txt");
2276 fs::write(&path, "original").unwrap();
2277 let mut original_permissions = fs::metadata(&path).unwrap().permissions();
2278 original_permissions.set_mode(0o600);
2279 fs::set_permissions(&path, original_permissions).unwrap();
2280
2281 let backup_store = BackupStore::new();
2282 let (mut store, _store_dir) = checkpoint_store();
2283 store
2284 .create(
2285 DEFAULT_SESSION_ID,
2286 "mode",
2287 vec![path.clone()],
2288 &backup_store,
2289 )
2290 .unwrap();
2291
2292 fs::write(&path, "changed").unwrap();
2293 let mut changed_permissions = fs::metadata(&path).unwrap().permissions();
2294 changed_permissions.set_mode(0o644);
2295 fs::set_permissions(&path, changed_permissions).unwrap();
2296
2297 store.restore(DEFAULT_SESSION_ID, "mode").unwrap();
2298
2299 assert_eq!(fs::read_to_string(&path).unwrap(), "original");
2300 let restored_mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
2301 assert_eq!(restored_mode, 0o600);
2302 }
2303
2304 #[cfg(unix)]
2305 #[test]
2306 fn checkpoint_restore_recreates_symlink() {
2307 let dir = tempfile::tempdir().unwrap();
2308 let target = dir.path().join("target.txt");
2309 let link = dir.path().join("link.txt");
2310 fs::write(&target, "target content").unwrap();
2311 std::os::unix::fs::symlink(&target, &link).unwrap();
2312
2313 let backup_store = BackupStore::new();
2314 let (mut store, _store_dir) = checkpoint_store();
2315 store
2316 .create(
2317 DEFAULT_SESSION_ID,
2318 "symlink",
2319 vec![link.clone()],
2320 &backup_store,
2321 )
2322 .unwrap();
2323
2324 fs::remove_file(&link).unwrap();
2325 fs::write(&link, "plain file").unwrap();
2326
2327 store.restore(DEFAULT_SESSION_ID, "symlink").unwrap();
2328
2329 assert!(fs::symlink_metadata(&link)
2330 .unwrap()
2331 .file_type()
2332 .is_symlink());
2333 assert_eq!(fs::read_link(&link).unwrap(), target);
2334 assert_eq!(fs::read_to_string(&link).unwrap(), "target content");
2335 }
2336
2337 #[test]
2338 fn captured_regular_file_is_shared_by_checkpoint_and_backup() {
2339 let (path, _dir) = temp_file("shared-capture.txt", "original bytes");
2340 crate::backup::reset_capture_read_count(&path);
2341 let mut capture = CapturedRegularFile::read(&path).unwrap().unwrap();
2342 assert_eq!(crate::backup::capture_read_count(&path), 1);
2343
2344 let checkpoint = CheckpointFile::from_captured(&path, &mut capture).unwrap();
2345 let mut backup = BackupStore::new();
2346 backup
2347 .snapshot_with_op_from_capture(
2348 DEFAULT_SESSION_ID,
2349 &path,
2350 "shared capture",
2351 Some("shared-op"),
2352 &capture,
2353 )
2354 .unwrap();
2355
2356 let history = backup.history(DEFAULT_SESSION_ID, &path);
2357 let CheckpointFileKind::Regular { bytes } = checkpoint.kind else {
2358 panic!("regular capture must create a regular checkpoint");
2359 };
2360 assert_eq!(bytes.as_ref(), b"original bytes");
2361 assert_eq!(history[0].content_bytes.as_ref(), b"original bytes");
2362 assert!(Arc::ptr_eq(&bytes, &history[0].content_bytes));
2363 assert_eq!(crate::backup::capture_read_count(&path), 1);
2364 }
2365
2366 #[test]
2367 fn stale_capture_refreshes_before_checkpoint_and_backup() {
2368 let (path, _dir) = temp_file("stale-capture.txt", "old");
2369 crate::backup::reset_capture_read_count(&path);
2370 let mut capture = CapturedRegularFile::read(&path).unwrap().unwrap();
2371 fs::write(&path, "fresh disk truth").unwrap();
2372
2373 let checkpoint = CheckpointFile::from_captured(&path, &mut capture).unwrap();
2374 let mut backup = BackupStore::new();
2375 backup
2376 .snapshot_with_op_from_capture(
2377 DEFAULT_SESSION_ID,
2378 &path,
2379 "freshened capture",
2380 Some("fresh-op"),
2381 &capture,
2382 )
2383 .unwrap();
2384
2385 let history = backup.history(DEFAULT_SESSION_ID, &path);
2386 let CheckpointFileKind::Regular { bytes } = checkpoint.kind else {
2387 panic!("regular capture must create a regular checkpoint");
2388 };
2389 assert_eq!(bytes.as_ref(), b"fresh disk truth");
2390 assert_eq!(history[0].content_bytes.as_ref(), b"fresh disk truth");
2391 assert!(Arc::ptr_eq(&bytes, &history[0].content_bytes));
2392 assert_eq!(crate::backup::capture_read_count(&path), 2);
2393 }
2394
2395 #[test]
2396 fn checkpoint_restore_failure_removes_created_parent_dirs() {
2397 let dir = tempfile::tempdir().unwrap();
2398 let missing_root = dir.path().join("created");
2399 let path_a = missing_root.join("nested").join("a.txt");
2400 let path_b = dir.path().join("blocking-dir");
2401 fs::create_dir(&path_b).unwrap();
2402
2403 let checkpoint = Checkpoint {
2404 name: "dir-cleanup".to_string(),
2405 file_contents: HashMap::from([
2406 (path_a.clone(), checkpoint_file("checkpoint-a")),
2407 (path_b.clone(), checkpoint_file("checkpoint-b")),
2408 ]),
2409 created_at: current_timestamp(),
2410 created_order: current_timestamp_nanos(),
2411 };
2412
2413 let result = restore_paths_atomically(&checkpoint, &[path_a.clone(), path_b.clone()]);
2414
2415 assert!(
2416 result.is_err(),
2417 "second restore write should fail on directory"
2418 );
2419 assert!(!path_a.exists(), "restored file should be rolled back");
2420 assert!(
2421 !missing_root.exists(),
2422 "new parent directories should be removed on rollback"
2423 );
2424 assert!(path_b.is_dir(), "pre-existing blocking directory remains");
2425 }
2426}