1use std::collections::{HashMap, HashSet};
2use std::fs;
3use std::io::{self, Write};
4use std::path::{Path, PathBuf};
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, LazyLock, Mutex};
7use std::time::Duration;
8
9use serde::{Deserialize, Serialize};
10
11use crate::backup::{hash_session, BackupStore, CapturedRegularFile};
12use crate::error::AftError;
13use crate::fs_lock;
14
15const CHECKPOINT_LOCK_TIMEOUT: Duration = Duration::from_secs(30);
16
17const MAX_NAMED_CHECKPOINTS_PER_SESSION: usize = 20;
20const NAMED_CHECKPOINT_RETENTION_DAYS: u64 = 14;
23const NAMED_CHECKPOINT_RETENTION_SECS: u64 = NAMED_CHECKPOINT_RETENTION_DAYS * 24 * 60 * 60;
24const CHECKPOINT_SCHEMA_VERSION: u32 = 1;
25const UNBOUND_HARNESS_SEGMENT: &str = "unbound";
26
27static CHECKPOINT_MAINTENANCE_KEYS: LazyLock<Mutex<HashSet<(PathBuf, String)>>> =
28 LazyLock::new(|| Mutex::new(HashSet::new()));
29
30pub const CHECKPOINT_RESTART_NOTICE: &str =
32 "no durable checkpoints found on disk; in-memory checkpoints do not survive restarts";
33pub const CHECKPOINT_HYDRATED_NOTICE: &str =
35 "durable checkpoints are hydrated from disk and survive restarts";
36
37pub fn checkpoint_durability(storage_path: &Path) -> String {
39 format!(
40 "durable on disk at {}; survives restarts",
41 storage_path.display()
42 )
43}
44
45#[derive(Debug, Clone)]
47pub struct CheckpointInfo {
48 pub name: String,
49 pub file_count: usize,
50 pub created_at: u64,
51 pub storage_path: Option<PathBuf>,
53 pub evicted: Vec<String>,
55 pub skipped: Vec<(PathBuf, String)>,
60}
61
62#[derive(Debug, Clone)]
64struct Checkpoint {
65 name: String,
66 file_contents: HashMap<PathBuf, CheckpointFile>,
67 created_at: u64,
68 created_order: u64,
71}
72
73#[derive(Debug, Clone)]
74struct CheckpointFile {
75 metadata: Option<fs::Metadata>,
78 mode: Option<u32>,
79 kind: CheckpointFileKind,
80}
81
82#[derive(Debug, Serialize, Deserialize)]
83struct DiskCheckpointMeta {
84 schema_version: u32,
85 session_id: String,
86 name: String,
87 created_at: u64,
88 created_order: u64,
89 files: Vec<DiskCheckpointFileMeta>,
90}
91
92#[derive(Debug, Serialize, Deserialize)]
93struct DiskCheckpointFileMeta {
94 original_path: String,
95 blob: String,
96 kind: DiskCheckpointFileKind,
97 mode: Option<u32>,
98 target_is_dir: bool,
99}
100
101#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
102#[serde(rename_all = "snake_case")]
103enum DiskCheckpointFileKind {
104 Regular,
105 Symlink,
106}
107
108#[derive(Debug, Clone)]
109enum CheckpointFileKind {
110 Regular {
111 bytes: Arc<[u8]>,
112 },
113 Symlink {
114 target: PathBuf,
115 target_is_dir: bool,
116 },
117}
118
119impl CheckpointFile {
120 fn read(path: &Path) -> io::Result<Self> {
121 let metadata = fs::symlink_metadata(path)?;
122 let file_type = metadata.file_type();
123 if file_type.is_symlink() {
124 let target = fs::read_link(path)?;
125 let target_is_dir = fs::metadata(path)
126 .map(|target_metadata| target_metadata.is_dir())
127 .unwrap_or(false);
128 return Ok(Self {
129 mode: checkpoint_mode(&metadata),
130 metadata: Some(metadata),
131 kind: CheckpointFileKind::Symlink {
132 target,
133 target_is_dir,
134 },
135 });
136 }
137
138 if metadata.is_file() {
139 let capture = CapturedRegularFile::read(path)?.ok_or_else(|| {
140 io::Error::new(
141 io::ErrorKind::InvalidInput,
142 "file changed while being captured",
143 )
144 })?;
145 return Ok(Self::from_fresh_capture(capture));
146 }
147
148 Err(io::Error::new(
149 io::ErrorKind::InvalidInput,
150 "not a regular file or symlink",
151 ))
152 }
153
154 fn from_captured(path: &Path, capture: &mut CapturedRegularFile) -> io::Result<Self> {
161 capture.refresh_if_stale(path)?;
162 let metadata = capture.metadata().clone();
163 Ok(Self {
164 mode: checkpoint_mode(&metadata),
165 metadata: Some(metadata),
166 kind: CheckpointFileKind::Regular {
167 bytes: capture.shared_bytes(),
168 },
169 })
170 }
171
172 fn from_fresh_capture(capture: CapturedRegularFile) -> Self {
173 let metadata = capture.metadata().clone();
174 Self {
175 mode: checkpoint_mode(&metadata),
176 metadata: Some(metadata),
177 kind: CheckpointFileKind::Regular {
178 bytes: capture.shared_bytes(),
179 },
180 }
181 }
182
183 fn read_optional(path: &Path) -> io::Result<Option<Self>> {
184 match Self::read(path) {
185 Ok(snapshot) => Ok(Some(snapshot)),
186 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
187 Err(error) => Err(error),
188 }
189 }
190
191 fn from_disk(meta: &DiskCheckpointFileMeta, bytes: Vec<u8>) -> Result<Self, String> {
192 let kind = match &meta.kind {
193 DiskCheckpointFileKind::Regular => CheckpointFileKind::Regular {
194 bytes: bytes.into(),
195 },
196 DiskCheckpointFileKind::Symlink => {
197 let target = String::from_utf8(bytes)
198 .map(PathBuf::from)
199 .map_err(|error| format!("checkpoint symlink target is not UTF-8: {error}"))?;
200 CheckpointFileKind::Symlink {
201 target,
202 target_is_dir: meta.target_is_dir,
203 }
204 }
205 };
206 Ok(Self {
207 metadata: None,
208 mode: meta.mode,
209 kind,
210 })
211 }
212}
213
214#[cfg(unix)]
215fn checkpoint_mode(metadata: &fs::Metadata) -> Option<u32> {
216 use std::os::unix::fs::PermissionsExt;
217 Some(metadata.permissions().mode())
218}
219
220#[cfg(not(unix))]
221fn checkpoint_mode(_metadata: &fs::Metadata) -> Option<u32> {
222 None
223}
224
225#[derive(Debug)]
233pub struct CheckpointStore {
234 checkpoints: HashMap<String, HashMap<String, Checkpoint>>,
236 lock_path: PathBuf,
237 lock_timeout: Duration,
238 storage_dir: Option<PathBuf>,
239 storage_harness: Option<String>,
240 blob_counter: AtomicU64,
241}
242
243struct CheckpointLockGuard {
247 guard: Option<fs_lock::LockGuard>,
248 scope_dir: Option<PathBuf>,
249}
250
251impl Drop for CheckpointLockGuard {
252 fn drop(&mut self) {
253 if let Some(guard) = self.guard.take() {
257 drop(guard);
258 }
259 if let Some(scope_dir) = &self.scope_dir {
260 remove_empty_scope_dir(scope_dir);
261 }
262 }
263}
264
265impl CheckpointStore {
266 pub fn new() -> Self {
267 let project_root = std::env::current_dir().unwrap_or_else(|_| std::env::temp_dir());
268 let project_key = crate::path_identity::project_scope_key(&project_root);
269 let storage_dir = crate::bash_background::storage_dir(None);
270 let lock_path = storage_dir
271 .join("checkpoints")
272 .join(project_key)
273 .join("checkpoint.lock");
274 let mut store = Self::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
275 store.storage_dir = Some(storage_dir);
278 store.storage_harness = Some(UNBOUND_HARNESS_SEGMENT.to_string());
279 store
280 }
281
282 #[cfg(test)]
286 pub(crate) fn set_lock_path_for_test(&mut self, lock_path: PathBuf) {
287 self.storage_dir = lock_path.parent().map(Path::to_path_buf);
288 self.storage_harness = Some("test".to_string());
289 self.lock_path = lock_path;
290 }
291
292 pub fn set_storage_dir_for_harness(&mut self, dir: PathBuf, harness: crate::harness::Harness) {
295 let harness = harness.storage_segment();
296 if self.storage_dir.as_ref() == Some(&dir)
297 && self.storage_harness.as_deref() == Some(&harness)
298 {
299 return;
300 }
301 if self.storage_dir.as_ref() == Some(&dir)
302 && self.storage_harness.as_deref() == Some(UNBOUND_HARNESS_SEGMENT)
303 {
304 match self.acquire_mutation_lock() {
305 Ok(_lock) => migrate_unbound_checkpoint_namespace(&dir, &harness),
306 Err(error) => crate::slog_warn!(
307 "could not migrate unbound durable checkpoints into {}: {}",
308 harness,
309 error
310 ),
311 }
312 }
313 self.storage_dir = Some(dir);
314 self.storage_harness = Some(harness);
315 self.checkpoints.clear();
316 }
317
318 fn with_lock_path(lock_path: PathBuf, lock_timeout: Duration) -> Self {
319 CheckpointStore {
320 checkpoints: HashMap::new(),
321 lock_path,
322 lock_timeout,
323 storage_dir: None,
324 storage_harness: None,
325 blob_counter: AtomicU64::new(0),
326 }
327 }
328
329 fn acquire_mutation_lock(&self) -> Result<CheckpointLockGuard, AftError> {
330 let scope_dir = self.lock_path.parent().map(Path::to_path_buf);
331 if let Some(parent) = scope_dir.as_deref() {
332 fs::create_dir_all(parent).map_err(|error| AftError::IoError {
333 path: parent.display().to_string(),
334 message: format!("failed to create checkpoint lock directory: {error}"),
335 })?;
336 }
337
338 let acquire_result = match fs_lock::try_acquire(&self.lock_path, self.lock_timeout) {
339 Err(fs_lock::AcquireError::Io(error)) if error.kind() == io::ErrorKind::NotFound => {
343 if let Some(parent) = scope_dir.as_deref() {
344 fs::create_dir_all(parent).map_err(|error| AftError::IoError {
345 path: parent.display().to_string(),
346 message: format!("failed to recreate checkpoint lock directory: {error}"),
347 })?;
348 }
349 fs_lock::try_acquire(&self.lock_path, self.lock_timeout)
350 }
351 result => result,
352 };
353 let guard = acquire_result.map_err(|error| match error {
354 fs_lock::AcquireError::Timeout => AftError::IoError {
355 path: self.lock_path.display().to_string(),
356 message: "timed out acquiring checkpoint mutation lock".to_string(),
357 },
358 fs_lock::AcquireError::Io(error) => AftError::IoError {
359 path: self.lock_path.display().to_string(),
360 message: format!("failed to acquire checkpoint mutation lock: {error}"),
361 },
362 })?;
363
364 Ok(CheckpointLockGuard {
365 guard: Some(guard),
366 scope_dir,
367 })
368 }
369
370 pub fn create(
384 &mut self,
385 session: &str,
386 name: &str,
387 files: Vec<PathBuf>,
388 backup_store: &BackupStore,
389 ) -> Result<CheckpointInfo, AftError> {
390 self.create_impl(session, name, files, backup_store, None)
391 }
392
393 pub(crate) fn create_from_captures(
394 &mut self,
395 session: &str,
396 name: &str,
397 files: Vec<PathBuf>,
398 backup_store: &BackupStore,
399 captures: &mut HashMap<PathBuf, CapturedRegularFile>,
400 ) -> Result<CheckpointInfo, AftError> {
401 self.create_impl(session, name, files, backup_store, Some(captures))
402 }
403
404 fn create_impl(
405 &mut self,
406 session: &str,
407 name: &str,
408 files: Vec<PathBuf>,
409 backup_store: &BackupStore,
410 mut captures: Option<&mut HashMap<PathBuf, CapturedRegularFile>>,
411 ) -> Result<CheckpointInfo, AftError> {
412 let _mutation_lock = self.acquire_mutation_lock()?;
413 validate_checkpoint_name(name)?;
414 self.run_process_maintenance_once_locked()?;
415 self.hydrate_session_locked(session)?;
416 let explicit_request = !files.is_empty();
417 let file_list = if files.is_empty() {
418 backup_store.tracked_files(session)
419 } else {
420 files
421 };
422
423 let mut file_contents = HashMap::new();
424 let mut skipped: Vec<(PathBuf, String)> = Vec::new();
425 for path in &file_list {
426 let seeded = captures
427 .as_deref_mut()
428 .and_then(|captures| captures.get_mut(path))
429 .map(|capture| CheckpointFile::from_captured(path, capture));
430 let snapshot = match seeded {
431 Some(Err(error)) if error.kind() == io::ErrorKind::InvalidInput => {
432 if let Some(captures) = captures.as_deref_mut() {
433 captures.remove(path);
434 }
435 CheckpointFile::read(path)
436 }
437 Some(result) => result,
438 None => CheckpointFile::read(path),
439 };
440 match snapshot {
441 Ok(snapshot) => {
442 file_contents.insert(path.clone(), snapshot);
443 }
444 Err(e) => {
445 crate::slog_warn!(
446 "checkpoint {}: skipping unreadable file {}: {}",
447 name,
448 path.display(),
449 e
450 );
451 skipped.push((path.clone(), e.to_string()));
452 }
453 }
454 }
455
456 if explicit_request && file_contents.is_empty() && !skipped.is_empty() {
462 let (path, err) = &skipped[0];
463 return Err(AftError::FileNotFound {
464 path: format!("{}: {}", path.display(), err),
465 });
466 }
467
468 let created_at = current_timestamp();
469 let created_order = current_timestamp_nanos()
470 .saturating_add(self.blob_counter.fetch_add(1, Ordering::Relaxed));
471 let file_count = file_contents.len();
472 let checkpoint = Checkpoint {
473 name: name.to_string(),
474 file_contents,
475 created_at,
476 created_order,
477 };
478 let storage_path = self.durable_checkpoint_dir(session, name);
479
480 self.persist_checkpoint_locked(session, &checkpoint)?;
481 self.checkpoints
482 .entry(session.to_string())
483 .or_default()
484 .insert(name.to_string(), checkpoint);
485
486 let evicted = self.evicted_checkpoint_names(session);
487 for evicted_name in &evicted {
488 self.remove_checkpoint_from_disk_locked(session, evicted_name)?;
489 }
490 if let Some(session_checkpoints) = self.checkpoints.get_mut(session) {
491 for evicted_name in &evicted {
492 session_checkpoints.remove(evicted_name);
493 }
494 }
495
496 if skipped.is_empty() {
497 crate::slog_info!("checkpoint created: {} ({} files)", name, file_count);
498 } else {
499 crate::slog_info!(
500 "checkpoint created: {} ({} files, {} skipped)",
501 name,
502 file_count,
503 skipped.len()
504 );
505 }
506
507 Ok(CheckpointInfo {
508 name: name.to_string(),
509 file_count,
510 created_at,
511 storage_path,
512 evicted,
513 skipped,
514 })
515 }
516
517 pub fn restore(&mut self, session: &str, name: &str) -> Result<CheckpointInfo, AftError> {
519 let _mutation_lock = self.acquire_mutation_lock()?;
520 self.run_process_maintenance_once_locked()?;
521 self.hydrate_session_locked(session)?;
522 let storage_path = self.durable_checkpoint_dir(session, name);
523 let checkpoint = self.get(session, name)?;
524 let mut paths = checkpoint.file_contents.keys().cloned().collect::<Vec<_>>();
525 paths.sort();
526
527 restore_paths_atomically(checkpoint, &paths)?;
528 crate::slog_info!("checkpoint restored: {}", name);
529
530 Ok(CheckpointInfo {
531 name: checkpoint.name.clone(),
532 file_count: checkpoint.file_contents.len(),
533 created_at: checkpoint.created_at,
534 storage_path,
535 evicted: Vec::new(),
536 skipped: Vec::new(),
537 })
538 }
539
540 pub fn restore_validated(
542 &mut self,
543 session: &str,
544 name: &str,
545 validated_paths: &[PathBuf],
546 ) -> Result<CheckpointInfo, AftError> {
547 let _mutation_lock = self.acquire_mutation_lock()?;
548 self.run_process_maintenance_once_locked()?;
549 self.hydrate_session_locked(session)?;
550 let storage_path = self.durable_checkpoint_dir(session, name);
551 let checkpoint = self.get(session, name)?;
552
553 for path in validated_paths {
554 checkpoint
555 .file_contents
556 .get(path)
557 .ok_or_else(|| AftError::FileNotFound {
558 path: path.display().to_string(),
559 })?;
560 }
561 restore_paths_atomically(checkpoint, validated_paths)?;
562 crate::slog_info!("checkpoint restored: {}", name);
563
564 Ok(CheckpointInfo {
565 name: checkpoint.name.clone(),
566 file_count: checkpoint.file_contents.len(),
567 created_at: checkpoint.created_at,
568 storage_path,
569 evicted: Vec::new(),
570 skipped: Vec::new(),
571 })
572 }
573
574 pub fn file_paths(&mut self, session: &str, name: &str) -> Result<Vec<PathBuf>, AftError> {
576 let _mutation_lock = self.acquire_mutation_lock()?;
577 self.run_process_maintenance_once_locked()?;
578 self.hydrate_session_locked(session)?;
579 let checkpoint = self.get(session, name)?;
580 Ok(checkpoint.file_contents.keys().cloned().collect())
581 }
582
583 pub fn absolute_file_paths(
585 &mut self,
586 session: &str,
587 name: &str,
588 ) -> Result<Vec<PathBuf>, AftError> {
589 let mut paths: Vec<PathBuf> = self
590 .file_paths(session, name)?
591 .into_iter()
592 .map(absolute_checkpoint_path)
593 .collect();
594 paths.sort();
595 Ok(paths)
596 }
597
598 pub fn delete(&mut self, session: &str, name: &str) -> bool {
600 let _mutation_lock = match self.acquire_mutation_lock() {
601 Ok(lock) => lock,
602 Err(error) => {
603 crate::slog_warn!("checkpoint delete lock failed for {}: {}", name, error);
604 return false;
605 }
606 };
607 if let Err(error) = self.run_process_maintenance_once_locked() {
608 crate::slog_warn!(
609 "checkpoint delete maintenance failed for {}: {}",
610 name,
611 error
612 );
613 return false;
614 }
615 if let Err(error) = self.hydrate_session_locked(session) {
616 crate::slog_warn!("checkpoint delete hydration failed for {}: {}", name, error);
617 return false;
618 }
619 if self
620 .checkpoints
621 .get(session)
622 .is_none_or(|checkpoints| !checkpoints.contains_key(name))
623 {
624 return false;
625 }
626 if let Err(error) = self.remove_checkpoint_from_disk_locked(session, name) {
627 crate::slog_warn!("checkpoint delete failed for {}: {}", name, error);
628 return false;
629 }
630 let Some(session_checkpoints) = self.checkpoints.get_mut(session) else {
631 return false;
632 };
633 let removed = session_checkpoints.remove(name).is_some();
634 if session_checkpoints.is_empty() {
635 self.checkpoints.remove(session);
636 }
637 removed
638 }
639
640 pub fn list(&mut self, session: &str) -> Result<Vec<CheckpointInfo>, AftError> {
643 let _mutation_lock = self.acquire_mutation_lock()?;
644 self.run_process_maintenance_once_locked()?;
645 self.hydrate_session_locked(session)?;
646 let mut list = self
647 .checkpoints
648 .get(session)
649 .map(|checkpoints| {
650 checkpoints
651 .values()
652 .map(|checkpoint| CheckpointInfo {
653 name: checkpoint.name.clone(),
654 file_count: checkpoint.file_contents.len(),
655 created_at: checkpoint.created_at,
656 storage_path: self.durable_checkpoint_dir(session, &checkpoint.name),
657 evicted: Vec::new(),
658 skipped: Vec::new(),
659 })
660 .collect::<Vec<_>>()
661 })
662 .unwrap_or_default();
663 list.sort_by(|left, right| left.name.cmp(&right.name));
664 Ok(list)
665 }
666
667 pub fn total_count(&self) -> usize {
669 self.checkpoints
670 .values()
671 .map(|checkpoints| checkpoints.len())
672 .sum()
673 }
674
675 pub fn cleanup(&mut self) {
679 let _mutation_lock = match self.acquire_mutation_lock() {
680 Ok(lock) => lock,
681 Err(error) => {
682 crate::slog_warn!("checkpoint cleanup lock failed: {}", error);
683 return;
684 }
685 };
686 if let Err(error) = self.cleanup_locked() {
687 crate::slog_warn!("checkpoint cleanup failed: {}", error);
688 }
689 }
690
691 fn get(&self, session: &str, name: &str) -> Result<&Checkpoint, AftError> {
692 self.checkpoints
693 .get(session)
694 .and_then(|checkpoints| checkpoints.get(name))
695 .ok_or_else(|| AftError::CheckpointNotFound {
696 name: name.to_string(),
697 })
698 }
699
700 fn durable_checkpoints_dir(&self) -> Option<PathBuf> {
701 self.storage_dir
702 .as_ref()
703 .zip(self.storage_harness.as_ref())
704 .map(|(storage_dir, harness)| storage_dir.join(harness).join("checkpoints"))
705 }
706
707 fn durable_session_dir(&self, session: &str) -> Option<PathBuf> {
708 self.durable_checkpoints_dir()
709 .map(|checkpoints_dir| checkpoints_dir.join(hash_session(session)))
710 }
711
712 fn durable_checkpoint_dir(&self, session: &str, name: &str) -> Option<PathBuf> {
713 self.durable_session_dir(session)
714 .map(|session_dir| session_dir.join(name))
715 }
716
717 fn run_process_maintenance_once_locked(&mut self) -> Result<(), AftError> {
718 let Some(storage_dir) = self.storage_dir.clone() else {
719 return Ok(());
720 };
721 let Some(harness) = self.storage_harness.clone() else {
722 return Ok(());
723 };
724 if !CHECKPOINT_MAINTENANCE_KEYS
725 .lock()
726 .unwrap()
727 .insert((storage_dir, harness))
728 {
729 return Ok(());
730 }
731 self.cleanup_locked()
732 }
733
734 fn cleanup_locked(&mut self) -> Result<(), AftError> {
735 let now = current_timestamp();
736 self.checkpoints.retain(|_, session_checkpoints| {
737 session_checkpoints.retain(|_, checkpoint| {
738 now.saturating_sub(checkpoint.created_at) < NAMED_CHECKPOINT_RETENTION_SECS
739 });
740 !session_checkpoints.is_empty()
741 });
742
743 if let Some(checkpoints_dir) = self.durable_checkpoints_dir() {
744 sweep_expired_durable_checkpoints(&checkpoints_dir, now);
745 }
746 if let Some(checkpoints_root) = self.lock_path.parent().and_then(Path::parent) {
747 if checkpoints_root.file_name() == Some(std::ffi::OsStr::new("checkpoints")) {
753 sweep_empty_scope_dirs(checkpoints_root);
754 }
755 }
756 Ok(())
757 }
758
759 fn hydrate_session_locked(&mut self, session: &str) -> Result<(), AftError> {
760 let Some(session_dir) = self.durable_session_dir(session) else {
761 return Ok(());
762 };
763 if !session_dir.exists() {
764 self.checkpoints.remove(session);
765 return Ok(());
766 }
767
768 let entries = fs::read_dir(&session_dir).map_err(|error| AftError::IoError {
769 path: session_dir.display().to_string(),
770 message: format!("failed to read durable checkpoint session: {error}"),
771 })?;
772 let mut hydrated = HashMap::new();
773 for entry in entries {
774 let entry = entry.map_err(|error| AftError::IoError {
775 path: session_dir.display().to_string(),
776 message: format!("failed to read durable checkpoint entry: {error}"),
777 })?;
778 let checkpoint_dir = entry.path();
779 if !entry
780 .file_type()
781 .map_err(|error| AftError::IoError {
782 path: checkpoint_dir.display().to_string(),
783 message: format!("failed to inspect durable checkpoint entry: {error}"),
784 })?
785 .is_dir()
786 {
787 continue;
788 }
789 let name = entry.file_name().to_string_lossy().into_owned();
790 if !is_safe_checkpoint_name(&name) {
791 continue;
792 }
793 let meta_path = checkpoint_dir.join("meta.json");
794 if !meta_path.exists() {
795 continue;
796 }
797 let checkpoint = read_checkpoint_from_disk(&checkpoint_dir, session, &name)?;
798 hydrated.insert(name, checkpoint);
799 }
800 if hydrated.is_empty() {
801 self.checkpoints.remove(session);
802 } else {
803 self.checkpoints.insert(session.to_string(), hydrated);
804 }
805 Ok(())
806 }
807
808 fn persist_checkpoint_locked(
809 &self,
810 session: &str,
811 checkpoint: &Checkpoint,
812 ) -> Result<(), AftError> {
813 let Some(checkpoint_dir) = self.durable_checkpoint_dir(session, &checkpoint.name) else {
814 return Ok(());
815 };
816 fs::create_dir_all(&checkpoint_dir).map_err(|error| AftError::IoError {
817 path: checkpoint_dir.display().to_string(),
818 message: format!("failed to create durable checkpoint directory: {error}"),
819 })?;
820
821 let mut files = Vec::with_capacity(checkpoint.file_contents.len());
822 for (index, (path, file)) in checkpoint.file_contents.iter().enumerate() {
823 let blob = format!(
824 "file_{}_{}_{}.blob",
825 checkpoint.created_order,
826 index,
827 self.blob_counter.fetch_add(1, Ordering::Relaxed)
828 );
829 let bytes = checkpoint_file_bytes(file);
830 write_temp_fsync_rename(&checkpoint_dir, &blob, &bytes).map_err(|error| {
831 AftError::IoError {
832 path: checkpoint_dir.join(&blob).display().to_string(),
833 message: format!("failed to write durable checkpoint blob: {error}"),
834 }
835 })?;
836 files.push(DiskCheckpointFileMeta {
837 original_path: path.display().to_string(),
838 blob,
839 kind: match &file.kind {
840 CheckpointFileKind::Regular { .. } => DiskCheckpointFileKind::Regular,
841 CheckpointFileKind::Symlink { .. } => DiskCheckpointFileKind::Symlink,
842 },
843 mode: file.mode,
844 target_is_dir: matches!(
845 &file.kind,
846 CheckpointFileKind::Symlink {
847 target_is_dir: true,
848 ..
849 }
850 ),
851 });
852 }
853 fsync_dir(&checkpoint_dir).map_err(|error| AftError::IoError {
854 path: checkpoint_dir.display().to_string(),
855 message: format!("failed to sync durable checkpoint blobs: {error}"),
856 })?;
857
858 let meta = DiskCheckpointMeta {
859 schema_version: CHECKPOINT_SCHEMA_VERSION,
860 session_id: session.to_string(),
861 name: checkpoint.name.clone(),
862 created_at: checkpoint.created_at,
863 created_order: checkpoint.created_order,
864 files,
865 };
866 let bytes = serde_json::to_vec_pretty(&meta).map_err(|error| AftError::IoError {
867 path: checkpoint_dir.join("meta.json").display().to_string(),
868 message: format!("failed to serialize durable checkpoint metadata: {error}"),
869 })?;
870 write_temp_fsync_rename(&checkpoint_dir, "meta.json", &bytes).map_err(|error| {
871 AftError::IoError {
872 path: checkpoint_dir.join("meta.json").display().to_string(),
873 message: format!("failed to write durable checkpoint metadata: {error}"),
874 }
875 })?;
876 fsync_dir(&checkpoint_dir).map_err(|error| AftError::IoError {
877 path: checkpoint_dir.display().to_string(),
878 message: format!("failed to sync durable checkpoint metadata: {error}"),
879 })?;
880 prune_unreferenced_checkpoint_blobs(&checkpoint_dir, &meta.files).map_err(|error| {
881 AftError::IoError {
882 path: checkpoint_dir.display().to_string(),
883 message: format!("failed to prune stale durable checkpoint blobs: {error}"),
884 }
885 })?;
886 Ok(())
887 }
888
889 fn remove_checkpoint_from_disk_locked(
890 &self,
891 session: &str,
892 name: &str,
893 ) -> Result<(), AftError> {
894 let Some(checkpoint_dir) = self.durable_checkpoint_dir(session, name) else {
895 return Ok(());
896 };
897 match fs::remove_dir_all(&checkpoint_dir) {
898 Ok(()) => {
899 if let Some(session_dir) = checkpoint_dir.parent() {
900 let _ = fs::remove_dir(session_dir);
901 }
902 Ok(())
903 }
904 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
905 Err(error) => Err(AftError::IoError {
906 path: checkpoint_dir.display().to_string(),
907 message: format!("failed to remove durable checkpoint: {error}"),
908 }),
909 }
910 }
911
912 fn evicted_checkpoint_names(&self, session: &str) -> Vec<String> {
913 let Some(checkpoints) = self.checkpoints.get(session) else {
914 return Vec::new();
915 };
916 let overflow = checkpoints
917 .len()
918 .saturating_sub(MAX_NAMED_CHECKPOINTS_PER_SESSION);
919 let mut checkpoints = checkpoints
920 .values()
921 .map(|checkpoint| (checkpoint.created_order, checkpoint.name.clone()))
922 .collect::<Vec<_>>();
923 checkpoints.sort();
924 checkpoints
925 .into_iter()
926 .take(overflow)
927 .map(|(_, name)| name)
928 .collect()
929 }
930
931 pub fn session_is_empty(&self, session: &str) -> bool {
932 self.checkpoints.get(session).is_none_or(HashMap::is_empty)
933 }
934}
935
936fn migrate_unbound_checkpoint_namespace(storage_dir: &Path, harness: &str) {
937 let source = storage_dir
938 .join(UNBOUND_HARNESS_SEGMENT)
939 .join("checkpoints");
940 if !source.exists() {
941 return;
942 }
943 let target = storage_dir.join(harness).join("checkpoints");
944 if !target.exists() {
945 if let Some(parent) = target.parent() {
946 if let Err(error) = fs::create_dir_all(parent) {
947 crate::slog_warn!(
948 "failed to create durable checkpoint harness directory {}: {}",
949 parent.display(),
950 error
951 );
952 return;
953 }
954 }
955 if let Err(error) = fs::rename(&source, &target) {
956 crate::slog_warn!(
957 "failed to move unbound durable checkpoints into {}: {}",
958 target.display(),
959 error
960 );
961 }
962 return;
963 }
964
965 let Ok(boundary) = crate::walk_boundary::DeviceBoundary::for_root(&source) else {
969 crate::slog_warn!(
970 "cannot establish filesystem boundary for checkpoint migration {}",
971 source.display()
972 );
973 return;
974 };
975 let mut skipped_foreign_mounts = 0usize;
976 let Ok(session_entries) = fs::read_dir(&source) else {
977 return;
978 };
979 for session_entry in session_entries.flatten() {
980 let source_session = session_entry.path();
981 if !source_session.is_dir() {
982 continue;
983 }
984 if !boundary.should_descend(&source_session).unwrap_or(false) {
985 skipped_foreign_mounts += 1;
986 continue;
987 }
988 let target_session = target.join(session_entry.file_name());
989 if !target_session.exists() {
990 let _ = fs::rename(&source_session, &target_session);
991 continue;
992 }
993 let Ok(checkpoint_entries) = fs::read_dir(&source_session) else {
994 continue;
995 };
996 for checkpoint_entry in checkpoint_entries.flatten() {
997 let source_checkpoint = checkpoint_entry.path();
998 let target_checkpoint = target_session.join(checkpoint_entry.file_name());
999 if !target_checkpoint.exists() {
1000 let _ = fs::rename(source_checkpoint, target_checkpoint);
1001 }
1002 }
1003 let _ = fs::remove_dir(&source_session);
1004 }
1005 let _ = fs::remove_dir(&source);
1006 if skipped_foreign_mounts > 0 {
1007 crate::slog_warn!(
1008 "checkpoint migration skipped {} foreign filesystem mount(s) below {}",
1009 skipped_foreign_mounts,
1010 source.display()
1011 );
1012 }
1013}
1014
1015fn validate_checkpoint_name(name: &str) -> Result<(), AftError> {
1016 if is_safe_checkpoint_name(name) {
1017 Ok(())
1018 } else {
1019 Err(AftError::InvalidRequest {
1020 message: "checkpoint name must be a single non-empty path component".to_string(),
1021 })
1022 }
1023}
1024
1025fn is_safe_checkpoint_name(name: &str) -> bool {
1026 matches!(
1027 Path::new(name).components().collect::<Vec<_>>().as_slice(),
1028 [std::path::Component::Normal(_)]
1029 ) && !name.chars().any(|character| {
1030 character.is_control()
1031 || matches!(character, '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|')
1032 })
1033}
1034
1035fn is_safe_blob_name(name: &str) -> bool {
1036 is_safe_checkpoint_name(name) && name.starts_with("file_") && name.ends_with(".blob")
1037}
1038
1039fn read_checkpoint_from_disk(
1040 checkpoint_dir: &Path,
1041 session: &str,
1042 expected_name: &str,
1043) -> Result<Checkpoint, AftError> {
1044 let meta_path = checkpoint_dir.join("meta.json");
1045 let bytes = fs::read(&meta_path).map_err(|error| AftError::IoError {
1046 path: meta_path.display().to_string(),
1047 message: format!("failed to read durable checkpoint metadata: {error}"),
1048 })?;
1049 let meta = serde_json::from_slice::<DiskCheckpointMeta>(&bytes).map_err(|error| {
1050 AftError::IoError {
1051 path: meta_path.display().to_string(),
1052 message: format!("failed to parse durable checkpoint metadata: {error}"),
1053 }
1054 })?;
1055 if meta.schema_version != CHECKPOINT_SCHEMA_VERSION {
1056 return Err(AftError::IoError {
1057 path: meta_path.display().to_string(),
1058 message: format!(
1059 "unsupported durable checkpoint metadata schema {}",
1060 meta.schema_version
1061 ),
1062 });
1063 }
1064 if meta.session_id != session
1065 || meta.name != expected_name
1066 || !is_safe_checkpoint_name(&meta.name)
1067 {
1068 return Err(AftError::IoError {
1069 path: meta_path.display().to_string(),
1070 message: "durable checkpoint metadata does not match its session or directory"
1071 .to_string(),
1072 });
1073 }
1074
1075 let mut file_contents = HashMap::with_capacity(meta.files.len());
1076 for file in &meta.files {
1077 if !is_safe_blob_name(&file.blob) {
1078 return Err(AftError::IoError {
1079 path: meta_path.display().to_string(),
1080 message: format!("invalid durable checkpoint blob name {}", file.blob),
1081 });
1082 }
1083 let blob_path = checkpoint_dir.join(&file.blob);
1084 let blob = fs::read(&blob_path).map_err(|error| AftError::IoError {
1085 path: blob_path.display().to_string(),
1086 message: format!("failed to read durable checkpoint blob: {error}"),
1087 })?;
1088 let path = PathBuf::from(&file.original_path);
1089 let checkpoint_file =
1090 CheckpointFile::from_disk(file, blob).map_err(|message| AftError::IoError {
1091 path: blob_path.display().to_string(),
1092 message,
1093 })?;
1094 if file_contents
1095 .insert(path.clone(), checkpoint_file)
1096 .is_some()
1097 {
1098 return Err(AftError::IoError {
1099 path: meta_path.display().to_string(),
1100 message: format!("duplicate durable checkpoint path {}", path.display()),
1101 });
1102 }
1103 }
1104
1105 Ok(Checkpoint {
1106 name: meta.name,
1107 file_contents,
1108 created_at: meta.created_at,
1109 created_order: meta.created_order,
1110 })
1111}
1112
1113fn checkpoint_file_bytes(file: &CheckpointFile) -> Vec<u8> {
1114 match &file.kind {
1115 CheckpointFileKind::Regular { bytes } => bytes.to_vec(),
1116 CheckpointFileKind::Symlink { target, .. } => {
1117 target.as_os_str().to_string_lossy().as_bytes().to_vec()
1118 }
1119 }
1120}
1121
1122fn write_temp_fsync_rename(dir: &Path, final_name: &str, bytes: &[u8]) -> io::Result<()> {
1123 let tmp_name = format!(
1124 ".{}.{}.{}.tmp",
1125 final_name,
1126 std::process::id(),
1127 current_timestamp_nanos()
1128 );
1129 let tmp_path = dir.join(tmp_name);
1130 let final_path = dir.join(final_name);
1131 {
1132 let mut file = fs::OpenOptions::new()
1133 .write(true)
1134 .create_new(true)
1135 .open(&tmp_path)?;
1136 file.write_all(bytes)?;
1137 file.sync_all()?;
1138 }
1139 fs::rename(tmp_path, final_path)
1140}
1141
1142#[cfg(unix)]
1143fn fsync_dir(path: &Path) -> io::Result<()> {
1144 fs::File::open(path)?.sync_all()
1145}
1146
1147#[cfg(not(unix))]
1148fn fsync_dir(_path: &Path) -> io::Result<()> {
1149 Ok(())
1150}
1151
1152fn prune_unreferenced_checkpoint_blobs(
1153 checkpoint_dir: &Path,
1154 files: &[DiskCheckpointFileMeta],
1155) -> io::Result<()> {
1156 let referenced = files
1157 .iter()
1158 .map(|file| file.blob.as_str())
1159 .collect::<HashSet<_>>();
1160 for entry in fs::read_dir(checkpoint_dir)? {
1161 let entry = entry?;
1162 let path = entry.path();
1163 if !path.is_file() {
1164 continue;
1165 }
1166 let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
1167 continue;
1168 };
1169 if (name.starts_with("file_") && name.ends_with(".blob") && !referenced.contains(name))
1170 || name.contains(".tmp.")
1171 || name.ends_with(".tmp")
1172 {
1173 let _ = fs::remove_file(path);
1174 }
1175 }
1176 Ok(())
1177}
1178
1179fn sweep_expired_durable_checkpoints(checkpoints_dir: &Path, now: u64) {
1180 let Ok(boundary) = crate::walk_boundary::DeviceBoundary::for_root(checkpoints_dir) else {
1184 crate::slog_warn!(
1185 "cannot establish filesystem boundary for checkpoint sweep {}",
1186 checkpoints_dir.display()
1187 );
1188 return;
1189 };
1190 let mut skipped_foreign_mounts = 0usize;
1191 let Ok(session_entries) = fs::read_dir(checkpoints_dir) else {
1192 return;
1193 };
1194 for session_entry in session_entries.flatten() {
1195 let session_dir = session_entry.path();
1196 if !session_dir.is_dir() {
1197 continue;
1198 }
1199 if !boundary.should_descend(&session_dir).unwrap_or(false) {
1200 skipped_foreign_mounts += 1;
1201 continue;
1202 }
1203 let Ok(checkpoint_entries) = fs::read_dir(&session_dir) else {
1204 continue;
1205 };
1206 for checkpoint_entry in checkpoint_entries.flatten() {
1207 let checkpoint_dir = checkpoint_entry.path();
1208 if !checkpoint_dir.is_dir() {
1209 continue;
1210 }
1211 if !boundary.should_descend(&checkpoint_dir).unwrap_or(false) {
1212 skipped_foreign_mounts += 1;
1213 continue;
1214 }
1215 let meta_path = checkpoint_dir.join("meta.json");
1216 let Ok(bytes) = fs::read(&meta_path) else {
1217 continue;
1218 };
1219 let Ok(meta) = serde_json::from_slice::<DiskCheckpointMeta>(&bytes) else {
1220 continue;
1221 };
1222 if now.saturating_sub(meta.created_at) < NAMED_CHECKPOINT_RETENTION_SECS {
1223 continue;
1224 }
1225 if let Err(error) = fs::remove_dir_all(&checkpoint_dir) {
1226 crate::slog_warn!(
1227 "failed to remove expired durable checkpoint {}: {}",
1228 checkpoint_dir.display(),
1229 error
1230 );
1231 }
1232 }
1233 let _ = fs::remove_dir(&session_dir);
1234 }
1235 if skipped_foreign_mounts > 0 {
1236 crate::slog_warn!(
1237 "checkpoint sweep skipped {} foreign filesystem mount(s) below {}",
1238 skipped_foreign_mounts,
1239 checkpoints_dir.display()
1240 );
1241 }
1242}
1243
1244fn absolute_checkpoint_path(path: PathBuf) -> PathBuf {
1245 if path.is_absolute() {
1246 return normalize_checkpoint_path(&path);
1247 }
1248 let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
1249 normalize_checkpoint_path(&cwd.join(path))
1250}
1251
1252fn normalize_checkpoint_path(path: &Path) -> PathBuf {
1253 let mut normalized = PathBuf::new();
1254 for component in path.components() {
1255 match component {
1256 std::path::Component::CurDir => {}
1257 std::path::Component::ParentDir => {
1258 if !normalized.pop() {
1259 normalized.push(component.as_os_str());
1260 }
1261 }
1262 other => normalized.push(other.as_os_str()),
1263 }
1264 }
1265 normalized
1266}
1267
1268fn restore_paths_atomically(checkpoint: &Checkpoint, paths: &[PathBuf]) -> Result<(), AftError> {
1269 let mut pre_restore_snapshot: HashMap<PathBuf, Option<CheckpointFile>> = HashMap::new();
1270 for path in paths {
1271 let current = CheckpointFile::read_optional(path).map_err(|error| AftError::IoError {
1272 path: path.display().to_string(),
1273 message: format!("failed to snapshot pre-restore file metadata: {error}"),
1274 })?;
1275 pre_restore_snapshot.insert(path.clone(), current);
1276 }
1277
1278 let mut restored_paths: Vec<PathBuf> = Vec::new();
1279 let mut created_dirs: Vec<PathBuf> = Vec::new();
1280 for path in paths {
1281 let snapshot =
1282 checkpoint
1283 .file_contents
1284 .get(path)
1285 .ok_or_else(|| AftError::FileNotFound {
1286 path: path.display().to_string(),
1287 })?;
1288 if let Err(e) = write_restored_file(path, snapshot, &mut created_dirs) {
1289 let mut rollback_errors = Vec::new();
1290 if let Some(snapshot) = pre_restore_snapshot.get(path) {
1291 if let Err(rollback_error) = restore_snapshot_file(path, snapshot.as_ref()) {
1292 rollback_errors.push(format!("{}: {}", path.display(), rollback_error));
1293 }
1294 }
1295 for restored_path in restored_paths.iter().rev() {
1296 if let Some(snapshot) = pre_restore_snapshot.get(restored_path) {
1297 if let Err(rollback_error) =
1298 restore_snapshot_file(restored_path, snapshot.as_ref())
1299 {
1300 rollback_errors.push(format!(
1301 "{}: {}",
1302 restored_path.display(),
1303 rollback_error
1304 ));
1305 }
1306 }
1307 }
1308 let dirs_rollback_ok = rollback_created_dirs(&created_dirs);
1309 if rollback_errors.is_empty() && dirs_rollback_ok {
1310 return Err(e);
1311 }
1312 return Err(AftError::IoError {
1313 path: path.display().to_string(),
1314 message: format!(
1315 "{}; restore_checkpoint rollback_succeeded: {}; rollback_errors: {}",
1316 e,
1317 rollback_errors.is_empty() && dirs_rollback_ok,
1318 if rollback_errors.is_empty() {
1319 "none".to_string()
1320 } else {
1321 rollback_errors.join("; ")
1322 }
1323 ),
1324 });
1325 }
1326 restored_paths.push(path.clone());
1327 }
1328
1329 Ok(())
1330}
1331
1332fn restore_snapshot_file(path: &Path, snapshot: Option<&CheckpointFile>) -> Result<(), AftError> {
1333 match snapshot {
1334 Some(snapshot) => write_restored_file(path, snapshot, &mut Vec::new()),
1335 None => remove_file_if_exists(path).map_err(|error| AftError::IoError {
1336 path: path.display().to_string(),
1337 message: format!("failed to remove file during checkpoint restore rollback: {error}"),
1338 }),
1339 }
1340}
1341
1342fn write_restored_file(
1343 path: &Path,
1344 snapshot: &CheckpointFile,
1345 created_dirs: &mut Vec<PathBuf>,
1346) -> Result<(), AftError> {
1347 create_parent_dirs(path, created_dirs)?;
1348
1349 match &snapshot.kind {
1350 CheckpointFileKind::Regular { bytes } => {
1351 if path_is_symlink(path) {
1352 remove_file_if_exists(path).map_err(|error| AftError::IoError {
1353 path: path.display().to_string(),
1354 message: format!("failed to replace symlink with regular file: {error}"),
1355 })?;
1356 }
1357 fs::write(path, bytes).map_err(|error| AftError::IoError {
1358 path: path.display().to_string(),
1359 message: format!("failed to restore checkpoint file contents: {error}"),
1360 })?;
1361 restore_checkpoint_permissions(path, snapshot).map_err(|error| AftError::IoError {
1362 path: path.display().to_string(),
1363 message: format!("failed to restore checkpoint file permissions: {error}"),
1364 })
1365 }
1366 CheckpointFileKind::Symlink {
1367 target,
1368 target_is_dir,
1369 } => {
1370 remove_file_if_exists(path).map_err(|error| AftError::IoError {
1371 path: path.display().to_string(),
1372 message: format!("failed to replace file with checkpoint symlink: {error}"),
1373 })?;
1374 create_symlink(target, path, *target_is_dir).map_err(|error| AftError::IoError {
1375 path: path.display().to_string(),
1376 message: format!("failed to restore checkpoint symlink: {error}"),
1377 })
1378 }
1379 }
1380}
1381
1382fn restore_checkpoint_permissions(path: &Path, snapshot: &CheckpointFile) -> io::Result<()> {
1383 if let Some(metadata) = &snapshot.metadata {
1384 return fs::set_permissions(path, metadata.permissions());
1385 }
1386 restore_checkpoint_mode(path, snapshot.mode)
1387}
1388
1389#[cfg(unix)]
1390fn restore_checkpoint_mode(path: &Path, mode: Option<u32>) -> io::Result<()> {
1391 use std::os::unix::fs::PermissionsExt;
1392 if let Some(mode) = mode {
1393 fs::set_permissions(path, fs::Permissions::from_mode(mode))?;
1394 }
1395 Ok(())
1396}
1397
1398#[cfg(not(unix))]
1399fn restore_checkpoint_mode(_path: &Path, _mode: Option<u32>) -> io::Result<()> {
1400 Ok(())
1401}
1402
1403fn create_parent_dirs(path: &Path, created_dirs: &mut Vec<PathBuf>) -> Result<(), AftError> {
1404 if let Some(parent) = path.parent() {
1405 let missing_dirs = missing_parent_dirs(parent);
1406 fs::create_dir_all(parent).map_err(|error| AftError::IoError {
1407 path: parent.display().to_string(),
1408 message: format!("failed to create checkpoint restore parent directories: {error}"),
1409 })?;
1410 created_dirs.extend(missing_dirs);
1411 }
1412 Ok(())
1413}
1414
1415fn path_is_symlink(path: &Path) -> bool {
1416 fs::symlink_metadata(path)
1417 .map(|metadata| metadata.file_type().is_symlink())
1418 .unwrap_or(false)
1419}
1420
1421fn remove_file_if_exists(path: &Path) -> io::Result<()> {
1422 match fs::remove_file(path) {
1423 Ok(()) => Ok(()),
1424 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
1425 Err(error) => Err(error),
1426 }
1427}
1428
1429#[cfg(unix)]
1430fn create_symlink(target: &Path, link: &Path, target_is_dir: bool) -> io::Result<()> {
1431 let _ = target_is_dir;
1432 std::os::unix::fs::symlink(target, link)
1433}
1434
1435#[cfg(windows)]
1436fn create_symlink(target: &Path, link: &Path, target_is_dir: bool) -> io::Result<()> {
1437 if target_is_dir {
1438 std::os::windows::fs::symlink_dir(target, link)
1439 } else {
1440 std::os::windows::fs::symlink_file(target, link)
1441 }
1442}
1443
1444#[cfg(not(any(unix, windows)))]
1445fn create_symlink(_target: &Path, _link: &Path, _target_is_dir: bool) -> io::Result<()> {
1446 Err(io::Error::new(
1447 io::ErrorKind::Unsupported,
1448 "checkpoint symlink restore is unsupported on this platform",
1449 ))
1450}
1451
1452fn missing_parent_dirs(parent: &Path) -> Vec<PathBuf> {
1453 let mut dirs = Vec::new();
1454 let mut current = Some(parent);
1455
1456 while let Some(dir) = current {
1457 if dir.as_os_str().is_empty() || dir.exists() {
1458 break;
1459 }
1460 dirs.push(dir.to_path_buf());
1461 current = dir.parent();
1462 }
1463
1464 dirs
1465}
1466
1467fn rollback_created_dirs(dirs: &[PathBuf]) -> bool {
1468 let mut dirs = dirs.to_vec();
1469 dirs.sort_by_key(|dir| std::cmp::Reverse(dir.components().count()));
1470 dirs.dedup();
1471
1472 let mut ok = true;
1473 for dir in dirs {
1474 match std::fs::remove_dir(&dir) {
1475 Ok(()) => {}
1476 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1477 Err(_) => ok = false,
1478 }
1479 }
1480 ok
1481}
1482
1483fn remove_empty_scope_dir(scope_dir: &Path) {
1487 let _ = fs::remove_dir(scope_dir);
1488}
1489
1490fn sweep_empty_scope_dirs(checkpoints_root: &Path) {
1494 let entries = match fs::read_dir(checkpoints_root) {
1495 Ok(entries) => entries,
1496 Err(_) => return,
1497 };
1498
1499 for entry in entries.flatten() {
1500 let Ok(file_type) = entry.file_type() else {
1501 continue;
1502 };
1503 if file_type.is_dir() {
1504 remove_empty_scope_dir(&entry.path());
1505 }
1506 }
1507}
1508
1509fn current_timestamp() -> u64 {
1510 std::time::SystemTime::now()
1511 .duration_since(std::time::UNIX_EPOCH)
1512 .unwrap_or_default()
1513 .as_secs()
1514}
1515
1516fn current_timestamp_nanos() -> u64 {
1517 u64::try_from(
1518 std::time::SystemTime::now()
1519 .duration_since(std::time::UNIX_EPOCH)
1520 .unwrap_or_default()
1521 .as_nanos(),
1522 )
1523 .unwrap_or(u64::MAX)
1524}
1525
1526#[cfg(test)]
1527mod tests {
1528 use super::*;
1529 use crate::protocol::DEFAULT_SESSION_ID;
1530 use std::fs;
1531
1532 fn temp_file(name: &str, content: &str) -> (PathBuf, tempfile::TempDir) {
1533 let dir = tempfile::Builder::new()
1534 .prefix("aft_checkpoint_tests_")
1535 .tempdir()
1536 .expect("create checkpoint temp dir");
1537 let path = dir.path().join(name);
1538 fs::write(&path, content).unwrap();
1539 (path, dir)
1540 }
1541
1542 fn fresh_checkpoint_store(storage: &Path) -> CheckpointStore {
1543 let lock_path = storage
1544 .join("checkpoints")
1545 .join("test-project")
1546 .join("checkpoint.lock");
1547 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1548 store.set_storage_dir_for_harness(storage.to_path_buf(), crate::harness::Harness::Opencode);
1549 store
1550 }
1551
1552 fn checkpoint_store() -> (CheckpointStore, tempfile::TempDir) {
1553 let dir = tempfile::tempdir().unwrap();
1554 (fresh_checkpoint_store(dir.path()), dir)
1555 }
1556
1557 fn checkpoint_file(content: &str) -> CheckpointFile {
1558 let file = tempfile::NamedTempFile::new().unwrap();
1559 fs::write(file.path(), content).unwrap();
1560 CheckpointFile::read(file.path()).unwrap()
1561 }
1562
1563 #[test]
1564 fn create_and_restore_round_trip() {
1565 let (path1, _dir1) = temp_file("cp_rt1.txt", "hello");
1566 let (path2, _dir2) = temp_file("cp_rt2.txt", "world");
1567
1568 let backup_store = BackupStore::new();
1569 let (mut store, _store_dir) = checkpoint_store();
1570
1571 let info = store
1572 .create(
1573 DEFAULT_SESSION_ID,
1574 "snap1",
1575 vec![path1.clone(), path2.clone()],
1576 &backup_store,
1577 )
1578 .unwrap();
1579 assert_eq!(info.name, "snap1");
1580 assert_eq!(info.file_count, 2);
1581
1582 fs::write(&path1, "changed1").unwrap();
1584 fs::write(&path2, "changed2").unwrap();
1585
1586 let info = store.restore(DEFAULT_SESSION_ID, "snap1").unwrap();
1588 assert_eq!(info.file_count, 2);
1589 assert_eq!(fs::read_to_string(&path1).unwrap(), "hello");
1590 assert_eq!(fs::read_to_string(&path2).unwrap(), "world");
1591 }
1592
1593 #[cfg(unix)]
1594 #[test]
1595 fn durable_checkpoint_hydrates_after_restart_with_bytes_and_mode() {
1596 use std::os::unix::fs::PermissionsExt;
1597
1598 let files = tempfile::tempdir().unwrap();
1599 let path = files.path().join("durable-mode.bin");
1600 let original = b"draft decision\n\0byte exact\n";
1601 fs::write(&path, original).unwrap();
1602 let mut mode = fs::metadata(&path).unwrap().permissions();
1603 mode.set_mode(0o600);
1604 fs::set_permissions(&path, mode).unwrap();
1605
1606 let backup_store = BackupStore::new();
1607 let (mut first, storage) = checkpoint_store();
1608 let info = first
1609 .create(
1610 DEFAULT_SESSION_ID,
1611 "restart-mode",
1612 vec![path.clone()],
1613 &backup_store,
1614 )
1615 .unwrap();
1616 let durable_path = info.storage_path.expect("durable checkpoint path");
1617 assert!(durable_path.join("meta.json").is_file());
1618 assert!(
1619 fs::read_dir(&durable_path)
1620 .unwrap()
1621 .flatten()
1622 .any(|entry| entry.path().extension().is_some_and(|ext| ext == "blob")),
1623 "checkpoint must persist one or more file blobs"
1624 );
1625
1626 fs::write(&path, b"mutated\n").unwrap();
1627 let mut changed_mode = fs::metadata(&path).unwrap().permissions();
1628 changed_mode.set_mode(0o644);
1629 fs::set_permissions(&path, changed_mode).unwrap();
1630 drop(first);
1631
1632 let mut restarted = fresh_checkpoint_store(storage.path());
1633 let listed = restarted.list(DEFAULT_SESSION_ID).unwrap();
1634 assert_eq!(
1635 listed.len(),
1636 1,
1637 "fresh store must hydrate durable checkpoint"
1638 );
1639 assert_eq!(listed[0].name, "restart-mode");
1640 restarted
1641 .restore(DEFAULT_SESSION_ID, "restart-mode")
1642 .unwrap();
1643
1644 assert_eq!(fs::read(&path).unwrap(), original);
1645 assert_eq!(
1646 fs::metadata(&path).unwrap().permissions().mode() & 0o777,
1647 0o600
1648 );
1649 }
1650
1651 #[cfg(unix)]
1652 #[test]
1653 fn durable_checkpoint_hydrates_symlink_without_following_target() {
1654 let files = tempfile::tempdir().unwrap();
1655 let target = files.path().join("target.txt");
1656 let link = files.path().join("link.txt");
1657 fs::write(&target, "target content").unwrap();
1658 std::os::unix::fs::symlink(&target, &link).unwrap();
1659
1660 let backup_store = BackupStore::new();
1661 let (mut first, storage) = checkpoint_store();
1662 first
1663 .create(
1664 DEFAULT_SESSION_ID,
1665 "restart-symlink",
1666 vec![link.clone()],
1667 &backup_store,
1668 )
1669 .unwrap();
1670 fs::remove_file(&link).unwrap();
1671 fs::write(&link, "plain replacement").unwrap();
1672 drop(first);
1673
1674 let mut restarted = fresh_checkpoint_store(storage.path());
1675 restarted
1676 .restore(DEFAULT_SESSION_ID, "restart-symlink")
1677 .unwrap();
1678 assert!(fs::symlink_metadata(&link)
1679 .unwrap()
1680 .file_type()
1681 .is_symlink());
1682 assert_eq!(fs::read_link(&link).unwrap(), target);
1683 assert_eq!(fs::read_to_string(&target).unwrap(), "target content");
1684 }
1685
1686 #[test]
1687 fn checkpoint_retention_evicts_oldest_name_from_memory_and_disk() {
1688 let (path, _files) = temp_file("retention.txt", "version-0");
1689 let backup_store = BackupStore::new();
1690 let (mut store, storage) = checkpoint_store();
1691
1692 for index in 0..=MAX_NAMED_CHECKPOINTS_PER_SESSION {
1693 fs::write(&path, format!("version-{index}")).unwrap();
1694 let info = store
1695 .create(
1696 DEFAULT_SESSION_ID,
1697 &format!("checkpoint-{index:02}"),
1698 vec![path.clone()],
1699 &backup_store,
1700 )
1701 .unwrap();
1702 if index == MAX_NAMED_CHECKPOINTS_PER_SESSION {
1703 assert_eq!(info.evicted, vec!["checkpoint-00"]);
1704 } else {
1705 assert!(info.evicted.is_empty());
1706 }
1707 }
1708
1709 let listed = store.list(DEFAULT_SESSION_ID).unwrap();
1710 assert_eq!(listed.len(), MAX_NAMED_CHECKPOINTS_PER_SESSION);
1711 assert!(listed.iter().all(|info| info.name != "checkpoint-00"));
1712 let old_dir = storage
1713 .path()
1714 .join("opencode")
1715 .join("checkpoints")
1716 .join(hash_session(DEFAULT_SESSION_ID))
1717 .join("checkpoint-00");
1718 assert!(!old_dir.exists(), "evicted checkpoint must leave disk too");
1719 }
1720
1721 #[test]
1722 fn cleanup_refuses_to_sweep_scope_dirs_outside_a_checkpoints_root() {
1723 let temp_root = tempfile::tempdir().expect("temp root");
1730 let victim = temp_root.path().join("innocent-empty-sibling");
1733 fs::create_dir(&victim).expect("victim dir");
1734 let scope_dir = temp_root.path().join("scope");
1735 fs::create_dir(&scope_dir).expect("scope dir");
1736 let lock_path = scope_dir.join("checkpoint.lock");
1737 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1738 store.cleanup_locked().expect("cleanup");
1739 assert!(
1740 victim.exists(),
1741 "cleanup must not sweep empty dirs outside a `checkpoints` root"
1742 );
1743 }
1744
1745 #[test]
1746 fn cleanup_sweeps_durable_checkpoints_older_than_fourteen_days() {
1747 let (path, _files) = temp_file("durable-gc.txt", "original");
1748 let backup_store = BackupStore::new();
1749 let (mut store, _storage) = checkpoint_store();
1750 let info = store
1751 .create(
1752 DEFAULT_SESSION_ID,
1753 "expired-durable",
1754 vec![path],
1755 &backup_store,
1756 )
1757 .unwrap();
1758 let durable_path = info.storage_path.unwrap();
1759 let meta_path = durable_path.join("meta.json");
1760 let mut meta: DiskCheckpointMeta =
1761 serde_json::from_slice(&fs::read(&meta_path).unwrap()).unwrap();
1762 meta.created_at = current_timestamp()
1763 .saturating_sub(NAMED_CHECKPOINT_RETENTION_SECS)
1764 .saturating_sub(1);
1765 fs::write(&meta_path, serde_json::to_vec_pretty(&meta).unwrap()).unwrap();
1766
1767 store.cleanup();
1768 assert!(
1769 !durable_path.exists(),
1770 "fourteen-day cleanup must remove the durable checkpoint directory"
1771 );
1772 }
1773
1774 #[test]
1775 fn durable_hydration_fails_when_a_referenced_blob_is_missing() {
1776 let (path, _files) = temp_file("hydration-control.txt", "original");
1777 let backup_store = BackupStore::new();
1778 let (mut first, storage) = checkpoint_store();
1779 let info = first
1780 .create(
1781 DEFAULT_SESSION_ID,
1782 "hydration-control",
1783 vec![path],
1784 &backup_store,
1785 )
1786 .unwrap();
1787 let durable_path = info.storage_path.unwrap();
1788 let meta: DiskCheckpointMeta =
1789 serde_json::from_slice(&fs::read(durable_path.join("meta.json")).unwrap()).unwrap();
1790 fs::remove_file(durable_path.join(&meta.files[0].blob)).unwrap();
1791 drop(first);
1792
1793 let mut restarted = fresh_checkpoint_store(storage.path());
1794 let error = restarted.list(DEFAULT_SESSION_ID).unwrap_err();
1795 match error {
1796 AftError::IoError { message, .. } => {
1797 assert!(message.contains("failed to read durable checkpoint blob"));
1798 }
1799 other => panic!("expected durable hydration I/O error, got {other:?}"),
1800 }
1801 }
1802
1803 #[test]
1804 fn overwrite_existing_name() {
1805 let (path, _dir) = temp_file("cp_overwrite.txt", "v1");
1806 let backup_store = BackupStore::new();
1807 let (mut store, _store_dir) = checkpoint_store();
1808
1809 store
1810 .create(DEFAULT_SESSION_ID, "dup", vec![path.clone()], &backup_store)
1811 .unwrap();
1812 fs::write(&path, "v2").unwrap();
1813 store
1814 .create(DEFAULT_SESSION_ID, "dup", vec![path.clone()], &backup_store)
1815 .unwrap();
1816
1817 fs::write(&path, "v3").unwrap();
1819 store.restore(DEFAULT_SESSION_ID, "dup").unwrap();
1820 assert_eq!(fs::read_to_string(&path).unwrap(), "v2");
1821 }
1822
1823 #[test]
1824 fn list_returns_metadata_scoped_to_session() {
1825 let (path, _dir) = temp_file("cp_list.txt", "data");
1826 let backup_store = BackupStore::new();
1827 let (mut store, _store_dir) = checkpoint_store();
1828
1829 store
1830 .create(DEFAULT_SESSION_ID, "a", vec![path.clone()], &backup_store)
1831 .unwrap();
1832 store
1833 .create(DEFAULT_SESSION_ID, "b", vec![path.clone()], &backup_store)
1834 .unwrap();
1835 store
1836 .create("other_session", "c", vec![path.clone()], &backup_store)
1837 .unwrap();
1838
1839 let default_list = store.list(DEFAULT_SESSION_ID).unwrap();
1840 assert_eq!(default_list.len(), 2);
1841 let names: Vec<&str> = default_list.iter().map(|i| i.name.as_str()).collect();
1842 assert!(names.contains(&"a"));
1843 assert!(names.contains(&"b"));
1844
1845 let other_list = store.list("other_session").unwrap();
1846 assert_eq!(other_list.len(), 1);
1847 assert_eq!(other_list[0].name, "c");
1848 }
1849
1850 #[test]
1851 fn sessions_isolate_checkpoint_names() {
1852 let (path_a, _dir_a) = temp_file("cp_isolated_a.txt", "a-original");
1854 let (path_b, _dir_b) = temp_file("cp_isolated_b.txt", "b-original");
1855 let backup_store = BackupStore::new();
1856 let (mut store, _store_dir) = checkpoint_store();
1857
1858 store
1860 .create("session_a", "snap", vec![path_a.clone()], &backup_store)
1861 .unwrap();
1862 store
1863 .create("session_b", "snap", vec![path_b.clone()], &backup_store)
1864 .unwrap();
1865
1866 fs::write(&path_a, "a-modified").unwrap();
1867 fs::write(&path_b, "b-modified").unwrap();
1868
1869 store.restore("session_a", "snap").unwrap();
1871 assert_eq!(fs::read_to_string(&path_a).unwrap(), "a-original");
1872 assert_eq!(fs::read_to_string(&path_b).unwrap(), "b-modified");
1873
1874 fs::write(&path_a, "a-modified").unwrap();
1876 store.restore("session_b", "snap").unwrap();
1877 assert_eq!(fs::read_to_string(&path_a).unwrap(), "a-modified");
1878 assert_eq!(fs::read_to_string(&path_b).unwrap(), "b-original");
1879 }
1880
1881 #[test]
1882 fn checkpoint_lock_scope_is_removed_after_release() {
1883 let dir = tempfile::tempdir().unwrap();
1884 let scope_dir = dir.path().join("checkpoints").join("project-scope");
1885 let lock_path = scope_dir.join("checkpoint.lock");
1886 let path = dir.path().join("checkpoint.txt");
1887 fs::write(&path, "data").unwrap();
1888 let backup_store = BackupStore::new();
1889 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1890
1891 store
1892 .create(DEFAULT_SESSION_ID, "released", vec![path], &backup_store)
1893 .unwrap();
1894
1895 assert!(!scope_dir.exists(), "released lock scope should be removed");
1896 }
1897
1898 #[test]
1899 fn cleanup_removes_expired_across_sessions() {
1900 let (path, _dir) = temp_file("cp_cleanup.txt", "data");
1901 let backup_store = BackupStore::new();
1902 let (mut store, _store_dir) = checkpoint_store();
1903
1904 store
1905 .create(
1906 DEFAULT_SESSION_ID,
1907 "recent",
1908 vec![path.clone()],
1909 &backup_store,
1910 )
1911 .unwrap();
1912
1913 store
1915 .checkpoints
1916 .entry("other".to_string())
1917 .or_default()
1918 .insert(
1919 "old".to_string(),
1920 Checkpoint {
1921 name: "old".to_string(),
1922 file_contents: HashMap::new(),
1923 created_at: 1000, created_order: 1000,
1925 },
1926 );
1927
1928 assert_eq!(store.total_count(), 2);
1929 store.cleanup();
1930 assert_eq!(store.total_count(), 1);
1931 assert_eq!(store.list(DEFAULT_SESSION_ID).unwrap()[0].name, "recent");
1932 assert!(store.list("other").unwrap().is_empty());
1933 }
1934
1935 #[test]
1936 fn cleanup_sweeps_empty_scope_dirs_but_keeps_live_lock_scope() {
1937 let dir = tempfile::tempdir().unwrap();
1938 let checkpoints_root = dir.path().join("checkpoints");
1939 let empty_a = checkpoints_root.join("empty-a");
1940 let empty_b = checkpoints_root.join("empty-b");
1941 let live_scope = checkpoints_root.join("live-scope");
1942 fs::create_dir_all(&empty_a).unwrap();
1943 fs::create_dir_all(&empty_b).unwrap();
1944 fs::create_dir_all(&live_scope).unwrap();
1945 fs::write(live_scope.join("checkpoint.lock"), "live lock").unwrap();
1946
1947 let lock_path = checkpoints_root
1948 .join("current-scope")
1949 .join("checkpoint.lock");
1950 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1951 store.cleanup();
1952
1953 assert!(!empty_a.exists());
1954 assert!(!empty_b.exists());
1955 assert!(live_scope.is_dir());
1956 assert!(live_scope.join("checkpoint.lock").is_file());
1957 }
1958
1959 #[test]
1960 fn cleanup_ignores_non_empty_scope_dir_removal_failure() {
1961 let dir = tempfile::tempdir().unwrap();
1962 let checkpoints_root = dir.path().join("checkpoints");
1963 let scope_dir = checkpoints_root.join("racing-scope");
1964 fs::create_dir_all(&scope_dir).unwrap();
1965 fs::write(scope_dir.join("checkpoint.lock"), "lock appeared").unwrap();
1968
1969 let lock_path = checkpoints_root
1970 .join("current-scope")
1971 .join("checkpoint.lock");
1972 let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT);
1973 store.cleanup();
1974
1975 assert!(scope_dir.is_dir());
1976 assert!(scope_dir.join("checkpoint.lock").is_file());
1977 }
1978
1979 #[test]
1980 fn restore_nonexistent_returns_error() {
1981 let (mut store, _store_dir) = checkpoint_store();
1982 let result = store.restore(DEFAULT_SESSION_ID, "nope");
1983 assert!(result.is_err());
1984 match result.unwrap_err() {
1985 AftError::CheckpointNotFound { name } => {
1986 assert_eq!(name, "nope");
1987 }
1988 other => panic!("expected CheckpointNotFound, got: {:?}", other),
1989 }
1990 }
1991
1992 #[test]
1993 fn restore_nonexistent_in_other_session_returns_error() {
1994 let (path, _dir) = temp_file("cp_cross_session.txt", "data");
1996 let backup_store = BackupStore::new();
1997 let (mut store, _store_dir) = checkpoint_store();
1998 store
1999 .create("session_a", "only_a", vec![path], &backup_store)
2000 .unwrap();
2001 assert!(store.restore("session_b", "only_a").is_err());
2002 }
2003
2004 #[test]
2005 fn create_skips_missing_files_from_backup_tracked_set() {
2006 let (readable, _readable_dir) = temp_file("cp_skip_readable.txt", "still_here");
2012 let (deleted, _deleted_dir) = temp_file("cp_skip_deleted.txt", "about_to_vanish");
2013
2014 let deleted_canonical = fs::canonicalize(&deleted).unwrap();
2017
2018 let mut backup_store = BackupStore::new();
2019 backup_store
2020 .snapshot(DEFAULT_SESSION_ID, &readable, "auto")
2021 .unwrap();
2022 backup_store
2023 .snapshot(DEFAULT_SESSION_ID, &deleted, "auto")
2024 .unwrap();
2025
2026 fs::remove_file(&deleted).unwrap();
2027
2028 let (mut store, _store_dir) = checkpoint_store();
2029 let info = store
2030 .create(DEFAULT_SESSION_ID, "partial", vec![], &backup_store)
2031 .expect("checkpoint should succeed despite one missing file");
2032 assert_eq!(info.file_count, 1);
2033 assert_eq!(info.skipped.len(), 1);
2034 assert_eq!(info.skipped[0].0, deleted_canonical);
2035 assert!(!info.skipped[0].1.is_empty());
2036 }
2037
2038 #[test]
2039 fn create_with_explicit_single_missing_file_errors() {
2040 let dir = tempfile::tempdir().unwrap();
2043 let missing = dir.path().join("cp_explicit_missing_does_not_exist.txt");
2044
2045 let backup_store = BackupStore::new();
2046 let (mut store, _store_dir) = checkpoint_store();
2047 let result = store.create(
2048 DEFAULT_SESSION_ID,
2049 "explicit",
2050 vec![missing.clone()],
2051 &backup_store,
2052 );
2053
2054 assert!(result.is_err());
2055 match result.unwrap_err() {
2056 AftError::FileNotFound { path } => {
2057 assert!(path.contains(&missing.display().to_string()));
2058 }
2059 other => panic!("expected FileNotFound, got: {:?}", other),
2060 }
2061 }
2062
2063 #[test]
2064 fn create_with_explicit_mixed_files_keeps_readable_and_reports_skipped() {
2065 let (good, _good_dir) = temp_file("cp_mixed_good.txt", "ok");
2069 let missing_dir = tempfile::tempdir().unwrap();
2070 let missing = missing_dir.path().join("cp_mixed_missing.txt");
2071
2072 let backup_store = BackupStore::new();
2073 let (mut store, _store_dir) = checkpoint_store();
2074 let info = store
2075 .create(
2076 DEFAULT_SESSION_ID,
2077 "mixed",
2078 vec![good.clone(), missing.clone()],
2079 &backup_store,
2080 )
2081 .expect("mixed checkpoint should succeed when any file is readable");
2082 assert_eq!(info.file_count, 1);
2083 assert_eq!(info.skipped.len(), 1);
2084 assert_eq!(info.skipped[0].0, missing);
2085 }
2086
2087 #[test]
2088 fn create_with_empty_files_uses_backup_tracked() {
2089 let (path, _dir) = temp_file("cp_tracked.txt", "tracked_content");
2090 let mut backup_store = BackupStore::new();
2091 backup_store
2092 .snapshot(DEFAULT_SESSION_ID, &path, "auto")
2093 .unwrap();
2094
2095 let (mut store, _store_dir) = checkpoint_store();
2096 let info = store
2097 .create(DEFAULT_SESSION_ID, "from_tracked", vec![], &backup_store)
2098 .unwrap();
2099 assert!(info.file_count >= 1);
2100
2101 fs::write(&path, "modified").unwrap();
2103 store.restore(DEFAULT_SESSION_ID, "from_tracked").unwrap();
2104 assert_eq!(fs::read_to_string(&path).unwrap(), "tracked_content");
2105 }
2106
2107 #[test]
2108 fn restore_recreates_missing_parent_directories() {
2109 let dir = tempfile::tempdir().unwrap();
2110 let path = dir.path().join("nested").join("deeper").join("file.txt");
2111 fs::create_dir_all(path.parent().unwrap()).unwrap();
2112 fs::write(&path, "original nested content").unwrap();
2113
2114 let backup_store = BackupStore::new();
2115 let (mut store, _store_dir) = checkpoint_store();
2116 store
2117 .create(
2118 DEFAULT_SESSION_ID,
2119 "nested",
2120 vec![path.clone()],
2121 &backup_store,
2122 )
2123 .unwrap();
2124
2125 fs::remove_dir_all(dir.path().join("nested")).unwrap();
2126
2127 store.restore(DEFAULT_SESSION_ID, "nested").unwrap();
2128 assert_eq!(
2129 fs::read_to_string(&path).unwrap(),
2130 "original nested content"
2131 );
2132 }
2133
2134 #[cfg(unix)]
2135 #[test]
2136 fn checkpoint_restore_rolls_back_on_partial_failure() {
2137 use std::os::unix::fs::PermissionsExt;
2138
2139 let dir = tempfile::tempdir().unwrap();
2140 let path_a = dir.path().join("a.txt");
2141 let path_b = dir.path().join("b.txt");
2142 fs::write(&path_a, "checkpoint-a").unwrap();
2143 fs::write(&path_b, "checkpoint-b").unwrap();
2144
2145 let backup_store = BackupStore::new();
2146 let (mut store, _store_dir) = checkpoint_store();
2147 store
2148 .create(
2149 DEFAULT_SESSION_ID,
2150 "partial_failure",
2151 vec![path_a.clone(), path_b.clone()],
2152 &backup_store,
2153 )
2154 .unwrap();
2155
2156 fs::write(&path_a, "pre-restore-a").unwrap();
2157 fs::write(&path_b, "pre-restore-b").unwrap();
2158 let mut readonly = fs::metadata(&path_b).unwrap().permissions();
2159 readonly.set_mode(0o444);
2160 fs::set_permissions(&path_b, readonly).unwrap();
2161
2162 let result = store.restore(DEFAULT_SESSION_ID, "partial_failure");
2163 let mut writable = fs::metadata(&path_b).unwrap().permissions();
2164 writable.set_mode(0o644);
2165 fs::set_permissions(&path_b, writable).unwrap();
2166
2167 assert!(result.is_err(), "restore should surface write failure");
2168 assert_eq!(fs::read_to_string(&path_a).unwrap(), "pre-restore-a");
2169 assert_eq!(fs::read_to_string(&path_b).unwrap(), "pre-restore-b");
2170 }
2171
2172 #[test]
2173 fn checkpoint_create_and_restore_use_mutation_lock() {
2174 let dir = tempfile::tempdir().unwrap();
2175 let lock_path = dir.path().join("locks").join("checkpoint.lock");
2176 fs::create_dir_all(lock_path.parent().unwrap()).unwrap();
2177 let mut store =
2178 CheckpointStore::with_lock_path(lock_path.clone(), Duration::from_millis(50));
2179 let backup_store = BackupStore::new();
2180 let path = dir.path().join("locked.txt");
2181 fs::write(&path, "original").unwrap();
2182
2183 let held_lock =
2184 fs_lock::try_acquire(&lock_path, Duration::from_secs(1)).expect("hold checkpoint lock");
2185 let create_result = store.create(
2186 DEFAULT_SESSION_ID,
2187 "locked",
2188 vec![path.clone()],
2189 &backup_store,
2190 );
2191 assert!(matches!(create_result, Err(AftError::IoError { .. })));
2192 drop(held_lock);
2193
2194 store
2195 .create(
2196 DEFAULT_SESSION_ID,
2197 "locked",
2198 vec![path.clone()],
2199 &backup_store,
2200 )
2201 .unwrap();
2202 fs::write(&path, "changed").unwrap();
2203
2204 fs::create_dir_all(lock_path.parent().unwrap()).unwrap();
2205 let held_lock =
2206 fs_lock::try_acquire(&lock_path, Duration::from_secs(1)).expect("hold checkpoint lock");
2207 let restore_result = store.restore(DEFAULT_SESSION_ID, "locked");
2208 assert!(matches!(restore_result, Err(AftError::IoError { .. })));
2209 drop(held_lock);
2210
2211 store.restore(DEFAULT_SESSION_ID, "locked").unwrap();
2212 assert_eq!(fs::read_to_string(&path).unwrap(), "original");
2213 }
2214
2215 #[cfg(unix)]
2216 #[test]
2217 fn checkpoint_restore_preserves_regular_file_permissions() {
2218 use std::os::unix::fs::PermissionsExt;
2219
2220 let dir = tempfile::tempdir().unwrap();
2221 let path = dir.path().join("mode.txt");
2222 fs::write(&path, "original").unwrap();
2223 let mut original_permissions = fs::metadata(&path).unwrap().permissions();
2224 original_permissions.set_mode(0o600);
2225 fs::set_permissions(&path, original_permissions).unwrap();
2226
2227 let backup_store = BackupStore::new();
2228 let (mut store, _store_dir) = checkpoint_store();
2229 store
2230 .create(
2231 DEFAULT_SESSION_ID,
2232 "mode",
2233 vec![path.clone()],
2234 &backup_store,
2235 )
2236 .unwrap();
2237
2238 fs::write(&path, "changed").unwrap();
2239 let mut changed_permissions = fs::metadata(&path).unwrap().permissions();
2240 changed_permissions.set_mode(0o644);
2241 fs::set_permissions(&path, changed_permissions).unwrap();
2242
2243 store.restore(DEFAULT_SESSION_ID, "mode").unwrap();
2244
2245 assert_eq!(fs::read_to_string(&path).unwrap(), "original");
2246 let restored_mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
2247 assert_eq!(restored_mode, 0o600);
2248 }
2249
2250 #[cfg(unix)]
2251 #[test]
2252 fn checkpoint_restore_recreates_symlink() {
2253 let dir = tempfile::tempdir().unwrap();
2254 let target = dir.path().join("target.txt");
2255 let link = dir.path().join("link.txt");
2256 fs::write(&target, "target content").unwrap();
2257 std::os::unix::fs::symlink(&target, &link).unwrap();
2258
2259 let backup_store = BackupStore::new();
2260 let (mut store, _store_dir) = checkpoint_store();
2261 store
2262 .create(
2263 DEFAULT_SESSION_ID,
2264 "symlink",
2265 vec![link.clone()],
2266 &backup_store,
2267 )
2268 .unwrap();
2269
2270 fs::remove_file(&link).unwrap();
2271 fs::write(&link, "plain file").unwrap();
2272
2273 store.restore(DEFAULT_SESSION_ID, "symlink").unwrap();
2274
2275 assert!(fs::symlink_metadata(&link)
2276 .unwrap()
2277 .file_type()
2278 .is_symlink());
2279 assert_eq!(fs::read_link(&link).unwrap(), target);
2280 assert_eq!(fs::read_to_string(&link).unwrap(), "target content");
2281 }
2282
2283 #[test]
2284 fn captured_regular_file_is_shared_by_checkpoint_and_backup() {
2285 let (path, _dir) = temp_file("shared-capture.txt", "original bytes");
2286 crate::backup::reset_capture_read_count(&path);
2287 let mut capture = CapturedRegularFile::read(&path).unwrap().unwrap();
2288 assert_eq!(crate::backup::capture_read_count(&path), 1);
2289
2290 let checkpoint = CheckpointFile::from_captured(&path, &mut capture).unwrap();
2291 let mut backup = BackupStore::new();
2292 backup
2293 .snapshot_with_op_from_capture(
2294 DEFAULT_SESSION_ID,
2295 &path,
2296 "shared capture",
2297 Some("shared-op"),
2298 &capture,
2299 )
2300 .unwrap();
2301
2302 let history = backup.history(DEFAULT_SESSION_ID, &path);
2303 let CheckpointFileKind::Regular { bytes } = checkpoint.kind else {
2304 panic!("regular capture must create a regular checkpoint");
2305 };
2306 assert_eq!(bytes.as_ref(), b"original bytes");
2307 assert_eq!(history[0].content_bytes.as_ref(), b"original bytes");
2308 assert!(Arc::ptr_eq(&bytes, &history[0].content_bytes));
2309 assert_eq!(crate::backup::capture_read_count(&path), 1);
2310 }
2311
2312 #[test]
2313 fn stale_capture_refreshes_before_checkpoint_and_backup() {
2314 let (path, _dir) = temp_file("stale-capture.txt", "old");
2315 crate::backup::reset_capture_read_count(&path);
2316 let mut capture = CapturedRegularFile::read(&path).unwrap().unwrap();
2317 fs::write(&path, "fresh disk truth").unwrap();
2318
2319 let checkpoint = CheckpointFile::from_captured(&path, &mut capture).unwrap();
2320 let mut backup = BackupStore::new();
2321 backup
2322 .snapshot_with_op_from_capture(
2323 DEFAULT_SESSION_ID,
2324 &path,
2325 "freshened capture",
2326 Some("fresh-op"),
2327 &capture,
2328 )
2329 .unwrap();
2330
2331 let history = backup.history(DEFAULT_SESSION_ID, &path);
2332 let CheckpointFileKind::Regular { bytes } = checkpoint.kind else {
2333 panic!("regular capture must create a regular checkpoint");
2334 };
2335 assert_eq!(bytes.as_ref(), b"fresh disk truth");
2336 assert_eq!(history[0].content_bytes.as_ref(), b"fresh disk truth");
2337 assert!(Arc::ptr_eq(&bytes, &history[0].content_bytes));
2338 assert_eq!(crate::backup::capture_read_count(&path), 2);
2339 }
2340
2341 #[test]
2342 fn checkpoint_restore_failure_removes_created_parent_dirs() {
2343 let dir = tempfile::tempdir().unwrap();
2344 let missing_root = dir.path().join("created");
2345 let path_a = missing_root.join("nested").join("a.txt");
2346 let path_b = dir.path().join("blocking-dir");
2347 fs::create_dir(&path_b).unwrap();
2348
2349 let checkpoint = Checkpoint {
2350 name: "dir-cleanup".to_string(),
2351 file_contents: HashMap::from([
2352 (path_a.clone(), checkpoint_file("checkpoint-a")),
2353 (path_b.clone(), checkpoint_file("checkpoint-b")),
2354 ]),
2355 created_at: current_timestamp(),
2356 created_order: current_timestamp_nanos(),
2357 };
2358
2359 let result = restore_paths_atomically(&checkpoint, &[path_a.clone(), path_b.clone()]);
2360
2361 assert!(
2362 result.is_err(),
2363 "second restore write should fail on directory"
2364 );
2365 assert!(!path_a.exists(), "restored file should be rolled back");
2366 assert!(
2367 !missing_root.exists(),
2368 "new parent directories should be removed on rollback"
2369 );
2370 assert!(path_b.is_dir(), "pre-existing blocking directory remains");
2371 }
2372}