use anyhow::{bail, Context, Result};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use std::fs;
use std::io::Write;
use std::path::PathBuf;
use crate::api::types::Message;
use crate::redact;
#[derive(Debug, Serialize, Deserialize)]
struct CheckpointEnvelope {
sha256: String,
payload: serde_json::Value,
}
impl CheckpointEnvelope {
fn get_hmac_key() -> Vec<u8> {
let path = dirs::data_local_dir()
.unwrap_or_else(|| PathBuf::from("."))
.join("selfware")
.join("checkpoint_hmac_key");
if let Ok(key) = std::fs::read(&path) {
if key.len() == 32 {
return key;
}
tracing::warn!(
"Existing HMAC key at {:?} has invalid length (expected 32, got {}). Generating new key.",
path,
key.len()
);
}
let mut key = vec![0u8; 32];
rand::Rng::fill_bytes(&mut rand::rng(), &mut key);
if let Err(e) = Self::persist_hmac_key(&path, &key) {
tracing::warn!(
"Failed to persist HMAC key to {:?}: {}. Key will be ephemeral for this session.",
path,
e
);
}
key
}
fn persist_hmac_key(path: &PathBuf, key: &[u8]) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).with_context(|| {
format!(
"Failed to create HMAC key directory {:?}. Check permissions and disk space.",
parent
)
})?;
}
#[cfg(unix)]
{
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut file = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(path)
.with_context(|| {
format!(
"Failed to create HMAC key file {:?} with secure permissions (0o600). Check file permissions.",
path
)
})?;
file.write_all(key).with_context(|| {
format!(
"Failed to write HMAC key to {:?}. Check disk space and permissions.",
path
)
})?;
file.sync_all()
.with_context(|| format!("Failed to sync HMAC key file {:?} to disk", path))?;
}
#[cfg(not(unix))]
{
std::fs::write(path, key).with_context(|| {
format!(
"Failed to write HMAC key to {:?}. Check disk space and permissions.",
path
)
})?;
}
Ok(())
}
fn wrap(payload: serde_json::Value) -> Result<Self> {
use hmac::{Hmac, Mac};
let canonical =
serde_json::to_string(&payload).context("Failed to serialize payload for hashing")?;
let mut mac = Hmac::<Sha256>::new_from_slice(&Self::get_hmac_key())
.expect("HMAC can take key of any size");
mac.update(canonical.as_bytes());
let hash = hex::encode(mac.finalize().into_bytes());
Ok(Self {
sha256: hash,
payload,
})
}
fn verify(&self) -> Result<()> {
use hmac::{Hmac, Mac};
let canonical = serde_json::to_string(&self.payload)
.context("Failed to serialize payload for verification")?;
let mut mac = Hmac::<Sha256>::new_from_slice(&Self::get_hmac_key())
.expect("HMAC can take key of any size");
mac.update(canonical.as_bytes());
let expected = hex::encode(mac.finalize().into_bytes());
if expected != self.sha256 {
bail!(
"Checkpoint integrity check failed: expected HMAC {}, got {}",
expected,
self.sha256
);
}
Ok(())
}
}
pub const CURRENT_CHECKPOINT_VERSION: u32 = 1;
fn default_version() -> u32 {
0 }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum TaskStatus {
InProgress,
Completed,
Failed,
Paused,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryEntry {
pub timestamp: String,
pub role: String,
pub content: String,
pub token_estimate: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolCallLog {
pub timestamp: DateTime<Utc>,
pub tool_name: String,
pub arguments: String,
pub result: Option<String>,
pub success: bool,
pub duration_ms: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct VerificationResult {
pub passed: bool,
pub confidence: f32, pub explanation: String, pub screenshot_hash: String, }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct VisualAssertion {
pub id: String, pub description: String, pub screenshot_path: Option<PathBuf>, pub verified: bool, pub verification_result: Option<VerificationResult>,
pub created_at: DateTime<Utc>,
pub verified_at: Option<DateTime<Utc>>,
pub step: Option<usize>,
pub tool_name: Option<String>,
pub expected: Option<String>,
pub observed: Option<String>,
pub passed: Option<bool>,
pub confidence: Option<f64>,
pub screenshot_hash_legacy: Option<String>,
pub timestamp: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ErrorLog {
pub timestamp: DateTime<Utc>,
pub step: usize,
pub error: String,
pub recovered: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct GitCheckpointInfo {
pub branch: String,
pub commit_hash: String,
pub dirty: bool,
pub staged_files: Vec<String>,
pub modified_files: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
pub struct GuardCounters {
#[serde(default)]
pub consecutive_no_action_prompts: usize,
#[serde(default)]
pub mutation_gate_rejections: usize,
#[serde(default)]
pub prefill_400_count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CheckpointDelta {
pub task_id: String,
pub base_version: u32,
pub target_version: u32,
pub updated_at: DateTime<Utc>,
pub status: Option<TaskStatus>,
pub current_step: Option<usize>,
pub current_iteration: Option<usize>,
pub new_messages: Vec<Message>,
pub new_memory_entries: Vec<MemoryEntry>,
pub new_tool_calls: Vec<ToolCallLog>,
pub new_errors: Vec<ErrorLog>,
pub new_visual_assertions: Vec<VisualAssertion>,
pub updated_tokens: Option<usize>,
#[serde(default)]
pub cumulative_tokens: Option<usize>,
#[serde(default)]
pub elapsed_wall_secs: Option<u64>,
#[serde(default)]
pub cumulative_cost_usd: Option<f64>,
#[serde(default)]
pub guard_counters: Option<GuardCounters>,
pub git_checkpoint: Option<GitCheckpointInfo>,
pub pending_visual_assertion: Option<Option<VisualAssertion>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TaskCheckpoint {
#[serde(default = "default_version")]
pub version: u32,
pub task_id: String,
pub task_description: String,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub status: TaskStatus,
pub current_step: usize,
#[serde(default)]
pub current_iteration: usize,
pub messages: Vec<Message>,
pub memory_entries: Vec<MemoryEntry>,
pub estimated_tokens: usize,
pub tool_calls: Vec<ToolCallLog>,
pub errors: Vec<ErrorLog>,
#[serde(default)]
pub visual_assertions: Vec<VisualAssertion>,
#[serde(default)]
pub pending_visual_assertion: Option<VisualAssertion>,
pub git_checkpoint: Option<GitCheckpointInfo>,
#[serde(default)]
pub cumulative_tokens: usize,
#[serde(default)]
pub elapsed_wall_secs: u64,
#[serde(default)]
pub cumulative_cost_usd: f64,
#[serde(default)]
pub guard_counters: GuardCounters,
#[serde(default)]
pub max_budget_tokens: Option<usize>,
#[serde(default)]
pub max_wall_secs: Option<u64>,
#[serde(default)]
pub max_cost_usd: Option<f64>,
}
impl TaskCheckpoint {
fn touch(&mut self) {
self.version = self.version.saturating_add(1);
self.updated_at = Utc::now();
}
pub fn compute_delta(&self, base: &TaskCheckpoint) -> Option<CheckpointDelta> {
if self.task_id != base.task_id || self.version <= base.version {
return None;
}
let status = (self.status != base.status).then_some(self.status.clone());
let current_step = (self.current_step != base.current_step).then_some(self.current_step);
let current_iteration =
(self.current_iteration != base.current_iteration).then_some(self.current_iteration);
let updated_tokens =
(self.estimated_tokens != base.estimated_tokens).then_some(self.estimated_tokens);
let cumulative_tokens =
(self.cumulative_tokens != base.cumulative_tokens).then_some(self.cumulative_tokens);
let elapsed_wall_secs =
(self.elapsed_wall_secs != base.elapsed_wall_secs).then_some(self.elapsed_wall_secs);
let cumulative_cost_usd = (self.cumulative_cost_usd != base.cumulative_cost_usd)
.then_some(self.cumulative_cost_usd);
let guard_counters =
(self.guard_counters != base.guard_counters).then(|| self.guard_counters.clone());
if self.git_checkpoint != base.git_checkpoint && self.git_checkpoint.is_none() {
return None;
}
let git_checkpoint = (self.git_checkpoint != base.git_checkpoint)
.then(|| self.git_checkpoint.clone())
.flatten();
let new_messages = if self.messages.len() >= base.messages.len() {
self.messages[base.messages.len()..].to_vec()
} else {
return None;
};
let new_memory_entries = if self.memory_entries.len() >= base.memory_entries.len() {
self.memory_entries[base.memory_entries.len()..].to_vec()
} else {
return None;
};
let new_tool_calls = if self.tool_calls.len() >= base.tool_calls.len() {
self.tool_calls[base.tool_calls.len()..].to_vec()
} else {
return None;
};
let new_errors = if self.errors.len() >= base.errors.len() {
self.errors[base.errors.len()..].to_vec()
} else {
return None;
};
let new_visual_assertions = if self.visual_assertions.len() >= base.visual_assertions.len()
{
self.visual_assertions[base.visual_assertions.len()..].to_vec()
} else {
return None;
};
let pending_changed = self.pending_visual_assertion != base.pending_visual_assertion;
let pending_visual_assertion =
pending_changed.then_some(self.pending_visual_assertion.clone());
let has_changes = status.is_some()
|| current_step.is_some()
|| current_iteration.is_some()
|| !new_messages.is_empty()
|| !new_memory_entries.is_empty()
|| !new_tool_calls.is_empty()
|| !new_errors.is_empty()
|| !new_visual_assertions.is_empty()
|| updated_tokens.is_some()
|| cumulative_tokens.is_some()
|| elapsed_wall_secs.is_some()
|| cumulative_cost_usd.is_some()
|| guard_counters.is_some()
|| git_checkpoint.is_some()
|| pending_changed;
if !has_changes {
return None;
}
Some(CheckpointDelta {
task_id: self.task_id.clone(),
base_version: base.version,
target_version: self.version,
updated_at: self.updated_at,
status,
current_step,
current_iteration,
new_messages,
new_memory_entries,
new_tool_calls,
new_errors,
new_visual_assertions,
updated_tokens,
cumulative_tokens,
elapsed_wall_secs,
cumulative_cost_usd,
guard_counters,
git_checkpoint,
pending_visual_assertion,
})
}
pub fn apply_delta(&mut self, delta: &CheckpointDelta) -> Result<()> {
if self.task_id != delta.task_id {
return Err(anyhow::anyhow!("Delta task ID mismatch"));
}
if self.version != delta.base_version {
return Err(anyhow::anyhow!(
"Delta base version mismatch: expected {}, got {}",
self.version,
delta.base_version
));
}
self.version = delta.target_version;
self.updated_at = delta.updated_at;
if let Some(ref status) = delta.status {
self.status = status.clone();
}
if let Some(step) = delta.current_step {
self.current_step = step;
}
if let Some(iter) = delta.current_iteration {
self.current_iteration = iter;
}
self.messages.extend(delta.new_messages.clone());
self.memory_entries.extend(delta.new_memory_entries.clone());
self.tool_calls.extend(delta.new_tool_calls.clone());
self.errors.extend(delta.new_errors.clone());
self.visual_assertions
.extend(delta.new_visual_assertions.clone());
if let Some(ref pending) = delta.pending_visual_assertion {
self.pending_visual_assertion = pending.clone();
}
if let Some(tokens) = delta.updated_tokens {
self.estimated_tokens = tokens;
}
if let Some(tokens) = delta.cumulative_tokens {
self.cumulative_tokens = tokens;
}
if let Some(secs) = delta.elapsed_wall_secs {
self.elapsed_wall_secs = secs;
}
if let Some(cost) = delta.cumulative_cost_usd {
self.cumulative_cost_usd = cost;
}
if let Some(ref gc) = delta.guard_counters {
self.guard_counters = gc.clone();
}
if let Some(ref git) = delta.git_checkpoint {
self.git_checkpoint = Some(git.clone());
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TaskSummary {
pub task_id: String,
pub task_description: String,
pub status: TaskStatus,
pub current_step: usize,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub tool_call_count: usize,
pub error_count: usize,
}
impl TaskCheckpoint {
pub fn new(task_id: String, task_description: String) -> Self {
let now = Utc::now();
Self {
version: CURRENT_CHECKPOINT_VERSION,
task_id,
task_description,
created_at: now,
updated_at: now,
status: TaskStatus::InProgress,
current_step: 0,
current_iteration: 0,
messages: Vec::new(),
memory_entries: Vec::new(),
estimated_tokens: 0,
tool_calls: Vec::new(),
errors: Vec::new(),
visual_assertions: Vec::new(),
pending_visual_assertion: None,
git_checkpoint: None,
cumulative_tokens: 0,
elapsed_wall_secs: 0,
cumulative_cost_usd: 0.0,
guard_counters: GuardCounters::default(),
max_budget_tokens: None,
max_wall_secs: None,
max_cost_usd: None,
}
}
pub fn to_summary(&self) -> TaskSummary {
TaskSummary {
task_id: self.task_id.clone(),
task_description: self.task_description.clone(),
status: self.status.clone(),
current_step: self.current_step,
created_at: self.created_at,
updated_at: self.updated_at,
tool_call_count: self.tool_calls.len(),
error_count: self.errors.len(),
}
}
pub fn log_tool_call(&mut self, log: ToolCallLog) {
self.tool_calls.push(log);
self.touch();
}
pub fn log_visual_assertion(&mut self, assertion: VisualAssertion) {
self.visual_assertions.push(assertion);
self.touch();
}
pub fn set_pending_visual_assertion(&mut self, assertion: VisualAssertion) {
self.pending_visual_assertion = Some(assertion);
self.touch();
}
pub fn log_error(&mut self, step: usize, error: String, recovered: bool) {
self.errors.push(ErrorLog {
timestamp: Utc::now(),
step,
error,
recovered,
});
self.touch();
}
pub fn set_step(&mut self, step: usize) {
self.current_step = step;
self.touch();
}
pub fn set_iteration(&mut self, iteration: usize) {
self.current_iteration = iteration;
self.touch();
}
pub fn set_status(&mut self, status: TaskStatus) {
self.status = status;
self.touch();
}
pub fn set_messages(&mut self, messages: Vec<Message>) {
self.messages = messages;
self.touch();
}
pub fn set_estimated_tokens(&mut self, estimated_tokens: usize) {
self.estimated_tokens = estimated_tokens;
self.touch();
}
}
pub struct CheckpointManager {
checkpoints_dir: PathBuf,
}
const MAX_DELTA_ENTRIES_BEFORE_COMPACT: usize = 24;
const MAX_DELTA_FILE_BYTES: u64 = 512 * 1024;
const MAX_CHECKPOINT_FILES: usize = 500;
fn sanitize_task_id(task_id: &str) -> Result<String> {
if task_id.split('/').any(|seg| seg == "..")
|| task_id.split('\\').any(|seg| seg == "..")
|| task_id == ".."
{
bail!(
"task_id contains a '..' traversal segment and is rejected: {:?}",
task_id
);
}
let sanitized = task_id.replace(['/', '\\'], "_");
let trimmed = sanitized
.trim_matches(['.', ' ', '\t', '\n', '\r'])
.to_string();
if trimmed.is_empty() {
bail!("task_id is empty after sanitization");
}
Ok(trimmed)
}
impl CheckpointManager {
pub fn new(checkpoints_dir: PathBuf) -> Result<Self> {
if !checkpoints_dir.exists() {
fs::create_dir_all(&checkpoints_dir).with_context(|| {
format!(
"Failed to create checkpoints directory: {:?}",
checkpoints_dir
)
})?;
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = fs::set_permissions(&checkpoints_dir, fs::Permissions::from_mode(0o700));
}
Ok(Self { checkpoints_dir })
}
pub fn default_path() -> Result<Self> {
let home = dirs_home();
let checkpoints_dir = home.join(".selfware").join("checkpoints");
Self::new(checkpoints_dir)
}
fn checkpoint_path(&self, task_id: &str) -> Result<PathBuf> {
let safe_id = sanitize_task_id(task_id)?;
let path = self.checkpoints_dir.join(format!("{}.json", safe_id));
self.verify_path_in_dir(&path)?;
Ok(path)
}
fn checkpoint_delta_path(&self, task_id: &str) -> Result<PathBuf> {
let safe_id = sanitize_task_id(task_id)?;
let path = self
.checkpoints_dir
.join(format!("{}.delta.jsonl", safe_id));
self.verify_path_in_dir(&path)?;
Ok(path)
}
fn verify_path_in_dir(&self, path: &std::path::Path) -> Result<()> {
if let (Ok(canon_dir), Ok(canon_path)) = (
std::fs::canonicalize(&self.checkpoints_dir),
std::fs::canonicalize(path),
) {
if !canon_path.starts_with(&canon_dir) {
bail!(
"checkpoint path {:?} escapes checkpoints_dir {:?}",
path,
self.checkpoints_dir
);
}
return Ok(());
}
let dir = self
.checkpoints_dir
.canonicalize()
.unwrap_or_else(|_| self.checkpoints_dir.clone());
let resolved_path = match (path.parent(), path.file_name()) {
(Some(parent), Some(name)) => match parent.canonicalize() {
Ok(canon_parent) => canon_parent.join(name),
Err(_) => path.to_path_buf(),
},
_ => path.to_path_buf(),
};
if !resolved_path.starts_with(&dir) {
bail!(
"checkpoint path {:?} escapes checkpoints_dir {:?}",
path,
self.checkpoints_dir
);
}
Ok(())
}
pub fn save(&self, checkpoint: &TaskCheckpoint) -> Result<()> {
let full_path = self.checkpoint_path(&checkpoint.task_id)?;
if full_path.exists() {
if let Ok(mut base) = self.try_load_from_path(&full_path) {
if let Err(e) = self.apply_deltas(&checkpoint.task_id, &mut base) {
tracing::warn!(
"Failed to hydrate checkpoint with deltas before save ({}). Falling back to full save.",
e
);
self.save_full_checkpoint(checkpoint)?;
self.clear_delta_log(&checkpoint.task_id)?;
self.prune_old_checkpoints();
return Ok(());
}
if let Some(delta) = checkpoint.compute_delta(&base) {
if self.delta_is_efficient(checkpoint, &delta)? {
match self.append_delta(&checkpoint.task_id, &delta) {
Ok(()) => {
if self.should_compact_deltas(&checkpoint.task_id)? {
self.save_full_checkpoint(checkpoint)?;
self.clear_delta_log(&checkpoint.task_id)?;
}
self.prune_old_checkpoints();
return Ok(());
}
Err(e) => {
tracing::warn!(
"Failed to append checkpoint delta: {}. Falling back to full save.",
e
);
}
}
}
}
}
}
self.save_full_checkpoint(checkpoint)?;
self.clear_delta_log(&checkpoint.task_id)?;
self.prune_old_checkpoints();
Ok(())
}
pub fn save_final(&self, checkpoint: &TaskCheckpoint) -> Result<()> {
self.save_full_checkpoint(checkpoint)?;
self.clear_delta_log(&checkpoint.task_id)?;
self.prune_old_checkpoints();
Ok(())
}
fn delta_is_efficient(
&self,
checkpoint: &TaskCheckpoint,
delta: &CheckpointDelta,
) -> Result<bool> {
let full_size = serde_json::to_vec(checkpoint)
.context("Failed to estimate full checkpoint size")?
.len();
let delta_size = serde_json::to_vec(delta)
.context("Failed to estimate checkpoint delta size")?
.len();
Ok(delta_size + 128 < full_size)
}
fn append_delta(&self, task_id: &str, delta: &CheckpointDelta) -> Result<()> {
let path = self.checkpoint_delta_path(task_id)?;
let mut json_value =
serde_json::to_value(delta).context("Failed to serialize checkpoint delta")?;
redact::redact_json(&mut json_value);
let envelope = CheckpointEnvelope::wrap(json_value)
.context("Failed to create checkpoint delta envelope")?;
let line = serde_json::to_string(&envelope)
.context("Failed to serialize checkpoint delta envelope")?;
let mut file = fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)
.with_context(|| format!("Failed to open checkpoint delta log {:?}", path))?;
file.write_all(line.as_bytes())
.with_context(|| format!("Failed to write checkpoint delta log {:?}", path))?;
file.write_all(b"\n")
.with_context(|| format!("Failed to write checkpoint delta newline {:?}", path))?;
file.sync_all()
.with_context(|| format!("Failed to fsync checkpoint delta log {:?}", path))?;
Ok(())
}
fn should_compact_deltas(&self, task_id: &str) -> Result<bool> {
let path = self.checkpoint_delta_path(task_id)?;
if !path.exists() {
return Ok(false);
}
let metadata = fs::metadata(&path)
.with_context(|| format!("Failed to stat checkpoint delta log {:?}", path))?;
if metadata.len() > MAX_DELTA_FILE_BYTES {
return Ok(true);
}
let content = fs::read_to_string(&path)
.with_context(|| format!("Failed to read checkpoint delta log {:?}", path))?;
let line_count = content
.lines()
.filter(|line| !line.trim().is_empty())
.count();
Ok(line_count >= MAX_DELTA_ENTRIES_BEFORE_COMPACT)
}
fn clear_delta_log(&self, task_id: &str) -> Result<()> {
let delta_path = self.checkpoint_delta_path(task_id)?;
if delta_path.exists() {
fs::remove_file(&delta_path).with_context(|| {
format!("Failed to delete checkpoint delta log {:?}", delta_path)
})?;
}
Ok(())
}
fn prune_old_checkpoints(&self) {
self.prune_old_task_dirs();
let entries = match fs::read_dir(&self.checkpoints_dir) {
Ok(e) => e,
Err(e) => {
tracing::warn!("prune_old_checkpoints: failed to read dir: {}", e);
return;
}
};
let mut json_files: Vec<(PathBuf, std::time::SystemTime)> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|s| s.to_str()) != Some("json") {
continue;
}
if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) {
if stem.ends_with(".bak") || stem.ends_with(".tmp") {
continue;
}
}
let mtime = entry
.metadata()
.and_then(|m| m.modified())
.unwrap_or(std::time::SystemTime::UNIX_EPOCH);
json_files.push((path, mtime));
}
if json_files.len() <= MAX_CHECKPOINT_FILES {
return;
}
json_files.sort_by_key(|b| std::cmp::Reverse(b.1));
let to_delete = &json_files[MAX_CHECKPOINT_FILES..];
for (path, _) in to_delete {
let stem = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or_default();
if let Err(e) = fs::remove_file(path) {
tracing::warn!("prune_old_checkpoints: failed to delete {:?}: {}", path, e);
}
let delta_path = self.checkpoints_dir.join(format!("{}.delta.jsonl", stem));
if delta_path.exists() {
if let Err(e) = fs::remove_file(&delta_path) {
tracing::warn!(
"prune_old_checkpoints: failed to delete delta {:?}: {}",
delta_path,
e
);
}
}
let bak_path = path.with_extension("json.bak");
if bak_path.exists() {
if let Err(e) = fs::remove_file(&bak_path) {
tracing::warn!(
"prune_old_checkpoints: failed to delete backup {:?}: {}",
bak_path,
e
);
}
}
}
tracing::debug!(
"prune_old_checkpoints: pruned {} checkpoint(s) exceeding cap of {}",
to_delete.len(),
MAX_CHECKPOINT_FILES,
);
}
fn prune_old_task_dirs(&self) {
let entries = match fs::read_dir(&self.checkpoints_dir) {
Ok(e) => e,
Err(_) => return,
};
let mut dirs: Vec<(PathBuf, std::time::SystemTime)> = Vec::new();
for entry in entries.flatten() {
if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
continue;
}
let mtime = entry
.metadata()
.and_then(|m| m.modified())
.unwrap_or(std::time::SystemTime::UNIX_EPOCH);
dirs.push((entry.path(), mtime));
}
if dirs.len() <= MAX_CHECKPOINT_FILES {
return;
}
dirs.sort_by_key(|b| std::cmp::Reverse(b.1)); for (path, _) in &dirs[MAX_CHECKPOINT_FILES..] {
if let Err(e) = fs::remove_dir_all(path) {
tracing::warn!("prune_old_task_dirs: failed to remove {:?}: {}", path, e);
}
}
}
fn save_full_checkpoint(&self, checkpoint: &TaskCheckpoint) -> Result<()> {
let path = self.checkpoint_path(&checkpoint.task_id)?;
let mut json_value =
serde_json::to_value(checkpoint).context("Failed to serialize checkpoint")?;
redact::redact_json(&mut json_value);
let envelope =
CheckpointEnvelope::wrap(json_value).context("Failed to create checkpoint envelope")?;
let json =
serde_json::to_string_pretty(&envelope).context("Failed to format checkpoint JSON")?;
let suffix = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let tmp_path = path.with_extension(format!(
"json.tmp.{}.{}.{}",
checkpoint.task_id,
std::process::id(),
suffix
));
{
let mut open_opts = fs::OpenOptions::new();
open_opts.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
open_opts.mode(0o600);
}
let mut tmp_file = open_opts
.open(&tmp_path)
.with_context(|| format!("Failed to create checkpoint temp file {:?}", tmp_path))?;
tmp_file
.write_all(json.as_bytes())
.with_context(|| format!("Failed to write checkpoint temp file {:?}", tmp_path))?;
tmp_file
.sync_all()
.with_context(|| format!("Failed to fsync checkpoint temp file {:?}", tmp_path))?;
}
if path.exists() {
let backup_path = path.with_extension("json.bak");
if let Err(e) = fs::rename(&path, &backup_path) {
tracing::warn!("Failed to create checkpoint backup: {}", e);
}
}
if let Err(first_err) = fs::rename(&tmp_path, &path) {
if path.exists() {
if let Err(remove_err) = fs::remove_file(&path) {
let _ = fs::remove_file(&tmp_path);
return Err(remove_err).with_context(|| {
format!(
"Failed to remove existing checkpoint {:?} for atomic replace (original rename error: {})",
path, first_err
)
});
}
if let Err(retry_err) = fs::rename(&tmp_path, &path) {
let _ = fs::remove_file(&tmp_path);
return Err(retry_err).with_context(|| {
format!(
"Failed to rename checkpoint {:?} from {:?} after removing target",
path, tmp_path
)
});
}
} else {
let _ = fs::remove_file(&tmp_path);
return Err(first_err).with_context(|| {
format!(
"Failed to atomically replace checkpoint {:?} from {:?}",
path, tmp_path
)
});
}
}
#[cfg(unix)]
{
if let Some(parent) = path.parent() {
let dir = fs::OpenOptions::new()
.read(true)
.open(parent)
.with_context(|| {
format!("Failed to open checkpoint directory for fsync {:?}", parent)
})?;
dir.sync_all().with_context(|| {
format!("Failed to fsync checkpoint directory {:?}", parent)
})?;
}
}
Ok(())
}
pub fn load(&self, task_id: &str) -> Result<TaskCheckpoint> {
let path = self.checkpoint_path(task_id)?;
match self.try_load_from_path(&path).and_then(|mut checkpoint| {
self.apply_deltas(task_id, &mut checkpoint)?;
Ok(checkpoint)
}) {
Ok(checkpoint) => Ok(checkpoint),
Err(primary_err) => {
tracing::warn!(
"Primary checkpoint load failed for {:?}: {}. Attempting recovery.",
path,
primary_err
);
self.recover_from_corruption(task_id).with_context(|| {
format!(
"Recovery also failed for task '{}'. Original error: {}",
task_id, primary_err
)
})
}
}
}
fn apply_deltas(&self, task_id: &str, checkpoint: &mut TaskCheckpoint) -> Result<()> {
let path = self.checkpoint_delta_path(task_id)?;
if !path.exists() {
return Ok(());
}
let content = fs::read_to_string(&path)
.with_context(|| format!("Failed to read checkpoint delta log {:?}", path))?;
for (line_no, line) in content.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
let delta = if let Ok(envelope) = serde_json::from_str::<CheckpointEnvelope>(line) {
envelope.verify().with_context(|| {
format!(
"Checkpoint delta integrity check failed for {:?} line {}",
path,
line_no + 1
)
})?;
serde_json::from_value::<CheckpointDelta>(envelope.payload).with_context(|| {
format!(
"Failed to deserialize checkpoint delta from {:?} line {}",
path,
line_no + 1
)
})?
} else {
serde_json::from_str::<CheckpointDelta>(line).with_context(|| {
format!(
"Failed to deserialize legacy checkpoint delta from {:?} line {}",
path,
line_no + 1
)
})?
};
checkpoint.apply_delta(&delta).with_context(|| {
format!(
"Failed to apply checkpoint delta from {:?} line {}",
path,
line_no + 1
)
})?;
}
Ok(())
}
fn try_load_from_path(&self, path: &std::path::Path) -> Result<TaskCheckpoint> {
let json = fs::read_to_string(path)
.with_context(|| format!("Failed to read checkpoint from {:?}", path))?;
if let Ok(envelope) = serde_json::from_str::<CheckpointEnvelope>(&json) {
envelope
.verify()
.with_context(|| format!("Checkpoint integrity check failed for {:?}", path))?;
let checkpoint: TaskCheckpoint = serde_json::from_value(envelope.payload)
.context("Failed to deserialize checkpoint from envelope payload")?;
return Ok(checkpoint);
}
let checkpoint: TaskCheckpoint =
serde_json::from_str(&json).context("Failed to deserialize checkpoint")?;
Ok(checkpoint)
}
pub fn recover_from_corruption(&self, task_id: &str) -> Result<TaskCheckpoint> {
let backup_path = self.checkpoint_path(task_id)?.with_extension("json.bak");
if backup_path.exists() {
match self.try_load_from_path(&backup_path) {
Ok(checkpoint) => {
tracing::info!(
"Recovered checkpoint for task '{}' from backup {:?}",
task_id,
backup_path
);
if let Err(e) = self.save(&checkpoint) {
tracing::warn!(
"Failed to re-save recovered checkpoint for '{}': {}",
task_id,
e
);
}
return Ok(checkpoint);
}
Err(e) => {
tracing::warn!("Backup checkpoint {:?} is also corrupt: {}", backup_path, e);
}
}
}
tracing::warn!(
"DATA LOSS: checkpoint for task '{}' and its backup are both unreadable; \
creating a blank fresh checkpoint. Prior messages/audit are lost and any \
uncommitted file changes from before the crash are now untracked — review \
the working tree manually.",
task_id
);
let fresh = TaskCheckpoint::new(task_id.to_string(), String::new());
self.save(&fresh)
.with_context(|| format!("Failed to save fresh checkpoint for '{}'", task_id))?;
Ok(fresh)
}
pub fn save_with_retry(&self, checkpoint: &TaskCheckpoint) -> Result<()> {
const DELAYS_MS: [u64; 3] = [100, 500, 2000];
let mut last_err: Option<anyhow::Error> = None;
for (attempt, delay_ms) in DELAYS_MS.iter().enumerate() {
if attempt > 0 {
if let Some(ref e) = last_err {
tracing::warn!(
"Checkpoint save attempt {}/3 failed for task '{}': {}. Retrying in {} ms.",
attempt,
checkpoint.task_id,
e,
delay_ms
);
}
std::thread::sleep(std::time::Duration::from_millis(*delay_ms));
}
match self.save(checkpoint) {
Ok(()) => return Ok(()),
Err(e) => {
last_err = Some(e);
}
}
}
Err(last_err.map_or_else(
|| {
anyhow::anyhow!(
"Checkpoint save failed: all {} retry attempts exhausted",
DELAYS_MS.len()
)
},
|e| {
anyhow::anyhow!(
"Checkpoint save failed after {} attempts: {}",
DELAYS_MS.len(),
e
)
},
))
}
pub fn list_tasks(&self) -> Result<Vec<TaskSummary>> {
let mut summaries = Vec::new();
if !self.checkpoints_dir.exists() {
return Ok(summaries);
}
for entry in fs::read_dir(&self.checkpoints_dir)? {
let entry = entry?;
let path = entry.path();
if path.extension().and_then(|s| s.to_str()) == Some("json") {
if let Ok(mut checkpoint) = self.try_load_from_path(&path) {
if let Some(task_id) = path.file_stem().and_then(|s| s.to_str()) {
if let Err(e) = self.apply_deltas(task_id, &mut checkpoint) {
tracing::warn!(
"Skipping checkpoint {:?} due to invalid deltas: {}",
path,
e
);
continue;
}
}
summaries.push(checkpoint.to_summary());
}
}
}
summaries.sort_by_key(|x| std::cmp::Reverse(x.updated_at));
Ok(summaries)
}
pub fn delete(&self, task_id: &str) -> Result<()> {
let path = self.checkpoint_path(task_id)?;
if path.exists() {
fs::remove_file(&path)
.with_context(|| format!("Failed to delete checkpoint: {:?}", path))?;
}
let backup_path = path.with_extension("json.bak");
if backup_path.exists() {
fs::remove_file(&backup_path).with_context(|| {
format!("Failed to delete checkpoint backup: {:?}", backup_path)
})?;
}
let delta_path = self.checkpoint_delta_path(task_id)?;
if delta_path.exists() {
fs::remove_file(&delta_path).with_context(|| {
format!("Failed to delete checkpoint delta log: {:?}", delta_path)
})?;
}
Ok(())
}
#[cfg(test)]
pub fn exists(&self, task_id: &str) -> bool {
self.checkpoint_path(task_id)
.map(|p| p.exists())
.unwrap_or(false)
}
#[cfg(test)]
pub fn checkpoints_dir(&self) -> &PathBuf {
&self.checkpoints_dir
}
}
fn dirs_home() -> PathBuf {
std::env::var("HOME")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("."))
}
pub fn capture_git_state(repo_path: &str) -> Option<GitCheckpointInfo> {
let repo = git2::Repository::open(repo_path).ok()?;
let head = repo.head().ok()?;
let branch = head
.shorthand()
.map(|s| s.to_string())
.unwrap_or_else(|_| "HEAD".to_string());
let commit = head.peel_to_commit().ok()?;
let commit_hash = commit.id().to_string();
let statuses = repo.statuses(None).ok()?;
let mut staged_files = Vec::new();
let mut modified_files = Vec::new();
for entry in statuses.iter() {
let status = entry.status();
let path = entry.path().unwrap_or("").to_string();
if status.is_index_new()
|| status.is_index_modified()
|| status.is_index_deleted()
|| status.is_index_renamed()
{
staged_files.push(path.clone());
}
if status.is_wt_new()
|| status.is_wt_modified()
|| status.is_wt_deleted()
|| status.is_wt_renamed()
{
modified_files.push(path);
}
}
let dirty = !staged_files.is_empty() || !modified_files.is_empty();
Some(GitCheckpointInfo {
branch,
commit_hash,
dirty,
staged_files,
modified_files,
})
}
#[cfg(test)]
#[path = "../../tests/unit/session/checkpoint/checkpoint_test.rs"]
mod tests;