use chrono::Utc;
use prodigy::cook::execution::mapreduce::checkpoint::{
AgentInfo, AgentState, CheckpointConfig, CheckpointManager, CheckpointMetadata,
CheckpointReason, CompletedWorkItem, ErrorState, ExecutionState, FailedWorkItem,
FileCheckpointStorage, MapReduceCheckpoint, PhaseType, ResourceState, ResumeStrategy,
VariableState, WorkItem, WorkItemProgress, WorkItemState,
};
use prodigy::cook::execution::mapreduce::{AgentResult, AgentStatus};
use serde_json::Value;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tempfile::TempDir;
use tokio::time::timeout;
fn create_test_checkpoint(
job_id: &str,
items_completed: usize,
total_items: usize,
) -> MapReduceCheckpoint {
let mut work_item_state = WorkItemState {
pending_items: vec![],
in_progress_items: HashMap::new(),
completed_items: vec![],
failed_items: vec![],
current_batch: None,
};
for i in 0..items_completed {
work_item_state.completed_items.push(CompletedWorkItem {
work_item: WorkItem {
id: format!("item-{}", i),
data: Value::String(format!("data-{}", i)),
},
result: AgentResult {
item_id: format!("item-{}", i),
status: AgentStatus::Success,
output: Some(format!("Processed item {}", i)),
commits: vec![format!("commit-{}", i)],
files_modified: vec![format!("file-{}.rs", i)],
duration: Duration::from_secs(10),
error: None,
worktree_path: Some(PathBuf::from(format!("/tmp/worktree-{}", i))),
branch_name: Some(format!("branch-{}", i)),
worktree_session_id: Some(format!("session-{}", i)),
json_log_location: None,
cleanup_status: None,
},
completed_at: Utc::now(),
});
}
for i in items_completed..total_items {
work_item_state.pending_items.push(WorkItem {
id: format!("item-{}", i),
data: Value::String(format!("data-{}", i)),
});
}
let mut active_agents = HashMap::new();
let mut agent_assignments = HashMap::new();
if items_completed < total_items {
for i in 0..2.min(total_items - items_completed) {
let agent_id = format!("agent-{}", i);
let item_id = format!("item-{}", items_completed + i);
active_agents.insert(
agent_id.clone(),
AgentInfo {
agent_id: agent_id.clone(),
worktree_path: PathBuf::from(format!("/tmp/worktree-agent-{}", i)),
started_at: Utc::now(),
last_heartbeat: Utc::now(),
status: AgentStatus::Running,
},
);
agent_assignments.insert(agent_id.clone(), vec![item_id.clone()]);
work_item_state.in_progress_items.insert(
item_id.clone(),
WorkItemProgress {
work_item: WorkItem {
id: item_id.clone(),
data: Value::String(format!("data-{}", items_completed + i)),
},
agent_id: agent_id.clone(),
started_at: Utc::now(),
last_update: Utc::now(),
},
);
}
}
MapReduceCheckpoint {
metadata: CheckpointMetadata {
checkpoint_id: format!("checkpoint-{}", uuid::Uuid::new_v4()),
job_id: job_id.to_string(),
version: 1,
created_at: Utc::now(),
phase: if items_completed == 0 {
PhaseType::Setup
} else if items_completed < total_items {
PhaseType::Map
} else {
PhaseType::Reduce
},
total_work_items: total_items,
completed_items: items_completed,
checkpoint_reason: CheckpointReason::Interval,
integrity_hash: String::new(),
},
execution_state: ExecutionState {
current_phase: if items_completed == 0 {
PhaseType::Setup
} else if items_completed < total_items {
PhaseType::Map
} else {
PhaseType::Reduce
},
phase_start_time: Utc::now() - chrono::Duration::minutes(10),
setup_results: if items_completed > 0 {
Some(
prodigy::cook::execution::mapreduce::checkpoint::PhaseResult {
success: true,
outputs: vec!["Setup complete".to_string()],
duration: Duration::from_secs(60),
},
)
} else {
None
},
map_results: if items_completed > 0 {
Some(
prodigy::cook::execution::mapreduce::checkpoint::MapPhaseResults {
successful_count: items_completed,
failed_count: 0,
total_duration: Duration::from_secs(items_completed as u64 * 10),
},
)
} else {
None
},
reduce_results: None,
workflow_variables: HashMap::new(),
},
work_item_state,
agent_state: AgentState {
active_agents: active_agents.clone(),
agent_assignments,
agent_results: HashMap::new(),
resource_allocation: HashMap::new(),
},
variable_state: VariableState {
workflow_variables: HashMap::from([
("job_id".to_string(), job_id.to_string()),
("total_items".to_string(), total_items.to_string()),
]),
captured_outputs: HashMap::new(),
environment_variables: HashMap::from([(
"PRODIGY_MODE".to_string(),
"test".to_string(),
)]),
item_variables: HashMap::new(),
},
resource_state: ResourceState {
total_agents_allowed: 10,
current_agents_active: active_agents.len(),
worktrees_created: (0..items_completed)
.map(|i| format!("worktree-{}", i))
.collect(),
worktrees_cleaned: vec![],
disk_usage_bytes: Some((items_completed as u64) * 1024 * 1024),
},
error_state: ErrorState {
error_count: 0,
dlq_items: vec![],
error_threshold_reached: false,
last_error: None,
},
}
}
#[tokio::test]
async fn test_checkpoint_save_and_load_after_interruption() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let config = CheckpointConfig::default();
let job_id = "interrupt-test-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let checkpoint_before = create_test_checkpoint(job_id, 5, 10);
let checkpoint_id = manager
.create_checkpoint(&checkpoint_before, CheckpointReason::BeforeShutdown)
.await
.unwrap();
drop(manager);
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let config = CheckpointConfig::default();
let new_manager = CheckpointManager::new(storage, config, job_id.to_string());
let resume_state = new_manager
.resume_from_checkpoint(Some(checkpoint_id))
.await
.unwrap();
assert_eq!(resume_state.checkpoint.metadata.job_id, job_id);
assert_eq!(resume_state.checkpoint.metadata.completed_items, 5);
assert_eq!(resume_state.checkpoint.metadata.total_work_items, 10);
assert_eq!(resume_state.work_items.completed_items.len(), 5);
assert!(resume_state.work_items.in_progress_items.is_empty());
assert_eq!(resume_state.work_items.pending_items.len(), 7); }
#[tokio::test]
async fn test_checkpoint_periodic_saves_during_execution() {
let temp_dir = TempDir::new().unwrap();
let _storage = Arc::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let config = CheckpointConfig {
interval_items: Some(3), interval_duration: Some(Duration::from_secs(5)), ..Default::default()
};
let job_id = "periodic-checkpoint-job";
let storage_clone = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let manager = Arc::new(CheckpointManager::new(
storage_clone,
config,
job_id.to_string(),
));
let mut last_checkpoint_time = Utc::now();
let mut items_since_checkpoint = 0;
for i in 0..10 {
let checkpoint = create_test_checkpoint(job_id, i + 1, 10);
items_since_checkpoint += 1;
if manager.should_checkpoint(items_since_checkpoint, last_checkpoint_time) {
let _checkpoint_id = manager
.create_checkpoint(&checkpoint, CheckpointReason::Interval)
.await
.unwrap();
items_since_checkpoint = 0;
last_checkpoint_time = Utc::now();
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
let checkpoints = manager.list_checkpoints().await.unwrap();
assert!(
checkpoints.len() >= 3,
"Should have created multiple checkpoints"
);
}
#[tokio::test]
async fn test_resume_after_agent_failure() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let config = CheckpointConfig::default();
let job_id = "agent-failure-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let mut checkpoint = create_test_checkpoint(job_id, 3, 10);
checkpoint.work_item_state.failed_items = vec![
FailedWorkItem {
work_item: WorkItem {
id: "failed-1".to_string(),
data: Value::String("failed-data-1".to_string()),
},
error: "Agent crashed".to_string(),
failed_at: Utc::now(),
retry_count: 1,
},
FailedWorkItem {
work_item: WorkItem {
id: "failed-2".to_string(),
data: Value::String("failed-data-2".to_string()),
},
error: "Timeout".to_string(),
failed_at: Utc::now(),
retry_count: 2,
},
];
checkpoint.error_state.error_count = 2;
checkpoint.error_state.last_error = Some("Agent timeout".to_string());
let checkpoint_id = manager
.create_checkpoint(&checkpoint, CheckpointReason::ErrorRecovery)
.await
.unwrap();
let resume_state = manager
.resume_from_checkpoint_with_strategy(
Some(checkpoint_id),
ResumeStrategy::ValidateAndContinue,
)
.await
.unwrap();
assert_eq!(
resume_state.checkpoint.work_item_state.failed_items.len(),
2
);
assert_eq!(resume_state.checkpoint.error_state.error_count, 2);
}
#[tokio::test]
async fn test_concurrent_checkpoint_updates() {
let temp_dir = TempDir::new().unwrap();
let _storage = Arc::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
false,
));
let config = CheckpointConfig::default();
let job_id = "concurrent-job";
let storage_clone = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let manager = Arc::new(CheckpointManager::new(
storage_clone,
config,
job_id.to_string(),
));
let mut tasks = vec![];
for i in 0..5 {
let manager_clone = manager.clone();
let job_id_clone = job_id.to_string();
tasks.push(tokio::spawn(async move {
let checkpoint = create_test_checkpoint(&job_id_clone, i * 2, 20);
manager_clone
.create_checkpoint(&checkpoint, CheckpointReason::Interval)
.await
}));
}
let results: Vec<_> = futures::future::join_all(tasks).await;
for result in results {
assert!(
result.unwrap().is_ok(),
"Checkpoint creation should succeed"
);
}
let checkpoints = manager.list_checkpoints().await.unwrap();
assert_eq!(checkpoints.len(), 5, "Should have 5 checkpoints");
}
#[tokio::test]
async fn test_checkpoint_compression_and_decompression() {
let temp_dir = TempDir::new().unwrap();
let algorithms = vec![
prodigy::cook::execution::mapreduce::checkpoint::CompressionAlgorithm::None,
prodigy::cook::execution::mapreduce::checkpoint::CompressionAlgorithm::Gzip,
prodigy::cook::execution::mapreduce::checkpoint::CompressionAlgorithm::Zstd,
prodigy::cook::execution::mapreduce::checkpoint::CompressionAlgorithm::Lz4,
];
for algo in algorithms {
let storage = Box::new(FileCheckpointStorage::with_compression(
temp_dir.path().join(format!("{:?}", algo)),
algo,
));
let config = CheckpointConfig::default();
let job_id = format!("compress-test-{:?}", algo);
let manager = CheckpointManager::new(storage, config, job_id.clone());
let mut checkpoint = create_test_checkpoint(&job_id, 50, 100);
for i in 0..50 {
checkpoint.variable_state.captured_outputs.insert(
format!("output-{}", i),
format!("This is a long output string for item {} that contains lots of repeated data to test compression effectiveness", i),
);
}
let checkpoint_id = manager
.create_checkpoint(&checkpoint, CheckpointReason::Manual)
.await
.unwrap();
let resume_state = manager
.resume_from_checkpoint(Some(checkpoint_id))
.await
.unwrap();
assert_eq!(resume_state.checkpoint.metadata.completed_items, 50);
assert_eq!(
resume_state
.checkpoint
.variable_state
.captured_outputs
.len(),
50
);
}
}
#[tokio::test]
async fn test_phase_transition_checkpoints() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let config = CheckpointConfig::default();
let job_id = "phase-transition-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let mut setup_checkpoint = create_test_checkpoint(job_id, 0, 10);
setup_checkpoint.metadata.phase = PhaseType::Setup;
setup_checkpoint.execution_state.current_phase = PhaseType::Setup;
let setup_id = manager
.create_checkpoint(&setup_checkpoint, CheckpointReason::PhaseTransition)
.await
.unwrap();
let setup_resume = manager
.resume_from_checkpoint(Some(setup_id))
.await
.unwrap();
assert!(matches!(
setup_resume.resume_strategy,
ResumeStrategy::RestartCurrentPhase
));
let mut map_checkpoint = create_test_checkpoint(job_id, 10, 10);
map_checkpoint.metadata.phase = PhaseType::Map;
map_checkpoint.execution_state.current_phase = PhaseType::Map;
let map_id = manager
.create_checkpoint(&map_checkpoint, CheckpointReason::PhaseTransition)
.await
.unwrap();
let map_resume = manager.resume_from_checkpoint(Some(map_id)).await.unwrap();
assert!(matches!(
map_resume.resume_strategy,
ResumeStrategy::ContinueFromCheckpoint
));
}
#[tokio::test]
async fn test_checkpoint_export_import_across_environments() {
let temp_dir1 = TempDir::new().unwrap();
let temp_dir2 = TempDir::new().unwrap();
let export_path = temp_dir1.path().join("export.json");
let storage1 = Box::new(FileCheckpointStorage::new(
temp_dir1.path().join("checkpoints"),
true,
));
let config1 = CheckpointConfig::default();
let job_id = "export-job";
let manager1 = CheckpointManager::new(storage1, config1, job_id.to_string());
let checkpoint = create_test_checkpoint(job_id, 7, 15);
let checkpoint_id = manager1
.create_checkpoint(&checkpoint, CheckpointReason::Manual)
.await
.unwrap();
manager1
.export_checkpoint(&checkpoint_id, export_path.clone())
.await
.unwrap();
assert!(export_path.exists(), "Export file should exist");
let storage2 = Box::new(FileCheckpointStorage::new(
temp_dir2.path().join("checkpoints"),
true,
));
let config2 = CheckpointConfig::default();
let manager2 = CheckpointManager::new(storage2, config2, job_id.to_string());
let imported_id = manager2.import_checkpoint(export_path).await.unwrap();
let resume_state = manager2
.resume_from_checkpoint(Some(imported_id))
.await
.unwrap();
assert_eq!(resume_state.checkpoint.metadata.completed_items, 7);
assert_eq!(resume_state.checkpoint.metadata.total_work_items, 15);
}
#[tokio::test]
async fn test_checkpoint_timeout_handling() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
false,
));
let config = CheckpointConfig::default();
let job_id = "timeout-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let checkpoint = create_test_checkpoint(job_id, 100, 200);
let save_future = manager.create_checkpoint(&checkpoint, CheckpointReason::Manual);
let result = timeout(Duration::from_secs(10), save_future).await;
assert!(result.is_ok(), "Should complete within timeout");
let checkpoint_id = result.unwrap().unwrap();
let load_future = manager.resume_from_checkpoint(Some(checkpoint_id));
let load_result = timeout(Duration::from_secs(10), load_future).await;
assert!(load_result.is_ok(), "Should load within timeout");
}
#[tokio::test]
async fn test_checkpoint_retention_and_cleanup() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
false,
));
let config = CheckpointConfig {
retention_policy: Some(
prodigy::cook::execution::mapreduce::checkpoint::RetentionPolicy {
max_checkpoints: Some(3),
max_age: Some(Duration::from_secs(60)),
keep_final: true,
},
),
..Default::default()
};
let job_id = "retention-test-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let mut checkpoint_ids = vec![];
for i in 0..5 {
let checkpoint = create_test_checkpoint(job_id, i * 2, 10);
let id = manager
.create_checkpoint(&checkpoint, CheckpointReason::Interval)
.await
.unwrap();
checkpoint_ids.push(id);
tokio::time::sleep(Duration::from_millis(100)).await;
}
let remaining = manager.list_checkpoints().await.unwrap();
assert!(remaining.len() <= 5, "Checkpoints should be tracked");
}
#[tokio::test]
async fn test_resume_strategy_selection() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let config = CheckpointConfig::default();
let job_id = "strategy-test-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let strategies = vec![
ResumeStrategy::ContinueFromCheckpoint,
ResumeStrategy::RestartCurrentPhase,
ResumeStrategy::RestartFromMapPhase,
ResumeStrategy::ValidateAndContinue,
];
let checkpoint = create_test_checkpoint(job_id, 5, 10);
let checkpoint_id = manager
.create_checkpoint(&checkpoint, CheckpointReason::Manual)
.await
.unwrap();
for strategy in strategies {
let resume_state = manager
.resume_from_checkpoint_with_strategy(Some(checkpoint_id.clone()), strategy.clone())
.await
.unwrap();
match strategy {
ResumeStrategy::ContinueFromCheckpoint => {
assert_eq!(resume_state.work_items.completed_items.len(), 5);
}
ResumeStrategy::RestartCurrentPhase => {
assert!(resume_state.work_items.completed_items.is_empty());
}
ResumeStrategy::RestartFromMapPhase => {
assert!(resume_state.work_items.completed_items.is_empty());
assert!(resume_state.work_items.pending_items.len() >= 10);
}
ResumeStrategy::ValidateAndContinue => {
assert!(resume_state.work_items.in_progress_items.is_empty());
}
}
}
}
#[tokio::test]
async fn test_checkpoint_integrity_verification() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
false,
));
let config = CheckpointConfig {
validate_on_save: true,
validate_on_load: true,
..Default::default()
};
let job_id = "integrity-test-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let checkpoint = create_test_checkpoint(job_id, 3, 10);
let checkpoint_id = manager
.create_checkpoint(&checkpoint, CheckpointReason::Manual)
.await
.unwrap();
let resume_state = manager
.resume_from_checkpoint(Some(checkpoint_id))
.await
.unwrap();
assert!(!resume_state.checkpoint.metadata.integrity_hash.is_empty());
}
#[tokio::test]
async fn test_worktree_resource_tracking_across_resume() {
let temp_dir = TempDir::new().unwrap();
let storage = Box::new(FileCheckpointStorage::new(
temp_dir.path().to_path_buf(),
true,
));
let config = CheckpointConfig::default();
let job_id = "worktree-tracking-job";
let manager = CheckpointManager::new(storage, config, job_id.to_string());
let mut checkpoint = create_test_checkpoint(job_id, 4, 10);
checkpoint.resource_state.worktrees_created = vec![
"wt-1".to_string(),
"wt-2".to_string(),
"wt-3".to_string(),
"wt-4".to_string(),
];
checkpoint.resource_state.worktrees_cleaned = vec!["wt-1".to_string()];
checkpoint.resource_state.current_agents_active = 3;
checkpoint.resource_state.disk_usage_bytes = Some(50 * 1024 * 1024);
let checkpoint_id = manager
.create_checkpoint(&checkpoint, CheckpointReason::Manual)
.await
.unwrap();
let resume_state = manager
.resume_from_checkpoint(Some(checkpoint_id))
.await
.unwrap();
assert_eq!(resume_state.resources.worktrees_created.len(), 4);
assert_eq!(resume_state.resources.worktrees_cleaned.len(), 1);
assert_eq!(resume_state.resources.current_agents_active, 3);
assert_eq!(
resume_state.resources.disk_usage_bytes,
Some(50 * 1024 * 1024)
);
}