use std::collections::BTreeMap;
#[derive(Clone)]
pub(super) struct LifecycleTracker {
remaining: u32,
outcomes: BTreeMap<u32, LifecycleOutcome>,
}
#[derive(Clone)]
pub(super) struct LifecycleOutcome {
pub(super) status: String,
pub(super) exit_code: Option<i32>,
}
impl LifecycleTracker {
pub(super) fn new(replicas: u32) -> Self {
Self {
remaining: replicas,
outcomes: BTreeMap::new(),
}
}
pub(super) fn finish(
&mut self,
replica: u32,
status: &str,
exit_code: Option<i32>,
) -> Option<LifecycleOutcome> {
if self.outcomes.contains_key(&replica) {
return None;
}
self.outcomes.insert(
replica,
LifecycleOutcome {
status: status.to_string(),
exit_code,
},
);
self.remaining = self.remaining.saturating_sub(1);
if self.remaining > 0 {
return None;
}
self.outcomes
.iter()
.min_by_key(|(replica, outcome)| (outcome_rank(&outcome.status), **replica))
.map(|(_, outcome)| outcome.clone())
}
}
fn outcome_rank(status: &str) -> u8 {
if status.starts_with("log_redaction_failed:") {
0
} else if status.starts_with("context_record_failed:") {
1
} else if status.starts_with("restart_failed:") {
2
} else {
match status {
"failed" => 3,
"interrupted" => 4,
"timed_out" => 5,
"stopped" => 6,
"completed" => 7,
_ => 8,
}
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
use tempfile::tempdir;
use super::super::*;
use super::outcome_rank;
#[test]
fn waits_for_every_replica_and_keeps_the_strongest_outcome() {
let mut tracker = LifecycleTracker::new(2);
assert!(tracker.finish(1, "completed", Some(0)).is_none());
let outcome = tracker.finish(0, "failed", Some(9)).unwrap();
assert_eq!(outcome.status, "failed");
assert_eq!(outcome.exit_code, Some(9));
assert!(tracker.finish(0, "completed", Some(0)).is_none());
}
#[test]
fn ranks_every_terminal_outcome() {
assert_eq!(outcome_rank("log_redaction_failed:rewrite"), 0);
assert_eq!(outcome_rank("context_record_failed:store"), 1);
assert_eq!(outcome_rank("restart_failed:spawn"), 2);
assert_eq!(outcome_rank("failed"), 3);
assert_eq!(outcome_rank("interrupted"), 4);
assert_eq!(outcome_rank("timed_out"), 5);
assert_eq!(outcome_rank("stopped"), 6);
assert_eq!(outcome_rank("completed"), 7);
assert_eq!(outcome_rank("unknown"), 8);
}
#[tokio::test]
async fn terminal_run_waits_for_every_replica_before_redacting_shared_logs() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let marker = directory.path().join("first-replica");
let secret = "synthetic-replica-secret";
let spec = RunSpec {
name: "replica-redaction".to_string(),
purpose: Some("verify terminal replica ownership".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec![
"/bin/sh".to_string(),
"-c".to_string(),
concat!(
"if mkdir \"$MARKER\" 2>/dev/null; then ",
"printf '%s\\n' \"$TEST_SECRET\"; else sleep 0.35; ",
"printf '%s\\n' \"$TEST_SECRET\"; fi"
)
.to_string(),
],
cwd: None,
env: BTreeMap::from([("MARKER".to_string(), marker.to_string_lossy().into_owned())]),
resolved_env: BTreeMap::from([("TEST_SECRET".to_string(), secret.to_string())]),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 2,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
};
let details = launch_details(&spec).unwrap().to_string();
assert!(!details.contains(secret));
let started = tokio::time::Instant::now();
let run = manager.start(spec).await.unwrap();
assert!(run.spec.resolved_env.is_empty());
assert!(!serde_json::to_string(&run).unwrap().contains(secret));
assert!(
manager
.logs(&run.id, 1024)
.unwrap_err()
.contains("until secret redaction completes")
);
let finished = manager
.wait(&run.id, Some(Duration::from_secs(3)))
.await
.unwrap();
let logs = manager.logs(&run.id, 1024).unwrap();
assert_eq!(finished.status, "completed");
assert!(finished.processes.is_empty());
assert!(!manager.is_active(&run.id).await);
assert!(started.elapsed() >= Duration::from_millis(300));
assert!(!logs.stdout.contains(secret));
assert_eq!(logs.stdout.matches("[REDACTED]").count(), 2);
}
#[tokio::test]
async fn recovery_without_secret_values_deletes_pending_logs_and_fails_the_run() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store.clone()).unwrap();
let logs = directory.path().join("pending-secret-logs");
std::fs::create_dir_all(&logs).unwrap();
let stdout = logs.join("stdout.log");
let stderr = logs.join("stderr.log");
std::fs::write(&stdout, "synthetic-recovery-secret").unwrap();
std::fs::write(&stderr, "synthetic-recovery-secret").unwrap();
crate::log_file::require_redaction(&logs).unwrap();
let created_at = now_ms();
store
.put_run(&ManagedRun {
id: "recovered-secret-run".to_string(),
spec: RunSpec {
name: "recovered-secret".to_string(),
purpose: Some("verify fail-closed secret recovery".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::from([(
"TEST_SECRET".to_string(),
"synthetic-recovery-secret".to_string(),
)]),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
},
status: "running".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at,
updated_at: created_at,
exit_code: None,
stdout_path: Some(stdout.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
})
.unwrap();
manager.recover().await.unwrap();
let recovered = manager.get("recovered-secret-run").unwrap();
assert!(recovered.status.starts_with("log_redaction_failed:"));
assert!(recovered.spec.resolved_env.is_empty());
assert!(!stdout.exists());
assert!(!stderr.exists());
}
}