apoc 0.9.1

Agentic process control for LLMs
Documentation
//! Managed-run lifecycle aggregation across replicas.

use std::collections::BTreeMap;

#[derive(Clone)]
pub(super) struct LifecycleTracker {
    remaining: u32,
    outcomes: BTreeMap<u32, LifecycleOutcome>,
}

#[derive(Clone)]
pub(super) struct LifecycleOutcome {
    pub(super) status: String,
    pub(super) exit_code: Option<i32>,
}

impl LifecycleTracker {
    pub(super) fn new(replicas: u32) -> Self {
        Self {
            remaining: replicas,
            outcomes: BTreeMap::new(),
        }
    }

    pub(super) fn finish(
        &mut self,
        replica: u32,
        status: &str,
        exit_code: Option<i32>,
    ) -> Option<LifecycleOutcome> {
        if self.outcomes.contains_key(&replica) {
            return None;
        }
        self.outcomes.insert(
            replica,
            LifecycleOutcome {
                status: status.to_string(),
                exit_code,
            },
        );
        self.remaining = self.remaining.saturating_sub(1);
        if self.remaining > 0 {
            return None;
        }
        self.outcomes
            .iter()
            .min_by_key(|(replica, outcome)| (outcome_rank(&outcome.status), **replica))
            .map(|(_, outcome)| outcome.clone())
    }
}

fn outcome_rank(status: &str) -> u8 {
    if status.starts_with("log_redaction_failed:") {
        0
    } else if status.starts_with("context_record_failed:") {
        1
    } else if status.starts_with("restart_failed:") {
        2
    } else {
        match status {
            "failed" => 3,
            "interrupted" => 4,
            "timed_out" => 5,
            "stopped" => 6,
            "completed" => 7,
            _ => 8,
        }
    }
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::sync::Arc;
    use std::time::Duration;

    use tempfile::tempdir;

    use super::super::*;
    use super::outcome_rank;

    #[test]
    fn waits_for_every_replica_and_keeps_the_strongest_outcome() {
        let mut tracker = LifecycleTracker::new(2);

        assert!(tracker.finish(1, "completed", Some(0)).is_none());
        let outcome = tracker.finish(0, "failed", Some(9)).unwrap();

        assert_eq!(outcome.status, "failed");
        assert_eq!(outcome.exit_code, Some(9));
        assert!(tracker.finish(0, "completed", Some(0)).is_none());
    }

    #[test]
    fn ranks_every_terminal_outcome() {
        assert_eq!(outcome_rank("log_redaction_failed:rewrite"), 0);
        assert_eq!(outcome_rank("context_record_failed:store"), 1);
        assert_eq!(outcome_rank("restart_failed:spawn"), 2);
        assert_eq!(outcome_rank("failed"), 3);
        assert_eq!(outcome_rank("interrupted"), 4);
        assert_eq!(outcome_rank("timed_out"), 5);
        assert_eq!(outcome_rank("stopped"), 6);
        assert_eq!(outcome_rank("completed"), 7);
        assert_eq!(outcome_rank("unknown"), 8);
    }

    #[tokio::test]
    async fn terminal_run_waits_for_every_replica_before_redacting_shared_logs() {
        let directory = tempdir().unwrap();
        let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
        let manager = RunManager::new(store).unwrap();
        let marker = directory.path().join("first-replica");
        let secret = "synthetic-replica-secret";
        let spec = RunSpec {
            name: "replica-redaction".to_string(),
            purpose: Some("verify terminal replica ownership".to_string()),
            labels: BTreeMap::new(),
            requested_by: None,
            argv: vec![
                "/bin/sh".to_string(),
                "-c".to_string(),
                concat!(
                    "if mkdir \"$MARKER\" 2>/dev/null; then ",
                    "printf '%s\\n' \"$TEST_SECRET\"; else sleep 0.35; ",
                    "printf '%s\\n' \"$TEST_SECRET\"; fi"
                )
                .to_string(),
            ],
            cwd: None,
            env: BTreeMap::from([("MARKER".to_string(), marker.to_string_lossy().into_owned())]),
            resolved_env: BTreeMap::from([("TEST_SECRET".to_string(), secret.to_string())]),
            suppressed_env: Vec::new(),
            shell: false,
            io: IoMode::Pipes,
            lifetime: crate::model::RunLifetime::Persistent,
            restart: RestartPolicy::Never,
            timeout_ms: None,
            replicas: 2,
            max_attempts: 1,
            retry_backoff_ms: 500,
            retry_max_backoff_ms: 30_000,
        };
        let details = launch_details(&spec).unwrap().to_string();
        assert!(!details.contains(secret));

        let started = tokio::time::Instant::now();
        let run = manager.start(spec).await.unwrap();
        assert!(run.spec.resolved_env.is_empty());
        assert!(!serde_json::to_string(&run).unwrap().contains(secret));
        assert!(
            manager
                .logs(&run.id, 1024)
                .unwrap_err()
                .contains("until secret redaction completes")
        );
        let finished = manager
            .wait(&run.id, Some(Duration::from_secs(3)))
            .await
            .unwrap();
        let logs = manager.logs(&run.id, 1024).unwrap();

        assert_eq!(finished.status, "completed");
        assert!(finished.processes.is_empty());
        assert!(!manager.is_active(&run.id).await);
        assert!(started.elapsed() >= Duration::from_millis(300));
        assert!(!logs.stdout.contains(secret));
        assert_eq!(logs.stdout.matches("[REDACTED]").count(), 2);
    }

    #[tokio::test]
    async fn recovery_without_secret_values_deletes_pending_logs_and_fails_the_run() {
        let directory = tempdir().unwrap();
        let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
        let manager = RunManager::new(store.clone()).unwrap();
        let logs = directory.path().join("pending-secret-logs");
        std::fs::create_dir_all(&logs).unwrap();
        let stdout = logs.join("stdout.log");
        let stderr = logs.join("stderr.log");
        std::fs::write(&stdout, "synthetic-recovery-secret").unwrap();
        std::fs::write(&stderr, "synthetic-recovery-secret").unwrap();
        crate::log_file::require_redaction(&logs).unwrap();
        let created_at = now_ms();
        store
            .put_run(&ManagedRun {
                id: "recovered-secret-run".to_string(),
                spec: RunSpec {
                    name: "recovered-secret".to_string(),
                    purpose: Some("verify fail-closed secret recovery".to_string()),
                    labels: BTreeMap::new(),
                    requested_by: None,
                    argv: vec!["/bin/true".to_string()],
                    cwd: None,
                    env: BTreeMap::new(),
                    resolved_env: BTreeMap::from([(
                        "TEST_SECRET".to_string(),
                        "synthetic-recovery-secret".to_string(),
                    )]),
                    suppressed_env: Vec::new(),
                    shell: false,
                    io: IoMode::Pipes,
                    lifetime: crate::model::RunLifetime::Persistent,
                    restart: RestartPolicy::Never,
                    timeout_ms: None,
                    replicas: 1,
                    max_attempts: 1,
                    retry_backoff_ms: 500,
                    retry_max_backoff_ms: 30_000,
                },
                status: "running".to_string(),
                processes: Vec::new(),
                process_history: Vec::new(),
                created_at,
                updated_at: created_at,
                exit_code: None,
                stdout_path: Some(stdout.to_string_lossy().into_owned()),
                stderr_path: Some(stderr.to_string_lossy().into_owned()),
            })
            .unwrap();

        manager.recover().await.unwrap();
        let recovered = manager.get("recovered-secret-run").unwrap();

        assert!(recovered.status.starts_with("log_redaction_failed:"));
        assert!(recovered.spec.resolved_env.is_empty());
        assert!(!stdout.exists());
        assert!(!stderr.exists());
    }
}