apoc 0.9.1

Agentic process control for LLMs
Documentation
//! Reproducible process-workflow performance comparisons.

use std::sync::Arc;
use std::time::Instant;

use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;

use crate::model::{ProcessController, now_ms};
use crate::process::ProcessManager;

/// Measurements for one apoc-versus-shell workflow.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct BenchmarkCase {
    /// Stable workflow name.
    pub name: String,
    /// Total apoc time in microseconds.
    pub apoc_total_us: u64,
    /// Total shell-tool time in microseconds.
    pub shell_total_us: u64,
    /// Mean apoc time per iteration in microseconds.
    pub apoc_mean_us: u64,
    /// Mean shell-tool time per iteration in microseconds.
    pub shell_mean_us: u64,
    /// `shell_total / apoc_total`; values above one favor apoc.
    pub speedup: f64,
    /// Whether apoc was faster in this sample.
    pub apoc_faster: bool,
    /// Important semantic differences between the compared outputs.
    pub context: String,
}

/// Structured process benchmark report.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct ProcessBenchmark {
    /// Report timestamp in Unix milliseconds.
    pub measured_at: u64,
    /// Timed iterations per case.
    pub iterations: usize,
    /// Warmup iterations excluded from timings.
    pub warmup: usize,
    /// Whether every representative agent workflow favored apoc.
    pub all_apoc_faster: bool,
    /// Individual workflow measurements.
    pub cases: Vec<BenchmarkCase>,
    /// Scope and interpretation of this benchmark.
    pub methodology: String,
}

/// Measures persistent apoc process workflows against spawned macOS `ps`.
pub async fn process_benchmark(
    manager: Arc<Mutex<ProcessManager>>,
    iterations: usize,
    warmup: usize,
    controller: ProcessController,
) -> Result<ProcessBenchmark, String> {
    if iterations == 0 || iterations > 1000 {
        return Err("iterations must be between 1 and 1000".to_string());
    }
    if warmup > 100 {
        return Err("warmup cannot exceed 100".to_string());
    }
    let task = tokio::task::spawn_blocking(move || {
        let mut manager = manager.blocking_lock();
        let controller_pid = std::process::id();
        for _ in 0..warmup {
            let processes = manager.list(None, "pid", 50)?;
            manager.record_with_controller(
                &processes,
                "benchmark.process.inventory",
                "observed",
                None,
                &controller,
                "observed",
                None,
                &serde_json::json!({ "warmup": true }),
            )?;
            let process = manager.get(&format!("pid:{controller_pid}"))?;
            manager.record_with_controller(
                std::slice::from_ref(&process),
                "benchmark.process.target",
                "observed",
                Some(&format!("pid:{controller_pid}")),
                &controller,
                "observed",
                None,
                &serde_json::json!({ "warmup": true }),
            )?;
        }

        let apoc_inventory = elapsed_us(|| {
            for _ in 0..iterations {
                let processes = manager.list(None, "pid", 50)?;
                manager.record_with_controller(
                    &processes,
                    "benchmark.process.inventory",
                    "observed",
                    None,
                    &controller,
                    "observed",
                    None,
                    &serde_json::json!({ "warmup": false }),
                )?;
            }
            Ok(())
        })?;
        let shell_inventory = elapsed_us(|| {
            for _ in 0..iterations {
                shell_ps(&[
                    "-axo",
                    "pid=,ppid=,state=,%cpu=,rss=,vsz=,etime=,comm=,args=",
                ])?;
            }
            Ok(())
        })?;

        let selector = format!("pid:{controller_pid}");
        let apoc_target = elapsed_us(|| {
            for _ in 0..iterations {
                let process = manager.get(&selector)?;
                manager.record_with_controller(
                    std::slice::from_ref(&process),
                    "benchmark.process.target",
                    "observed",
                    Some(&selector),
                    &controller,
                    "observed",
                    None,
                    &serde_json::json!({ "warmup": false }),
                )?;
            }
            Ok(())
        })?;
        let pid = controller_pid.to_string();
        let shell_target = elapsed_us(|| {
            for _ in 0..iterations {
                shell_ps(&[
                    "-p",
                    &pid,
                    "-o",
                    "pid=,ppid=,state=,%cpu=,rss=,vsz=,etime=,comm=,args=",
                ])?;
            }
            Ok(())
        })?;

        let cases = vec![
            benchmark_case(
                "inventory",
                apoc_inventory,
                shell_inventory,
                iterations,
                "apoc also returns start-time-qualified identities, executable paths, cwd, disk I/O, byte-normalized memory, and a durable controller event",
            ),
            benchmark_case(
                "target_context",
                apoc_target,
                shell_target,
                iterations,
                "apoc also validates PID reuse and persists exact argv, cwd, controller, actor, operation phase, purpose, labels, and execution ID",
            ),
        ];
        Ok(ProcessBenchmark {
            measured_at: now_ms(),
            iterations,
            warmup,
            all_apoc_faster: cases.iter().all(|case| case.apoc_faster),
            cases,
            methodology: "Measures the persistent in-process path used by Code Mode and the daemon. Each apoc iteration includes durable SQLite context recording; each shell iteration spawns /bin/ps. It does not claim cold standalone CLI startup is faster.".to_string(),
        })
    });
    blocking_result(task).await
}

async fn blocking_result<T>(task: tokio::task::JoinHandle<Result<T, String>>) -> Result<T, String> {
    task.await.map_err(|error| error.to_string())?
}

fn elapsed_us(operation: impl FnOnce() -> Result<(), String>) -> Result<u64, String> {
    let started = Instant::now();
    operation()?;
    Ok(started.elapsed().as_micros().min(u64::MAX as u128) as u64)
}

fn shell_ps(arguments: &[&str]) -> Result<(), String> {
    shell_command("/bin/ps", arguments)
}

fn shell_command(program: &str, arguments: &[&str]) -> Result<(), String> {
    let status = std::process::Command::new(program)
        .args(arguments)
        .stdout(std::process::Stdio::null())
        .stderr(std::process::Stdio::null())
        .status()
        .map_err(|error| error.to_string())?;
    if status.success() {
        Ok(())
    } else {
        Err(format!("/bin/ps exited with {status}"))
    }
}

fn benchmark_case(
    name: &str,
    apoc_total_us: u64,
    shell_total_us: u64,
    iterations: usize,
    context: &str,
) -> BenchmarkCase {
    BenchmarkCase {
        name: name.to_string(),
        apoc_total_us,
        shell_total_us,
        apoc_mean_us: apoc_total_us / iterations as u64,
        shell_mean_us: shell_total_us / iterations as u64,
        speedup: shell_total_us as f64 / apoc_total_us.max(1) as f64,
        apoc_faster: apoc_total_us < shell_total_us,
        context: context.to_string(),
    }
}

#[cfg(test)]
mod tests {
    use tempfile::tempdir;

    use super::*;
    use crate::store::Store;

    #[tokio::test]
    async fn measures_context_recording_against_spawned_ps() {
        let directory = tempdir().unwrap();
        let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
        let manager = Arc::new(Mutex::new(ProcessManager::with_store(store.clone())));
        let report = process_benchmark(
            manager,
            1,
            1,
            ProcessController::capture(true, Some("benchmark-test".to_string())),
        )
        .await
        .unwrap();
        assert_eq!(report.cases.len(), 2);
        assert!(report.cases.iter().all(|case| case.apoc_total_us > 0));
        assert!(report.cases.iter().all(|case| case.shell_total_us > 0));
        assert!(
            store
                .list_process_events(20)
                .unwrap()
                .iter()
                .all(|event| event.controller.execution_id.as_deref() == Some("benchmark-test"))
        );
    }

    #[tokio::test]
    async fn rejects_out_of_range_workloads_before_spawning() {
        let directory = tempdir().unwrap();
        let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
        let manager = Arc::new(Mutex::new(ProcessManager::with_store(store)));
        let controller = ProcessController::capture(true, None);

        assert!(
            process_benchmark(Arc::clone(&manager), 0, 0, controller.clone())
                .await
                .unwrap_err()
                .contains("iterations")
        );
        assert!(
            process_benchmark(Arc::clone(&manager), 1001, 0, controller.clone())
                .await
                .unwrap_err()
                .contains("iterations")
        );
        assert!(
            process_benchmark(manager, 1, 101, controller)
                .await
                .unwrap_err()
                .contains("warmup")
        );
    }

    #[test]
    fn reports_unsuccessful_ps_status() {
        let error = shell_ps(&["--apoc-invalid-option"]).unwrap_err();
        assert!(error.contains("/bin/ps exited with"));

        let error = shell_command("/apoc/nonexistent/ps", &[]).unwrap_err();
        assert!(!error.is_empty());
    }

    #[tokio::test]
    async fn reports_a_blocking_task_fault() {
        let task = tokio::task::spawn_blocking(|| -> Result<(), String> {
            panic!("synthetic benchmark task fault")
        });
        let error = blocking_result(task).await.unwrap_err();
        assert!(error.contains("synthetic benchmark task fault"));
    }
}