agent-effects-sqlite 0.2.1

SQLite store for agent-effects
Documentation
//! Several processes sharing one database file must still run each effect
//! once.
//!
//! The test re-invokes its own binary as worker processes: `worker` is a
//! no-op unless `AGENT_EFFECTS_SQLITE_WORKER` is set, which only the parent
//! does for its children.

use std::fs::OpenOptions;
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Duration;

use agent_effects::store::{ExecutionContract, NewEffect};
use agent_effects::{
    EffectContext, EffectFailure, EffectHandler, EffectKey, EffectKind, EffectName, EffectOutcome,
    EffectStatus, EffectStore, Handler, LogicalKey, RecoveryConfig, Runtime, WorkerId,
};
use agent_effects_sqlite::SqliteStore;

const WORKER_ENV: &str = "AGENT_EFFECTS_SQLITE_WORKER";
const RECOVERY_ENV: &str = "AGENT_EFFECTS_RECOVERY_ONLY";
/// Effects each recovery process works on at once; unset is sequential.
const CONCURRENCY_ENV: &str = "AGENT_EFFECTS_RECOVERY_CONCURRENCY";
const EFFECTS: u32 = 150;
const WORKERS: usize = 3;

struct Paths {
    db: PathBuf,
    log: PathBuf,
    go: PathBuf,
}

impl Paths {
    fn new(dir: &Path) -> Self {
        Self {
            db: dir.join("effects.db"),
            log: dir.join("side-effects.log"),
            go: dir.join("go"),
        }
    }
}

#[derive(Default)]
struct RecoveryChildren(Vec<std::process::Child>);

impl RecoveryChildren {
    async fn finish(&mut self) {
        tokio::time::timeout(Duration::from_secs(60), async {
            for child in &mut self.0 {
                loop {
                    if let Some(status) = child.try_wait().unwrap() {
                        assert!(status.success(), "recovery process failed: {status}");
                        break;
                    }
                    tokio::time::sleep(Duration::from_millis(5)).await;
                }
            }
        })
        .await
        .expect("recovery processes timed out");
    }
}
impl Drop for RecoveryChildren {
    fn drop(&mut self) {
        for child in &mut self.0 {
            let _ = child.kill();
            let _ = child.wait();
        }
    }
}

struct RecoveryJob {
    log: PathBuf,
    runner: String,
}

impl EffectHandler for RecoveryJob {
    const NAME: &'static str = "job.recover";
    type Input = u32;
    type Output = u32;
    type Error = EffectFailure;
    async fn execute(&self, _: &EffectContext, input: &u32) -> Result<u32, EffectFailure> {
        let dir = self.log.parent().unwrap();
        std::fs::write(dir.join(format!("claimed-{}", self.runner)), b"").unwrap();
        tokio::time::timeout(Duration::from_secs(30), async {
            while std::fs::read_dir(dir)
                .unwrap()
                .filter(|entry| {
                    entry
                        .as_ref()
                        .unwrap()
                        .file_name()
                        .to_string_lossy()
                        .starts_with("claimed-")
                })
                .count()
                < WORKERS
            {
                tokio::time::sleep(Duration::from_millis(5)).await;
            }
        })
        .await
        .expect("all recovery processes must acquire an effect before proceeding");
        tokio::time::sleep(Duration::from_millis(2)).await;
        let mut file = OpenOptions::new()
            .create(true)
            .append(true)
            .open(&self.log)
            .unwrap();
        file.write_all(format!("{input} {}\n", self.runner).as_bytes())
            .unwrap();
        Ok(*input)
    }
}

/// One worker process: runs every effect, in the same order as the others
/// to maximise contention. The action is the "side effect": appending a line
/// to a shared log. A short sleep keeps attempts overlapping.
#[test]
fn worker() {
    let Ok(spec) = std::env::var(WORKER_ENV) else {
        return;
    };
    let (dir, name) = spec.split_once('|').expect("dir|name");
    let paths = Paths::new(Path::new(dir));
    let name = name.to_owned();

    tokio::runtime::Builder::new_multi_thread()
        .worker_threads(2)
        .enable_all()
        .build()
        .unwrap()
        .block_on(async move {
            let store = SqliteStore::open(&paths.db).await.unwrap();
            let rt = Runtime::builder(store)
                .worker_id(WorkerId::new(name.clone()))
                .register(Handler::new(RecoveryJob {
                    log: paths.log.clone(),
                    runner: name.clone(),
                }))
                .build();
            std::fs::write(paths.go.with_file_name(format!("ready-{name}")), b"").unwrap();
            while !paths.go.exists() {
                tokio::time::sleep(Duration::from_millis(5)).await;
            }
            if std::env::var_os(RECOVERY_ENV).is_some() {
                let config =
                    std::env::var(CONCURRENCY_ENV).map_or(RecoveryConfig::SEQUENTIAL, |n| {
                        RecoveryConfig::SEQUENTIAL
                            .max_concurrency(n.parse().unwrap())
                            .batch_size(16)
                    });
                tokio::time::timeout(Duration::from_secs(60), async {
                    loop {
                        let report = rt.recover_with(config).await.unwrap();
                        assert!(report.resume_errors.is_empty(), "{report:?}");
                        if paths.go.with_file_name(format!("claimed-{name}")).exists() {
                            break;
                        }
                        // Another scan can temporarily lock an entire page.
                        tokio::time::sleep(Duration::from_millis(5)).await;
                    }
                })
                .await
                .expect("recovery worker did not acquire an effect");
                return;
            }
            for n in 0..EFFECTS {
                let (log, runner) = (paths.log.clone(), name.clone());
                let outcome = rt
                    .effect("job.run", n)
                    .run(move |_| {
                        let (log, name) = (log.clone(), runner.clone());
                        async move {
                            tokio::time::sleep(Duration::from_millis(2)).await;
                            // One small O_APPEND write per line: atomic.
                            let mut file = OpenOptions::new()
                                .create(true)
                                .append(true)
                                .open(&log)
                                .unwrap();
                            file.write_all(format!("{n} {name}\n").as_bytes()).unwrap();
                            Ok::<_, EffectFailure>(n)
                        }
                    })
                    .await
                    .unwrap();
                assert!(
                    matches!(
                        outcome,
                        EffectOutcome::Committed(_) | EffectOutcome::InProgress { .. }
                    ),
                    "{name}: effect {n}: {outcome:?}"
                );
            }
        });
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn processes_sharing_a_database_run_each_effect_once() {
    let dir = tempfile::tempdir().unwrap();
    let paths = Paths::new(dir.path());
    // Create and migrate up front, so the workers only contend on effects.
    SqliteStore::open(&paths.db).await.unwrap();

    let exe = std::env::current_exe().unwrap();
    let mut children: Vec<_> = (0..WORKERS)
        .map(|i| {
            Command::new(&exe)
                .args(["worker", "--exact", "--nocapture", "--test-threads=1"])
                .env(WORKER_ENV, format!("{}|process-{i}", dir.path().display()))
                .spawn()
                .unwrap()
        })
        .collect();
    std::fs::write(&paths.go, b"").unwrap();
    for child in &mut children {
        assert!(child.wait().unwrap().success(), "a worker process failed");
    }

    let log = std::fs::read_to_string(&paths.log).unwrap();
    let mut runs = vec![0u32; EFFECTS as usize];
    let mut runners = std::collections::BTreeSet::new();
    for line in log.lines() {
        let (n, name) = line.split_once(' ').unwrap();
        runs[n.parse::<usize>().unwrap()] += 1;
        runners.insert(name.to_owned());
    }
    let duplicated: Vec<_> = (0..EFFECTS).filter(|&n| runs[n as usize] > 1).collect();
    let missing: Vec<_> = (0..EFFECTS).filter(|&n| runs[n as usize] == 0).collect();
    assert!(duplicated.is_empty(), "ran more than once: {duplicated:?}");
    assert!(missing.is_empty(), "never ran: {missing:?}");
    assert!(
        runners.len() > 1,
        "only {runners:?} did any work: no real contention"
    );

    let store = SqliteStore::open(&paths.db).await.unwrap();
    let records = store
        .list(agent_effects::store::ListQuery::statuses([]).limit(1000))
        .await
        .unwrap();
    assert_eq!(records.len(), EFFECTS as usize);
    assert!(records.iter().all(|r| r.status == EffectStatus::Committed));
    assert!(records.iter().all(|r| r.attempt_count == 1));
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn recovery_processes_share_durable_pending_intents_without_duplicate_actions() {
    recovery_processes(None).await;
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concurrent_recovery_processes_share_durable_pending_intents_without_duplicate_actions() {
    recovery_processes(Some(8)).await;
}

/// Several recovery processes, each working on `concurrency` effects at
/// once (sequentially for `None`), share one backlog of durable intents.
async fn recovery_processes(concurrency: Option<usize>) {
    let dir = tempfile::tempdir().unwrap();
    let paths = Paths::new(dir.path());
    let store = SqliteStore::open(&paths.db).await.unwrap();
    for n in 0..EFFECTS {
        let key = EffectKey::new(
            EffectName::new(RecoveryJob::NAME).unwrap(),
            LogicalKey::new(n.to_string()).unwrap(),
        );
        let mut new = NewEffect::new(
            key,
            EffectKind::IrreversibleWrite,
            std::time::SystemTime::now(),
        )
        .with_contract(ExecutionContract::new(EffectKind::IrreversibleWrite, 5));
        new.input = Some(serde_json::json!(n));
        store.insert_or_get(new).await.unwrap();
    }
    let mut children = RecoveryChildren::default();
    for index in 0..WORKERS {
        children.0.push(
            Command::new(std::env::current_exe().unwrap())
                .args(["worker", "--exact", "--nocapture", "--test-threads=1"])
                .env(
                    WORKER_ENV,
                    format!("{}|recover-{index}", dir.path().display()),
                )
                .env(RECOVERY_ENV, "1")
                .envs(concurrency.map(|n| (CONCURRENCY_ENV, n.to_string())))
                .spawn()
                .unwrap(),
        );
    }
    tokio::time::timeout(Duration::from_secs(30), async {
        while (0..WORKERS).any(|i| !dir.path().join(format!("ready-recover-{i}")).exists()) {
            tokio::time::sleep(Duration::from_millis(5)).await;
        }
    })
    .await
    .expect("recovery workers did not initialize");
    std::fs::write(&paths.go, b"").unwrap();
    children.finish().await;
    let log = std::fs::read_to_string(&paths.log).unwrap();
    let mut counts = vec![0; EFFECTS as usize];
    let mut runners = std::collections::BTreeSet::new();
    for line in log.lines() {
        let (n, runner) = line.split_once(' ').unwrap();
        counts[n.parse::<usize>().unwrap()] += 1;
        runners.insert(runner);
    }
    assert!(counts.iter().all(|count| *count == 1));
    assert_eq!(
        runners.len(),
        WORKERS,
        "every process must actually execute a handler"
    );
    let records = store
        .list(agent_effects::store::ListQuery::statuses([]).limit(1000))
        .await
        .unwrap();
    assert_eq!(records.len(), EFFECTS as usize);
    assert!(
        records
            .iter()
            .all(|record| record.status == EffectStatus::Committed && record.attempt_count == 1)
    );
}