use kranz_engine::backend::{AgentBackend, AgentSession, SessionSpec};
use kranz_engine::backend_mock::{mock_init, mock_result_text, mock_text, MockBackend, MockScript};
use kranz_engine::error::Result as EngineResult;
use kranz_engine::event_log::{EventLog, LockForce};
use kranz_engine::events::{Event, EventKind};
use kranz_engine::git_ops::GitRepo;
use kranz_engine::orchestrator::MissionEngine;
use kranz_engine::paths::MissionPaths;
use kranz_engine::reducer;
use kranz_engine::types::*;
use serde_json::json;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::{Arc, Once};
use std::time::Duration;
use tempfile::TempDir;
use tokio::time::timeout;
const TEST_TIMEOUT: Duration = Duration::from_secs(60);
static ENV_ISOLATION: Once = Once::new();
fn isolate_git_env() {
ENV_ISOLATION.call_once(|| {
let missing =
std::env::temp_dir().join(format!("kranz-soak-test-no-config-{}", std::process::id()));
std::env::set_var("GIT_CONFIG_GLOBAL", &missing);
std::env::set_var("GIT_CONFIG_SYSTEM", &missing);
if let Ok(ceiling) = std::fs::canonicalize(std::env::temp_dir()) {
std::env::set_var("GIT_CEILING_DIRECTORIES", ceiling);
}
});
}
fn git_available() -> bool {
Command::new("git")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
fn setup() -> bool {
isolate_git_env();
if git_available() {
true
} else {
kranz_engine::test_capability::skip(
kranz_engine::test_capability::capability::GIT,
"git is not on PATH",
);
false
}
}
fn raw_git(dir: &Path, args: &[&str]) -> String {
let out = Command::new("git")
.args(args)
.current_dir(dir)
.output()
.expect("spawn git");
assert!(
out.status.success(),
"git {args:?} failed: {}",
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8_lossy(&out.stdout).into_owned()
}
fn init_repo() -> (TempDir, PathBuf) {
let dir = tempfile::tempdir().expect("create tempdir");
let init = Command::new("git")
.args(["init", "-b", "main"])
.current_dir(dir.path())
.output()
.expect("spawn git init");
if !init.status.success() {
raw_git(dir.path(), &["init"]);
raw_git(dir.path(), &["symbolic-ref", "HEAD", "refs/heads/main"]);
}
raw_git(dir.path(), &["config", "user.name", "test"]);
raw_git(dir.path(), &["config", "user.email", "test@example.com"]);
std::fs::write(dir.path().join("README.md"), "seed\n").unwrap();
raw_git(dir.path(), &["add", "-A"]);
raw_git(dir.path(), &["commit", "-m", "seed"]);
let root = std::fs::canonicalize(dir.path()).expect("canonicalize repo root");
(dir, root)
}
const GOAL: &str = "ship the demo feature";
fn test_cfg() -> MissionConfig {
MissionConfig {
skip_scrutiny: true,
skip_functional: true,
worker_isolation: WorkerIsolation::Checkout,
..MissionConfig::default()
}
}
fn make_engine(backend: Arc<dyn AgentBackend>, root: &Path, cfg: MissionConfig) -> MissionEngine {
MissionEngine::create(backend, root, GOAL, cfg).expect("create mission engine")
}
fn worker_pass() -> MockScript {
static COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let path = format!("delivered-{n}.txt");
MockScript::single_shot_json(&json!({
"result": "pass",
"summary": "implemented and tested",
"filesTouched": [path],
"testsAdded": [],
"testEvidence": "all green",
"commits": []
}))
.writes_file(&path, "delivered by the mock worker\n")
}
fn orch_script(replies: Vec<String>) -> MockScript {
MockScript::streaming(vec![mock_init("orch-session"), mock_result_text("ready")]).responding(
replies
.iter()
.map(|reply| vec![mock_text(reply), mock_result_text(reply)])
.collect(),
)
}
fn judgement(decision: &str, guidance: &str) -> String {
json!({ "decision": decision, "guidance": guidance, "summary": format!("worker judged: {decision}") })
.to_string()
}
fn dirty_tree_commit_as_is() -> String {
json!({ "action": "commit-as-is", "note": "worker delivered files" }).to_string()
}
fn parallel_plan(ids: &[&str]) -> String {
json!({
"independent": ids,
"mergeOrder": ids,
"summary": format!("{} features are independent", ids.len())
})
.to_string()
}
fn read_log(paths: &MissionPaths) -> Vec<Event> {
EventLog::read_events(&paths.events_file()).expect("read events.jsonl")
}
fn soak_plan(milestones: usize, features: usize) -> Plan {
Plan {
goal: GOAL.to_string(),
validation_contract: vec![],
milestones: (1..=milestones)
.map(|m| PlanMilestone {
title: format!("M{m}"),
features: (1..=features)
.map(|i| PlanFeature {
title: format!("feature {m}.{i}"),
spec: format!("build part {m}.{i}"),
validation_criteria: vec![format!("part {m}.{i} works")],
})
.collect(),
})
.collect(),
considered_alternatives: None,
command_grants: vec![],
touch_set: vec![],
standards_manifest: None,
reviewer_independence: None,
}
}
struct ConflictBackend {
inner: MockBackend,
}
#[async_trait::async_trait]
impl AgentBackend for ConflictBackend {
async fn start(&self, spec: SessionSpec) -> EngineResult<Box<dyn AgentSession>> {
let worktree_name = spec
.cwd
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.filter(|n| n.starts_with("kranz-wt-"));
if let Some(name) = worktree_name {
std::fs::write(spec.cwd.join("CONFLICT.txt"), format!("edit from {name}\n"))
.expect("write conflicting file into the parallel worktree");
}
self.inner.start(spec).await
}
}
struct SoakCtx {
iter: usize,
variant: &'static str,
paths: MissionPaths,
}
impl SoakCtx {
fn fail(&self, msg: impl std::fmt::Display) -> ! {
let tail = std::fs::read_to_string(self.paths.events_file())
.map(|s| {
let lines: Vec<&str> = s.lines().collect();
let start = lines.len().saturating_sub(12);
format!(
"(last {} of {} events)\n{}",
lines.len() - start,
lines.len(),
lines[start..].join("\n")
)
})
.unwrap_or_else(|e| format!("<events.jsonl unreadable: {e}>"));
panic!(
"soak iteration {} [{}] FAILED: {}\n--- events.jsonl tail ---\n{}",
self.iter, self.variant, msg, tail
);
}
fn ensure(&self, cond: bool, msg: impl std::fmt::Display) {
if !cond {
self.fail(msg);
}
}
}
fn json_diff(a: &serde_json::Value, b: &serde_json::Value, path: &str) -> Option<String> {
use serde_json::Value;
match (a, b) {
(Value::Number(x), Value::Number(y)) => {
let (x, y) = (
x.as_f64().unwrap_or(f64::NAN),
y.as_f64().unwrap_or(f64::NAN),
);
if (x - y).abs() > 1e-9 {
Some(format!("{path}: {x} != {y}"))
} else {
None
}
}
(Value::Array(xs), Value::Array(ys)) => {
if xs.len() != ys.len() {
return Some(format!(
"{path}: array lengths {} != {}",
xs.len(),
ys.len()
));
}
xs.iter()
.zip(ys)
.enumerate()
.find_map(|(i, (x, y))| json_diff(x, y, &format!("{path}[{i}]")))
}
(Value::Object(xs), Value::Object(ys)) => {
for key in xs.keys().chain(ys.keys()) {
match (xs.get(key), ys.get(key)) {
(Some(x), Some(y)) => {
if let Some(d) = json_diff(x, y, &format!("{path}.{key}")) {
return Some(d);
}
}
(x, y) => {
return Some(format!(
"{path}.{key}: {} != {}",
x.map(|v| v.to_string())
.unwrap_or_else(|| "<absent>".into()),
y.map(|v| v.to_string())
.unwrap_or_else(|| "<absent>".into())
))
}
}
}
None
}
_ => {
if a == b {
None
} else {
Some(format!("{path}: {a} != {b}"))
}
}
}
}
fn assert_invariants(ctx: &SoakCtx, root: &Path, mission_id: &str, allowed_failed: &[&str]) {
let events = match EventLog::read_events(&ctx.paths.events_file()) {
Ok(events) => events,
Err(e) => ctx.fail(format!("event log unreadable/corrupt: {e}")),
};
ctx.ensure(!events.is_empty(), "event log is empty");
for (i, event) in events.iter().enumerate() {
ctx.ensure(
event.seq == (i + 1) as u64,
format!(
"seq gap: position {i} carries seq {} (want {})",
event.seq,
i + 1
),
);
}
let state = match reducer::fold(&events) {
Ok(state) => state,
Err(e) => ctx.fail(format!("reducer::fold failed on the final log: {e}")),
};
ctx.ensure(
state.mission.status == MissionStatus::Complete,
format!("mission status {:?}, want Complete", state.mission.status),
);
for ms in &state.mission.milestones {
for f in &ms.features {
let ok = if allowed_failed.contains(&f.id.as_str()) {
f.status == FeatureStatus::Failed
} else {
f.status == FeatureStatus::Complete
};
ctx.ensure(
ok,
format!(
"feature {} ended {:?} (allowed_failed: {allowed_failed:?})",
f.id, f.status
),
);
}
}
let snapshot = match reducer::read_snapshot(&ctx.paths.state_file()) {
Ok(snapshot) => snapshot,
Err(e) => ctx.fail(format!("state.json snapshot unreadable: {e}")),
};
let snapshot_json = serde_json::to_value(&snapshot).expect("serialize snapshot");
let fold_json = serde_json::to_value(&state).expect("serialize fold");
if let Some(diff) = json_diff(&snapshot_json, &fold_json, "$") {
ctx.fail(format!(
"state.json snapshot diverges from a fresh fold at {diff}"
));
}
let repo = match GitRepo::open(root) {
Ok(repo) => repo,
Err(e) => ctx.fail(format!("GitRepo::open failed: {e}")),
};
match repo.list_worktrees() {
Ok(worktrees) => ctx.ensure(
worktrees.len() == 1,
format!("leaked worktrees: {worktrees:?}"),
),
Err(e) => ctx.fail(format!("git worktree list failed: {e}")),
}
let leak_prefix = format!("kranz-wt-{mission_id}-");
for entry in std::fs::read_dir(std::env::temp_dir())
.expect("read temp dir")
.flatten()
{
let name = entry.file_name();
let name = name.to_string_lossy();
ctx.ensure(
!name.starts_with(&leak_prefix),
format!("parallel worktree dir leaked into temp: {name}"),
);
}
let wt_branches = raw_git(root, &["branch", "--list", "kranz/wt/*"]);
ctx.ensure(
wt_branches.trim().is_empty(),
format!("leftover worktree branches: {}", wt_branches.trim()),
);
let status = raw_git(root, &["status", "--porcelain"]);
ctx.ensure(
status.trim().is_empty(),
format!("repo left dirty: {}", status.trim()),
);
}
async fn run_to_status(ctx: &SoakCtx, engine: &mut MissionEngine) -> MissionStatus {
match timeout(TEST_TIMEOUT, engine.run()).await {
Err(_) => ctx.fail("engine.run() hung past the test timeout"),
Ok(Err(e)) => ctx.fail(format!("engine.run() errored: {e}")),
Ok(Ok(status)) => status,
}
}
async fn run_clean_iteration(iter: usize) {
let (_dir, root) = init_repo();
let backend = Arc::new(MockBackend::with_scripts(vec![
orch_script(vec![
parallel_plan(&["f-1-1", "f-1-2"]),
judgement("complete", ""),
judgement("complete", ""),
parallel_plan(&["f-2-1", "f-2-2"]),
judgement("complete", ""),
judgement("complete", ""),
]),
worker_pass(),
worker_pass(),
worker_pass(),
worker_pass(),
]));
let cfg = MissionConfig {
max_parallel_workers: 2,
..test_cfg()
};
let mut engine = make_engine(backend, &root, cfg);
engine.approve_plan(soak_plan(2, 2)).expect("approve plan");
let mission_id = engine.mission_id().to_string();
let ctx = SoakCtx {
iter,
variant: "CLEAN",
paths: engine.paths().clone(),
};
let status = run_to_status(&ctx, &mut engine).await;
ctx.ensure(
status == MissionStatus::Complete,
format!("run() returned {status:?}, want Complete"),
);
drop(engine);
assert_invariants(&ctx, &root, &mission_id, &[]);
}
async fn run_crash_resume_iteration(iter: usize) {
let (_dir, root) = init_repo();
let backend1 = Arc::new(MockBackend::with_scripts(vec![
orch_script(vec![parallel_plan(&["f-1-1", "f-1-2"])]),
worker_pass(), ]));
let cfg = MissionConfig {
max_parallel_workers: 2,
..test_cfg()
};
let mut engine = make_engine(backend1, &root, cfg.clone());
engine.approve_plan(soak_plan(1, 2)).expect("approve plan");
let mission_id = engine.mission_id().to_string();
let ctx = SoakCtx {
iter,
variant: "CRASH+RESUME",
paths: engine.paths().clone(),
};
match timeout(TEST_TIMEOUT, engine.run()).await {
Err(_) => ctx.fail("crash phase hung instead of erroring"),
Ok(Ok(status)) => ctx.fail(format!(
"crash phase unexpectedly succeeded with {status:?} (starved worker should abort)"
)),
Ok(Err(_)) => {} }
drop(engine);
let phase1 = match EventLog::read_events(&ctx.paths.events_file()) {
Ok(events) => events,
Err(e) => ctx.fail(format!("post-crash log unreadable/corrupt: {e}")),
};
if let Err(e) = reducer::fold(&phase1) {
ctx.fail(format!("post-crash log does not fold: {e}"));
}
let backend2: Arc<dyn AgentBackend> = Arc::new(MockBackend::with_scripts(vec![
worker_pass(), orch_script(vec![
dirty_tree_commit_as_is(),
judgement("complete", ""),
dirty_tree_commit_as_is(),
judgement("complete", ""),
]),
worker_pass(), ]));
let mut engine = match MissionEngine::resume(backend2, &root, &mission_id, LockForce::No) {
Ok(engine) => engine,
Err(e) => ctx.fail(format!("resume after crash failed: {e}")),
};
let status = run_to_status(&ctx, &mut engine).await;
ctx.ensure(
status == MissionStatus::Complete,
format!("resumed run() returned {status:?}, want Complete"),
);
drop(engine);
assert_invariants(&ctx, &root, &mission_id, &[]);
}
async fn run_conflict_iteration(iter: usize) {
let (_dir, root) = init_repo();
let inner = MockBackend::with_scripts(vec![
orch_script(vec![
parallel_plan(&["f-1-1", "f-1-2"]),
judgement("complete", ""),
judgement("complete", ""),
dirty_tree_commit_as_is(),
judgement("complete", ""),
]),
worker_pass(), worker_pass(), worker_pass(), ]);
let backend: Arc<dyn AgentBackend> = Arc::new(ConflictBackend { inner });
let cfg = MissionConfig {
max_parallel_workers: 2,
..test_cfg()
};
let mut engine = make_engine(backend, &root, cfg);
engine.approve_plan(soak_plan(1, 2)).expect("approve plan");
let mission_id = engine.mission_id().to_string();
let ctx = SoakCtx {
iter,
variant: "CONFLICT",
paths: engine.paths().clone(),
};
let status = run_to_status(&ctx, &mut engine).await;
ctx.ensure(
status == MissionStatus::Complete,
format!("run() returned {status:?}, want Complete"),
);
drop(engine);
let events = read_log(&ctx.paths);
ctx.ensure(
events.iter().any(|e| matches!(
&e.kind,
EventKind::FeatureFailed { feature_id, reason, .. }
if feature_id == "f-1-2" && reason.contains("conflicted")
)),
"no feature.failed(f-1-2) with a conflicted-merge reason — the merge conflict never happened",
);
ctx.ensure(
events.iter().any(|e| {
matches!(
&e.kind,
EventKind::FixFeatureCreated { feature, .. } if feature.id == "ms-1-conflict-1"
)
}),
"no fixfeature.created for ms-1-conflict-1 — the resolution feature was never synthesized",
);
assert_invariants(&ctx, &root, &mission_id, &["f-1-2"]);
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "corruption-soak harness; run explicitly via scripts/soak.sh"]
async fn soak_parallel_corruption() {
if !setup() {
return;
}
let iters: usize = std::env::var("KRANZ_SOAK_ITERS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(20);
let (mut clean, mut crash, mut conflict) = (0usize, 0usize, 0usize);
for i in 0..iters {
match i % 3 {
0 => {
run_clean_iteration(i).await;
clean += 1;
eprintln!("soak iter {}/{iters} [CLEAN] ok", i + 1);
}
1 => {
run_crash_resume_iteration(i).await;
crash += 1;
eprintln!("soak iter {}/{iters} [CRASH+RESUME] ok", i + 1);
}
_ => {
run_conflict_iteration(i).await;
conflict += 1;
eprintln!("soak iter {}/{iters} [CONFLICT] ok", i + 1);
}
}
}
eprintln!(
"soak PASS: {iters} iterations, zero event-log corruption \
({clean} clean, {crash} crash+resume, {conflict} conflict)"
);
}