use crate::harness::{agent, human, plan_of, World, REFUSED, REPORTING_MEMBER};
use serde_json::{json, Value};
#[cfg(unix)]
use crate::harness::end_process;
pub const OBSERVER_RESTARTS_ENV: &str = "ONEPIPELINE_OBSERVER_RESTARTS";
fn configs_of(world: &World, run: &str, member: &str) -> Vec<(String, String)> {
let events = world.journal(run);
let started: Vec<&Value> = events
.iter()
.filter(|event| event["kind"] == "member-started")
.filter(|event| event["labels"]["member"] == member)
.collect();
assert!(
!started.is_empty(),
"no member '{member}' started in {run}: {events:#?}"
);
started
.into_iter()
.map(|event| {
let path = event["payload"]["config"]
.as_str()
.expect("the sibling publishes the config it launched the member with");
let text =
std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path} unreadable: {e}"));
let node = event["labels"]["onepipeline.node"]
.as_str()
.unwrap_or_default()
.to_string();
(node, text)
})
.collect()
}
fn config_of(world: &World, run: &str, member: &str) -> String {
configs_of(world, run, member).swap_remove(0).1
}
fn ready_and_undriven(world: &World, run: &str, node: Value) {
let path = world.plan(run, &plan_of(run, vec![human("approve", &[]), node]));
world.run(&["start", &path, "--attach"]).exited(0);
world.run(&["attest", run, "approve"]).exited(0);
}
#[test]
fn relative_default_graphs_dispatch_from_the_launch_directory() {
let world = World::new("real-relative-defaults");
world.write_graphs();
let path = world.plan(
"relative-defaults",
&plan_of("relative-defaults", vec![agent("build", &[])]),
);
let mut command = world.agentgraph_cmd(&[
"start",
&path,
"--attach",
"--dag-graph",
"graphs/dag-scope.yaml",
]);
command
.current_dir(&world.root)
.env_remove("ONEPIPELINE_NODE_GRAPH");
let started = world.run_on(command, "start relative defaults");
started.exited(0).settled();
assert!(
world
.turns()
.iter()
.any(|turn| turn.prompt.contains("Do build.")),
"the node-scope graph did not dispatch its member: {}",
world.dump()
);
let launch = world.run_json("relative-defaults", "launch.json");
for field in ["graph", "node_graph"] {
assert!(
std::path::Path::new(launch[field].as_str().expect("a graph path")).is_absolute(),
"{field} was not resolved at launch: {launch}"
);
}
assert_eq!(launch["dir"], json!(world.root));
let announced: Vec<String> = world
.journal("relative-defaults")
.into_iter()
.filter(|event| event["kind"] == "graph-started" && event["labels"]["node"].is_null())
.filter_map(|event| event["labels"]["run_id"].as_str().map(str::to_string))
.collect();
assert!(
!announced.is_empty(),
"no observer announced itself into the merged store"
);
let watched: Vec<String> = launch["observer_runs"]
.as_array()
.expect("the record names the graphs that have watched")
.iter()
.filter_map(|run| run.as_str().map(str::to_string))
.collect();
assert!(
watched.starts_with(&announced),
"the record does not name the graphs that watched this run, in order: \
{watched:?} against {announced:?}"
);
assert_eq!(
launch["graph_run"],
json!(watched.last()),
"the run addresses a graph that is not the last one it started: {launch}"
);
}
#[test]
fn relative_node_and_step_graph_overrides_dispatch_from_the_launch_directory() {
let world = World::new("real-relative-plan-overrides");
world.write_graphs();
world.repository("local-direct", &[]);
for (source, target) in [
("node-scope.yaml", "node-override.yaml"),
("node-scope.yaml", "step-override.yaml"),
] {
std::fs::copy(world.graphs().join(source), world.root.join(target))
.expect("the relative graph override is written");
}
let worker_config = "oneharness-worker.toml";
std::fs::copy(
world.graphs().join(worker_config),
world.root.join(worker_config),
)
.expect("the relative graphs' harness config is written");
world.script("harness.work", "the engineer wrote this");
let node = json!({
"id": "service",
"repo": "service",
"title": "feat: land the workstream",
"agent_graph": "node-override.yaml",
"steps": [
{"id": "implement", "persona": "engineer", "task": "## What\nimplement"},
{
"id": "review",
"persona": "reviewer",
"task": "## What\nreview",
"deps": ["implement"],
"agent_graph": "step-override.yaml",
},
],
});
let path = world.plan(
"relative-plan-overrides",
&plan_of("relative-plan-overrides", vec![node]),
);
let mut command = world.agentgraph_cmd(&["start", &path, "--attach"]);
command.current_dir(&world.root);
world
.run_on(command, "start relative plan graph overrides")
.exited(0)
.settled();
for (step, graph) in [
("implement", world.root.join("node-override.yaml")),
("review", world.root.join("step-override.yaml")),
] {
let graph = graph
.canonicalize()
.expect("the expected relative graph path resolves");
assert!(
world
.journal("relative-plan-overrides")
.iter()
.any(|event| {
event["kind"] == "graph-started"
&& event["labels"]["node"] == "service"
&& event["labels"]["step"] == step
&& event["payload"]["graph"].as_str().is_some_and(|actual| {
std::fs::canonicalize(actual)
.map(|actual| actual == graph)
.unwrap_or(false)
})
}),
"{step} did not dispatch with its resolved graph: {}",
world.dump()
);
}
}
#[test]
fn a_lifecycle_nodes_two_graphs_are_the_ones_its_launch_resolved() {
let world = World::new("lifecycle-recorded-default-graph");
world.repository("local-direct", &[]);
world.script("driver.wait", "hold");
world.script("service.work", "the worker wrote this\n");
let launch_graph = crate::harness::repo_file("graphs/node-scope.yaml");
let later_graph = world.root.join("later-node-scope.yaml");
std::fs::copy(&launch_graph, &later_graph).expect("the later graph is written");
let drafting = world.pr_author_graph();
let mut service = crate::harness::lifecycle("service", &["approve"]);
service["deps"] = json!(["approve"]);
let path = world.plan(
"recorded-lifecycle-graph",
&plan_of(
"recorded-lifecycle-graph",
vec![human("approve", &[]), service],
),
);
let mut start = world.cmd(&["start", &path, "--attach", "--pr-author-graph", &drafting]);
start.env("ONEPIPELINE_NODE_GRAPH", &launch_graph);
world
.run_on(start, "start recorded lifecycle graph")
.exited(0);
assert_eq!(
world.run_json("recorded-lifecycle-graph", "launch.json")["pr_author_graph"],
json!(drafting),
"the launch record does not name the graph the launch was given"
);
world
.run(&["attest", "recorded-lifecycle-graph", "approve"])
.exited(0);
let mut adopted = world.cmd(&["adopt", "recorded-lifecycle-graph"]);
adopted.env("ONEPIPELINE_NODE_GRAPH", &later_graph);
world
.run_on(adopted, "adopt with a changed live node graph")
.exited(0);
let invocations = world.invocations();
let relevant: Vec<&Value> = invocations
.iter()
.filter(|call| {
call["tool"] == "oneagentgraph"
&& call["args"]
.as_array()
.is_some_and(|args| args.iter().any(|arg| arg == "onepipeline.node=service"))
})
.collect();
let under = |persona: &str| -> Vec<&Value> {
relevant
.iter()
.filter(|call| {
call["args"]
.as_array()
.is_some_and(|args| args.iter().any(|arg| arg == persona))
})
.copied()
.collect()
};
let drafts = under("onepipeline.persona=pr-author");
assert_eq!(
drafts.len(),
1,
"the body drafting dispatch did not run after adoption: {relevant:?}"
);
assert_eq!(
drafts[0]["args"][1], drafting,
"the drafting dispatch ran a graph the launch did not record: {drafts:?}"
);
let worked = under("onepipeline.persona=engineer");
assert!(
!worked.is_empty(),
"the node never dispatched: {relevant:?}"
);
assert!(
worked
.iter()
.all(|call| call["args"][1] == launch_graph.to_string_lossy().as_ref()),
"a lifecycle dispatch re-read the live graph instead of launch state: {worked:?}"
);
}
#[test]
fn an_unreadable_relative_graph_names_its_launch_base() {
let world = World::new("relative-graph-error");
let path = world.plan(
"relative-error",
&plan_of("relative-error", vec![agent("build", &[])]),
);
let mut command = world.agentgraph_cmd(&[
"start",
&path,
"--attach",
"--dag-graph",
"graphs/missing-dag.yaml",
]);
command.current_dir(&world.root);
let failed = world.run_on(command, "start missing relative graph");
failed.exited(crate::harness::REFUSED);
failed.err_has("graphs/missing-dag.yaml");
failed.err_has(&world.root.to_string_lossy());
}
#[test]
fn a_blank_graph_reference_is_refused_before_any_path_is_read() {
let world = World::new("blank-graph-reference");
world.write_graphs();
let cases: [(&str, Vec<String>, Value); 2] = [
(
"blank-observer",
vec!["--dag-graph".to_string(), String::new()],
agent("build", &[]),
),
(
"blank-node-override",
Vec::new(),
json!({
"id": "build",
"persona": "engineer",
"task": "## What\nbuild",
"agent_graph": "",
}),
),
];
for (name, extra, node) in cases {
let path = world.plan(name, &plan_of(name, vec![node]));
let mut args = vec!["start".to_string(), path.clone(), "--attach".to_string()];
args.extend(extra);
let mut command =
world.agentgraph_cmd(&args.iter().map(String::as_str).collect::<Vec<_>>());
command.current_dir(&world.root);
let failed = world.run_on(command, "start with a blank graph reference");
failed.exited(REFUSED);
failed.err_has("graph reference is blank");
assert!(
!world.run_file(name, "launch.json").exists(),
"{name} minted a run for a reference nothing could resolve"
);
}
}
#[test]
fn an_unreadable_relative_node_graph_names_its_launch_base() {
let world = World::new("relative-node-graph-error");
world.write_graphs();
let path = world.plan(
"relative-node-error",
&plan_of("relative-node-error", vec![agent("build", &[])]),
);
let mut command = world.agentgraph_cmd(&["start", &path, "--attach"]);
command
.current_dir(&world.root)
.env("ONEPIPELINE_NODE_GRAPH", "graphs/missing-node.yaml");
let failed = world.run_on(command, "start missing relative node graph");
failed.exited(crate::harness::REFUSED);
failed.err_has("graphs/missing-node.yaml");
failed.err_has(&world.root.to_string_lossy());
}
#[test]
fn unreadable_relative_plan_graphs_name_their_path_and_launch_base() {
let world = World::new("relative-plan-graph-errors");
world.write_graphs();
world.repository("local-direct", &[]);
let cases = [
(
"missing-node-override",
json!({
"id": "build",
"persona": "engineer",
"task": "## What\nbuild",
"agent_graph": "graphs/missing-node-override.yaml",
}),
"graphs/missing-node-override.yaml",
),
(
"missing-step-override",
json!({
"id": "service",
"repo": "service",
"title": "feat: land the workstream",
"steps": [{
"id": "implement",
"persona": "engineer",
"task": "## What\nimplement",
"agent_graph": "graphs/missing-step-override.yaml",
}],
}),
"graphs/missing-step-override.yaml",
),
];
for (name, node, missing) in cases {
let path = world.plan(name, &plan_of(name, vec![node]));
let mut command = world.agentgraph_cmd(&["start", &path, "--attach"]);
command.current_dir(&world.root);
let failed = world.run_on(command, &format!("start {name}"));
failed.exited(crate::harness::REFUSED);
failed.err_has(missing);
failed.err_has(&world.root.to_string_lossy());
}
}
#[test]
fn broken_launch_records_refuse_the_adoption_before_direct_or_lifecycle_dispatch() {
let direct = World::new("corrupt-launch-direct");
let mut build = agent("build", &["approve"]);
build["deps"] = json!(["approve"]);
ready_and_undriven(&direct, "corrupt-direct", build);
std::fs::write(direct.run_file("corrupt-direct", "launch.json"), "not json")
.expect("the launch record is corrupted");
direct
.run(&["adopt", "corrupt-direct"])
.exited(crate::harness::REFUSED)
.err_has("launch.json");
let lifecycle_world = World::new("missing-launch-lifecycle");
lifecycle_world.repository("local-direct", &[]);
let mut service = crate::harness::lifecycle("service", &["approve"]);
service["deps"] = json!(["approve"]);
ready_and_undriven(&lifecycle_world, "missing-lifecycle", service);
std::fs::remove_file(lifecycle_world.run_file("missing-lifecycle", "launch.json"))
.expect("the launch record is removed");
lifecycle_world
.run(&["adopt", "missing-lifecycle"])
.exited(crate::harness::REFUSED)
.err_has("launch.json");
}
#[test]
fn a_legacy_launch_without_a_node_graph_fails_instead_of_reading_live_environment() {
let world = World::new("legacy-empty-node-graph");
let mut build = agent("build", &["approve"]);
build["deps"] = json!(["approve"]);
ready_and_undriven(&world, "legacy-empty", build);
let path = world.run_file("legacy-empty", "launch.json");
let mut launch: Value =
serde_json::from_str(&std::fs::read_to_string(&path).expect("the launch record reads"))
.expect("the launch record parses");
launch["node_graph"] = json!("");
std::fs::write(&path, serde_json::to_vec_pretty(&launch).unwrap())
.expect("the legacy launch record is written");
let mut driving = world.cmd(&["adopt", "legacy-empty"]);
driving.env(
"ONEPIPELINE_NODE_GRAPH",
world.graphs().join("node-scope.yaml"),
);
world
.run_on(driving, "adopt legacy-empty")
.exited(crate::harness::REFUSED)
.err_has("has no resolved node graph");
}
#[test]
fn a_legacy_plan_path_launch_record_is_still_reportable_and_adoptable() {
let world = World::new("legacy-plan-launch-record");
let mut build = agent("build", &["approve"]);
build["deps"] = json!(["approve"]);
ready_and_undriven(&world, "legacy-plan", build);
let path = world.run_file("legacy-plan", "launch.json");
let mut launch = world.run_json("legacy-plan", "launch.json");
launch
.as_object_mut()
.expect("a launch record")
.remove("project");
launch["plan"] = json!("/retired/plan.json");
std::fs::write(&path, serde_json::to_vec_pretty(&launch).unwrap())
.expect("the historical launch record is installed");
world.run(&["status", "legacy-plan"]).exited(0);
world
.run(&["adopt", "legacy-plan"])
.exited(0)
.out_has("\"settlement\":\"complete\"");
}
#[test]
fn launch_overrides_reach_the_graphs_that_actually_run() {
let world = World::new("real-overrides");
world.write_graphs();
std::fs::write(
world.graphs().join("dag-override.toml"),
"run_mode = \"fallback\"\nharnesses = [\"claude-code\"]\n# DAG_OVERRIDE\n",
)
.expect("the dag override config is written");
std::fs::write(
world.graphs().join("node-override.toml"),
"run_mode = \"fallback\"\nharnesses = [\"claude-code\"]\n# NODE_OVERRIDE\n",
)
.expect("the node override config is written");
let path = world.plan(
"overrides",
&plan_of("overrides", vec![agent("build", &[])]),
);
let started = world.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
"--set",
"members.monitor.oneharness_config=./dag-override.toml",
"--node-set",
"members.worker.oneharness_config=./node-override.toml",
]);
started.exited(0).settled();
let turns = world.turns();
for (member, marker, job) in [
("monitor", "DAG_OVERRIDE", "Observe this run"),
("worker", "NODE_OVERRIDE", "Do build."),
] {
let config = config_of(&world, "overrides", member);
assert!(
config.contains(marker),
"the {member} member did not receive its override: {config}"
);
assert!(
turns.iter().any(|turn| turn.prompt.contains(job)),
"the {member} member never ran its turn: {turns:?}"
);
}
}
#[test]
fn a_plan_persona_reaches_the_member_that_actually_runs() {
let world = World::new("real-plan-persona");
world.write_graphs();
std::fs::write(
world.graphs().join("requested-reviewer.yaml"),
"name: requested-reviewer\nsystem_prompt: Review the change.\nuser:\n persona: Demand evidence.\n",
)
.expect("the requested persona is written");
let mut node = agent("review", &[]);
node["persona"] = Value::from("./requested-reviewer.yaml");
let path = world.plan("plan-persona", &plan_of("plan-persona", vec![node]));
world.script("turn.hold", "");
let launch = world.agentgraph_cmd(&["start", &path, "--detach"]);
world.run_on(launch, "start plan-persona").exited(0);
world.until("the node's member to take its turn", |world| {
world
.turns()
.iter()
.any(|turn| turn.prompt.contains("Do review."))
});
let state = world.graph_state();
let sibling = |args: &[&str]| -> std::process::Output {
std::process::Command::new(crate::harness::oneagentgraph_binary())
.args(args)
.env("ONEAGENTGRAPH_STATE_DIR", &state)
.output()
.expect("the real oneagentgraph runs")
};
let listed = sibling(&["history"]);
let runs: Vec<String> = String::from_utf8_lossy(&listed.stdout)
.lines()
.filter_map(|line| line.split('\t').next().map(str::to_string))
.collect();
assert!(
!runs.is_empty(),
"the sibling lists no run at all while its member is taking a turn: it exited {:?} \
saying {:?}",
listed.status.code(),
String::from_utf8_lossy(&listed.stderr)
);
let records: Vec<Value> = runs
.iter()
.map(|run| {
let shown = sibling(&["history", "show", run]);
serde_json::from_slice(&shown.stdout).unwrap_or_else(|error| {
panic!(
"the sibling cannot print the record it listed for {run}: {error}; it exited \
{:?} saying {:?}",
shown.status.code(),
String::from_utf8_lossy(&shown.stderr)
)
})
})
.collect();
assert!(
records.iter().any(|record| {
record["refs"].as_array().is_some_and(|refs| {
refs.iter()
.any(|reference| reference["origin"] == "./requested-reviewer.yaml")
})
}),
"the graph that dispatched the member did not resolve the plan's persona: {records:?}"
);
world.release("turn.go");
world.release("turn.settle");
}
#[test]
fn adoption_retains_node_overrides_for_later_dispatches() {
let world = World::new("real-adopted-node-override");
world.write_graphs();
world.script("harness.fail", "");
std::fs::write(
world.graphs().join("adopted-node.toml"),
"run_mode = \"fallback\"\nharnesses = [\"claude-code\"]\n# ADOPTED_NODE_OVERRIDE\n",
)
.expect("the adopted node config is written");
let path = world.plan(
"adopted-override",
&plan_of("adopted-override", vec![agent("build", &[])]),
);
let mut start = world.agentgraph_cmd(&[
"start",
&path,
"--attach",
"--node-set",
"members.worker.oneharness_config=./adopted-node.toml",
]);
start
.current_dir(&world.root)
.env("ONEPIPELINE_NODE_GRAPH", "graphs/node-scope.yaml");
world.run_on(start, "start adopted-override");
world.until("the run to settle on the failure", |world| {
world.run_file("adopted-override", "result.json").is_file()
});
std::fs::remove_file(world.fakes.join("harness.fail")).expect("the failure is cleared");
world
.run_with_stdin(
&["reply", "adopted-override"],
&json!({
"version": 2,
"commands": [{
"op": "retry",
"id": "build",
"node": {"id": "build-2", "persona": "engineer",
"task": "## What\nDo build.\n\n## Why\nIt failed.\n\n\
## Acceptance criteria\n- build is done."},
}],
})
.to_string(),
)
.exited(0);
let mut adopt = world.agentgraph_cmd(&["adopt", "adopted-override"]);
adopt
.current_dir(&world.project)
.env("ONEPIPELINE_NODE_GRAPH", "missing-node.yaml");
let adopted = world.run_on(adopt, "adopt adopted-override");
adopted.exited(0).settled();
let configs = configs_of(&world, "adopted-override", "worker");
let retried = configs
.iter()
.find(|(node, _)| node == "build-2")
.unwrap_or_else(|| panic!("the replacement node was never dispatched: {configs:?}"));
assert!(
retried.1.contains("ADOPTED_NODE_OVERRIDE"),
"the node dispatched after adoption did not run under its retained override: {}",
retried.1
);
let turns = world.turns();
assert!(
turns.iter().any(|turn| turn.prompt.contains("It failed.")),
"the replacement node's turn never ran: {turns:?}"
);
}
#[test]
fn a_dispatchs_scratch_directory_reaches_the_turn_the_library_backend_runs() {
let world = World::new("real-scratch");
world.write_graphs();
let path = world.plan("scratch", &plan_of("scratch", vec![agent("build", &[])]));
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0)
.settled();
let turns = world.turns();
let worker = turns
.iter()
.find(|turn| turn.member == "worker")
.unwrap_or_else(|| panic!("the node's own member never ran a turn: {turns:?}"));
let at = std::path::Path::new(&worker.scratch);
assert!(
at.is_absolute(),
"the turn was handed {:?}, which is not an absolute path\n{}",
worker.scratch,
world.dump()
);
assert!(
at.is_dir(),
"the turn was handed {}, which is not a directory that exists\n{}",
at.display(),
world.dump()
);
std::fs::write(
at.join("written"),
"by a journey standing where the turn stood",
)
.unwrap_or_else(|error| panic!("{} is not writable: {error}", at.display()));
}
fn scratch_readings(world: &World) -> Vec<(String, String, String)> {
world
.invocations()
.into_iter()
.filter(|call| call["tool"] == "claude-scratch")
.map(|call| {
let at = |n: usize| {
call["args"][n]
.as_str()
.unwrap_or_else(|| panic!("a scratch reading is three strings: {call}"))
.to_string()
};
(at(0), at(1), at(2))
})
.collect()
}
#[test]
fn concurrent_dispatches_each_hold_their_own_scratch_directory_throughout() {
let world = World::new("real-scratch-concurrent");
world.write_graphs();
world.script("turn.concurrent", "2");
let path = world.plan(
"concurrent",
&plan_of(
"concurrent",
vec![agent("first", &[]), agent("second", &[])],
),
);
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0)
.settled();
let arrived = std::fs::read_to_string(world.fakes.join("turn.concurrent.arrived"))
.unwrap_or_else(|error| panic!("no barrier was reached: {error}\n{}", world.dump()));
let live: std::collections::BTreeSet<&str> = arrived
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.collect();
assert_eq!(
live.len(),
2,
"two dispatches in flight at one instant were not holding two directories: \
{arrived:?}\n{}",
world.dump()
);
let readings = scratch_readings(&world);
let held = |job: &str| -> Vec<(String, String)> {
readings
.iter()
.filter(|(prompt, _, _)| prompt.contains(job))
.map(|(_, phase, at)| (phase.clone(), at.clone()))
.collect()
};
let mut each = Vec::new();
for job in ["Do first.", "Do second."] {
let taken = held(job);
assert_eq!(
taken.len(),
2,
"{job} did not read its scratch directory on both sides of the barrier: \
{readings:?}\n{}",
world.dump()
);
assert_eq!(taken[0].0, "entered");
assert_eq!(taken[1].0, "beside the others");
assert_eq!(
taken[0].1, taken[1].1,
"{job} was holding one scratch directory when it started and another while \
its sibling dispatch ran: {taken:?}"
);
let at = std::path::PathBuf::from(&taken[1].1);
assert!(
at.is_absolute(),
"{job} was given {at:?}, which is not absolute"
);
assert!(
at.is_dir(),
"{job}'s scratch directory {} is gone\n{}",
at.display(),
world.dump()
);
std::fs::write(at.join("read-back"), job)
.unwrap_or_else(|error| panic!("{} is not writable: {error}", at.display()));
each.push(at);
}
assert_ne!(
each[0], each[1],
"two dispatches running at once were handed one directory between them: {each:?}"
);
assert_eq!(
each.iter()
.map(|at| at.display().to_string())
.collect::<std::collections::BTreeSet<String>>(),
live.iter().map(|at| (*at).to_owned()).collect(),
);
let settled = world.events_of("concurrent", "node-settled");
assert_eq!(settled.len(), 2, "{settled:?}\n{}", world.dump());
}
#[test]
fn a_plan_dispatches_through_the_real_oneagentgraph_and_its_members_run() {
let world = World::new("real-dispatch");
world.write_graphs();
let path = world.plan("real", &plan_of("real", vec![agent("build", &[])]));
let started = world.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
]);
started.exited(0).settled();
let run = started.json()["run_id"]
.as_str()
.expect("the launch named its run")
.to_string();
for (member, job) in [("monitor", "Observe this run"), ("worker", "Do build.")] {
let prompt = world.turn_of(member);
assert!(
prompt.contains(job),
"the {member} member was not given its own job: {prompt}"
);
assert!(
!config_of(&world, &run, member).is_empty(),
"the {member} member was started with an empty configuration"
);
}
assert!(
world
.journal(&run)
.iter()
.any(|event| event["source"] == "agentgraph"
&& event["kind"] == "graph-started"
&& event["labels"]["node"].is_null()),
"the driver's own start never reached the merged store: {}",
world.dump()
);
assert!(
world
.journal(&run)
.iter()
.any(|event| event["kind"] == "turn-activity"),
"no member reported a turn: {}",
world.dump()
);
assert_eq!(
world.run_json(&run, "result.json")["state"],
"complete",
"the run did not settle: {}",
world.dump()
);
let journal = world.journal(&run);
let graph_settled = journal
.iter()
.position(|event| {
event["source"] == "agentgraph"
&& event["kind"] == "graph-settled"
&& event["labels"]["node"] == "build"
})
.expect("the linked graph published its terminal event");
let node_settled = journal
.iter()
.position(|event| event["kind"] == "node-settled" && event["labels"]["node"] == "build")
.expect("the terminal graph event settled its node");
assert!(
graph_settled < node_settled,
"the node settled without relaying the linked graph's terminal answer: {}",
world.dump()
);
let relayed: Vec<serde_json::Value> = world
.journal(&run)
.into_iter()
.filter(|event| event["source"] == "agentgraph" && event["labels"]["node"] == "build")
.collect();
assert!(
!relayed.is_empty(),
"no relayed envelope belongs to the node: {}",
world.dump()
);
for event in relayed {
assert_eq!(
event["labels"]["onepipeline.run_id"],
run.as_str(),
"{event}"
);
assert_ne!(
event["labels"]["run_id"],
run.as_str(),
"the graph run's own id was overwritten by this run's: {event}"
);
}
}
#[cfg(unix)]
#[test]
fn a_dispatch_settles_on_its_terminal_event_while_the_graphs_final_reaper_runs() {
let world = World::new("real-terminal-before-reap");
world.write_graphs();
world.script("harness.outlives-graph", "");
let path = world.plan(
"terminal-before-reap",
&plan_of("terminal-before-reap", vec![agent("build", &[])]),
);
let mut command = world.agentgraph_cmd(&["start", &path, "--attach"]);
let mut launch = command
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::piped())
.spawn()
.expect("the attached launch starts");
world.until("the node to settle", |world| {
!world
.events_of("terminal-before-reap", "node-settled")
.is_empty()
});
let pid: u32 = std::fs::read_to_string(world.fakes.join("harness.outlives-graph.pid"))
.expect("the process held for graph-final teardown recorded its pid")
.trim()
.parse()
.expect("the process recorded a pid");
let still_running = std::process::Command::new("kill")
.args(["-0", &pid.to_string()])
.status()
.expect("the host answers about the fixture process")
.success();
if still_running {
end_process(pid);
}
assert!(
still_running,
"the node waited for graph-final reaping instead of settling on graph-settled"
);
let graph_run = world.events_of("terminal-before-reap", "graph-started")[0]["labels"]["run_id"]
.as_str()
.expect("the sibling's own run id is on its announcement")
.to_string();
let recorded = oneagentgraph::history::show(&world.graph_state(), &graph_run)
.unwrap_or_else(|error| panic!("the settled node's record does not read back: {error}"));
assert!(
recorded.finished_ms.is_some(),
"the node settled ahead of its record's ending: {recorded:?}"
);
let status = launch.wait().expect("the attached launch exits");
assert!(status.success(), "the attached launch failed: {status}");
}
fn events_reported(status: &str, node: &str) -> u64 {
let line = status
.lines()
.find(|line| line.trim_start().starts_with(&format!("{node}: running")))
.unwrap_or_else(|| panic!("`status` has no in-flight line for {node}:\n{status}"));
let at = line
.find(" event(s)")
.unwrap_or_else(|| panic!("`{line}` carries no event count"));
let digits: String = line[..at]
.chars()
.rev()
.take_while(char::is_ascii_digit)
.collect();
digits
.chars()
.rev()
.collect::<String>()
.parse()
.unwrap_or_else(|e| panic!("`{line}` carries no readable count: {e}"))
}
#[cfg(unix)]
#[test]
fn two_dispatches_of_one_run_are_stopped_as_one_run() {
let world = World::new("real-shared-process");
world.write_graphs();
world.script("turn.hold", "hold");
let path = world.plan(
"shared",
&plan_of(
"shared",
vec![
agent("first", &[]),
human("approve", &[]),
agent("second", &["approve"]),
],
),
);
world
.run_on_agentgraph(&["start", &path, "--detach"])
.exited(0);
let dispatched = |world: &World| -> Vec<String> {
world
.events_of("shared", "node-dispatched")
.iter()
.filter_map(|event| event["labels"]["node"].as_str().map(str::to_string))
.collect()
};
world.until(
"the first node to be in flight beside the person",
|world| {
dispatched(world).contains(&"first".to_string())
&& !world.events_of("shared", "node-settled").is_empty()
},
);
world.run(&["attest", "shared", "approve"]).exited(0);
world.until("both nodes to be in flight", |world| {
dispatched(world).contains(&"second".to_string())
});
world
.run(&["status", "shared"])
.exited(0)
.out_has("first: running")
.out_has("second: running");
let stopped = world.run(&["stop", "shared"]);
stopped.exited(0).out_has("\"stopped\":true");
assert_eq!(
stopped.json()["teardown"],
json!("signalled"),
"a stop over two dispatches in one driver did not report reaching them:\n{}",
stopped.stdout
);
world.release("turn.go");
world.release("turn.settle");
}
#[test]
fn status_says_what_a_live_dispatch_is_doing_and_the_readout_advances() {
let world = World::new("real-activity");
world.write_graphs();
world.script("turn.hold", "hold");
let path = world.plan("watched", &plan_of("watched", vec![agent("build", &[])]));
world
.run_on_agentgraph(&["start", &path, "--detach"])
.exited(0);
world.until("the dispatch to report a turn", |world| {
!world.events_of("watched", "turn-activity").is_empty()
});
let first = world.run(&["status", "watched"]);
first
.exited(0)
.out_has("build: running")
.out_has("now bash echo the turn ran")
.out_has("event(s)")
.out_has("ago");
let before = events_reported(&first.stdout, "build");
world.release("turn.go");
world.until("the dispatch to report a second tool call", |world| {
world
.events_of("watched", "turn-activity")
.iter()
.filter(|event| event["payload"]["kind"] == "tool_call")
.count()
> 1
});
let second = world.run(&["status", "watched"]);
second
.exited(0)
.out_has("build: running")
.out_has("now bash cargo llvm-cov --workspace");
assert!(
events_reported(&second.stdout, "build") > before,
"the readout did not advance while the node was still in flight:\n{}",
second.stdout
);
world.release("turn.settle");
world.until("the run to settle", |world| {
world.run_file("watched", "result.json").is_file()
});
}
#[test]
fn a_drafted_body_reaches_the_change_request_through_the_real_siblings() {
let world = World::new("real-pr-author");
world.write_graphs();
world.repository("change-open", &[]);
world.script("harness.work", "the worker wrote this");
let drafted = "## What\nRead off the branch's own diff.\n\n## Why\nSo a reviewer knows.";
world.script("harness.body", drafted);
let drafting = world.pr_author_graph();
let node = json!({
"id": "service",
"repo": "service",
"persona": "engineer",
"title": "feat: land what the member made",
"task": "## What\nship the thing",
});
let path = world.plan("authored", &plan_of("authored", vec![node]));
let launched =
world.run_on_agentgraph(&["start", &path, "--attach", "--pr-author-graph", &drafting]);
launched.settled();
let opened = world.changes_opened();
assert_eq!(opened.len(), 1, "{opened:?}\n{}", world.dump());
assert_eq!(
opened[0]["body"],
drafted,
"the drafted body did not reach the change request: {opened:?}\n{}",
world.dump()
);
let kept: Vec<serde_json::Value> = std::fs::read_dir(world.run_file("authored", "reports"))
.expect("the run kept the reports its dispatches settled with")
.filter_map(Result::ok)
.filter_map(|entry| std::fs::read_to_string(entry.path()).ok())
.filter_map(|text| serde_json::from_str(&text).ok())
.collect();
assert!(
kept.iter().any(|report| {
report["results"]
.as_array()
.is_some_and(|results| results.iter().any(|result| {
result["schema_valid"] == json!(true) && result["structured"]["body"] == drafted
}))
}),
"no report this run retained carries the validated answer the body was read from: {kept:#?}"
);
}
#[test]
fn a_validated_answer_carrying_no_body_publishes_the_change_request_without_one() {
let world = World::new("blank-pr-author");
world.write_graphs();
world.repository("change-open", &[]);
world.script("harness.work", "the worker wrote this");
world.script("harness.body", " \n");
let drafting = world.pr_author_graph();
let node = json!({
"id": "service",
"repo": "service",
"persona": "engineer",
"title": "feat: land it with a blank draft",
"task": "## What\nship the thing",
});
let path = world.plan("blankdraft", &plan_of("blankdraft", vec![node]));
let launched =
world.run_on_agentgraph(&["start", &path, "--attach", "--pr-author-graph", &drafting]);
launched.settled();
let opened = world.changes_opened();
assert_eq!(opened.len(), 1, "{opened:?}\n{}", world.dump());
assert_eq!(opened[0]["title"], "feat: land it with a blank draft");
assert_eq!(
opened[0]["body"], "",
"a validated answer with no body in it still put one on the change request: {opened:?}"
);
assert_eq!(
world.run_json("blankdraft", "result.json")["state"],
"complete",
"a drafting dispatch that answered blank took the publication with it:\n{}",
world.dump()
);
let kept: Vec<serde_json::Value> = std::fs::read_dir(world.run_file("blankdraft", "reports"))
.expect("the run kept the reports its dispatches settled with")
.filter_map(Result::ok)
.filter_map(|entry| std::fs::read_to_string(entry.path()).ok())
.filter_map(|text| serde_json::from_str(&text).ok())
.collect();
assert!(
kept.iter().any(|report| {
report["results"].as_array().is_some_and(|results| {
results.iter().any(|result| {
result["schema_valid"] == json!(true) && result["structured"]["body"] == ""
})
})
}),
"no report this run retained carries a validated answer with a blank body: {kept:#?}"
);
let undrafted = world.events_of("blankdraft", "body-not-drafted");
assert_eq!(undrafted.len(), 1, "{undrafted:?}\n{}", world.dump());
assert_eq!(undrafted[0]["payload"]["ending"], "no-body");
assert_eq!(undrafted[0]["labels"]["node"], "service");
}
#[test]
fn a_drafting_graph_the_runner_refuses_still_publishes_the_change_request() {
let world = World::new("real-pr-author-refused");
world.write_graphs();
world.repository("change-open", &[]);
world.script("harness.work", "the worker wrote this");
let refused = world.graphs().join("unrunnable.yaml");
std::fs::write(
&refused,
format!(
"version: {}\nname: pr-author\nmembers:\n author:\n kind: nonesuch\n",
oneagentgraph::config::SCHEMA_VERSION
),
)
.expect("the unrunnable graph is written");
let node = json!({
"id": "service",
"repo": "service",
"persona": "engineer",
"title": "feat: land it with no body",
"task": "## What\nship the thing",
});
let path = world.plan("refuseddraft", &plan_of("refuseddraft", vec![node]));
let launched = world.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--pr-author-graph",
&refused.to_string_lossy(),
]);
launched.settled();
let opened = world.changes_opened();
assert_eq!(opened.len(), 1, "{opened:?}\n{}", world.dump());
assert_eq!(opened[0]["title"], "feat: land it with no body");
assert_eq!(opened[0]["body"], "", "{opened:?}");
assert_eq!(
world.run_json("refuseddraft", "result.json")["state"],
"complete",
"a drafting graph the runner refused took the publication with it:\n{}",
world.dump()
);
for said in [
"node 'service'",
"the drafting dispatch settled without succeeding",
"so it publishes with no body",
] {
assert!(
launched.stderr.contains(said),
"the refusal never reached the operator, which lacks {said:?}:\n{}",
launched.stderr
);
}
let undrafted = world.events_of("refuseddraft", "body-not-drafted");
assert_eq!(undrafted.len(), 1, "{undrafted:?}\n{}", world.dump());
assert_eq!(undrafted[0]["payload"]["ending"], "dispatch-failed");
assert_eq!(undrafted[0]["labels"]["node"], "service");
let detail = undrafted[0]["payload"]["detail"]
.as_str()
.unwrap_or_default();
assert!(
detail.contains("the drafting dispatch settled without succeeding"),
"the recorded ending does not carry the sibling's refusal: {detail}"
);
assert!(
detail.contains("nonesuch") || detail.contains("kind"),
"the recorded ending does not carry the runner's own words: {detail}"
);
let settled = world.events_of("refuseddraft", "node-settled");
assert_eq!(
settled[0]["payload"]["detail"], undrafted[0]["payload"]["detail"],
"the settlement of a node whose drafter would not start did not name it"
);
}
#[test]
fn transcript_renders_a_real_dispatched_turns_tools_and_words() {
let world = World::new("real-transcript");
world.write_graphs();
let path = world.plan("read", &plan_of("read", vec![agent("build", &[])]));
world
.run_on_agentgraph(&["start", &path, "--attach"])
.exited(0)
.settled();
let transcript = world.run(&["transcript", "read", "build"]);
transcript.exited(0).out_has("read build");
transcript.out_has("tool_call bash echo the turn ran");
transcript.out_has("report ");
transcript.out_has("Ran what the task asked for.");
let (from_the_store, from_the_report) = transcript
.stdout
.split_once("\n report ")
.expect("the transcript renders the store's summaries and then the report");
assert!(
from_the_store
.lines()
.any(|line| line == " tool_result the turn ran"),
"what the tool returned is a blank column in the store's own summaries:\n{}",
transcript.stdout
);
assert!(
from_the_report
.lines()
.any(|line| line == " tool_result the turn ran"),
"what the tool returned is a blank column in the retained report:\n{}",
transcript.stdout
);
assert!(
!transcript.stdout.contains("unreadable from this host"),
"the retained report was named and not read:\n{}",
transcript.stdout
);
world
.run(&["transcript", "read", "nowhere"])
.exited(crate::harness::REFUSED)
.err_has("has recorded nothing for node 'nowhere'")
.err_has("build");
}
#[test]
fn a_transcript_names_the_harness_that_answered_and_skips_the_ones_it_stepped_past() {
let world = World::new("real-fallback-transcript");
world.write_graphs();
std::fs::write(
world.graphs().join("chain.toml"),
"run_mode = \"fallback\"\nharnesses = [\"codex\", \"claude-code\"]\n",
)
.expect("the two-candidate chain is written");
let path = world.plan("chained", &plan_of("chained", vec![agent("build", &[])]));
let mut launch = world.agentgraph_cmd(&[
"start",
&path,
"--attach",
"--node-set",
"members.worker.oneharness_config=./chain.toml",
]);
launch.env(
"ONEHARNESS_BIN_CODEX",
world.graphs().join("no-codex-on-this-host"),
);
launch.env("PATH", world.path_with_only_what_a_dispatch_resolves());
if let Some(found) = World::resolved_on(&launch, "codex") {
panic!(
"this launch can resolve codex at {}, so its first candidate would run rather than \
be stepped past and the fall-through below would never happen",
found.display()
);
}
world.run_on(launch, "start --attach").exited(0).settled();
let advanced = world.events_of("chained", "fallback-advanced");
assert!(
advanced
.iter()
.any(|event| event["payload"]["identity"] == "codex"),
"the chain never stepped past its first candidate: {advanced:#?}"
);
let transcript = world.run(&["transcript", "chained", "build"]);
transcript
.exited(0)
.out_has("claude-code")
.out_has("Ran what the task asked for.");
assert!(
!transcript.stdout.contains("codex"),
"a candidate the chain stepped past was rendered as a turn:\n{}",
transcript.stdout
);
}
#[test]
fn a_launch_the_graph_refuses_fails_with_the_graphs_own_words() {
let world = World::new("real-refusal");
let path = world.plan("refused", &plan_of("refused", vec![agent("build", &[])]));
for form in ["--detach", "--attach"] {
let started =
world.run_on_agentgraph(&["start", &path, form, "--dag-graph", &world.dag_graph()]);
started.exited(crate::harness::REFUSED);
started.err_has("oneagentgraph");
started.err_has("dag-scope.yaml");
assert!(
!started.stdout.contains("\"pid\""),
"`start {form}` still printed a pid to drive:\n{}",
started.stdout
);
}
}
#[test]
fn an_adoption_the_graph_refuses_fails_rather_than_leaving_the_run_undriven() {
let world = World::new("real-adopt-refusal");
world.write_graphs();
let path = world.plan(
"orphaned",
&plan_of("orphaned", vec![human("approve", &[])]),
);
world
.run_on_agentgraph(&[
"start",
&path,
"--detach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0);
world.until("the driver to be gone", |world| {
world
.run_on_agentgraph(&["status", "orphaned"])
.stdout
.contains("DRIVER DEAD")
});
std::fs::remove_file(world.graphs().join("dag-scope.yaml")).expect("the graph is removed");
let adopted = world.run_on_agentgraph(&["adopt", "orphaned"]);
adopted.exited(crate::harness::REFUSED);
adopted.err_has("oneagentgraph");
assert!(
world.events_of("orphaned", "driver-adopted").len() == 1,
"the adoption was recorded more than once: {:?}",
world.events_of("orphaned", "driver-adopted")
);
}
#[test]
fn the_run_state_this_crate_places_is_where_the_sibling_looks_for_it() {
let world = World::new("state-dir-drift");
world.write_graphs();
let state = world.root.join("graph-state");
let path = world.plan(
"state-drift",
&plan_of("state-drift", vec![agent("build", &[])]),
);
world
.run_on_agentgraph(&["start", &path, "--attach"])
.exited(0)
.settled();
let history = std::process::Command::new(crate::harness::oneagentgraph_binary())
.arg("history")
.env("ONEAGENTGRAPH_STATE_DIR", &state)
.output()
.expect("the real oneagentgraph runs");
assert!(
history.status.success(),
"the sibling refused to list the directory this crate placed a run in, so this gate \
learned nothing about drift — it exited {} saying:\n{}\n{}",
history.status.code().unwrap_or(-1),
String::from_utf8_lossy(&history.stderr).trim(),
world.dump()
);
let listed = String::from_utf8_lossy(&history.stdout);
assert!(
listed.lines().any(|line| line.contains("node-scope")),
"the sibling found no run where this crate placed one — the state-directory variable, \
or the fallback around it, has drifted on one side. It listed:\n{listed}\nand the \
directory it was asked about holds:\n{}\n{}",
placed(&state),
world.dump()
);
}
fn placed(state: &std::path::Path) -> String {
let Ok(entries) = std::fs::read_dir(state) else {
return " (no state directory at all)".into();
};
let mut held: Vec<String> = entries
.flatten()
.map(|entry| {
let name = entry.file_name().to_string_lossy().to_string();
let record = entry.path().join(oneagentgraph::run::RECORD_FILE);
match std::fs::read_to_string(&record) {
Ok(text) => format!(
" {name}: {}",
text.split_whitespace().collect::<Vec<_>>().join(" ")
),
Err(error) => format!(
" {name}: no readable {}: {error}",
oneagentgraph::run::RECORD_FILE
),
}
})
.collect();
held.sort();
if held.is_empty() {
return " (the state directory is empty)".into();
}
held.join("\n")
}
#[test]
fn the_siblings_own_refusals_still_exit_with_the_codes_this_crate_maps_onto() {
let world = World::new("exit-code-drift");
let missing = world.root.join("no-such-graph.yaml");
let refused = std::process::Command::new(crate::harness::oneagentgraph_binary())
.args(["run", &missing.to_string_lossy(), "--task", "anything"])
.env("ONEAGENTGRAPH_STATE_DIR", world.root.join("graph-state"))
.output()
.expect("the real oneagentgraph runs");
assert_eq!(
refused.status.code(),
Some(oneagentgraph::error::EXIT_INVALID_CONFIG),
"an unreadable graph is no longer the invalid-config exit this crate maps \
`Error::InvalidConfig` onto: {}",
String::from_utf8_lossy(&refused.stderr)
);
}
#[test]
fn the_model_turn_double_refuses_an_argument_the_real_claude_does_not_take() {
let world = World::new("claude-argv");
let sent = |extra: &[&str]| {
let mut args = vec![
"-p",
"Do build.",
"--permission-mode",
"acceptEdits",
"--output-format",
"json",
];
args.extend_from_slice(extra);
std::process::Command::new(crate::harness::double("fake-claude"))
.args(&args)
.env(onepipeline_testfakes::SCRIPT_DIR_ENV, &world.fakes)
.output()
.expect("the double runs")
};
let refused = sent(&["--dangerously-skip-permissions"]);
let said = String::from_utf8_lossy(&refused.stderr).to_string();
assert_eq!(
refused.status.code(),
Some(i32::from(onepipeline_testfakes::USAGE)),
"an argv the real claude exits on ran a turn instead: {said}"
);
assert!(
said.contains("--dangerously-skip-permissions"),
"the refusal does not name what it refused: {said}"
);
let truncated = sent(&["--input-format"]);
let said = String::from_utf8_lossy(&truncated.stderr).to_string();
assert_eq!(
truncated.status.code(),
Some(i32::from(onepipeline_testfakes::USAGE)),
"an option sent with no value after it ran a turn instead: {said}"
);
assert!(
said.contains("--input-format"),
"the refusal does not name the option that was left without a value: {said}"
);
let ran = sent(&[]);
assert_eq!(
ran.status.code(),
Some(0),
"the argv `oneharness` really sends was refused: {}",
String::from_utf8_lossy(&ran.stderr)
);
}
#[test]
fn the_sibling_still_takes_its_harness_from_the_variable_this_crate_restates() {
let world = World::new("harness-bin-drift");
world.write_graphs();
world.write_supervised_node_graph();
write_persona(&world, "engineer");
let mut node = agent("build", &[]);
node["persona"] = Value::from("./engineer.yaml");
let path = world.plan("harness-bin", &plan_of("harness-bin", vec![node]));
let named = "oneharness-that-is-not-installed";
let mut command = world.agentgraph_cmd(&["start", &path, "--attach"]);
command.env("ONEAGENTGRAPH_ONEHARNESS_BIN", named);
world.run_on(command, "start --attach").settled();
let config = config_of(&world, "harness-bin", "worker");
assert!(
config.contains(named),
"the config the sibling composed does not name the harness it was told to \
drive, so the variable was not read — it has drifted:\n{config}"
);
}
#[test]
fn a_note_delivered_through_the_real_sibling_records_what_the_conversation_answered() {
let world = World::new("real-note");
world.write_graphs();
world.script("turn.hold", "hold");
let path = world.plan("noted", &plan_of("noted", vec![agent("build", &[])]));
world
.run_on_agentgraph(&["start", &path, "--detach"])
.exited(0);
world.until("the dispatch to report a turn", |world| {
!world.events_of("noted", "turn-activity").is_empty()
});
let note = "the fixture moved to tests/data; stop editing src/old.rs";
let submitted = world.run_with_stdin_on(
world.agentgraph_cmd(&["reply", "noted"]),
&json!({
"version": 2,
"commands": [{
"op": "note", "id": "build", "addressee": "worker", "text": note
}],
})
.to_string(),
);
submitted.exited(0);
world.until("the note to be reconciled", |world| {
!world.events_of("noted", "edit-committed").is_empty()
});
let committed = world.events_of("noted", "edit-committed");
assert_eq!(
committed[0]["payload"]["operations"][0]["reached"],
json!("carried"),
"a note no turn of the real conversation took was not carried to the next \
dispatch: {committed:?}"
);
assert_eq!(
committed[0]["payload"]["operations"][0]["text"],
json!(note),
"the record does not carry what the note said: {committed:?}"
);
assert!(
world.events_of("noted", "turn-interrupted").is_empty(),
"a note reached for the interrupt lever: {:?}",
world.kinds("noted")
);
world.release("turn.go");
world.release("turn.settle");
}
#[test]
fn a_cancel_against_a_real_dispatch_asks_its_lever_and_reaps_it_at_the_deadline() {
let world = World::new("real-cancel");
world.write_graphs();
world.script("turn.hold", "hold");
let path = world.plan("stopped", &plan_of("stopped", vec![agent("build", &[])]));
let mut launch = world.agentgraph_cmd(&["start", &path, "--detach"]);
launch.env(crate::harness::CANCEL_GRACE_ENV, "1");
world.run_on(launch, "start --detach").exited(0);
world.until("the dispatch to report a turn", |world| {
!world.events_of("stopped", "turn-activity").is_empty()
});
world
.run_with_stdin(
&["reply", "stopped"],
&json!({"version": 2, "commands": [{"op": "cancel", "id": "build"}]}).to_string(),
)
.exited(0);
world.until("the interrupt to be recorded", |world| {
!world.events_of("stopped", "turn-interrupted").is_empty()
});
let interrupted = world.events_of("stopped", "turn-interrupted");
assert_eq!(interrupted[0]["payload"]["delivered"], json!(false));
assert_eq!(
interrupted[0]["labels"]["node"], "build",
"the envelope is not stamped with the node it is about: {}",
interrupted[0]
);
assert!(
interrupted[0]["payload"]["input_bytes"]
.as_u64()
.is_some_and(|bytes| bytes > 0),
"the cancellation offered the turn no redirection at all: {}",
interrupted[0]
);
world.until("the deadline to expire", |world| {
world
.events_of("stopped", "planner-surface-queued")
.iter()
.any(|event| event["payload"]["kind"] == "dispatch-killed")
});
world.until("the cancelled node to settle", |world| {
world
.events_of("stopped", "node-settled")
.iter()
.any(|event| event["labels"]["node"] == "build")
});
let settled = world
.events_of("stopped", "node-settled")
.into_iter()
.find(|event| event["labels"]["node"] == "build")
.expect("the settlement was just seen");
assert_eq!(settled["payload"]["status"], "cancelled", "{settled}");
world.release("turn.go");
world.release("turn.settle");
}
#[test]
fn consuming_a_surface_restarts_the_real_pacemakers_clock() {
let world = World::new("real-pacemaker");
world.write_graphs_with_pacemaker();
let path = world.plan("paced", &plan_of("paced", vec![human("approve", &[])]));
world
.run_on_agentgraph(&[
"start",
&path,
"--detach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0);
let graph_run = world.run_json("paced", "launch.json")["graph_run"]
.as_str()
.expect("the launch record names the graph run driving this run")
.to_string();
assert_ne!(graph_run, "paced");
world
.run_on_agentgraph(&[
"surface",
"paced",
"--kind",
"check-in",
"--message",
"steady",
])
.exited(0);
let read = world.run_on(world.agentgraph_cmd(&["next", "paced"]), "next paced");
read.exited(0).out_has("\"surface\"");
assert!(
!read
.stderr
.contains("could not reset the check-in pacemaker"),
"the real sibling refused the reset: {}",
read.stderr
);
world.until(
"the reset to reach a check-in clock or signal directory of this launch's graph runs",
|world| {
let Some(launch) = std::fs::read_to_string(world.run_file("paced", "launch.json"))
.ok()
.and_then(|text| serde_json::from_str::<Value>(&text).ok())
else {
return false;
};
let recorded: Vec<String> = launch["observer_runs"]
.as_array()
.into_iter()
.flatten()
.chain(std::iter::once(&launch["graph_run"]))
.filter_map(|run| run.as_str().map(str::to_string))
.collect();
recorded.iter().any(|run| {
world
.graph_state()
.join(run)
.join("signals")
.join("check-in.reset")
.is_file()
|| world.graph_journal(run).iter().any(|event| {
event["kind"] == "cron-reset" && event["labels"]["member"] == "check-in"
})
})
},
);
}
#[test]
fn a_view_renders_with_the_health_block_read_through_the_library() {
let world = World::new("real-health");
world.write_graphs();
let path = world.plan("probed", &plan_of("probed", vec![agent("build", &[])]));
world
.run_on_agentgraph(&["start", &path, "--attach"])
.exited(0)
.settled();
let status = world.run_on(world.agentgraph_cmd(&["status", "probed"]), "status probed");
status.exited(0).out_has("probed").out_has("SETTLED");
assert!(
!status.stdout.contains("fake-provider"),
"the view carried the override's health block on the default path:\n{}",
status.stdout
);
}
#[test]
fn a_launchs_own_environment_reaches_the_member_the_library_backend_runs() {
let world = World::new("real-launch-env");
world.write_graphs();
let path = world.plan("carried", &plan_of("carried", vec![agent("build", &[])]));
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0)
.settled();
let saw = world.observer_saw();
assert_eq!(
saw.first().map(|saw| saw["run"].clone()),
Some(json!("carried")),
"the observer was not told which run it was started for: {saw:?}\n{}",
world.dump()
);
assert_eq!(
saw[0]["launch_record"],
json!(true),
"the observer was not told where the run's ledger lives: {saw:?}"
);
}
#[test]
fn a_document_the_runner_accepts_launches_whichever_way_it_is_asked_for() {
for form in ["--attach", "--detach"] {
let world = World::new(&format!("runner-schema-{}", form.trim_start_matches("--")));
world.write_graphs_at_the_runners_schema();
let path = world.plan("schema", &plan_of("schema", vec![agent("build", &[])]));
let mut command =
world.agentgraph_cmd(&["start", &path, form, "--dag-graph", &world.dag_graph()]);
command.env("PATH", world.empty_path());
let started = world.run_on(command, &format!("start {form}"));
assert!(
!started.stderr.contains("schema_version"),
"the launch refused the document the runner accepts:\n{}",
started.stderr
);
world.until("the graph the launch named to run", |world| {
!world.observer_saw().is_empty()
});
world.until("the run to settle", |world| {
world.run_file("schema", "result.json").is_file()
});
assert!(
!world.events_of("schema", "node-dispatched").is_empty(),
"the loop never dispatched the node:\n{}",
world.dump()
);
let settled = world.events_of("schema", "node-settled");
#[cfg(all(unix, not(target_os = "linux")))]
assert_eq!(
settled[0]["payload"]["outcome"],
json!("infrastructure-failure"),
"a dispatch nothing could stamp settled as something else: {}",
settled[0]
);
#[cfg(any(target_os = "linux", windows))]
assert_eq!(
settled[0]["payload"]["status"],
json!("done"),
"a dispatch this host could stamp did not run: {}",
settled[0]
);
let results = world.run(&["results", "schema"]);
results.exited(0).out_has("build");
}
}
#[test]
fn every_dag_scope_member_is_given_the_runs_description_and_its_own_job() {
let world = World::new("neutral-run-task");
world.write_graphs_at_the_runners_schema();
let path = world.plan("neutral", &plan_of("neutral", vec![agent("build", &[])]));
world
.run_on(
world.agentgraph_cmd(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
]),
"start neutral",
)
.exited(0)
.settled();
let monitor = world.turn_of("monitor");
let reporter = world.turn_of(REPORTING_MEMBER);
for (member, prompt) in [("monitor", &monitor), (REPORTING_MEMBER, &reporter)] {
for expected in ["neutral", "Deliver neutral"] {
assert!(
prompt.contains(expected),
"member '{member}' was not told {expected:?}: {prompt}"
);
}
}
assert!(
monitor.contains("Observe this run"),
"the monitor was not given its own job: {monitor}"
);
assert!(
!reporter.contains("Observe this run"),
"a member whose job is not the monitor's was given it: {reporter}"
);
assert!(
reporter.contains("Report on this run"),
"the reporter was not given its own job: {reporter}"
);
}
#[test]
fn the_retained_driver_relays_its_graphs_stream_and_exits_with_its_code() {
let world = World::new("drive-relay");
world.write_graphs();
let graph = world.graphs().join("node-scope.yaml");
let dir = world.root.join("driven");
std::fs::create_dir_all(&dir).expect("a directory for the driven graph");
let driven = world.run_on(
world.agentgraph_cmd(&[
"drive",
&graph.to_string_lossy(),
"--task",
"Do the work and settle.",
"--dir",
&dir.to_string_lossy(),
]),
"drive node-scope",
);
driven.exited(0);
let relayed: Vec<Value> = driven
.stdout
.lines()
.filter(|line| !line.trim().is_empty())
.map(|line| {
serde_json::from_str(line).unwrap_or_else(|error| {
panic!("`drive` wrote a line that is not an envelope: {error}\n{line}")
})
})
.collect();
assert!(
relayed
.iter()
.any(|event| event["kind"] == "member-started"),
"the relay carried no member-started:\n{}",
driven.stdout
);
assert!(
relayed
.iter()
.all(|event| event["source"] == "agentgraph" && event["v"] == 1),
"the relay rewrote the envelopes it was given:\n{}",
driven.stdout
);
}
#[cfg(target_os = "linux")]
#[test]
fn a_retained_driver_that_cannot_write_its_relay_refuses() {
let world = World::new("drive-nospace");
world.write_graphs();
let graph = world.graphs().join("node-scope.yaml");
let dir = world.root.join("driven");
std::fs::create_dir_all(&dir).expect("a directory for the driven graph");
let mut command = world.agentgraph_cmd(&[
"drive",
&graph.to_string_lossy(),
"--task",
"Do the work and settle.",
"--dir",
&dir.to_string_lossy(),
]);
command.stdout(
std::fs::OpenOptions::new()
.write(true)
.open("/dev/full")
.expect("/dev/full"),
);
let refused = world.run_on(command, "drive onto a full disk");
assert_ne!(
refused.code, 0,
"a driver that could not relay its own stream reported success:\n{}",
refused.stderr
);
refused.err_has("relaying graph event");
}
#[test]
fn a_retained_driver_carries_a_failing_graphs_own_exit_code() {
let world = World::new("drive-failed");
world.write_graphs();
let graph = world.graphs().join("node-scope.yaml");
let dir = world.root.join("driven-failed");
std::fs::create_dir_all(&dir).expect("a directory for the driven graph");
world.script("harness.fail", "the turn did not get there");
let failed = world.run_on(
world.agentgraph_cmd(&[
"drive",
&graph.to_string_lossy(),
"--task",
"Do the work and settle.",
"--dir",
&dir.to_string_lossy(),
]),
"drive a graph whose member fails",
);
assert_eq!(
failed.code,
oneagentgraph::error::EXIT_MEMBER_FAILED,
"a driver did not carry its graph's own exit code:\nstdout: {}\nstderr: {}",
failed.stdout,
failed.stderr
);
assert!(
failed
.stdout
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.any(|event| event["kind"] == "member-started"),
"the graph never started a member, so its code is not a settlement:\n{}",
failed.stdout
);
}
#[test]
fn a_classification_the_harness_record_contradicts_settles_rather_than_dies() {
let world = World::new("drive-reconciled");
world.write_graphs();
world.write_supervised_node_graph();
let graph = world.graphs().join("node-scope.yaml");
let dir = world.root.join("driven-reconciled");
std::fs::create_dir_all(&dir).expect("a directory for the driven graph");
world.script("harness.rejects", "");
let driven = world.run_on(
world.agentgraph_cmd(&[
"drive",
&graph.to_string_lossy(),
"--task",
"Be billed for a turn the provider then rejects.",
"--dir",
&dir.to_string_lossy(),
]),
"drive a graph whose classification its own record contradicts",
);
let published: Vec<Value> = driven
.stdout
.lines()
.filter_map(|line| serde_json::from_str(line).ok())
.collect();
assert_eq!(
driven.code,
oneagentgraph::error::EXIT_MEMBER_FAILED,
"a driver did not carry its graph's own exit code:\nstdout: {}\nstderr: {}",
driven.stdout,
driven.stderr
);
assert!(
!published.iter().any(|event| event["kind"] == "member-died"),
"a turn the harness recorded as completed and billed was published as a \
death, which is what a node destroys finished work on:\n{}",
driven.stdout
);
let settled: Vec<&Value> = published
.iter()
.filter(|event| event["kind"] == "member-settled")
.collect();
assert_eq!(
settled.len(),
1,
"the carried turn did not settle exactly once:\n{}",
driven.stdout
);
assert_eq!(
settled[0]["payload"]["completed"],
json!(false),
"a turn that never reached its bar settled as one that did: {}",
settled[0]
);
let stored = settled[0]["payload"]["report_path"]
.as_str()
.unwrap_or_else(|| panic!("the settle named no stored report: {}", settled[0]));
let report: Value = serde_json::from_str(
&std::fs::read_to_string(stored).expect("the stored report is readable"),
)
.expect("the stored report is JSON");
let why = report["settled_reason"]
.as_str()
.unwrap_or_else(|| panic!("the carried turn said nothing about why: {report}"));
for said in ["rate_limit", "status ok", "exit code 0"] {
assert!(
why.contains(said),
"the carried turn's reason does not name {said:?}: {why}"
);
}
}
#[test]
fn a_candidate_served_under_the_wrong_model_is_stepped_past_and_named_on_the_record() {
use oneharness_core::domain::fallback::FallThroughReason;
use oneharness_core::domain::signals::FailureKind;
let world = World::new("real-misrouted-model");
world.write_graphs();
world.write_supervised_node_graph();
let requested = "gpt-5.5";
let served = "gpt-5.5-mini";
std::fs::write(
world.graphs().join("chain.toml"),
format!(
"run_mode = \"fallback\"\nharnesses = [\"codex\", \"claude-code\"]\n\n\
[harness.codex]\nmodel = \"{requested}\"\n"
),
)
.expect("the two-candidate chain is written");
world.script("harness.serves", served);
world.script("judge.unmet", "the change builds nothing");
let path = world.plan(
"misrouted",
&plan_of("misrouted", vec![agent("build", &[])]),
);
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--node-set",
"members.worker.agent.oneharness_config=./chain.toml",
])
.settled();
world.until("the run to settle on the refused criterion", |world| {
world.run_file("misrouted", "result.json").is_file()
});
let node = world.run_json("misrouted", "result.json")["nodes"][0].clone();
assert_eq!(
node["status"], "failed",
"the node fails on its own criterion over the turn the honouring candidate ran, \
not on the chain: {node}"
);
assert_eq!(
node["outcome"], "task-failed",
"a chain that recovered was settled as something other than its own \
conversation's verdict: {node}"
);
let advanced: Vec<Value> = world
.journal("misrouted")
.into_iter()
.filter(|event| {
event["source"] == "agentgraph"
&& event["kind"] == "fallback-advanced"
&& event["labels"]["onepipeline.node"] == "build"
})
.collect();
assert_eq!(
advanced.len(),
1,
"one candidate was stepped past once: {advanced:#?}"
);
let advanced: oneagentgraph::event::FallbackAdvanced =
serde_json::from_value(advanced[0]["payload"].clone())
.expect("the relayed advance is the sibling's own payload");
assert_eq!(advanced.identity, "codex", "{advanced:?}");
assert_eq!(
advanced.reason,
FallThroughReason::ModelMismatch.as_str(),
"the chain stepped past the misrouted candidate for some other reason: {advanced:?}"
);
let settled = world.events_of("misrouted", "member-settled");
let stored = settled[0]["payload"]["report_path"]
.as_str()
.unwrap_or_else(|| panic!("the settle named no stored report: {settled:#?}"));
let report: Value = serde_json::from_str(
&std::fs::read_to_string(stored).expect("the stored report is readable"),
)
.expect("the stored report is JSON");
let telemetry: onejudge::Telemetry = serde_json::from_value(report["telemetry"].clone())
.expect("the retained report carries onejudge's telemetry");
let attribution = telemetry
.attribution
.iter()
.find(|attribution| {
attribution.role == onejudge::TelemetryRole::Agent && attribution.turn_index == 1
})
.unwrap_or_else(|| {
panic!("the agent side's first turn attributes nothing: {telemetry:#?}")
});
assert_eq!(
attribution.ran.as_deref(),
Some("claude-code"),
"{attribution:#?}"
);
let [fell] = attribution.fell_through.as_slice() else {
panic!("one candidate fell through: {attribution:#?}");
};
assert_eq!(fell.harness, "codex", "{attribution:#?}");
assert_eq!(
fell.reason,
FallThroughReason::ModelMismatch.as_str(),
"{attribution:#?}"
);
let [refused, ran] = attribution.candidates.as_slice() else {
panic!("two candidates were attempted, in order: {attribution:#?}");
};
assert_eq!(refused.harness, "codex", "{attribution:#?}");
assert!(!refused.ran, "{attribution:#?}");
assert_eq!(
refused.failure_kind.as_deref(),
Some(FailureKind::ModelMismatch.as_str()),
"the refused candidate is attributed under some other kind: {attribution:#?}"
);
assert_eq!(
refused.model.as_deref(),
Some(requested),
"{attribution:#?}"
);
assert_eq!(ran.harness, "claude-code", "{attribution:#?}");
assert!(ran.ran, "{attribution:#?}");
let session = world
.journal("misrouted")
.into_iter()
.filter(|event| {
event["source"] == "agentgraph"
&& event["kind"] == "oneharness-session"
&& event["labels"]["onepipeline.node"] == "build"
})
.find_map(|event| {
serde_json::from_value::<oneagentgraph::event::OneharnessSession>(
event["payload"].clone(),
)
.ok()
.filter(|session| {
session.role == oneagentgraph::event::Role::Agent && session.turn == 1
})
})
.expect("the agent side's first turn names the record it wrote");
let file = oneharness_core::io::history::find_session_path(
std::path::Path::new(&session.history_dir),
Some(&session.history_project),
&session.history_session,
)
.expect("the history store is readable")
.unwrap_or_else(|| panic!("the pointer names a session the store does not hold: {session:?}"));
let records = oneharness_core::io::history::read_session(&file)
.expect("the session file is oneharness's");
let record_of = |identity: &str| {
records
.iter()
.find(|record| record.harness_id == identity)
.unwrap_or_else(|| {
panic!(
"no record for '{identity}' in {}: {records:#?}",
file.display()
)
})
};
let misrouted = record_of("codex");
assert_eq!(
misrouted.model.as_deref(),
Some(requested),
"{misrouted:#?}"
);
assert_eq!(
misrouted.observed_model.as_deref(),
Some(served),
"the record does not carry the model the server said it would run under: {misrouted:#?}"
);
assert_eq!(
misrouted.failure_kind,
Some(FailureKind::ModelMismatch),
"{misrouted:#?}"
);
let honoured = record_of("claude-code");
assert_eq!(
honoured.observed_model, None,
"an observation was invented for a harness that reported none: {honoured:#?}"
);
assert_eq!(
honoured.history_id.to_string(),
session.history_id,
"the pointer names a record other than the invocation that ran: {session:?}"
);
let line = format!(
"fallback: the agent side fell through 'codex' ({}) → served by 'claude-code'",
FallThroughReason::ModelMismatch.as_str()
);
let results = world.run(&["results", "misrouted"]);
results.exited(0).out_has(&line);
assert!(
!results.stdout.contains("provider:") && !results.stdout.contains("refused"),
"a chain that recovered was reported as a refusal:\n{}",
results.stdout
);
let status = world.run(&["status", "misrouted"]);
status
.exited(0)
.out_has("build: fallback — the agent side fell through 'codex'")
.out_has("served by 'claude-code'");
assert!(
!status.stdout.contains("build: failed — the agent side"),
"a chain that recovered was reported as the node's failure:\n{}",
status.stdout
);
}
fn turns_dispatched(world: &World, run: &str, node: &str, step: Option<&str>) -> u64 {
let events = world.journal(run);
let started = events
.iter()
.filter(|event| event["kind"] == "member-started")
.find(|event| {
event["labels"]["onepipeline.node"] == node
&& step.is_none_or(|step| event["labels"]["onepipeline.step"] == step)
})
.unwrap_or_else(|| panic!("no member started for {node}/{step:?}: {events:?}"));
let config = started["payload"]["config"]
.as_str()
.expect("the sibling publishes the config it launched the member with");
let text = std::fs::read_to_string(config).expect("that configuration is on disk");
let effective: Value = serde_norway::from_str(&text).expect("it parses");
effective["user"]["max_turns"]
.as_u64()
.unwrap_or_else(|| panic!("{config} states no turn ceiling: {text}"))
}
fn write_persona(world: &World, name: &str) {
std::fs::write(
world.graphs().join(format!("{name}.yaml")),
format!("name: {name}\nsystem_prompt: Ship it.\nuser:\n persona: Review it.\n"),
)
.expect("the persona is written");
}
#[test]
fn a_nodes_turn_budget_reaches_its_dispatch_and_outranks_the_run_wide_one() {
let world = World::new("real-turn-budget");
world.write_graphs();
world.write_supervised_node_graph();
for persona in ["budgeted", "plain"] {
write_persona(&world, persona);
}
let dispatched = |run: &str, node: Value| {
let path = world.plan(run, &plan_of(run, vec![node]));
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--node-set",
"members.worker.max_turns=9",
])
.settled();
turns_dispatched(&world, run, run, None)
};
let mut budgeted = agent("budgeted", &[]);
budgeted["persona"] = Value::from("./budgeted.yaml");
budgeted["max_turns"] = json!(45);
let mut plain = agent("plain", &[]);
plain["persona"] = Value::from("./plain.yaml");
assert_eq!(
dispatched("budgeted", budgeted),
45,
"the node's own turn budget did not reach the member that runs its work"
);
assert_eq!(
dispatched("plain", plain),
9,
"the operator's run-wide override did not reach a node that declared none"
);
}
fn two_party_worktree(world: &World, run: &str) -> String {
let started: Vec<Value> = world
.journal(run)
.into_iter()
.filter(|event| event["kind"] == "member-started")
.collect();
started
.iter()
.find(|event| event["payload"]["engine"] == "onejudge")
.and_then(|event| event["payload"]["worktree"].as_str().map(str::to_string))
.unwrap_or_else(|| panic!("no two-party member was started in {run}: {started:#?}"))
}
#[test]
fn a_two_party_member_is_started_in_the_directory_the_graph_was_given() {
let world = World::new("real-two-party-cwd");
world.write_graphs();
world.write_supervised_node_graph();
write_persona(&world, "engineer");
world.repository("local-direct", &[]);
let node = json!({
"id": "service",
"repo": "service",
"persona": "./engineer.yaml",
"task": "## What\nship the thing",
"title": "feat: land what the member made",
});
let path = world.plan("twoparty", &plan_of("twoparty", vec![node]));
world
.run_on_agentgraph(&["start", &path, "--attach"])
.settled();
let session_worktree = world
.journal("twoparty")
.into_iter()
.filter(|event| event["source"] == "vcs" && event["kind"] == "session-opened")
.find_map(|event| event["payload"]["worktree"].as_str().map(str::to_string))
.expect("the lifecycle node's session opened a worktree");
assert_eq!(
two_party_worktree(&world, "twoparty"),
session_worktree,
"the two-party member was started somewhere other than the directory the graph was \
given. A member started in its own scratch has no repository to work in, and the work \
it leaves there is discarded at publication as `no-changes`."
);
}
#[test]
fn a_steps_turn_budget_reaches_that_steps_own_dispatch() {
let world = World::new("real-step-budget");
world.write_graphs();
world.write_supervised_node_graph();
write_persona(&world, "implementer");
world.repository("local-direct", &[]);
let node = json!({
"id": "service",
"repo": "service",
"title": "feat: land what the step made",
"steps": [
{"id": "implement", "persona": "./implementer.yaml", "task": "## What\nimplement",
"max_turns": 45},
],
});
let path = world.plan("stepbudget", &plan_of("stepbudget", vec![node]));
world
.run_on_agentgraph(&["start", &path, "--attach"])
.settled();
assert_eq!(
turns_dispatched(&world, "stepbudget", "service", Some("implement")),
45,
"the step's own turn budget did not reach the dispatch that ran it; the graph's \
own default is 12"
);
}
#[test]
fn a_launchs_agentgraph_filter_reaches_the_real_sibling_and_narrows_what_it_relays() {
let world = World::new("real-agentgraph-filter");
world.write_graphs();
let relayed = |run: &str| -> String { world.run(&["monitor", run, "--all"]).stdout };
let path = world.plan(
"unfiltered",
&plan_of("unfiltered", vec![agent("build", &[])]),
);
world
.run_on_agentgraph(&["start", &path, "--attach"])
.settled();
let ingested = relayed("unfiltered");
for kind in ["turn-activity", "member-settled"] {
assert!(
ingested.contains(kind),
"a launch naming no filters did not ingest {kind}:\n{ingested}"
);
}
let path = world.plan("filtered", &plan_of("filtered", vec![agent("build", &[])]));
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--filter-agentgraph",
r#"{"exclude": [{"kind": "turn-*"}]}"#,
])
.settled();
let kinds = relayed("filtered");
assert!(
!kinds.contains("turn-"),
"the source filter did not reach `oneagentgraph`:\n{kinds}"
);
assert!(
kinds.contains("member-settled"),
"the source filter dropped the settlement, which it admits:\n{kinds}"
);
world
.run(&["results", "filtered"])
.exited(0)
.out_has("build")
.out_has("done");
}
#[test]
fn the_observer_graphs_own_stream_is_filtered_too_and_the_spec_may_be_a_file() {
let world = World::new("real-observer-filter");
world.write_graphs();
let spec = world.root.join("relay.json");
std::fs::write(&spec, r#"{"exclude": [{"kind": "turn-*"}]}"#).expect("the spec is written");
let observed = |run: &str| -> Vec<String> {
world
.journal(run)
.iter()
.filter(|event| event["source"] == "agentgraph" && event["labels"]["node"].is_null())
.filter_map(|event| event["kind"].as_str().map(str::to_string))
.collect()
};
let path = world.plan("watched", &plan_of("watched", vec![agent("build", &[])]));
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
])
.settled();
let ingested = observed("watched");
assert!(
ingested.iter().any(|kind| kind.starts_with("turn-")),
"the observer graph relayed no turn of its own, so this journey could not \
tell a filtered observer from a quiet one: {ingested:?}\n{}",
world.dump()
);
let path = world.plan("quiet", &plan_of("quiet", vec![agent("build", &[])]));
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
"--filter-agentgraph",
&spec.to_string_lossy(),
])
.settled();
let kinds = observed("quiet");
assert!(
!kinds.iter().any(|kind| kind.starts_with("turn-")),
"the source filter did not reach the observer graph's own launch: {kinds:?}"
);
assert!(
!kinds.is_empty(),
"the observer graph relayed nothing at all, so nothing here is about the filter"
);
}
#[cfg(unix)]
#[test]
fn a_run_whose_observer_graph_is_watching_and_then_is_killed_reads_as_each() {
let world = World::new("real-observer-dead").with_env(OBSERVER_RESTARTS_ENV, "0");
world.write_graphs();
world.script("turn.hold", "hold");
world.script("observer.wait", "hold");
for (run, dag_graph) in [("watched", true), ("bare", false)] {
let path = world.plan(run, &plan_of(run, vec![agent("build", &[])]));
let mut launch = vec!["start".to_string(), path];
launch.push("--detach".into());
if dag_graph {
launch.push("--dag-graph".into());
launch.push(world.dag_graph());
}
world
.run_on_agentgraph(&launch.iter().map(String::as_str).collect::<Vec<_>>())
.exited(0);
}
let graph_run = || {
world.run_json("watched", "launch.json")["graph_run"]
.as_str()
.unwrap_or_default()
.to_string()
};
world.until("the observer graph to be recorded", |_| {
!graph_run().is_empty()
});
let graph_dir = world.graph_state().join(graph_run());
let record = || {
std::fs::read_to_string(graph_dir.join(oneagentgraph::run::RECORD_FILE)).unwrap_or_default()
};
let line = |run: &str, view: &[&str]| -> String {
let rendered = world.run_on_agentgraph(view);
rendered.exited(0);
rendered
.stdout
.lines()
.find(|line| line.contains(run))
.unwrap_or_else(|| panic!("no line for {run} in:\n{}", rendered.stdout))
.to_string()
};
for view in [vec!["runs"], vec!["status"]] {
assert!(
!record().contains("finished_ms"),
"the observer settled before it was read"
);
let watching = line("watched", &view);
assert!(
watching.contains("ACTIVE") && !watching.contains("OBSERVER"),
"a run whose observer is still taking its turn is reported unwatched: {watching}"
);
let bare = line("bare", &view);
assert!(
bare.contains("NO OBSERVER") && !bare.contains("OBSERVER DEAD"),
"a run launched with no observer reads as one whose observer died: {bare}"
);
assert_ne!(watching, bare);
}
let lock = std::fs::read_to_string(graph_dir.join(oneagentgraph::liveness::OWNER_LOCK_FILE))
.expect("the graph run records who owns its state");
let owner: i32 = lock
.split_whitespace()
.next()
.and_then(|pid| pid.parse().ok())
.unwrap_or_else(|| panic!("the owner lock names no process: {lock:?}"));
assert!(
oneagentgraph::scratch::reclaimable(&graph_dir).is_err(),
"the graph run's state was already unowned, so killing its owner proves nothing"
);
assert_eq!(
unsafe { libc::kill(owner, libc::SIGKILL) },
0,
"could not end the process the graph run's own lock names"
);
world.until("the driver to notice its observer is gone", |_| {
line("watched", &["runs"]).contains("OBSERVER DEAD")
});
assert!(
!record().contains("finished_ms"),
"the observer wrote an ending after being killed, so this proves the \
record rather than the lock: {}",
record()
);
for view in [vec!["runs"], vec!["status"]] {
let dead = line("watched", &view);
assert!(
dead.contains("ACTIVE") && dead.contains("OBSERVER DEAD"),
"a run whose observer was killed is still reported as watched: {dead}"
);
let bare = line("bare", &view);
assert!(
bare.contains("NO OBSERVER") && !bare.contains("OBSERVER DEAD"),
"the run that never had an observer changed when another run's died: {bare}"
);
assert_ne!(dead, bare);
}
world.until_run_file_holds("watched", "driver.log", "has stopped watching");
world.release("observer.go");
world.release("turn.go");
world.release("turn.settle");
}
#[cfg(unix)]
#[test]
fn a_killed_observer_is_replaced_and_the_run_goes_on_being_watched() {
let world = World::new("real-observer-restarted");
world.write_graphs();
world.script("turn.hold", "hold");
world.script("observer.wait", "hold");
let path = world.plan(
"rewatched",
&plan_of("rewatched", vec![agent("build", &[])]),
);
world
.run_on_agentgraph(&[
"start",
&path,
"--detach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0);
let recorded = || world.run_json("rewatched", "launch.json");
let graph_run = || {
recorded()["graph_run"]
.as_str()
.unwrap_or_default()
.to_string()
};
world.until("the observer graph to be recorded", |_| {
!graph_run().is_empty()
});
let killed = graph_run();
let graph_dir = world.graph_state().join(&killed);
let lock = std::fs::read_to_string(graph_dir.join(oneagentgraph::liveness::OWNER_LOCK_FILE))
.expect("the graph run records who owns its state");
let owner: i32 = lock
.split_whitespace()
.next()
.and_then(|pid| pid.parse().ok())
.unwrap_or_else(|| panic!("the owner lock names no process: {lock:?}"));
assert_eq!(
unsafe { libc::kill(owner, libc::SIGKILL) },
0,
"could not end the process the graph run's own lock names"
);
world.until("the run to be watched by another graph", |_| {
let now = graph_run();
!now.is_empty() && now != killed
});
let record = recorded();
assert_eq!(
record["observer_runs"],
json!([killed, graph_run()]),
"the run does not name the graphs that have watched it: {record}"
);
assert!(
record["observer_ending"].is_null(),
"a run that is being watched again says nothing is watching it: {record}"
);
for view in [vec!["runs"], vec!["status"]] {
let rendered = world.run_on_agentgraph(&view);
rendered.exited(0);
let line = rendered
.stdout
.lines()
.find(|line| line.contains("rewatched"))
.unwrap_or_else(|| panic!("no line for the run in:\n{}", rendered.stdout));
assert!(
line.contains("ACTIVE") && !line.contains("OBSERVER"),
"a run whose observer was replaced still reads as unwatched: {line}"
);
}
world.until_run_file_holds("rewatched", "driver.log", "started another observer graph");
world.release("observer.go");
world.release("turn.go");
world.release("turn.settle");
}
#[test]
fn a_settlement_record_is_still_attributable_to_the_observer_that_wrote_it() {
let world = World::new("real-observer-attributed").with_env(OBSERVER_RESTARTS_ENV, "1");
world.write_graphs();
world.script("turn.hold", "hold");
let path = world.plan(
"attributed",
&plan_of("attributed", vec![agent("build", &[])]),
);
let mut command = world.agentgraph_cmd(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
]);
let mut launch = command
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.expect("the attached launch starts");
world.until("the run to be recorded", |world| {
world.run_file("attributed", "launch.json").is_file()
});
let watched = || -> Vec<String> {
world.run_json("attributed", "launch.json")["observer_runs"]
.as_array()
.map(|runs| {
runs.iter()
.filter_map(|run| run.as_str().map(str::to_string))
.collect()
})
.unwrap_or_default()
};
world.until("the driver to spend its restart bound", |world| {
watched().len() == 2
&& world.run_json("attributed", "launch.json")["observer_ending"].is_string()
});
let observers = watched();
let record = world.run_json("attributed", "launch.json");
assert_eq!(
record["graph_run"],
json!(observers[1]),
"the run addresses a graph that is not the last one it started: {record}"
);
let of = |run: &str, kind: &str| -> Vec<Value> {
world
.journal("attributed")
.into_iter()
.filter(|event| event["kind"] == json!(kind) && event["labels"]["run_id"] == json!(run))
.collect()
};
world.until("the first observer's settlement to reach the store", |_| {
!of(&observers[0], "graph-settled").is_empty()
});
let settled = of(&observers[0], "graph-settled");
assert_eq!(
settled.len(),
1,
"the graph that stopped watching settled more than once: {settled:#?}"
);
assert!(
settled[0]["labels"]["node"].is_null(),
"the observer's settlement is labelled as a node dispatch's: {}",
settled[0]
);
assert!(
observers.contains(
&settled[0]["labels"]["run_id"]
.as_str()
.unwrap_or_default()
.to_string()
),
"the run does not name the graph that wrote its observer's settlement: {record}"
);
assert!(
!of(&observers[0], "graph-started").is_empty(),
"the store says the observer settled without ever having started"
);
let dispatched: Vec<String> = world
.journal("attributed")
.into_iter()
.filter(|event| event["labels"]["node"] == json!("build"))
.filter_map(|event| event["labels"]["run_id"].as_str().map(str::to_string))
.collect();
assert!(
!dispatched.is_empty(),
"the node was never dispatched, so there is nothing to tell the observer from"
);
assert!(
dispatched.iter().all(|run| !observers.contains(run)),
"a node dispatch's graph is named as one of this run's observers: {dispatched:?}"
);
world.release("turn.go");
world.release("turn.settle");
let status = launch.wait().expect("the attached launch exits");
assert!(status.success(), "the attached launch failed: {status}");
}
#[test]
fn a_run_whose_observer_graph_finished_is_reported_unwatched() {
let world = World::new("real-observer-finished").with_env(OBSERVER_RESTARTS_ENV, "0");
world.write_graphs();
world.script("turn.hold", "hold");
let path = world.plan("outlived", &plan_of("outlived", vec![agent("build", &[])]));
world
.run_on_agentgraph(&[
"start",
&path,
"--detach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0);
let graph_run = || {
world.run_json("outlived", "launch.json")["graph_run"]
.as_str()
.unwrap_or_default()
.to_string()
};
world.until("the observer graph to be recorded", |_| {
!graph_run().is_empty()
});
world.until("the observer graph to write its ending", |world| {
std::fs::read_to_string(
world
.graph_state()
.join(graph_run())
.join(oneagentgraph::run::RECORD_FILE),
)
.is_ok_and(|record| record.contains("finished_ms"))
});
for view in [vec!["runs"], vec!["status"]] {
let rendered = world.run_on_agentgraph(&view);
rendered.exited(0);
let line = rendered
.stdout
.lines()
.find(|line| line.contains("outlived"))
.unwrap_or_else(|| panic!("no line for the run in:\n{}", rendered.stdout));
assert!(
line.contains("ACTIVE") && line.contains("OBSERVER DEAD"),
"a run whose observer finished is still reported as watched: {line}"
);
}
world.release("turn.go");
world.release("turn.settle");
}
#[cfg(unix)]
#[test]
fn an_observer_records_its_ending_before_anything_reads_that_it_has_gone() {
let world = World::new("real-observer-ending").with_env(OBSERVER_RESTARTS_ENV, "0");
world.write_graphs();
world.script("turn.hold", "hold");
world.script("observer.wait", "hold");
let path = world.plan(
"outlasted",
&plan_of("outlasted", vec![agent("build", &[])]),
);
world
.run_on_agentgraph(&[
"start",
&path,
"--detach",
"--dag-graph",
&world.dag_graph(),
])
.exited(0);
let graph_run = || {
world.run_json("outlasted", "launch.json")["graph_run"]
.as_str()
.unwrap_or_default()
.to_string()
};
world.until("the observer graph to be recorded", |_| {
!graph_run().is_empty()
});
world.until("the observer to take its turn", |world| {
!world.observer_saw().is_empty()
});
let graph_dir = world.graph_state().join(graph_run());
let record = graph_dir.join(oneagentgraph::run::RECORD_FILE);
interpose_a_fifo(&record);
world.release("observer.go");
world.until("the observer graph to announce it has settled", |world| {
std::fs::read_to_string(world.run_file("outlasted", "driver.log"))
.unwrap_or_default()
.lines()
.any(|line| line.contains("\"kind\":\"graph-settled\"") && line.contains(&graph_run()))
});
let gone = |world: &World| -> Option<String> {
if oneagentgraph::scratch::reclaimable(&graph_dir).is_ok() {
return Some(format!(
"the graph run's state is already unowned: {}",
graph_dir.display()
));
}
let log =
std::fs::read_to_string(world.run_file("outlasted", "driver.log")).unwrap_or_default();
log.contains("has stopped watching")
.then(|| format!("the driver said its observer had stopped watching:\n{log}"))
};
let mut said = None;
let until = std::time::Instant::now() + std::time::Duration::from_secs(2);
while std::time::Instant::now() < until && said.is_none() {
said = gone(&world);
std::thread::sleep(std::time::Duration::from_millis(50));
}
let held = record.clone();
let (drained, drain) = std::sync::mpsc::channel();
std::thread::spawn(move || drop(drained.send(std::fs::read_to_string(&held))));
let written = drain
.recv_timeout(std::time::Duration::from_secs(30))
.ok()
.and_then(Result::ok);
std::fs::remove_file(&record).expect("the interposed FIFO is removed");
if let Some(written) = &written {
std::fs::write(&record, written).expect("the record is put back where a view reads it");
}
assert!(
said.is_none(),
"the observer was reported gone with its ending unwritten: {}",
said.unwrap_or_default()
);
let written = written.expect("the graph wrote the ending this journey was holding");
assert!(
written.contains("finished_ms"),
"the graph settled without recording an ending: {written}"
);
world.until_run_file_holds("outlasted", "driver.log", "has stopped watching");
for view in [vec!["runs"], vec!["status"]] {
let rendered = world.run_on_agentgraph(&view);
rendered.exited(0);
let line = rendered
.stdout
.lines()
.find(|line| line.contains("outlasted"))
.unwrap_or_else(|| panic!("no line for the run in:\n{}", rendered.stdout));
assert!(
line.contains("ACTIVE") && line.contains("OBSERVER DEAD"),
"a run whose observer finished is still reported as watched: {line}"
);
}
world.release("turn.go");
world.release("turn.settle");
}
#[cfg(unix)]
#[test]
fn a_dispatch_records_its_ending_before_its_launch_can_exit() {
let world = World::new("real-dispatch-ending");
world.write_graphs();
world.script("turn.hold", "hold");
let path = world.plan(
"recorded-ending",
&plan_of("recorded-ending", vec![agent("build", &[])]),
);
let mut command = world.agentgraph_cmd(&["start", &path, "--attach"]);
let mut launch = command
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::piped())
.spawn()
.expect("the attached launch starts");
world.until("the dispatch's graph to start", |world| {
!world
.events_of("recorded-ending", "graph-started")
.is_empty()
});
let graph_run = world.events_of("recorded-ending", "graph-started")[0]["labels"]["run_id"]
.as_str()
.expect("the sibling's own run id is on its announcement")
.to_string();
let record = world
.graph_state()
.join(&graph_run)
.join(oneagentgraph::run::RECORD_FILE);
interpose_a_fifo(&record);
world.release("turn.go");
world.release("turn.settle");
world.until("the dispatch's graph to announce it has settled", |world| {
!world
.events_of("recorded-ending", "graph-settled")
.is_empty()
});
let settled_inside = |world: &World| {
!world
.events_of("recorded-ending", "node-settled")
.is_empty()
};
let mut settled_early = false;
let until = std::time::Instant::now() + std::time::Duration::from_secs(1);
while std::time::Instant::now() < until && !settled_early {
settled_early = settled_inside(&world);
std::thread::sleep(std::time::Duration::from_millis(20));
}
let held = record.clone();
let (drained, drain) = std::sync::mpsc::channel();
std::thread::spawn(move || drop(drained.send(std::fs::read_to_string(&held))));
let written = drain
.recv_timeout(std::time::Duration::from_secs(10))
.ok()
.and_then(Result::ok);
std::fs::remove_file(&record).expect("the interposed FIFO is removed");
let written = written.unwrap_or_else(|| {
let _ = launch.wait();
panic!(
"the graph's ending was never written: its launch {} inside the interval and left \
nobody to write it:\n{}",
if settled_early {
"settled the node"
} else {
"held, but the write did not come through"
},
world.dump()
)
});
assert!(
!settled_early,
"the node settled inside the interval, ahead of the ending's write: {}",
world.dump()
);
assert!(
written.contains("finished_ms"),
"the graph settled without recording an ending: {written}"
);
let status = launch.wait().expect("the attached launch exits");
assert!(status.success(), "the attached launch failed: {status}");
world.until("the node to settle", |world| {
world
.events_of("recorded-ending", "node-settled")
.iter()
.any(|event| event["payload"]["status"] == "done")
});
std::fs::write(&record, &written).expect("the record is put back where the sibling reads it");
let history = std::process::Command::new(crate::harness::oneagentgraph_binary())
.arg("history")
.env("ONEAGENTGRAPH_STATE_DIR", world.graph_state())
.output()
.expect("the real oneagentgraph runs");
let listed = String::from_utf8_lossy(&history.stdout);
assert!(
history.status.success() && listed.lines().any(|line| line.contains(&graph_run)),
"the sibling does not list the run whose ending it wrote — exited {} with:\n{listed}\n{}",
history.status.code().unwrap_or(-1),
String::from_utf8_lossy(&history.stderr).trim(),
);
}
#[cfg(unix)]
fn interpose_a_fifo(path: &std::path::Path) {
use std::os::unix::ffi::OsStrExt;
std::fs::remove_file(path).expect("the file the FIFO stands in for is there to replace");
let raw = std::ffi::CString::new(path.as_os_str().as_bytes()).expect("a path with no NUL");
assert_eq!(
unsafe { libc::mkfifo(raw.as_ptr(), 0o600) },
0,
"could not put a FIFO at {}: {}",
path.display(),
std::io::Error::last_os_error()
);
}
#[test]
fn an_adoption_relaunches_the_observer_under_the_launchs_own_filter() {
let world = World::new("real-adopt-filter");
world.write_graphs();
let observed = |run: &str| -> Vec<String> {
world
.journal(run)
.iter()
.filter(|event| event["source"] == "agentgraph" && event["labels"]["node"].is_null())
.filter_map(|event| event["kind"].as_str().map(str::to_string))
.collect()
};
let path = world.plan(
"readopted",
&plan_of(
"readopted",
vec![human("approve", &[]), agent("build", &["approve"])],
),
);
world
.run_on_agentgraph(&[
"start",
&path,
"--attach",
"--dag-graph",
&world.dag_graph(),
"--filter-agentgraph",
r#"{"exclude": [{"kind": "turn-*"}]}"#,
])
.exited(0);
world.run(&["attest", "readopted", "approve"]).exited(0);
let before = observed("readopted").len();
world
.run_on_agentgraph(&["adopt", "readopted"])
.exited(0)
.settled();
let kinds = observed("readopted");
assert!(
kinds.len() > before,
"the adoption relaunched no observer, so nothing here is about its filter: {kinds:?}"
);
assert!(
!kinds.iter().any(|kind| kind.starts_with("turn-")),
"the adoption relaunched the observer without the launch's own filter: {kinds:?}"
);
}
#[test]
fn the_retained_driver_reads_its_own_event_filter_and_refuses_an_unusable_one() {
let world = World::new("drive-filter");
world.write_graphs();
let graph = world.graphs().join("node-scope.yaml");
let dir = world.root.join("driven");
std::fs::create_dir_all(&dir).expect("a directory for the driven graph");
let drive = |spec: &str| {
world.run_on(
world.agentgraph_cmd(&[
"drive",
&graph.to_string_lossy(),
"--task",
"Do the work and settle.",
"--dir",
&dir.to_string_lossy(),
"--event-filter",
spec,
]),
"drive with an event filter",
)
};
let refused = drive(r#"{"include": [{"role": "agent"}]}"#);
assert_eq!(refused.code, REFUSED, "{}", refused.stderr);
assert!(
refused.stderr.contains("role"),
"the refusal does not name the offending field:\n{}",
refused.stderr
);
let driven = drive(r#"{"exclude": [{"kind": "turn-*"}]}"#);
assert_eq!(driven.code, 0, "{}", driven.stderr);
assert!(
!driven.stdout.contains("turn-activity"),
"the retained driver relayed what its filter excluded:\n{}",
driven.stdout
);
assert!(
driven.stdout.contains("member-settled"),
"the retained driver relayed nothing at all, so nothing here is about the \
filter:\n{}",
driven.stdout
);
}
#[test]
fn a_detached_runs_worker_turn_can_ask_its_manager() {
let world = World::new("real-detached-run-id");
world.write_graphs();
world.script(
"harness.asks",
"the worker: this repository has two mains. Which?",
);
let path = world.plan("askable", &plan_of("askable", vec![agent("build", &[])]));
let started = world.run_on(
world.agentgraph_cmd(&["start", &path, "--detach"]),
"start --detach with no dag-scope graph",
);
started.exited(0);
let run = started.json()["run_id"]
.as_str()
.expect("a detached launch names its run")
.to_string();
world.until("the run to settle", |world| {
world.run_file(&run, "result.json").is_file()
});
assert_eq!(
world.question_for_the_manager_on(world.agentgraph_cmd(&["next", &run]), &run),
"the worker: this repository has two mains. Which?"
);
assert_eq!(
world.run_json(&run, "result.json")["state"],
"complete",
"the run did not settle: {}",
world.dump()
);
}