mod support;
use std::fs;
use std::path::{Path, PathBuf};
use serde_json::Value;
use support::{World, wait_until, wait_until_slow};
fn configure(world: &World, flow: &str, script: &Path) {
let flow_dir = world.root().join(".agents/sloop/flows");
fs::create_dir_all(&flow_dir).expect("create flow directory");
fs::write(flow_dir.join("default.yaml"), flow).expect("write flow");
fs::write(
world.root().join(".agents/sloop/config.yaml"),
format!(
"version: 1\nscheduler:\n max_parallel_tasks: 1\nagent:\n default_target: fake\n targets:\n fake:\n cmd: [\"sh\", {}, \"{{prompt}}\"]\n",
serde_json::to_string(&script.to_string_lossy()).expect("serialize script path"),
),
)
.expect("write config");
}
fn write_script(world: &World, name: &str, body: &str) -> PathBuf {
let path = world.root().join(name);
fs::write(&path, format!("#!/bin/sh\nset -eu\n{body}")).expect("write fake agent script");
path
}
fn post(world: &World, name: &str) -> String {
let ticket = world.write_ticket(name, "# Fail action scenario\n");
let output = world.sloop(&["post", ticket.to_str().unwrap(), "--manual"]);
assert!(
output.status.success(),
"post failed: {}",
String::from_utf8_lossy(&output.stderr)
);
World::json_stdout(&output)["data"]["ticket"]["id"]
.as_str()
.expect("ticket id")
.to_owned()
}
fn status(world: &World) -> Value {
let output = world.sloop(&["status"]);
assert!(output.status.success());
World::json_stdout(&output)["data"].clone()
}
fn commit(identity: &str) -> String {
format!("git -c user.name={identity} -c user.email={identity}@example.invalid commit --quiet")
}
fn shell_quote(value: &str) -> String {
format!("'{}'", value.replace('\'', "'\"'\"'"))
}
fn prompts(log: &Path) -> Vec<String> {
fs::read_to_string(log)
.unwrap_or_default()
.split("\u{1}PROMPT\u{1}\n")
.skip(1)
.map(str::to_owned)
.collect()
}
fn recording_agent(world: &World, prompt_log: &Path) -> PathBuf {
fs::write(prompt_log, b"").expect("create prompt log");
write_script(
world,
"fake-agent.sh",
&format!(
"printf '\\001PROMPT\\001\\n%s\\n' \"$1\" >> {log}\n{commit} --allow-empty -m build\nexit 0\n",
log = shell_quote(&prompt_log.to_string_lossy()),
commit = commit("agent"),
),
)
}
fn barren_agent(world: &World, prompt_log: &Path) -> PathBuf {
fs::write(prompt_log, b"").expect("create prompt log");
write_script(
world,
"fake-agent.sh",
&format!(
"printf '\\001PROMPT\\001\\n%s\\n' \"$1\" >> {log}\nexit 0\n",
log = shell_quote(&prompt_log.to_string_lossy()),
),
)
}
fn flaky_test_command(world: &World, name: &str, failures: usize) -> PathBuf {
let counter = world.root().join(format!("{name}.count"));
fs::write(&counter, b"").expect("create counter");
write_script(
world,
name,
&format!(
"printf x >> {counter}\nruns=$(wc -c < {counter} | tr -d ' ')\nif [ \"$runs\" -le {failures} ]; then\n echo \"assertion failed: the widget is not blue\"\n echo \"run $runs of the test command\"\n exit 3\nfi\necho \"tests passed on run $runs\"\nexit 0\n",
counter = shell_quote(&counter.to_string_lossy()),
),
)
}
fn shown_stages(world: &World, run: &str) -> Vec<(String, String)> {
let value = world.show_snapshot(run);
value["stages"]
.as_array()
.expect("stages array")
.iter()
.map(|stage| {
let name = stage["stage"].as_str().unwrap_or("?").to_owned();
let label = match stage["attempt"].as_u64() {
Some(attempt) if attempt > 1 => format!("{name}#{attempt}"),
_ => name,
};
(label, stage["state"].as_str().unwrap_or("?").to_owned())
})
.collect()
}
#[test]
fn a_return_to_loop_re_runs_the_span_with_the_failure_in_the_prompt() {
let world = World::configured();
let prompt_log = world.root().join("prompts.log");
let script = recording_agent(&world, &prompt_log);
let test_command = flaky_test_command(&world, "flaky-test.sh", 1);
configure(
&world,
&format!(
"- {{ name: build, action: agent, result_check: {{ exec: ['true'] }} }}\n- name: test\n action: {{ exec: [\"sh\", {command}] }}\n result_check: none\n fail_action: {{ return_to: build, attempts: 2 }}\n- {{ name: merge, action: {{ builtin: merge }} }}\n",
command = serde_json::to_string(&test_command.to_string_lossy()).unwrap(),
),
&script,
);
world.commit_all("initial");
world.start_daemon();
let ticket = post(&world, "return-to-loop.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow("the looping run merges", || {
status(&world)["tickets"]["merged"] == 1
});
let prompts = prompts(&prompt_log);
assert_eq!(prompts.len(), 2, "{prompts:?}");
assert!(
!prompts[0].contains("previous attempt failed"),
"{}",
prompts[0]
);
let rerun = &prompts[1];
assert!(rerun.contains("--- previous attempt failed ---"), "{rerun}");
assert!(
rerun.contains("Stage `test` (attempt 1) failed: exit 3"),
"{rerun}"
);
assert!(
rerun.contains("assertion failed: the widget is not blue"),
"{rerun}"
);
assert!(rerun.contains("--- end previous attempt ---"), "{rerun}");
assert!(rerun.trim_end().ends_with(FIRST_FAILURE_BLOCK), "{rerun}");
let bootstrap = rerun
.find("sloop brief")
.expect("the bootstrap prompt is present");
assert!(
bootstrap < rerun.find("previous attempt failed").expect("the block"),
"{rerun}"
);
assert_eq!(
shown_stages(&world, &world.run_alias(1)),
[
("build".to_owned(), "passed".to_owned()),
("build#2".to_owned(), "passed".to_owned()),
("test".to_owned(), "failed".to_owned()),
("test#2".to_owned(), "passed".to_owned()),
("merge".to_owned(), "passed".to_owned()),
]
);
}
#[test]
fn an_exhausted_return_budget_fails_with_every_attempt_visible() {
let world = World::configured();
let prompt_log = world.root().join("prompts.log");
let script = barren_agent(&world, &prompt_log);
let test_command = flaky_test_command(&world, "always-failing.sh", 99);
configure(
&world,
&format!(
"- {{ name: build, action: agent, result_check: {{ exec: ['true'] }} }}\n- name: test\n action: {{ exec: [\"sh\", {command}] }}\n result_check: none\n fail_action: {{ return_to: build, attempts: 1 }}\n- {{ name: merge, action: {{ builtin: merge }} }}\n",
command = serde_json::to_string(&test_command.to_string_lossy()).unwrap(),
),
&script,
);
world.commit_all("initial");
world.start_daemon();
let ticket = post(&world, "return-to-exhausted.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow("the exhausted run fails", || {
status(&world)["tickets"]["failed"] == 1
});
assert_eq!(status(&world)["tickets"]["merged"], 0);
assert_eq!(prompts(&prompt_log).len(), 2);
assert_eq!(
shown_stages(&world, &world.run_alias(1)),
[
("build".to_owned(), "passed".to_owned()),
("build#2".to_owned(), "passed".to_owned()),
("test".to_owned(), "failed".to_owned()),
("test#2".to_owned(), "failed".to_owned()),
("merge".to_owned(), "pending".to_owned()),
]
);
let reason = world.show_snapshot(&world.run_alias(1))["reason"]
.as_str()
.expect("a derived reason")
.to_owned();
assert!(reason.contains("stage `test` failed"), "{reason}");
assert!(reason.contains("on attempt 2"), "{reason}");
assert!(reason.contains("return_to budget spent"), "{reason}");
assert_eq!(
world.show_snapshot(&world.run_alias(1))["halt"],
"return_budget_exhausted"
);
}
#[test]
fn an_advisory_failure_is_visible_but_does_not_block_the_merge() {
let world = World::configured();
let prompt_log = world.root().join("prompts.log");
let script = recording_agent(&world, &prompt_log);
configure(
&world,
"- { name: build, action: agent, result_check: { exec: ['true'] } }\n- name: lint\n action: { exec: [\"false\"] }\n result_check: none\n fail_action: continue\n- { name: merge, action: { builtin: merge } }\n",
&script,
);
world.commit_all("initial");
world.start_daemon();
let ticket = post(&world, "advisory-lint.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow("the run merges past its advisory lint", || {
status(&world)["tickets"]["merged"] == 1
});
assert_eq!(
shown_stages(&world, &world.run_alias(1)),
[
("build".to_owned(), "passed".to_owned()),
("lint".to_owned(), "failed".to_owned()),
("merge".to_owned(), "passed".to_owned()),
]
);
assert_eq!(prompts(&prompt_log).len(), 1);
let stages = world.show_snapshot(&world.run_alias(1))["stages"].clone();
let advisory = |name: &str| {
stages
.as_array()
.expect("stages")
.iter()
.find(|stage| stage["stage"] == name)
.unwrap_or_else(|| panic!("no stage `{name}`"))["advisory"]
.clone()
};
assert_eq!(advisory("lint"), true);
assert_eq!(advisory("build"), false);
assert_eq!(advisory("merge"), false);
let text = String::from_utf8_lossy(&world.sloop_plain(&["show", &ticket]).stdout).into_owned();
assert!(
text.contains("build:ok lint:warn merge:ok"),
"advisory strip missing: {text}"
);
}
#[test]
fn the_derived_reason_names_the_halting_failure_not_the_advisory_one() {
let world = World::configured();
let prompt_log = world.root().join("prompts.log");
let script = recording_agent(&world, &prompt_log);
configure(
&world,
"- { name: build, action: agent, result_check: { exec: ['true'] } }\n- name: lint\n action: { exec: [\"false\"] }\n result_check: none\n fail_action: continue\n- name: test\n action: { exec: [\"false\"] }\n result_check: none\n fail_action: fail\n- { name: merge, action: { builtin: merge } }\n",
&script,
);
world.commit_all("initial");
world.start_daemon();
let ticket = post(&world, "advisory-then-halt.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow("the run stops for review", || {
status(&world)["tickets"]["needs_review"] == 1
});
assert_eq!(
shown_stages(&world, &world.run_alias(1)),
[
("build".to_owned(), "passed".to_owned()),
("lint".to_owned(), "failed".to_owned()),
("test".to_owned(), "failed".to_owned()),
("merge".to_owned(), "pending".to_owned()),
]
);
let run = world.show_snapshot(&world.run_alias(1));
let reason = run["reason"].as_str().expect("a derived reason").to_owned();
assert!(reason.contains("stage `test` failed"), "{reason}");
assert!(!reason.contains("lint"), "{reason}");
assert_eq!(run["halt"], "fail_action");
}
const FIRST_FAILURE_BLOCK: &str = concat!(
"--- previous attempt failed ---\n",
"Stage `test` (attempt 1) failed: exit 3\n",
"You are running again to address that failure.\n",
"\n",
"The last 100 lines of that stage's output:\n",
"assertion failed: the widget is not blue\n",
"run 1 of the test command\n",
"--- end previous attempt ---",
);
#[test]
fn a_restart_mid_loop_resumes_the_same_attempt_with_the_same_prompt() {
let world = World::configured();
let prompt_log = world.root().join("prompts.log");
let script = recording_agent(&world, &prompt_log);
let test_command = flaky_test_command(&world, "flaky-test.sh", 1);
configure(
&world,
&format!(
"- {{ name: build, action: agent, result_check: {{ exec: ['true'] }} }}\n- name: test\n action: {{ exec: [\"sh\", {command}] }}\n result_check: none\n fail_action: {{ return_to: build, attempts: 2 }}\n- {{ name: merge, action: {{ builtin: merge }} }}\n",
command = serde_json::to_string(&test_command.to_string_lossy()).unwrap(),
),
&script,
);
world.commit_all("initial");
world.arm_test_hook("after-stage-test");
let first = World::json_stdout(&world.sloop(&["daemon"]))["data"]["pid"]
.as_u64()
.unwrap() as u32;
let ticket = post(&world, "restart-mid-loop.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow(
"the walk records the failure that triggers the jump",
|| world.test_hook_reached("after-stage-test"),
);
world.kill_daemon(first);
assert_eq!(prompts(&prompt_log).len(), 1);
world.release_test_hook("after-stage-test");
world.start_daemon();
wait_until_slow("the resumed run merges", || {
status(&world)["tickets"]["merged"] == 1
});
let prompts = prompts(&prompt_log);
assert_eq!(prompts.len(), 2, "{prompts:?}");
assert!(
prompts[1].trim_end().ends_with(FIRST_FAILURE_BLOCK),
"{}",
prompts[1]
);
assert_eq!(
shown_stages(&world, &world.run_alias(1)),
[
("build".to_owned(), "passed".to_owned()),
("build#2".to_owned(), "passed".to_owned()),
("test".to_owned(), "failed".to_owned()),
("test#2".to_owned(), "passed".to_owned()),
("merge".to_owned(), "passed".to_owned()),
]
);
}
#[test]
fn a_re_entered_reported_stage_is_judged_again() {
let world = World::configured();
let counter = world.root().join("review.count");
fs::write(&counter, b"").unwrap();
let review = write_script(
&world,
"review.sh",
&format!(
"printf x >> {counter}\nruns=$(wc -c < {counter} | tr -d ' ')\nif [ \"$runs\" -le 1 ]; then\n {sloop} --json verdict fail --reason 'needs another pass' >/dev/null\nelse\n {sloop} --json verdict pass --reason 'looks right now' >/dev/null\nfi\nexit 0\n",
counter = shell_quote(&counter.to_string_lossy()),
sloop = shell_quote(env!("CARGO_BIN_EXE_sloop")),
),
);
let prompt_log = world.root().join("prompts.log");
let script = recording_agent(&world, &prompt_log);
configure(
&world,
&format!(
"- {{ name: build, action: agent, result_check: {{ exec: ['true'] }} }}\n- name: review\n action: {{ exec: [\"sh\", {command}] }}\n result_check: reported\n fail_action: {{ return_to: build, attempts: 2 }}\n- {{ name: merge, action: {{ builtin: merge }} }}\n",
command = serde_json::to_string(&review.to_string_lossy()).unwrap(),
),
&script,
);
world.commit_all("initial");
world.start_daemon();
let ticket = post(&world, "reported-loop.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow("the re-reviewed run merges", || {
status(&world)["tickets"]["merged"] == 1
});
assert_eq!(
shown_stages(&world, &world.run_alias(1)),
[
("build".to_owned(), "passed".to_owned()),
("build#2".to_owned(), "passed".to_owned()),
("review".to_owned(), "failed".to_owned()),
("review#2".to_owned(), "passed".to_owned()),
("merge".to_owned(), "passed".to_owned()),
]
);
let rerun = &prompts(&prompt_log)[1];
assert!(
rerun.contains("Stage `review` (attempt 1) failed: needs another pass"),
"{rerun}"
);
}
#[test]
fn a_re_entered_exec_stage_is_handed_no_context() {
let world = World::configured();
let prompt_log = world.root().join("prompts.log");
let script = recording_agent(&world, &prompt_log);
let argv_log = world.root().join("probe-argv.log");
fs::write(&argv_log, b"").unwrap();
let probe = write_script(
&world,
"probe.sh",
&format!(
"printf '%s\\n' \"$*\" >> {log}\nenv >> {log}\nexit 0\n",
log = shell_quote(&argv_log.to_string_lossy()),
),
);
let test_command = flaky_test_command(&world, "flaky-test.sh", 1);
configure(
&world,
&format!(
"- {{ name: build, action: agent, result_check: {{ exec: ['true'] }} }}\n- name: probe\n action: {{ exec: [\"sh\", {probe}, \"fixed-argument\"] }}\n result_check: none\n- name: test\n action: {{ exec: [\"sh\", {command}] }}\n result_check: none\n fail_action: {{ return_to: probe, attempts: 2 }}\n- {{ name: merge, action: {{ builtin: merge }} }}\n",
probe = serde_json::to_string(&probe.to_string_lossy()).unwrap(),
command = serde_json::to_string(&test_command.to_string_lossy()).unwrap(),
),
&script,
);
world.commit_all("initial");
world.start_daemon();
let ticket = post(&world, "exec-reentry.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow("the run merges after re-running the probe", || {
status(&world)["tickets"]["merged"] == 1
});
let recorded = fs::read_to_string(&argv_log).unwrap();
assert_eq!(recorded.matches("fixed-argument").count(), 2, "{recorded}");
assert!(!recorded.contains("previous attempt failed"), "{recorded}");
assert!(!recorded.contains("the widget is not blue"), "{recorded}");
assert_eq!(prompts(&prompt_log).len(), 1);
}
#[test]
fn a_flow_whose_return_budgets_exceed_the_cap_is_rejected_at_post_time() {
let world = World::configured();
let script = write_script(&world, "fake-agent.sh", "exit 0\n");
let mut flow = String::from("- { name: build, action: agent }\n");
for index in 1..9 {
flow.push_str(&format!(
"- {{ name: s{index}, action: {{ exec: ['true'] }} }}\n"
));
}
flow.push_str(
"- { name: last, action: { exec: ['true'] }, fail_action: { return_to: build, attempts: 3 } }\n",
);
configure(&world, &flow, &script);
world.commit_all("initial");
let output = world.sloop(&["daemon"]);
let message = World::json_stdout_or_stderr(&output).to_string();
assert!(!output.status.success(), "{message}");
assert!(message.contains("at most 32 stages"), "{message}");
assert!(message.contains("imply 40"), "{message}");
}
#[test]
fn the_new_grammar_spellings_are_accepted_by_a_running_daemon() {
let world = World::configured();
let prompt_log = world.root().join("prompts.log");
let script = recording_agent(&world, &prompt_log);
configure(
&world,
"- name: build\n action: agent\n result_check: { builtin: commits }\n fail_action: fail\n- name: lint\n action: { exec: ['false'] }\n result_check: none\n fail_action: continue\n- name: merge\n action: { builtin: merge }\n result_check: none\n",
&script,
);
world.commit_all("initial");
world.start_daemon();
let ticket = post(&world, "new-grammar.md");
assert!(world.sloop(&["run", &ticket]).status.success());
wait_until_slow("the new-grammar flow merges", || {
status(&world)["tickets"]["merged"] == 1
});
wait_until("the run alias resolves", || world.run_alias(1) != "pending");
assert_eq!(
shown_stages(&world, &world.run_alias(1))
.into_iter()
.map(|(_, state)| state)
.collect::<Vec<_>>(),
["passed", "failed", "passed"]
);
}