use std::collections::BTreeSet;
use std::process::{Command, Output};
use std::sync::atomic::{AtomicU64, Ordering};
use serde_json::Value;
use tempfile::{TempDir, tempdir};
static TEST_IDEMPOTENCY_SEQUENCE: AtomicU64 = AtomicU64::new(1);
struct Fixture {
binary: String,
home: TempDir,
runtime: TempDir,
}
impl Fixture {
fn new() -> Self {
Self {
binary: env!("CARGO_BIN_EXE_apoc").to_string(),
home: tempdir().unwrap(),
runtime: tempdir().unwrap(),
}
}
fn run(&self, args: &[&str]) -> Output {
self.run_with_env(args, &[])
}
fn run_with_env(&self, args: &[&str], env: &[(&str, &str)]) -> Output {
let mut argv = args
.iter()
.map(|arg| (*arg).to_string())
.collect::<Vec<_>>();
let mutating = match argv.as_slice() {
[group, command, ..] if group == "execution" => matches!(
command.as_str(),
"start"
| "command"
| "batch"
| "pipeline"
| "input"
| "signal"
| "cancel"
| "remove"
),
[group, action, target, ..] if group == "poll" => {
matches!(action.as_str(), "run" | "start")
&& matches!(target.as_str(), "command" | "apoc")
}
_ => false,
};
let already_keyed = argv
.iter()
.any(|arg| arg == "--idempotency-key" || arg.starts_with("--idempotency-key="));
if mutating && !already_keyed {
let boundary = argv
.iter()
.position(|arg| arg == "--")
.unwrap_or(argv.len());
argv.splice(
boundary..boundary,
[
"--idempotency-key".to_string(),
format!(
"execution-verbosity-auto-{}",
TEST_IDEMPOTENCY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
),
],
);
}
let boundary = argv
.iter()
.position(|arg| arg == "--")
.unwrap_or(argv.len());
argv.splice(
boundary..boundary,
[
"--purpose".to_string(),
"Running the execution verbosity fixture.".to_string(),
],
);
let mut command = Command::new(&self.binary);
command
.args(argv)
.env("APOC_HOME", self.home.path())
.env("APOC_RUNTIME_DIR", self.runtime.path())
.env("XDG_DATA_HOME", self.home.path().join("xdg-data"))
.env_remove("APOC_VERBOSITY")
.env_remove("APOC_VIEW");
for (name, value) in env {
command.env(name, value);
}
command.output().unwrap()
}
fn json(&self, args: &[&str]) -> Value {
let output = self.run(args);
assert!(
output.status.success(),
"command failed:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
serde_json::from_slice(&output.stdout).unwrap()
}
fn verdict(&self, args: &[&str]) -> (Value, Option<i32>) {
let output = self.run(args);
let value = serde_json::from_slice(&output.stdout).unwrap_or_else(|error| {
panic!(
"unparsable response: {error}\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
)
});
(value, output.status.code())
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = self.run(&[
"daemon",
"stop",
"--idempotency-key",
"execution-verbosity-fixture-stop",
"--format",
"json",
]);
}
}
fn keys(value: &Value) -> BTreeSet<String> {
value
.as_object()
.unwrap()
.keys()
.map(ToString::to_string)
.collect()
}
fn set(values: &[&str]) -> BTreeSet<String> {
values.iter().map(ToString::to_string).collect()
}
#[test]
fn every_call_returns_the_same_minimal_verdict() {
let apoc = Fixture::new();
let passed = apoc.run(&[
"execution",
"command",
"/bin/echo",
"--format",
"json",
"--",
"quiet",
]);
let failed = apoc.run(&[
"execution",
"command",
"/bin/sh",
"--format",
"json",
"--",
"-c",
"echo noisy failure output >&2; exit 3",
]);
let passed_value: Value = serde_json::from_slice(&passed.stdout).unwrap();
let failed_value: Value = serde_json::from_slice(&failed.stdout).unwrap();
assert_eq!(
keys(&passed_value),
keys(&failed_value),
"a failure must not return a different shape than a success\n{failed_value:#}"
);
assert_eq!(keys(&passed_value), set(&["id", "outcome"]));
assert_eq!(passed_value["outcome"], "passed");
assert_eq!(failed_value["outcome"], "failed");
assert_eq!(passed.status.code(), Some(0));
assert_eq!(failed.status.code(), Some(3));
for response in [&passed.stdout, &failed.stdout] {
assert!(
response.len() < 80,
"minimal response grew to {} bytes: {}",
response.len(),
String::from_utf8_lossy(response)
);
}
let stdout = String::from_utf8_lossy(&failed.stdout);
assert!(!stdout.contains("noisy failure output"), "{stdout}");
}
#[test]
fn the_verdict_separates_a_failed_assertion_from_a_fault() {
let apoc = Fixture::new();
let (met, code) = apoc.verdict(&[
"execution",
"command",
"/bin/echo",
"--expect-stdout-regex",
"All tests passed",
"--format",
"json",
"--",
"All tests passed",
]);
assert_eq!(met["outcome"], "passed", "{met:#}");
assert_eq!(code, Some(0));
let (unmet, code) = apoc.verdict(&[
"execution",
"command",
"/bin/echo",
"--expect-stdout-regex",
"All tests passed",
"--format",
"json",
"--",
"1 test failed",
]);
assert_eq!(unmet["outcome"], "failed", "{unmet:#}");
assert_eq!(
code,
Some(1),
"an unmet assertion must exit nonzero, or the exit status contradicts the verdict"
);
let (nonzero, code) = apoc.verdict(&[
"execution",
"command",
"/bin/sh",
"--format",
"json",
"--",
"-c",
"exit 4",
]);
assert_eq!(nonzero["outcome"], "failed", "{nonzero:#}");
assert_eq!(code, Some(4));
let (fault, _) = apoc.verdict(&[
"execution",
"command",
"/nonexistent/program",
"--format",
"json",
]);
assert_eq!(
fault["outcome"], "error",
"a launch failure is a fault, not a failed assertion\n{fault:#}"
);
}
#[test]
fn an_unlaunched_batch_child_is_skipped() {
let apoc = Fixture::new();
let parent = apoc.verdict(&[
"execution",
"batch",
"--commands",
r#"[{"id":"first","argv":["/bin/sh","-c","exit 9"]},{"id":"second","argv":["/bin/echo","never"]}]"#,
"--max-parallel",
"1",
"--fail-fast",
"--format",
"json",
])
.0;
let parent_id = parent["id"].as_str().unwrap().to_string();
assert_eq!(
parent["outcome"], "failed",
"a group whose command failed did not fault; it failed\n{parent:#}"
);
let children = apoc.json(&[
"execution",
"list",
"--parent",
&parent_id,
"--format",
"json",
]);
let verdicts = children["executions"]
.as_array()
.unwrap()
.iter()
.map(|child| {
let id = child["id"].as_str().unwrap();
apoc.verdict(&["execution", "get", id, "--format", "json"])
.0["outcome"]
.as_str()
.unwrap()
.to_string()
})
.collect::<BTreeSet<_>>();
assert!(
verdicts.contains("skipped"),
"a declared command that never launched must be skipped, not failed: {verdicts:?}"
);
}
#[test]
fn raising_verbosity_adds_without_replacing() {
let apoc = Fixture::new();
let mut previous: Option<BTreeSet<String>> = None;
for level in ["error", "warn", "info", "debug"] {
let (value, _) = apoc.verdict(&[
"execution",
"command",
"/bin/sh",
"--verbosity",
level,
"--format",
"json",
"--",
"-c",
"echo out; echo err >&2; exit 5",
]);
let current = keys(&value);
assert_eq!(value["outcome"], "failed", "{level}: {value:#}");
if let Some(previous) = &previous {
assert!(
previous.is_subset(¤t),
"{level} dropped keys that a lower level returned: {previous:?} -> {current:?}"
);
}
previous = Some(current);
}
let debug = previous.unwrap();
assert!(
debug.contains("logs"),
"debug must carry log tails: {debug:?}"
);
assert!(debug.contains("inspect"), "{debug:?}");
let (traced, _) = apoc.verdict(&[
"execution",
"command",
"/bin/sh",
"--verbosity",
"trace",
"--format",
"json",
"--",
"-c",
"echo out; echo err >&2; exit 5",
]);
let traced = keys(&traced);
assert!(traced.contains("outcome"), "{traced:?}");
assert!(
traced.contains("execution"),
"trace must answer with the durable record: {traced:?}"
);
assert!(
!debug.is_subset(&traced),
"trace has become additive; make the ladder claim unconditional instead of noting an \
exception here"
);
let (silent, code) = apoc.verdict(&[
"execution",
"command",
"/bin/echo",
"--verbosity",
"off",
"--format",
"json",
"--",
"quiet",
]);
assert_eq!(keys_or_empty(&silent), BTreeSet::new(), "{silent:#}");
assert_eq!(code, Some(0));
}
fn keys_or_empty(value: &Value) -> BTreeSet<String> {
value
.as_object()
.map(|object| object.keys().map(ToString::to_string).collect())
.unwrap_or_default()
}
#[test]
fn progressive_disclosure_reveals_one_line_then_its_neighbours() {
let apoc = Fixture::new();
let (started, _) = apoc.verdict(&[
"execution",
"command",
"/bin/sh",
"--format",
"json",
"--",
"-c",
"i=1; while [ $i -le 400 ]; do echo \"line $i\"; i=$((i+1)); done; \
echo 'FATAL: disk unavailable' >&2; exit 2",
]);
let id = started["id"].as_str().unwrap().to_string();
let found = apoc.json(&[
"execution",
"logs",
&id,
"--grep",
"FATAL",
"--context",
"0",
"--format",
"json",
]);
assert_eq!(found["total_matches"], 1, "{found:#}");
assert_eq!(found["stdout_lines"], 400, "{found:#}");
let hit = &found["lines"].as_array().unwrap()[0];
assert_eq!(hit["stream"], "stderr", "{found:#}");
assert!(
hit["text"].as_str().unwrap().contains("disk unavailable"),
"{found:#}"
);
let expanded = apoc.json(&[
"execution",
"logs",
&id,
"--lines",
"200-202",
"--format",
"json",
]);
let lines = expanded["lines"].as_array().unwrap();
assert_eq!(lines.len(), 3, "{expanded:#}");
assert_eq!(lines[0]["number"], 200, "{expanded:#}");
assert_eq!(lines[2]["text"], "line 202", "{expanded:#}");
assert_eq!(expanded["stdout_lines"], 400, "{expanded:#}");
}
#[test]
fn grep_finds_a_failure_that_only_wrote_to_stdout() {
let apoc = Fixture::new();
let (started, _) = apoc.verdict(&[
"execution",
"command",
"/bin/sh",
"--format",
"json",
"--",
"-c",
"echo 'AssertionError: expected 2 to equal 3'; exit 1",
]);
let id = started["id"].as_str().unwrap().to_string();
let found = apoc.json(&[
"execution",
"logs",
&id,
"--grep",
"AssertionError",
"--format",
"json",
]);
assert_eq!(found["total_matches"], 1, "{found:#}");
let hit = &found["lines"].as_array().unwrap()[0];
assert_eq!(
hit["stream"], "stdout",
"diagnostics on stdout must be findable\n{found:#}"
);
}
#[test]
fn an_addressed_read_refuses_a_log_pending_redaction() {
let apoc = Fixture::new();
let (started, _) = apoc.verdict(&[
"execution",
"command",
"/bin/echo",
"--format",
"json",
"--",
"secret-bearing",
]);
let id = started["id"].as_str().unwrap().to_string();
let directory = apoc.home.path().join("runs").join(&id);
assert!(directory.is_dir(), "missing run directory {directory:?}");
std::fs::write(directory.join("redaction.required"), []).unwrap();
for query in [["--grep", "secret"], ["--lines", "1-2"]] {
let refused = apoc.run(&[
"execution",
"logs",
&id,
query[0],
query[1],
"--format",
"json",
]);
let message = format!(
"{}{}",
String::from_utf8_lossy(&refused.stdout),
String::from_utf8_lossy(&refused.stderr)
);
assert!(
!refused.status.success() && message.contains("redaction"),
"{query:?} read a log whose redaction had not completed: {message}"
);
}
}
#[test]
fn the_deprecated_view_alias_still_resolves() {
let apoc = Fixture::new();
let (aliased, _) = apoc.verdict(&[
"execution",
"command",
"/bin/echo",
"--view",
"full",
"--format",
"json",
"--",
"aliased",
]);
assert!(
aliased.get("execution").is_some() && aliased.get("logs").is_some(),
"--view full must still reach the complete record\n{aliased:#}"
);
let env = apoc.run_with_env(
&[
"execution",
"command",
"/bin/echo",
"--format",
"json",
"--",
"aliased",
],
&[("APOC_VIEW", "outcome")],
);
let value: Value = serde_json::from_slice(&env.stdout).unwrap();
assert_eq!(
keys(&value),
set(&["id", "outcome"]),
"APOC_VIEW=outcome asked for the collapsed verdict and must not widen past it\n{value:#}"
);
let preferred = apoc.run_with_env(
&[
"execution",
"command",
"/bin/echo",
"--format",
"json",
"--",
"preferred",
],
&[("APOC_VERBOSITY", "debug"), ("APOC_VIEW", "outcome")],
);
let preferred: Value = serde_json::from_slice(&preferred.stdout).unwrap();
assert!(
preferred.get("logs").is_some(),
"APOC_VERBOSITY must take precedence over APOC_VIEW\n{preferred:#}"
);
let blank = apoc.run_with_env(
&[
"execution",
"command",
"/bin/echo",
"--format",
"json",
"--",
"blank",
],
&[("APOC_VERBOSITY", ""), ("APOC_VIEW", "outcome")],
);
let blank: Value = serde_json::from_slice(&blank.stdout).unwrap();
assert_eq!(
keys(&blank),
set(&["id", "outcome"]),
"a blank APOC_VERBOSITY must fall back to APOC_VIEW\n{blank:#}"
);
}
#[test]
fn trace_carries_the_verdict_beside_the_complete_record() {
let apoc = Fixture::new();
let (traced, _) = apoc.verdict(&[
"execution",
"command",
"/bin/sh",
"--verbosity",
"trace",
"--format",
"json",
"--",
"-c",
"echo out; exit 6",
]);
assert_eq!(traced["outcome"], "failed", "{traced:#}");
assert_eq!(traced["execution"]["status"], "failed", "{traced:#}");
assert!(traced.get("logs").is_some(), "{traced:#}");
assert!(traced.get("replayed").is_some(), "{traced:#}");
}
#[test]
fn a_timeout_reaches_no_verdict() {
let apoc = Fixture::new();
let (timed_out, _) = apoc.verdict(&[
"execution",
"command",
"/bin/sh",
"--timeout-ms",
"200",
"--verbosity",
"trace",
"--format",
"json",
"--",
"-c",
"sleep 30",
]);
assert_eq!(
timed_out["outcome"], "error",
"a command apoc stopped never answered its assertion\n{timed_out:#}"
);
assert_eq!(
timed_out["execution"]["status"], "canceled",
"{timed_out:#}"
);
assert_eq!(
timed_out["execution"]["result"]["error_code"], "EXECUTION_TIMEOUT",
"{timed_out:#}"
);
let id = timed_out["execution"]["id"].as_str().unwrap();
let executions = apoc.json(&[
"execution",
"list",
"--error-code",
"EXECUTION_TIMEOUT",
"--format",
"json",
]);
assert_eq!(executions["executions"][0]["id"], id, "{executions:#}");
let history = apoc.json(&[
"history",
"list",
"--error-code",
"EXECUTION_TIMEOUT",
"--format",
"json",
]);
assert_eq!(history["invocations"][0]["execution_id"], id, "{history:#}");
}
#[test]
fn a_wait_boundary_returns_pending_without_failing_or_canceling_the_execution() {
let apoc = Fixture::new();
let started = apoc.json(&[
"execution",
"start",
"/bin/sleep",
"--format",
"json",
"--",
"2",
]);
let id = started["id"].as_str().unwrap();
let (pending, code) = apoc.verdict(&[
"execution",
"wait",
id,
"--timeout-ms",
"10",
"--format",
"json",
]);
assert_eq!(pending, serde_json::json!({"id": id, "outcome": "pending"}));
assert_eq!(
code,
Some(0),
"an observation boundary is not a command failure"
);
let (finished, code) = apoc.verdict(&[
"execution",
"wait",
id,
"--timeout-ms",
"5000",
"--format",
"json",
]);
assert_eq!(finished["outcome"], "passed", "{finished:#}");
assert_eq!(code, Some(0));
}
#[test]
fn a_terminal_wait_mirrors_the_verdict_exit_status() {
let apoc = Fixture::new();
let started = apoc.json(&["execution", "start", "/usr/bin/false", "--format", "json"]);
let id = started["id"].as_str().unwrap();
let (failed, code) = apoc.verdict(&[
"execution",
"wait",
id,
"--timeout-ms",
"2000",
"--format",
"json",
]);
assert_eq!(failed["outcome"], "failed", "{failed:#}");
assert_ne!(code, Some(0), "{failed:#}");
}
#[test]
fn every_declared_command_carries_its_own_assertion() {
let apoc = Fixture::new();
let (parent, code) = apoc.verdict(&[
"execution",
"batch",
"--commands",
r#"[{"id":"first","argv":["/bin/echo","ready"],"expect":{"kind":"stdout_regex","pattern":"ready"}},
{"id":"second","argv":["/bin/echo","degraded"],"expect":{"kind":"stdout_regex","pattern":"ready"}}]"#,
"--max-parallel",
"2",
"--format",
"json",
]);
assert_eq!(
parent["outcome"], "failed",
"a child's unmet assertion must reach its parent\n{parent:#}"
);
assert_ne!(code, Some(0), "the exit status must not contradict failed");
let parent_id = parent["id"].as_str().unwrap().to_string();
let children = apoc.json(&[
"execution",
"list",
"--parent",
&parent_id,
"--format",
"json",
]);
let verdicts = children["executions"]
.as_array()
.unwrap()
.iter()
.map(|child| {
let id = child["id"].as_str().unwrap();
apoc.verdict(&["execution", "get", id, "--format", "json"])
.0["outcome"]
.as_str()
.unwrap()
.to_string()
})
.collect::<BTreeSet<_>>();
assert_eq!(
verdicts,
set(&["passed", "failed"]),
"each declared command is judged against its own expectation: {verdicts:?}"
);
}
#[test]
fn command_groups_accept_declared_nonzero_exits() {
for options in [
vec!["batch", "--max-parallel", "1", "--fail-fast"],
vec!["pipeline", "--pipefail", "true"],
vec!["pipeline", "--pipefail", "false"],
] {
let apoc = Fixture::new();
let mut args = vec!["execution"];
args.extend(options);
args.extend([
"--commands",
r#"[{"id":"first","argv":["/usr/bin/false"],"expect":{"kind":"exit_code","codes":[1]}},
{"id":"second","argv":["/usr/bin/false"],"expect":{"kind":"exit_code","codes":[1]}}]"#,
"--idempotency-key", "accepted-group", "--verbosity", "trace", "--format", "json",
]);
let (parent, code) = apoc.verdict(&args);
assert_eq!(parent["outcome"], "passed", "{args:?}: {parent:#}");
assert_eq!(code, Some(0), "{parent:#}");
assert_eq!(
parent["execution"]["result"]["counts"],
serde_json::json!({
"total": 2, "completed": 2, "failed": 0, "canceled": 0, "skipped": 0
})
);
let id = parent["execution"]["id"].as_str().unwrap();
let children = apoc.json(&["execution", "list", "--parent", id, "--format", "json"]);
assert_eq!(children["executions"].as_array().unwrap().len(), 2);
for child in children["executions"].as_array().unwrap() {
let child_id = child["id"].as_str().unwrap();
let (record, code) = apoc.verdict(&[
"execution",
"get",
child_id,
"--verbosity",
"trace",
"--format",
"json",
]);
assert_eq!(record["outcome"], "passed", "{record:#}");
assert_eq!(code, Some(0));
assert_eq!(
record["status"], "failed",
"raw process status is preserved"
);
assert_eq!(record["result"]["exit_code"], 1);
let lease = apoc.json(&[
"lease",
"get",
&format!("execution/{child_id}"),
"--format",
"json",
]);
assert_eq!(lease["released"], true, "{lease:#}");
}
let (replay, code) = apoc.verdict(&args);
assert_eq!(code, Some(0));
assert_eq!(replay["execution"]["id"], id);
assert_eq!(replay["replayed"], true);
assert_eq!(replay["execution"]["result"], parent["execution"]["result"]);
assert_eq!(
apoc.json(&["execution", "list", "--parent", id, "--format", "json"]),
children
);
}
}
#[test]
fn unmet_pipeline_assertions_fail_the_parent_regardless_of_pipefail() {
for (pipefail, failed_stage, passes) in [
(true, 0, false),
(true, 1, false),
(false, 0, false),
(false, 1, false),
] {
let apoc = Fixture::new();
let commands = serde_json::to_string(&(0..2).map(|index| serde_json::json!({
"id": format!("stage-{index}"), "argv": ["/usr/bin/true"],
"expect": {"kind": "exit_code", "codes": [if index == failed_stage {1} else {0}]}
})).collect::<Vec<_>>()).unwrap();
let (parent, code) = apoc.verdict(&[
"execution",
"pipeline",
"--commands",
&commands,
"--pipefail",
if pipefail { "true" } else { "false" },
"--verbosity",
"trace",
"--format",
"json",
]);
assert_eq!(
parent["outcome"],
if passes { "passed" } else { "failed" },
"pipefail={pipefail}, stage={failed_stage}: {parent:#}"
);
assert_eq!(code == Some(0), passes);
assert_eq!(parent["execution"]["result"]["counts"]["failed"], 1);
assert_eq!(parent["execution"]["result"]["counts"]["completed"], 1);
}
}
#[test]
fn poll_conditions_keep_the_full_record_under_the_minimal_default() {
let apoc = Fixture::new();
let (started, _) = apoc.verdict(&[
"execution",
"command",
"/bin/echo",
"--format",
"json",
"--",
"polled",
]);
let id = started["id"].as_str().unwrap().to_string();
let arguments = serde_json::json!({
"id": id,
"purpose": "Reading a durable execution inside a poll condition."
})
.to_string();
let (poll, _) = apoc.verdict(&[
"poll",
"run",
"apoc",
"execution.get",
"--arguments",
&arguments,
"--until-json-pointer",
"/attempts/0/exit_code",
"--until-json-equals",
"0",
"--interval-ms",
"200",
"--max-iterations",
"3",
"--format",
"json",
]);
assert_eq!(poll["outcome"], "passed", "{poll:#}");
}