use std::os::unix::fs::PermissionsExt;
use std::process::{Command, Output};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Barrier};
use std::thread;
use serde_json::Value;
use tempfile::{TempDir, tempdir};
static TEST_IDEMPOTENCY_SEQUENCE: AtomicU64 = AtomicU64::new(1);
struct Fixture {
binary: String,
home: TempDir,
runtime: TempDir,
}
impl Fixture {
fn new() -> Self {
Self {
binary: env!("CARGO_BIN_EXE_apoc").to_string(),
home: tempdir().unwrap(),
runtime: tempdir().unwrap(),
}
}
fn run_from(&self, cwd: &std::path::Path, args: &[&str]) -> Output {
self.run_from_with_env(cwd, args, &[])
}
fn run_from_with_env(
&self,
cwd: &std::path::Path,
args: &[&str],
env: &[(&str, &std::path::Path)],
) -> Output {
let separator = args
.iter()
.position(|arg| *arg == "--")
.unwrap_or(args.len());
let mut command = Command::new(&self.binary);
command
.args(&args[..separator])
.args(["--purpose", "Running the execution contract fixture."])
.args(&args[separator..])
.current_dir(cwd)
.env("APOC_HOME", self.home.path())
.env("APOC_RUNTIME_DIR", self.runtime.path())
.env("APOC_VIEW", "full");
for (name, value) in env {
command.env(name, value);
}
command.output().unwrap()
}
fn json_from(&self, cwd: &std::path::Path, args: &[&str]) -> Value {
let owned_args = keyed_success_args(args);
let args = owned_args.iter().map(String::as_str).collect::<Vec<_>>();
let output = self.run_from(cwd, &args);
assert!(
output.status.success(),
"command failed:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
serde_json::from_slice(&output.stdout).unwrap()
}
}
fn keyed_success_args(args: &[&str]) -> Vec<String> {
let mut args = args
.iter()
.map(|arg| (*arg).to_string())
.collect::<Vec<_>>();
let already_keyed = args
.iter()
.any(|arg| arg == "--idempotency-key" || arg.starts_with("--idempotency-key="));
let mutating = match args.as_slice() {
[group, command, ..] if group == "execution" => matches!(
command.as_str(),
"start" | "command" | "batch" | "pipeline" | "input" | "signal" | "cancel" | "remove"
),
[group, command, ..] if group == "code" => {
matches!(command.as_str(), "run" | "start" | "cancel")
}
[group, action, target, ..] if group == "poll" => {
matches!(action.as_str(), "run" | "start")
&& matches!(target.as_str(), "command" | "apoc")
}
[group, command, ..] if group == "daemon" => {
matches!(command.as_str(), "start" | "serve" | "stop")
}
_ => false,
};
if mutating && !already_keyed {
let key = format!(
"execution-contract-auto-{}",
TEST_IDEMPOTENCY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
);
let separator = args
.iter()
.position(|arg| arg == "--")
.unwrap_or(args.len());
args.splice(separator..separator, ["--idempotency-key".to_string(), key]);
}
args
}
#[test]
fn isolated_execution_uses_the_builtin_gateway_without_provider_registration() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let provider_directory = tempdir().unwrap();
let provider = provider_directory.path().join("apoc-macos-fixture");
std::fs::write(
&provider,
r#"#!/usr/bin/env python3
import json
import sys
for line in sys.stdin:
request = json.loads(line)
if "id" not in request:
continue
method = request.get("method")
if method == "initialize":
result = {"protocolVersion":"2025-06-18","capabilities":{"tools":{}},"serverInfo":{"name":"apoc-macos-fixture","version":"1"}}
elif method == "tools/call":
name = request["params"]["name"]
arguments = request["params"]["arguments"]
if name == "execution_start":
result = {"structuredContent":{"provider_execution_id":arguments["request_id"]}}
elif name == "execution_events":
result = {"structuredContent":{"cursor":"3","events":[
{"type":"started","request_id":arguments["execution_id"]},
{"type":"stdout","data":"isolated fixture output\n"},
{"type":"exit","exit_code":0}
]}}
else:
result = {"isError":True,"content":[{"type":"text","text":"unexpected tool"}]}
else:
result = {}
print(json.dumps({"jsonrpc":"2.0","id":request["id"],"result":result}), flush=True)
"#,
)
.unwrap();
let mut permissions = std::fs::metadata(&provider).unwrap().permissions();
permissions.set_mode(0o755);
std::fs::set_permissions(&provider, permissions).unwrap();
let output = apoc.run_from_with_env(
cwd.path(),
&[
"execution",
"command",
"/bin/echo",
"--isolated",
"--image",
"local",
"--workspace-mode",
"none",
"--idempotency-key",
"isolated-contract-v1",
"--format",
"json",
"--",
"ignored-by-fixture",
],
&[("APOC_MACOS_PROVIDER", provider.as_path())],
);
assert!(
output.status.success(),
"command failed:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let result: Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(result["execution"]["spec"]["backend"]["kind"], "isolated");
assert_eq!(result["logs"]["stdout"], "isolated fixture output\n");
let providers = apoc.json_from(cwd.path(), &["provider", "list", "--format", "json"]);
assert_eq!(providers, serde_json::json!([]));
}
#[test]
fn execution_accepts_exact_argv_after_the_separator() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let result = apoc.json_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/printf",
"--format",
"json",
"--",
"%s|%s|%s\n",
"-u",
"value with spaces",
"--arg",
],
);
assert_eq!(
result["execution"]["spec"]["program"]["argv"],
serde_json::json!([
"/usr/bin/printf",
"%s|%s|%s\n",
"-u",
"value with spaces",
"--arg"
])
);
assert_eq!(result["logs"]["stdout"], "-u|value with spaces|--arg\n");
}
#[test]
fn native_resource_budget_cancels_the_execution_and_preserves_the_measurement() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/bin/zsh",
"--memory-limit-mb",
"1024",
"--cpu-percent",
"10000",
"--cpu-time-ms",
"1",
"--timeout-ms",
"5000",
"--idempotency-key",
"execution-native-resource-budget",
"--verbosity",
"trace",
"--format",
"json",
"--",
"-c",
"while :; do :; done",
],
);
assert_eq!(output.status.code(), Some(128));
let result: Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(
result["execution"]["spec"]["schema_version"], 8,
"fixture result: {result:#}"
);
assert_eq!(result["execution"]["spec"]["budget"]["cpu_time_ms"], 1);
assert_eq!(
result["execution"]["spec"]["budget"]["memory_bytes"],
1024 * 1024 * 1024_u64
);
assert_eq!(
result["execution"]["spec"]["budget"]["cpu_percent"],
10000.0
);
assert_eq!(result["execution"]["status"], "canceled");
assert_eq!(
result["execution"]["result"]["error_code"],
"EXECUTION_CPU_TIME_BUDGET"
);
assert_eq!(result["execution"]["result"]["limit"], 1);
assert!(result["execution"]["result"]["observed"].as_u64().unwrap() > 1);
}
#[test]
fn execution_forwards_builtin_flags_and_nested_separators() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let expected = [
"--help",
"-h",
"--version",
"--json",
"--format",
"child-format",
"--schema",
"--llms",
"--mcp",
"--token-limit",
"child-limit",
"--",
"path",
];
let result = apoc.json_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/printf",
"--format",
"json",
"--",
"%s\n",
"--help",
"-h",
"--version",
"--json",
"--format",
"child-format",
"--schema",
"--llms",
"--mcp",
"--token-limit",
"child-limit",
"--",
"path",
],
);
assert_eq!(
result["execution"]["spec"]["program"]["argv"],
serde_json::json!(
std::iter::once("/usr/bin/printf")
.chain(std::iter::once("%s\n"))
.chain(expected)
.collect::<Vec<_>>()
)
);
assert_eq!(
result["logs"]["stdout"],
expected
.iter()
.map(|value| format!("{value}\n"))
.collect::<String>()
);
}
#[test]
fn execution_rejects_legacy_arg_before_acceptance() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--arg=unexpected",
"--format",
"json",
],
);
assert!(!output.status.success());
let message = format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
assert!(
message.contains("must follow the `--` separator"),
"{message}"
);
assert!(
message.contains("The CLI has no `--arg` option"),
"{message}"
);
let executions = apoc.json_from(cwd.path(), &["execution", "list", "--format", "json"]);
assert_eq!(executions["executions"], serde_json::json!([]));
}
#[test]
fn execution_rejects_separator_free_child_argv_with_the_exact_recovery() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"git",
"worktree",
"add",
"-b",
"topic",
"/tmp/topic",
"origin/develop",
],
);
assert!(!output.status.success());
let message = format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
assert!(
message.contains("must follow the `--` separator"),
"{message}"
);
assert!(
message.contains("-- worktree add -b BRANCH PATH BASE"),
"{message}"
);
let executions = apoc.json_from(cwd.path(), &["execution", "list", "--format", "json"]);
assert_eq!(executions["executions"], serde_json::json!([]));
}
#[test]
fn execution_rejects_boolean_flag_values_before_the_separator() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/printf",
"--cancel-on-stall",
"true",
"--format",
"json",
"--",
"ok",
],
);
assert!(!output.status.success());
let message = format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
assert!(
message.contains("Boolean apoc flags do not take values"),
"{message}"
);
assert!(
message.contains("without a following `true` or `false`"),
"{message}"
);
let executions = apoc.json_from(cwd.path(), &["execution", "list", "--format", "json"]);
assert_eq!(executions["executions"], serde_json::json!([]));
}
#[test]
fn execution_preserves_unknown_apoc_option_diagnostics() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let output = apoc.run_from(
cwd.path(),
&["execution", "command", "git", "--purpos", "typo"],
);
assert!(!output.status.success());
let message = format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
assert!(message.contains("Unknown flag: --purpos"), "{message}");
assert!(!message.contains("child argv"), "{message}");
let executions = apoc.json_from(cwd.path(), &["execution", "list", "--format", "json"]);
assert_eq!(executions["executions"], serde_json::json!([]));
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = self.run_from(
self.home.path(),
&[
"daemon",
"stop",
"--idempotency-key",
"execution-fixture-stop",
"--format",
"json",
],
);
}
}
#[test]
fn execution_inherits_the_invoking_directory_not_the_daemon_directory() {
let apoc = Fixture::new();
let daemon_cwd = tempdir().unwrap();
let caller_cwd = tempdir().unwrap();
apoc.json_from(
daemon_cwd.path(),
&["execution", "command", "/usr/bin/true", "--format", "json"],
);
let result = apoc.json_from(
caller_cwd.path(),
&["execution", "command", "/bin/pwd", "--format", "json"],
);
let expected = caller_cwd.path().canonicalize().unwrap();
assert_eq!(
result["execution"]["spec"]["workspace"]["cwd"],
expected.to_string_lossy().as_ref()
);
assert_eq!(
result["logs"]["stdout"].as_str().unwrap().trim(),
expected.to_string_lossy().as_ref()
);
assert_eq!(result["replayed"], false);
}
#[test]
fn execution_tree_resolves_live_processes_from_the_execution_identity() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let started = apoc.json_from(
cwd.path(),
&[
"execution",
"start",
"/bin/sh",
"--format",
"json",
"--",
"-c",
"sleep 15 & wait",
],
);
let started_id = started["execution"]["id"].as_str().unwrap();
let listed = apoc.json_from(
cwd.path(),
&[
"execution",
"list",
"--status",
"running",
"--format",
"json",
],
);
let id = listed["executions"]
.as_array()
.unwrap()
.iter()
.find(|execution| execution["id"] == started_id)
.and_then(|execution| execution["id"].as_str())
.unwrap();
let tree = apoc.json_from(cwd.path(), &["execution", "tree", id, "--format", "json"]);
assert_eq!(tree["execution_id"], id);
assert_eq!(tree["state"], "active");
assert_eq!(tree["roots"].as_array().unwrap().len(), 1, "{tree:#}");
assert_eq!(tree["roots"][0]["execution_id"], id);
assert_eq!(tree["roots"][0]["live"], true);
assert_eq!(
tree["processes"][0]["identity"], tree["roots"][0]["identity"],
"{tree:#}"
);
assert!(
tree["processes"]
.as_array()
.unwrap()
.iter()
.any(|process| process["identity"]["name"] == "sleep"),
"{tree:#}"
);
let code = format!(
"return await apoc.execution_tree({{id:'{id}',purpose:'Inspecting the managed process tree.'}});"
);
let code_tree = apoc.json_from(
cwd.path(),
&[
"code",
"run",
code.as_str(),
"--idempotency-key",
"execution-tree-code-run",
"--format",
"json",
],
);
assert_eq!(code_tree["status"], "completed", "{code_tree:#}");
assert_eq!(code_tree["result"]["execution_id"], id);
assert_eq!(code_tree["result"]["state"], "active");
apoc.json_from(
cwd.path(),
&[
"execution",
"cancel",
id,
"--idempotency-key",
"execution-contract/tree-cancel",
"--timeout-ms",
"1000",
"--format",
"json",
],
);
let manual_tree_calls = apoc.json_from(
cwd.path(),
&[
"history",
"query",
"SELECT id, purpose FROM history WHERE command_path = 'process tree'",
"--format",
"json",
],
);
assert_eq!(manual_tree_calls["rows"], serde_json::json!([]));
}
#[test]
fn execution_tree_returns_an_explicit_terminal_empty_state() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let completed = apoc.json_from(
cwd.path(),
&["execution", "command", "/usr/bin/true", "--format", "json"],
);
let id = completed["execution"]["id"].as_str().unwrap();
let tree = apoc.json_from(cwd.path(), &["execution", "tree", id, "--format", "json"]);
assert_eq!(tree["execution_id"], id);
assert_eq!(tree["status"], "completed");
assert_eq!(tree["state"], "terminal");
assert_eq!(tree["roots"].as_array().unwrap().len(), 1, "{tree:#}");
assert_eq!(tree["roots"][0]["live"], false);
assert_eq!(tree["processes"], serde_json::json!([]));
}
#[test]
fn execution_tree_returns_terminal_for_an_accepted_launch_failure() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let failed = apoc.json_from(
cwd.path(),
&[
"execution",
"start",
"/apoc-fixture/does-not-exist",
"--format",
"json",
],
);
let id = failed["execution"]["id"].as_str().unwrap();
assert_eq!(failed["execution"]["status"], "failed", "{failed:#}");
let tree = apoc.json_from(cwd.path(), &["execution", "tree", id, "--format", "json"]);
assert_eq!(tree["status"], "failed");
assert_eq!(tree["state"], "terminal");
assert_eq!(tree["roots"], serde_json::json!([]));
assert_eq!(tree["processes"], serde_json::json!([]));
}
#[test]
fn relative_cwd_is_resolved_from_the_invoking_directory() {
let apoc = Fixture::new();
let caller_cwd = tempdir().unwrap();
let child = caller_cwd.path().join("child");
std::fs::create_dir(&child).unwrap();
let result = apoc.json_from(
caller_cwd.path(),
&[
"execution",
"command",
"/bin/pwd",
"--cwd",
"child",
"--format",
"json",
],
);
let expected = child.canonicalize().unwrap();
assert_eq!(
result["execution"]["spec"]["workspace"]["cwd"],
expected.to_string_lossy().as_ref()
);
assert_eq!(
result["logs"]["stdout"].as_str().unwrap().trim(),
expected.to_string_lossy().as_ref()
);
}
#[test]
fn synchronous_execution_returns_bounded_logs_and_the_managed_exit_code() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/python3",
"--idempotency-key",
"execution-sync-exit-seven",
"--format",
"json",
"--",
"-c",
"import sys; print('outcome'); print('diagnostic', file=sys.stderr); sys.exit(7)",
],
);
assert_eq!(output.status.code(), Some(7));
let result: Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(result["execution"]["result"]["exit_code"], 7);
assert_eq!(result["logs"]["stdout"], "outcome\n");
assert_eq!(result["logs"]["stderr"], "diagnostic\n");
assert_eq!(result["logs"]["stdout_truncated"], false);
assert_eq!(result["logs"]["stderr_truncated"], false);
}
#[test]
fn synchronous_execution_marks_each_truncated_stream() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let result = apoc.json_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/python3",
"--format",
"json",
"--",
"-c",
"import sys; print('o' * 40000); print('e' * 40000, file=sys.stderr)",
],
);
assert!(result["logs"]["stdout"].as_str().unwrap().len() <= 32 * 1024);
assert!(result["logs"]["stderr"].as_str().unwrap().len() <= 32 * 1024);
assert_eq!(result["logs"]["stdout_truncated"], true);
assert_eq!(result["logs"]["stderr_truncated"], true);
}
#[test]
fn invalid_working_directory_fails_before_accepting_an_execution() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--cwd",
"missing",
"--idempotency-key",
"execution-invalid-workspace",
"--format",
"json",
],
);
assert!(!output.status.success());
assert!(
String::from_utf8_lossy(&output.stdout).contains("INVALID_WORKSPACE")
|| String::from_utf8_lossy(&output.stderr).contains("INVALID_WORKSPACE")
);
}
#[test]
fn execution_idempotency_replays_one_side_effect_and_rejects_conflicts() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let side_effect = cwd.path().join("effect");
let script = format!("printf x >> '{}'", side_effect.display());
let args = [
"execution",
"command",
"/bin/zsh",
"--idempotency-key",
"execution-contract/replay",
"--format",
"json",
"--",
"-c",
script.as_str(),
];
let first = apoc.json_from(cwd.path(), &args);
let second = apoc.json_from(cwd.path(), &args);
assert_eq!(first["execution"]["id"], second["execution"]["id"]);
assert_eq!(first["replayed"], false);
assert_eq!(second["replayed"], true);
assert_eq!(std::fs::read_to_string(&side_effect).unwrap(), "x");
let conflict = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--idempotency-key",
"execution-contract/replay",
"--format",
"json",
],
);
assert!(!conflict.status.success());
let output = format!(
"{}{}",
String::from_utf8_lossy(&conflict.stdout),
String::from_utf8_lossy(&conflict.stderr)
);
assert!(
output.contains("EXECUTION_IDEMPOTENCY_CONFLICT"),
"{output}"
);
}
#[test]
fn command_group_idempotency_replays_the_original_parent() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let side_effect = cwd.path().join("batch-effect");
let file = cwd.path().join("batch.json");
std::fs::write(
&file,
serde_json::to_vec(&serde_json::json!({
"commands": [{
"id": "effect",
"argv": [
"/bin/zsh",
"-c",
format!("printf x >> '{}'", side_effect.display())
]
}]
}))
.unwrap(),
)
.unwrap();
let args = [
"execution",
"batch",
"--file",
file.to_str().unwrap(),
"--idempotency-key",
"execution-contract/batch",
"--format",
"json",
];
let first = apoc.json_from(cwd.path(), &args);
let second = apoc.json_from(cwd.path(), &args);
assert_eq!(first["execution"]["id"], second["execution"]["id"]);
assert_eq!(first["replayed"], false);
assert_eq!(second["replayed"], true);
assert_eq!(std::fs::read_to_string(&side_effect).unwrap(), "x");
}
#[test]
fn execution_batch_accepts_a_cli_json_command_array() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let commands = serde_json::json!([
{
"id": "first",
"argv": ["/usr/bin/printf", "first"]
},
{
"id": "second",
"argv": ["/usr/bin/printf", "second"]
}
])
.to_string();
let result = apoc.json_from(
cwd.path(),
&[
"execution",
"batch",
"--commands",
commands.as_str(),
"--max-parallel",
"2",
"--format",
"json",
],
);
assert_eq!(result["execution"]["status"], "completed", "{result:#}");
assert_eq!(
result["execution"]["spec"]["program"]["commands"]
.as_array()
.unwrap()
.len(),
2,
"{result:#}"
);
}
#[test]
fn command_groups_accept_repeated_cli_json_objects() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let first = r#"{"id":"first","argv":["/usr/bin/printf","first"]}"#;
let second = r#"{"id":"second","argv":["/usr/bin/tr","[:lower:]","[:upper:]"]}"#;
let result = apoc.json_from(
cwd.path(),
&[
"execution",
"pipeline",
"--commands",
first,
"--commands",
second,
"--format",
"json",
],
);
assert_eq!(result["execution"]["status"], "completed", "{result:#}");
assert_eq!(result["logs"]["stdout"], "FIRST");
assert_eq!(
result["execution"]["spec"]["program"]["commands"]
.as_array()
.unwrap()
.len(),
2,
"{result:#}"
);
}
#[test]
fn code_mode_batch_keeps_native_structured_commands() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let code = "return await apoc.execution_batch({commands:[{id:'first',argv:['/usr/bin/printf','first']},{id:'second',argv:['/usr/bin/printf','second']}],max_parallel:2,purpose:'Running a structured command batch.',idempotency_key:'execution-batch-code-nested'});";
let result = apoc.json_from(
cwd.path(),
&[
"code",
"run",
code,
"--idempotency-key",
"execution-batch-code-run",
"--format",
"json",
],
);
assert_eq!(result["status"], "completed", "{result:#}");
assert_eq!(
result["result"]["execution"]["spec"]["program"]["commands"]
.as_array()
.unwrap()
.len(),
2,
"{result:#}"
);
}
#[test]
fn code_mode_retries_replay_nested_execution_idempotency() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let side_effect = cwd.path().join("code-retry-effect");
let code = format!(
"return await apoc.execution_command({{executable:'/bin/zsh',purpose:'Checking nested execution replay.',arg:['-c',\"printf x >> '{}'\"],idempotency_key:'execution-contract/code-retry'}});",
side_effect.display()
);
let first = apoc.json_from(
cwd.path(),
&[
"code",
"run",
code.as_str(),
"--idempotency-key",
"execution-code-retry-outer-first",
"--format",
"json",
],
);
let second = apoc.json_from(
cwd.path(),
&[
"code",
"run",
code.as_str(),
"--idempotency-key",
"execution-code-retry-outer-second",
"--format",
"json",
],
);
assert_eq!(first["status"], "completed", "{first:#}");
assert_eq!(second["status"], "completed", "{second:#}");
assert_eq!(first["result"]["replayed"], false);
assert_eq!(second["result"]["replayed"], true);
assert_eq!(
first["result"]["execution"]["id"],
second["result"]["execution"]["id"]
);
assert_eq!(std::fs::read_to_string(&side_effect).unwrap(), "x");
}
#[test]
fn concurrent_idempotent_launches_create_one_execution() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let side_effect = cwd.path().join("concurrent-effect");
let script = format!("printf x >> '{}'; sleep 0.1", side_effect.display());
let workers = 8;
let barrier = Arc::new(Barrier::new(workers));
let outputs = (0..workers)
.map(|_| {
let binary = apoc.binary.clone();
let home = apoc.home.path().to_path_buf();
let runtime = apoc.runtime.path().to_path_buf();
let cwd = cwd.path().to_path_buf();
let script = script.clone();
let barrier = Arc::clone(&barrier);
thread::spawn(move || {
barrier.wait();
Command::new(binary)
.args([
"execution",
"command",
"/bin/zsh",
"--idempotency-key",
"execution-contract/concurrent",
"--purpose",
"Checking concurrent idempotent execution launch.",
"--format",
"json",
"--",
"-c",
&script,
])
.current_dir(cwd)
.env("APOC_HOME", home)
.env("APOC_RUNTIME_DIR", runtime)
.env("APOC_VIEW", "full")
.output()
.unwrap()
})
})
.collect::<Vec<_>>()
.into_iter()
.map(|worker| worker.join().unwrap())
.collect::<Vec<_>>();
assert!(
outputs.iter().all(|output| output.status.success()),
"{}",
outputs
.iter()
.map(|output| format!(
"status={:?} stdout={} stderr={}",
output.status.code(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
))
.collect::<Vec<_>>()
.join("\n")
);
let results = outputs
.iter()
.map(|output| serde_json::from_slice::<Value>(&output.stdout).unwrap())
.collect::<Vec<_>>();
assert!(
results
.iter()
.all(|result| result["execution"]["id"] == results[0]["execution"]["id"])
);
assert_eq!(
results
.iter()
.filter(|result| result["replayed"] == false)
.count(),
1
);
assert_eq!(std::fs::read_to_string(&side_effect).unwrap(), "x");
}
#[test]
fn concurrent_unique_poll_starts_complete_without_database_lock() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
apoc.json_from(
cwd.path(),
&["execution", "command", "/usr/bin/true", "--format", "json"],
);
let workers = 20;
let barrier = Arc::new(Barrier::new(workers));
let outputs = (0..workers)
.map(|worker| {
let binary = apoc.binary.clone();
let home = apoc.home.path().to_path_buf();
let runtime = apoc.runtime.path().to_path_buf();
let cwd = cwd.path().to_path_buf();
let barrier = Arc::clone(&barrier);
thread::spawn(move || {
let idempotency_key = format!("execution-contract/concurrent-poll-{worker}");
barrier.wait();
Command::new(binary)
.args([
"poll",
"start",
"command",
"/usr/bin/true",
"--until-exit-code",
"0",
"--max-iterations",
"1",
"--idempotency-key",
&idempotency_key,
"--purpose",
"Checking concurrent unique poll launches.",
"--format",
"json",
"--",
])
.current_dir(cwd)
.env("APOC_HOME", home)
.env("APOC_RUNTIME_DIR", runtime)
.env("APOC_VIEW", "full")
.output()
.unwrap()
})
})
.collect::<Vec<_>>()
.into_iter()
.map(|worker| worker.join().unwrap())
.collect::<Vec<_>>();
for output in &outputs {
assert!(
output.status.success(),
"poll start failed:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
for output in outputs {
let started: Value = serde_json::from_slice(&output.stdout).unwrap();
let id = started["execution"]["id"].as_str().unwrap();
let terminal = apoc.json_from(
cwd.path(),
&[
"execution",
"wait",
id,
"--timeout-ms",
"5000",
"--format",
"json",
],
);
assert_eq!(terminal["status"], "completed", "{terminal:#}");
assert_eq!(terminal["result"]["matched"], true, "{terminal:#}");
}
}
#[test]
fn execution_list_is_compact_bounded_and_cursor_paginated() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
for name in ["first", "second", "third"] {
apoc.json_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--name",
name,
"--format",
"json",
],
);
}
let first = apoc.json_from(
cwd.path(),
&["execution", "list", "--limit", "2", "--format", "json"],
);
let executions = first["executions"].as_array().unwrap();
assert_eq!(executions.len(), 2);
assert!(
executions
.iter()
.all(|execution| execution.get("spec").is_none())
);
assert!(
executions
.iter()
.all(|execution| execution.get("passes").is_none())
);
assert!(
executions
.iter()
.all(|execution| execution.get("program").is_some())
);
let cursor = first["next_cursor"].as_str().unwrap();
let second = apoc.json_from(
cwd.path(),
&[
"execution",
"list",
"--limit",
"2",
"--cursor",
cursor,
"--status",
"completed",
"--format",
"json",
],
);
assert_eq!(second["executions"].as_array().unwrap().len(), 1);
assert!(second["next_cursor"].is_null());
assert!(serde_json::to_vec(&first).unwrap().len() <= 128 * 1024);
}
#[test]
fn code_mode_inherits_and_preserves_the_invoking_directory() {
let apoc = Fixture::new();
let daemon_cwd = tempdir().unwrap();
let caller_cwd = tempdir().unwrap();
apoc.json_from(
daemon_cwd.path(),
&["execution", "command", "/usr/bin/true", "--format", "json"],
);
let state = apoc.json_from(
caller_cwd.path(),
&[
"code",
"run",
"return await apoc.execution_command({ executable: '/bin/pwd', purpose: 'Checking the inherited directory.', idempotency_key: 'execution-code-cwd-nested' });",
"--idempotency-key",
"execution-code-cwd-run",
"--format",
"json",
],
);
let expected = caller_cwd.path().canonicalize().unwrap();
assert_eq!(
state["result"]["execution"]["spec"]["workspace"]["cwd"],
expected.to_string_lossy().as_ref()
);
assert_eq!(
state["result"]["logs"]["stdout"].as_str().unwrap().trim(),
expected.to_string_lossy().as_ref()
);
let execution = apoc.json_from(
daemon_cwd.path(),
&[
"execution",
"get",
state["id"].as_str().unwrap(),
"--format",
"json",
],
);
assert_eq!(
execution["spec"]["workspace"]["cwd"],
expected.to_string_lossy().as_ref()
);
}
#[test]
fn daemon_reports_its_runtime_contract_version() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let status = apoc.json_from(cwd.path(), &["daemon", "start", "--format", "json"]);
assert_eq!(status["version"], env!("CARGO_PKG_VERSION"));
assert_eq!(status["execution_schema_version"], 8);
}
#[test]
fn poll_command_matches_and_progressively_exposes_iteration_evidence() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let result = apoc.json_from(
cwd.path(),
&[
"poll",
"run",
"command",
"/usr/bin/printf",
"--interval-ms",
"1",
"--until-stdout-regex",
"ready",
"--max-iterations",
"2",
"--idempotency-key",
"execution-bound-poll-failure",
"--format",
"json",
"--",
"ready",
],
);
assert_eq!(result["execution"]["status"], "completed");
assert_eq!(result["execution"]["result"]["matched"], true);
assert_eq!(result["execution"]["result"]["iterations"], 1);
let poll_id = result["execution"]["id"].as_str().unwrap();
let page = apoc.json_from(
cwd.path(),
&[
"poll",
"iterations",
poll_id,
"--limit",
"20",
"--format",
"json",
],
);
assert_eq!(page["iterations"].as_array().unwrap().len(), 1);
assert_eq!(page["iterations"][0]["matched"], true);
let iteration_id = page["iterations"][0]["id"].as_str().unwrap();
let detail = apoc.json_from(
cwd.path(),
&[
"poll",
"iteration",
poll_id,
iteration_id,
"--format",
"json",
],
);
assert_eq!(detail["observation"]["stdout"], "ready");
assert!(detail["execution_id"].as_str().is_some());
}
#[test]
fn bound_only_poll_completes_but_conditional_exhaustion_fails() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let samples = apoc.json_from(
cwd.path(),
&[
"poll",
"run",
"command",
"/usr/bin/true",
"--interval-mode",
"backoff",
"--initial-interval-ms",
"1",
"--max-interval-ms",
"2",
"--max-iterations",
"2",
"--format",
"json",
],
);
assert_eq!(samples["execution"]["status"], "completed");
assert_eq!(samples["execution"]["result"]["matched"], false);
assert_eq!(samples["execution"]["result"]["iterations"], 2);
assert_eq!(
samples["execution"]["spec"]["program"]["schedule"]["kind"],
"backoff"
);
let failed = apoc.run_from(
cwd.path(),
&[
"poll",
"run",
"command",
"/usr/bin/printf",
"--interval-mode",
"fixed-rate",
"--interval-ms",
"1",
"--until-stdout-regex",
"ready",
"--max-iterations",
"2",
"--idempotency-key",
"execution-conditional-poll-failure",
"--format",
"json",
"--",
"pending",
],
);
assert_eq!(failed.status.code(), Some(1));
let failed: Value = serde_json::from_slice(&failed.stdout).unwrap();
assert_eq!(failed["execution"]["status"], "failed");
assert_eq!(
failed["execution"]["result"]["error_code"],
"POLL_CONDITION_NOT_MET"
);
assert_eq!(failed["execution"]["result"]["iterations"], 2);
}
#[test]
fn poll_invokes_the_typed_apoc_graph_without_a_nested_cli() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let result = apoc.json_from(
cwd.path(),
&[
"poll",
"run",
"apoc",
"version",
"--arguments",
"{}",
"--until-json-pointer",
"/execution_schema_version",
"--until-json-equals",
"8",
"--max-iterations",
"1",
"--format",
"json",
],
);
assert_eq!(result["execution"]["status"], "completed");
assert_eq!(result["execution"]["result"]["matched"], true);
let poll_id = result["execution"]["id"].as_str().unwrap();
let children = apoc.json_from(
cwd.path(),
&["execution", "list", "--limit", "100", "--format", "json"],
);
assert!(
!children["executions"]
.as_array()
.unwrap()
.iter()
.any(|value| {
value["parent_id"].as_str() == Some(poll_id)
&& value["program"]["kind"] == "command"
})
);
}
#[test]
fn execution_cancel_requires_an_idempotency_key_before_mutation() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let started = apoc.json_from(
cwd.path(),
&[
"poll",
"start",
"command",
"/bin/sleep",
"--until-exit-code",
"0",
"--format",
"json",
"--",
"30",
],
);
let id = started["execution"]["id"].as_str().unwrap();
let missing = apoc.run_from(
cwd.path(),
&[
"execution",
"cancel",
id,
"--timeout-ms",
"5000",
"--format",
"json",
],
);
if !missing.status.success() {
apoc.json_from(
cwd.path(),
&[
"execution",
"cancel",
id,
"--idempotency-key",
"execution-contract/cancel-cleanup",
"--timeout-ms",
"5000",
"--format",
"json",
],
);
}
assert!(!missing.status.success());
let message = format!(
"{}{}",
String::from_utf8_lossy(&missing.stdout),
String::from_utf8_lossy(&missing.stderr)
);
assert!(message.contains("idempotency-key"), "{message}");
}
#[test]
fn execution_cancel_terminalizes_a_poll_and_its_active_child() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let started = apoc.json_from(
cwd.path(),
&[
"poll",
"start",
"command",
"/bin/sleep",
"--until-exit-code",
"0",
"--format",
"json",
"--",
"30",
],
);
let id = started["execution"]["id"].as_str().unwrap();
let cancel_args = [
"execution",
"cancel",
id,
"--idempotency-key",
"execution-contract/cancel-poll",
"--timeout-ms",
"5000",
"--format",
"json",
];
let canceled = apoc.json_from(cwd.path(), &cancel_args);
let replayed = apoc.json_from(cwd.path(), &cancel_args);
assert_eq!(canceled["status"], "canceled");
assert_eq!(canceled["result"]["stop_reason"], "canceled");
assert_eq!(replayed, canceled);
let conflict = apoc.run_from(
cwd.path(),
&[
"execution",
"cancel",
id,
"--idempotency-key",
"execution-contract/cancel-poll",
"--timeout-ms",
"4000",
"--format",
"json",
],
);
assert!(!conflict.status.success());
let conflict_message = format!(
"{}{}",
String::from_utf8_lossy(&conflict.stdout),
String::from_utf8_lossy(&conflict.stderr)
);
assert!(
conflict_message.contains("EXECUTION_IDEMPOTENCY_CONFLICT"),
"{conflict_message}"
);
let page = apoc.json_from(
cwd.path(),
&[
"poll",
"iterations",
id,
"--limit",
"20",
"--format",
"json",
],
);
assert!(
page["iterations"]
.as_array()
.unwrap()
.iter()
.all(|iteration| {
matches!(
iteration["status"].as_str(),
Some("observed" | "canceled" | "failed")
)
})
);
}
#[test]
fn releasing_an_execution_lease_terminalizes_a_poll_and_its_active_child() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let started = apoc.json_from(
cwd.path(),
&[
"poll",
"start",
"command",
"/bin/sleep",
"--until-exit-code",
"0",
"--format",
"json",
"--",
"30",
],
);
let id = started["execution"]["id"].as_str().unwrap();
let leases = apoc.json_from(cwd.path(), &["lease", "list", "--format", "json"]);
let lease_id = leases
.as_array()
.unwrap()
.iter()
.find(|lease| lease["owner_id"] == id)
.and_then(|lease| lease["id"].as_str())
.unwrap();
let released = apoc.json_from(
cwd.path(),
&[
"lease",
"release",
lease_id,
"--idempotency-key",
"release-owned-poll-lease",
"--format",
"json",
],
);
assert_eq!(released["released"], true);
let parent = apoc.json_from(
cwd.path(),
&[
"execution",
"get",
id,
"--verbosity",
"trace",
"--format",
"json",
],
);
assert_eq!(parent["status"], "canceled", "parent: {parent:#}");
let children = apoc.json_from(
cwd.path(),
&["execution", "list", "--limit", "100", "--format", "json"],
);
assert!(
children["executions"]
.as_array()
.unwrap()
.iter()
.filter(|child| child["parent_id"].as_str() == Some(id))
.all(|child| matches!(
child["status"].as_str(),
Some("completed" | "failed" | "canceled")
)),
"children: {children:#}"
);
}
#[test]
fn closing_a_session_terminalizes_its_poll_and_active_child() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let session = apoc.json_from(
cwd.path(),
&[
"session",
"open",
"--actor",
"session-close-regression",
"--idempotency-key",
"open-owned-poll-session",
"--format",
"json",
],
);
let session_id = session["id"].as_str().unwrap();
let started = apoc.json_from(
cwd.path(),
&[
"poll",
"start",
"command",
"/bin/sleep",
"--owner",
"session",
"--session",
session_id,
"--until-exit-code",
"0",
"--idempotency-key",
"start-session-owned-poll",
"--format",
"json",
"--",
"30",
],
);
let id = started["execution"]["id"].as_str().unwrap();
let closed = apoc.json_from(
cwd.path(),
&[
"session",
"close",
session_id,
"--idempotency-key",
"close-owned-poll-session",
"--format",
"json",
],
);
assert_eq!(closed["closed"], true);
let parent = apoc.json_from(
cwd.path(),
&[
"execution",
"get",
id,
"--verbosity",
"trace",
"--format",
"json",
],
);
assert_eq!(parent["status"], "canceled", "parent: {parent:#}");
let children = apoc.json_from(
cwd.path(),
&["execution", "list", "--limit", "100", "--format", "json"],
);
assert!(
children["executions"]
.as_array()
.unwrap()
.iter()
.filter(|child| child["parent_id"].as_str() == Some(id))
.all(|child| matches!(
child["status"].as_str(),
Some("completed" | "failed" | "canceled")
)),
"children: {children:#}"
);
}
#[test]
fn session_owner_rejects_inactive_sessions_and_preserves_replay() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let session = apoc.json_from(
cwd.path(),
&[
"session",
"open",
"--actor",
"closed-session-owner-regression",
"--idempotency-key",
"open-closed-owner-session",
"--format",
"json",
],
);
let session_id = session["id"].as_str().unwrap();
let closed = apoc.json_from(
cwd.path(),
&[
"session",
"close",
session_id,
"--idempotency-key",
"close-inactive-owner-session",
"--format",
"json",
],
);
assert_eq!(closed["closed"], true);
let output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--owner",
"session",
"--session",
session_id,
"--idempotency-key",
"reject-closed-session-owner",
"--format",
"json",
],
);
assert!(
!output.status.success(),
"closed session accepted an execution:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let error: Value = serde_json::from_slice(if output.stdout.is_empty() {
&output.stderr
} else {
&output.stdout
})
.unwrap();
assert!(
error["message"]
.as_str()
.is_some_and(|message| message.contains("closed or expired")),
"error: {error:#}"
);
let executions = apoc.json_from(
cwd.path(),
&["execution", "list", "--limit", "100", "--format", "json"],
);
assert!(
executions["executions"]
.as_array()
.unwrap()
.iter()
.all(|execution| execution["session_id"].as_str() != Some(session_id)),
"executions: {executions:#}"
);
let expired = apoc.json_from(
cwd.path(),
&[
"session",
"open",
"--actor",
"expired-session-owner-regression",
"--ttl-ms",
"1",
"--idempotency-key",
"open-expiring-owner-session",
"--format",
"json",
],
);
let expired_id = expired["id"].as_str().unwrap();
thread::sleep(std::time::Duration::from_millis(10));
let expired_output = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--owner",
"session",
"--session",
expired_id,
"--idempotency-key",
"expired-session-owner",
"--format",
"json",
],
);
assert!(
!expired_output.status.success(),
"expired session accepted an execution:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&expired_output.stdout),
String::from_utf8_lossy(&expired_output.stderr)
);
let replay_session = apoc.json_from(
cwd.path(),
&[
"session",
"open",
"--actor",
"closed-session-replay-regression",
"--idempotency-key",
"open-closed-session-replay",
"--format",
"json",
],
);
let replay_session_id = replay_session["id"].as_str().unwrap();
let replay_args = [
"execution",
"command",
"/usr/bin/true",
"--owner",
"session",
"--session",
replay_session_id,
"--idempotency-key",
"closed-session-replay",
"--format",
"json",
];
let original = apoc.json_from(cwd.path(), &replay_args);
let original_id = original["execution"]["id"].as_str().unwrap();
apoc.json_from(
cwd.path(),
&[
"session",
"close",
replay_session_id,
"--idempotency-key",
"close-replay-owner-session",
"--format",
"json",
],
);
let replay = apoc.json_from(cwd.path(), &replay_args);
assert_eq!(replay["execution"]["id"], original_id);
assert_eq!(replay["replayed"], true);
}
#[test]
fn session_close_rejects_work_after_cancellation_begins() {
let apoc = Arc::new(Fixture::new());
let cwd = tempdir().unwrap();
let session = apoc.json_from(
cwd.path(),
&[
"session",
"open",
"--actor",
"session-close-admission-regression",
"--idempotency-key",
"open-session-close-admission",
"--format",
"json",
],
);
let session_id = session["id"].as_str().unwrap().to_string();
apoc.json_from(
cwd.path(),
&[
"execution",
"start",
"/bin/sh",
"--owner",
"session",
"--session",
&session_id,
"--idempotency-key",
"start-session-close-admission-fixture",
"--format",
"json",
"--",
"-c",
"trap 'printf seen > close-entered' TERM; while :; do sleep 1; done",
],
);
let close_apoc = Arc::clone(&apoc);
let close_cwd = cwd.path().to_path_buf();
let close_session_id = session_id.clone();
let close = thread::spawn(move || {
close_apoc.run_from(
&close_cwd,
&[
"session",
"close",
&close_session_id,
"--timeout-ms",
"2000",
"--idempotency-key",
"close-session-admission-fixture",
"--format",
"json",
],
)
});
let marker = cwd.path().join("close-entered");
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while !marker.exists() && !close.is_finished() && std::time::Instant::now() < deadline {
thread::sleep(std::time::Duration::from_millis(10));
}
assert!(
marker.exists(),
"session close never signaled its owned process"
);
let closing = apoc.json_from(
cwd.path(),
&["session", "get", &session_id, "--format", "json"],
);
assert_eq!(closing["closing"], true, "session: {closing:#}");
assert_eq!(closing["closed"], false, "session: {closing:#}");
let heartbeat = apoc.run_from(
cwd.path(),
&[
"session",
"heartbeat",
&session_id,
"--ttl-ms",
"60000",
"--idempotency-key",
"heartbeat-closing-session",
"--format",
"json",
],
);
assert!(
!heartbeat.status.success(),
"closing session accepted a heartbeat:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&heartbeat.stdout),
String::from_utf8_lossy(&heartbeat.stderr)
);
let admission = apoc.run_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--owner",
"session",
"--session",
&session_id,
"--idempotency-key",
"reject-closing-session-owner",
"--format",
"json",
],
);
let closed = close.join().unwrap();
assert!(
closed.status.success(),
"session close failed:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&closed.stdout),
String::from_utf8_lossy(&closed.stderr)
);
assert!(
!admission.status.success(),
"session accepted work after close cancellation began:\nstdout: {}\nstderr: {}\nclose stdout: {}\nclose stderr: {}",
String::from_utf8_lossy(&admission.stdout),
String::from_utf8_lossy(&admission.stderr),
String::from_utf8_lossy(&closed.stdout),
String::from_utf8_lossy(&closed.stderr)
);
}
#[test]
fn canceling_a_parent_terminalizes_an_owned_poll_waiting_between_iterations() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let parent = apoc.json_from(
cwd.path(),
&[
"execution",
"start",
"/bin/sleep",
"--format",
"json",
"--",
"30",
],
);
let parent_id = parent["execution"]["id"].as_str().unwrap();
let poll = apoc.json_from(
cwd.path(),
&[
"poll",
"start",
"command",
"/usr/bin/true",
"--owner",
"execution",
"--parent",
parent_id,
"--until-exit-code",
"1",
"--interval-ms",
"60000",
"--format",
"json",
],
);
let poll_id = poll["execution"]["id"].as_str().unwrap();
let mut observed = None;
for _ in 0..200 {
let page = apoc.json_from(
cwd.path(),
&[
"poll",
"iterations",
poll_id,
"--limit",
"20",
"--format",
"json",
],
);
if page["iterations"][0]["status"] == "observed" {
observed = Some(page);
break;
}
thread::sleep(std::time::Duration::from_millis(10));
}
let observed = observed.unwrap_or_else(|| panic!("poll {poll_id} never observed an iteration"));
assert_eq!(observed["iterations"][0]["status"], "observed");
let canceled = apoc.json_from(
cwd.path(),
&[
"execution",
"cancel",
parent_id,
"--timeout-ms",
"5000",
"--format",
"json",
],
);
assert_eq!(canceled["status"], "canceled");
let child = apoc.json_from(
cwd.path(),
&[
"execution",
"get",
poll_id,
"--verbosity",
"trace",
"--format",
"json",
],
);
assert_eq!(child["status"], "canceled", "child poll: {child:#}");
assert_eq!(child["result"]["stop_reason"], "canceled");
}
#[test]
fn budget_cancellation_wakes_and_terminalizes_an_owned_poll() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let parent = apoc.json_from(
cwd.path(),
&[
"execution",
"start",
"/bin/sleep",
"--progress-timeout-ms",
"10000",
"--cancel-on-stall",
"--format",
"json",
"--",
"30",
],
);
let parent_id = parent["execution"]["id"].as_str().unwrap();
let poll = apoc.json_from(
cwd.path(),
&[
"poll",
"start",
"command",
"/usr/bin/true",
"--owner",
"execution",
"--parent",
parent_id,
"--until-exit-code",
"1",
"--interval-ms",
"60000",
"--format",
"json",
],
);
let poll_id = poll["execution"]["id"].as_str().unwrap();
let mut observed = false;
for _ in 0..200 {
let page = apoc.json_from(
cwd.path(),
&[
"poll",
"iterations",
poll_id,
"--limit",
"20",
"--format",
"json",
],
);
if page["iterations"][0]["status"] == "observed" {
observed = true;
break;
}
thread::sleep(std::time::Duration::from_millis(10));
}
assert!(observed, "poll {poll_id} never entered its interval wait");
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);
let parent = loop {
let parent = apoc.json_from(
cwd.path(),
&[
"execution",
"get",
parent_id,
"--verbosity",
"trace",
"--format",
"json",
],
);
if matches!(
parent["status"].as_str(),
Some("completed" | "failed" | "canceled")
) || std::time::Instant::now() >= deadline
{
break parent;
}
thread::sleep(std::time::Duration::from_millis(25));
};
let child = apoc.json_from(
cwd.path(),
&[
"execution",
"get",
poll_id,
"--verbosity",
"trace",
"--format",
"json",
],
);
assert_eq!(parent["status"], "canceled", "parent: {parent:#}");
assert_eq!(child["status"], "canceled", "child poll: {child:#}");
assert_eq!(child["result"]["stop_reason"], "canceled");
}
#[test]
fn execution_owner_rejects_a_terminal_parent_before_acceptance() {
let apoc = Fixture::new();
let cwd = tempdir().unwrap();
let parent = apoc.json_from(
cwd.path(),
&[
"execution",
"command",
"/usr/bin/true",
"--idempotency-key",
"terminal-parent-command",
"--format",
"json",
],
);
let parent_id = parent["execution"]["id"].as_str().unwrap();
let output = apoc.run_from(
cwd.path(),
&[
"poll",
"start",
"command",
"/bin/sleep",
"--owner",
"execution",
"--parent",
parent_id,
"--until-exit-code",
"0",
"--idempotency-key",
"reject-terminal-parent-child",
"--format",
"json",
"--",
"30",
],
);
assert!(
!output.status.success(),
"terminal parent accepted a child:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let error: Value = serde_json::from_slice(if output.stdout.is_empty() {
&output.stderr
} else {
&output.stdout
})
.unwrap();
assert!(
error["message"]
.as_str()
.is_some_and(|message| message.contains("terminal")),
"error: {error:#}"
);
let executions = apoc.json_from(
cwd.path(),
&["execution", "list", "--limit", "100", "--format", "json"],
);
assert!(
executions["executions"]
.as_array()
.unwrap()
.iter()
.all(|execution| execution["parent_id"].as_str() != Some(parent_id)),
"executions: {executions:#}"
);
let active_parent = apoc.json_from(
cwd.path(),
&[
"execution",
"start",
"/bin/sleep",
"--idempotency-key",
"start-active-parent-for-replay",
"--format",
"json",
"--",
"4",
],
);
let active_parent_id = active_parent["execution"]["id"].as_str().unwrap();
let child_args = [
"poll",
"start",
"command",
"/usr/bin/true",
"--owner",
"execution",
"--parent",
active_parent_id,
"--max-iterations",
"1",
"--idempotency-key",
"terminal-parent-replay",
"--format",
"json",
];
let child = apoc.json_from(cwd.path(), &child_args);
let child_id = child["execution"]["id"].as_str().unwrap();
apoc.json_from(
cwd.path(),
&[
"execution",
"wait",
active_parent_id,
"--timeout-ms",
"5000",
"--format",
"json",
],
);
let replay = apoc.json_from(cwd.path(), &child_args);
assert_eq!(replay["execution"]["id"], child_id);
assert_eq!(replay["replayed"], true);
}