use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::thread;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
static NEXT_FIXTURE: AtomicU64 = AtomicU64::new(0);
const AGENT_SOURCE: &str = r#"type SupportRequest {
prompt: String
}
type SupportResponse {
answer: String
}
type ToolCall {
name: String
capability: String
}
type ToolResult {
name: String
summary: String
ok: Boolean
}
type Permission {
capability: String
runId: String
}
agent Support {
model: Assistant
instructions: "server/agents/support.md"
maxTurns: 4
input SupportRequest
output SupportResponse
event Token(String)
event ToolStarted(ToolCall)
event ToolCompleted(ToolResult)
event PermissionRequired(Permission)
can [tickets.read]
cannot [billing.refund]
}
component Page {
state {
prompt: String = "look up 42"
}
agents {
session = Support(SupportRequest(prompt = prompt))
}
view {
<section>
#stream session {
Started { <p>starting</p> }
Token(text) { <p>{text}</p> }
ToolStarted(tool) { <p>{tool.name}</p> }
ToolCompleted(result) { <p>{result.summary}</p> }
PermissionRequired(permission) { <p>{permission.capability}</p> }
Paused(runId) { <p>paused {runId}</p> }
Completed(answer) { <p>{answer.answer}</p> }
Failed(error) { <p>{error.code}</p> }
Cancelled { <p>cancelled</p> }
}
</section>
}
}
"#;
const HOST: &str = r#"export const endpoints = Object.freeze({
"endpoint:ReadTickets@1": async ({ customer }) => ({ id: `T-${customer}`, subject: "Password reset" }),
"endpoint:RefundBilling@1": async ({ amount }) => `refunded ${amount}`,
});
// The authorizer is scripted per process through an environment variable, so a
// restarted server can answer differently from the one that paused the run.
export async function authorize({ capability }) {
const decisions = JSON.parse(process.env.PROBE_AUTHORIZER ?? "{}");
if (Object.hasOwn(decisions, capability)) return decisions[capability];
return capability === "agents.Support.run";
}
"#;
struct Fixture {
root: PathBuf,
}
impl Fixture {
fn new(label: &str, storage: &str) -> Self {
let ordinal = NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed);
let root = std::env::temp_dir().join(format!(
"noxid-wo31-{label}-{}-{ordinal}",
std::process::id()
));
let _ = fs::remove_dir_all(&root);
fs::create_dir_all(&root).expect("create WO-31 fixture");
let fixture = Self { root };
fixture.write("package.json", "{\"private\":true,\"type\":\"module\"}\n");
fixture.write(
"Noxid.toml",
&format!(
"[app]\ntitle = \"WO-31 agent engine\"\nroutes = \"src/routes\"\n\n[server]\nruntime = \"node\"\nstorage = \"{storage}\"\nsecrets = [\"ANTHROPIC_API_KEY\", \"MODEL_GATEWAY_URL\"]\ntracing = \"full\"\n"
),
);
fixture.write("src/routes/+page.nox", AGENT_SOURCE);
fixture.write(
"server/models/Assistant.nox",
"model Assistant {\n provider: anthropic\n id: \"claude-sonnet-5\"\n baseUrl: MODEL_GATEWAY_URL\n maxTokens: 512\n retries: 1\n secret: ANTHROPIC_API_KEY\n}\n",
);
fixture.write(
"server/api/tickets.get.nox",
"type Ticket {\n id: String\n subject: String\n}\n\nendpoint ReadTickets {\n version: 1\n capabilities [tickets.read]\n query { customer: String }\n result: Ticket\n}\n",
);
fixture.write(
"server/api/refund.post.nox",
"endpoint RefundBilling {\n version: 1\n capabilities [billing.refund]\n body { amount: Int }\n result: String\n}\n",
);
fixture.write(
"server/api/status.get.nox",
"endpoint OpenStatus {\n version: 1\n result: String\n}\n",
);
fixture.write(
"server/agents/support.md",
"You are a support agent. Use the tools you are given.\n",
);
fixture.write("server/host.js", HOST);
fixture
}
#[cfg(unix)]
fn link_postgres_driver(&self) -> bool {
let repository_modules = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../node_modules");
if !repository_modules.join("postgres").exists() {
eprintln!(
"WO-31 durable agent pause: SKIP (workspace postgres Node driver unavailable)"
);
return false;
}
let link = self.root.join("node_modules");
if link.exists() {
return true;
}
std::os::unix::fs::symlink(&repository_modules, link)
.expect("link the admitted postgres driver");
true
}
fn with_queue(self) -> Self {
self.write(
"server/queues/AuditTrail.nox",
"queue AuditTrail {\n payload { note: String }\n retry: 1\n backoff: 30s\n}\n",
);
self
}
fn write(&self, relative: &str, contents: &str) {
let path = self.root.join(relative);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).expect("create fixture parent");
}
fs::write(path, contents).expect("write fixture file");
}
fn build(&self) -> Output {
Command::new(env!("CARGO_BIN_EXE_noxid"))
.args(["build", ".", "--out-dir", "dist"])
.current_dir(&self.root)
.output()
.expect("build WO-31 fixture")
}
fn adapt(&self, out_dir: &str) -> Output {
Command::new(env!("CARGO_BIN_EXE_noxid"))
.args(["adapt", ".", "--adapter", "node", "--out-dir", out_dir])
.current_dir(&self.root)
.env_remove("VERCEL")
.env_remove("NETLIFY")
.env_remove("CF_PAGES")
.env_remove("DENO_DEPLOYMENT_ID")
.env_remove("RAILWAY_ENVIRONMENT")
.output()
.expect("adapt WO-31 fixture for node")
}
fn read(&self, relative: &str) -> String {
fs::read_to_string(self.root.join(relative))
.unwrap_or_else(|error| panic!("read {relative}: {error}"))
}
fn run(
&self,
name: &str,
script: &str,
authorizer: &str,
environment: &[(&str, &str)],
) -> Output {
let file = format!("probe-{name}.mjs");
self.write(&format!("dist/{file}"), &format!("{}{script}", preamble()));
let mut command = Command::new("node");
command
.arg(&file)
.current_dir(self.root.join("dist"))
.env("PROBE_AUTHORIZER", authorizer)
.env("ANTHROPIC_API_KEY", "wo31-test-key");
for (key, value) in environment {
command.env(key, value);
}
command.output().expect("execute agent probe")
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.root);
}
}
fn preamble() -> String {
r#"import http from "node:http";
const scripted = [];
const recorded = [];
function expect(frames) { scripted.push(frames); }
function sse(events) { return events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""); }
function textTurn(text) {
return sse([
{ type: "message_start", message: { usage: { input_tokens: 11 } } },
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
...[...text].map((character) => ({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: character } })),
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 3 } },
]);
}
function toolTurn(name, input, id = "toolu_1") {
return sse([
{ type: "message_start", message: { usage: { input_tokens: 12 } } },
{ type: "content_block_start", index: 0, content_block: { type: "tool_use", id, name } },
{ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: JSON.stringify(input) } },
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 7 } },
]);
}
let holdOpen = false;
const held = [];
const sink = http.createServer((request, response) => {
const chunks = [];
request.on("data", (chunk) => chunks.push(chunk));
request.on("end", () => {
let body = null;
try { body = JSON.parse(Buffer.concat(chunks).toString("utf8")); } catch {}
recorded.push(body);
if (holdOpen) { held.push(response); return; }
const next = scripted.shift();
if (next === undefined) {
response.writeHead(500, { "content-type": "application/json" });
response.end(JSON.stringify({ error: { type: "sink_unscripted", code: "sink_unscripted" } }));
return;
}
response.writeHead(200, { "content-type": "text/event-stream" });
response.end(next);
});
});
await new Promise((resolve) => sink.listen(0, "127.0.0.1", resolve));
process.env.MODEL_GATEWAY_URL = `http://127.0.0.1:${sink.address().port}`;
const handler = await import("./server/handler.js");
const { storage } = await import("./server/noxid-server.js");
function report(name, value) { console.log(`PROBE ${name} ${JSON.stringify(value)}`); }
async function post(path, body, init = {}) {
const response = await handler.fetch(new Request(`http://noxid.test${path}`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body ?? {}),
...init,
}), { sessionId: "session-1" }, {});
return { status: response.status, response, text: init.signal === undefined ? await response.text() : null };
}
function frames(text) {
const out = [];
for (const block of text.split("\n\n")) {
if (block.trim().length === 0) continue;
const lines = block.split("\n");
const event = lines.find((line) => line.startsWith("event: "))?.slice(7) ?? "message";
const data = lines.filter((line) => line.startsWith("data: ")).map((line) => line.slice(6)).join("\n");
out.push({ event, data: JSON.parse(data) });
}
return out;
}
const tags = (text) => frames(text).map((frame) => frame.event === "message" ? frame.data.tag : `!${frame.data.error.code}`);
const done = () => { for (const response of held) response.destroy(); sink.close(); };
// A Postgres-backed store keeps its pool open for the life of the process, so
// a probe that used one ends by flushing stdout and exiting rather than
// waiting for a handle that never closes.
async function exitProbe() {
done();
try { await handler.closeQueueDatabase(); } catch {}
await new Promise((resolve) => process.stdout.write("\n", resolve));
process.exit(0);
}
"#
.to_string()
}
fn assert_success(output: &Output, context: &str) {
assert!(
output.status.success(),
"{context}:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
fn stdout(output: &Output) -> String {
String::from_utf8_lossy(&output.stdout).into_owned()
}
fn probe(log: &str, name: &str) -> String {
log.lines()
.find_map(|line| line.strip_prefix(&format!("PROBE {name} ")))
.unwrap_or_else(|| panic!("probe `{name}` is missing from:\n{log}"))
.to_string()
}
#[test]
fn a_full_loop_dispatches_a_real_endpoint_tool_and_validates_the_final_answer() {
let fixture = Fixture::new("loop", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"loop",
r#"
expect(toolTurn("ReadTickets", { customer: "42" }));
expect(toolTurn("noxid_final_answer", { answer: "Ticket T-42 is a password reset" }, "toolu_2"));
const run = await post("/_noxid/agents/Support/runs", { input: { prompt: "look up 42" } });
report("status", run.status);
report("contentType", run.response.headers.get("content-type"));
report("tags", tags(run.text));
report("completed", frames(run.text).at(-1).data.value);
report("toolStarted", frames(run.text).find((frame) => frame.data.tag === "ToolStarted").data.value);
report("toolCompleted", frames(run.text).find((frame) => frame.data.tag === "ToolCompleted").data.value);
// The second provider request carries the first turn's tool result verbatim,
// which is how the model learns what the endpoint returned.
report("toolResultSeen", recorded[1].messages.at(-1).content[0].content);
report("systemPrompt", recorded[0].system);
const keys = await storage("agent_runs").list("");
const record = await storage("agent_runs").get(keys[0]);
report("persistedState", record.state);
report("persistedTurns", record.turns.length);
report("persistedPrincipal", record.principal);
done();
"#,
r#"{"agents.Support.run": true, "tickets.read": true}"#,
&[],
);
assert_success(&output, "run the full loop probe");
let log = stdout(&output);
assert_eq!(probe(&log, "status"), "200");
assert!(
probe(&log, "contentType").contains("text/event-stream"),
"{log}"
);
assert_eq!(
probe(&log, "tags"),
r#"["Started","ToolStarted","ToolCompleted","Completed"]"#
);
assert_eq!(
probe(&log, "completed"),
r#"{"answer":"Ticket T-42 is a password reset"}"#
);
assert_eq!(
probe(&log, "toolStarted"),
r#"{"name":"ReadTickets","capability":"tickets.read"}"#
);
assert_eq!(
probe(&log, "toolCompleted"),
r#"{"name":"ReadTickets","summary":"ReadTickets completed","ok":true}"#
);
assert!(
probe(&log, "toolResultSeen").contains("Password reset"),
"the endpoint's validated result must reach the next model turn: {log}"
);
assert!(
probe(&log, "systemPrompt").contains("support agent"),
"the embedded instructions asset is the system prompt: {log}"
);
assert_eq!(probe(&log, "persistedState"), "\"Completed\"");
assert_eq!(probe(&log, "persistedTurns"), "2");
assert_eq!(
probe(&log, "persistedPrincipal"),
"\"agent:Support:acting:session:session-1\"",
"the run executes under Principal.Agent {{ id: AgentId(Support), actingFor }}"
);
}
#[test]
fn a_denied_tool_is_never_described_and_never_reachable() {
let fixture = Fixture::new("denied", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"denied",
r#"
expect(toolTurn("RefundBilling", { amount: 100 }));
expect(toolTurn("noxid_final_answer", { answer: "no refund was issued" }, "toolu_2"));
const run = await post("/_noxid/agents/Support/runs", { input: { prompt: "refund me" } });
report("tags", tags(run.text));
report("offeredTools", recorded[0].tools.map((tool) => tool.name));
report("modelWasTold", recorded[1].messages.at(-1).content[0].content);
report("endpointRan", process.env.PROBE_REFUND_RAN ?? "no");
done();
"#,
r#"{"agents.Support.run": true, "billing.refund": true, "tickets.read": true}"#,
&[],
);
assert_success(&output, "run the denied-tool probe");
let log = stdout(&output);
assert_eq!(probe(&log, "tags"), r#"["Started","Completed"]"#);
assert_eq!(
probe(&log, "offeredTools"),
r#"["ReadTickets","noxid_final_answer"]"#,
"the denied endpoint must not appear in the provider request: {log}"
);
assert!(
probe(&log, "modelWasTold").contains("AGENT_TOOL_NOT_AVAILABLE"),
"{log}"
);
assert!(
!probe(&log, "modelWasTold").contains("refunded"),
"the denied endpoint must not have run: {log}"
);
let manifest = fixture.read("dist/server/security.manifest.json");
let registry = manifest
.split_once("\"agents\":[")
.expect("agents section")
.1;
assert!(
registry.contains("\"endpoint\":\"ReadTickets\""),
"{manifest}"
);
assert!(!registry.contains("RefundBilling"), "{manifest}");
assert!(!registry.contains("OpenStatus"), "{manifest}");
let handler = fixture.read("dist/server/handler.js");
assert!(
handler.contains("// noxid-runtime:feature-start:agents"),
"the agents section is behind its feature marker"
);
}
#[test]
fn the_provider_tool_list_equals_the_manifest_registry_exactly() {
let fixture = Fixture::new("registry", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"registry",
r#"
import nodeFs from "node:fs";
expect(textTurn(JSON.stringify({ answer: "done" })));
await post("/_noxid/agents/Support/runs", { input: { prompt: "list tools" } });
const manifest = JSON.parse(nodeFs.readFileSync("server/security.manifest.json", "utf8"));
report("registry", manifest.agents[0].tools.map((tool) => tool.endpoint).sort());
report("offered", recorded[0].tools.map((tool) => tool.name).sort());
// Every deployment endpoint, so the assertion is about a real subtraction.
const endpoints = JSON.parse(nodeFs.readFileSync("server/execution.manifest.json", "utf8")).endpoints ?? [];
report("deploymentEndpoints", endpoints.map((endpoint) => endpoint.name).sort());
// The schema bytes, not just the name: the tool the model sees must describe
// exactly the endpoint's declared inputs.
const readTickets = recorded[0].tools.find((tool) => tool.name === "ReadTickets");
report("toolSchema", readTickets.input_schema);
report("finalAnswerSchema", recorded[0].tools.find((tool) => tool.name === "noxid_final_answer").input_schema);
done();
"#,
r#"{"agents.Support.run": true}"#,
&[],
);
assert_success(&output, "run the registry-equality probe");
let log = stdout(&output);
assert_eq!(probe(&log, "registry"), r#"["ReadTickets"]"#);
assert_eq!(
probe(&log, "offered"),
r#"["ReadTickets","noxid_final_answer"]"#,
"the provider-visible tool list is the manifest registry plus the \
compiler-owned final-answer tool, and nothing else: {log}"
);
let deployment = probe(&log, "deploymentEndpoints");
assert!(
deployment.contains("RefundBilling") && deployment.contains("OpenStatus"),
"the deployment really does declare the endpoints the registry subtracts: {deployment}"
);
assert_eq!(
probe(&log, "toolSchema"),
r#"{"type":"object","properties":{"customer":{"type":"string"}},"required":["customer"],"additionalProperties":false}"#,
"the tool schema describes the endpoint's declared inputs exactly"
);
assert_eq!(
probe(&log, "finalAnswerSchema"),
r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"],"additionalProperties":false}"#,
"the final-answer tool describes the agent's declared output type"
);
}
#[test]
fn max_turns_bounds_a_runaway_loop() {
let fixture = Fixture::new("maxturns", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"maxturns",
r#"
for (let index = 0; index < 12; index += 1) expect(toolTurn("ReadTickets", { customer: String(index) }, `toolu_${index}`));
const run = await post("/_noxid/agents/Support/runs", { input: { prompt: "loop forever" } });
report("tags", tags(run.text));
report("failure", frames(run.text).at(-1).data.value);
report("providerCalls", recorded.length);
const keys = await storage("agent_runs").list("");
const record = await storage("agent_runs").get(keys[0]);
report("persistedState", record.state);
report("declaredCeiling", 4);
done();
"#,
r#"{"agents.Support.run": true, "tickets.read": true}"#,
&[],
);
assert_success(&output, "run the maxTurns probe");
let log = stdout(&output);
assert_eq!(
probe(&log, "tags"),
r#"["Started","ToolStarted","ToolCompleted","ToolStarted","ToolCompleted","ToolStarted","ToolCompleted","ToolStarted","ToolCompleted","Failed"]"#
);
let failure = probe(&log, "failure");
assert!(failure.contains("\"code\":\"AGENT_MAX_TURNS\""), "{log}");
assert!(failure.contains("ceiling of 4 turns"), "{log}");
assert_eq!(
probe(&log, "providerCalls"),
"4",
"the declared ceiling is a hard stop on provider calls: {log}"
);
assert_eq!(probe(&log, "persistedState"), "\"Failed\"");
}
#[test]
fn the_wo18_deadline_bounds_a_wedged_model_call() {
let fixture = Fixture::new("timeout", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"timeout",
r#"
report("declaredTimeoutMs", JSON.parse(process.env.PROBE_EXPECTED_TIMEOUT));
holdOpen = true; // the provider accepts the request and never answers
const started = Date.now();
const run = await post("/_noxid/agents/Support/runs", { input: { prompt: "hang" } });
report("elapsedMs", Date.now() - started);
report("tags", tags(run.text));
report("failure", frames(run.text).at(-1).data.value);
const keys = await storage("agent_runs").list("");
report("persistedState", (await storage("agent_runs").get(keys[0])).state);
done();
"#,
r#"{"agents.Support.run": true}"#,
&[("PROBE_EXPECTED_TIMEOUT", "30000")],
);
assert_success(&output, "run the deadline probe");
let log = stdout(&output);
let handler = fixture.read("dist/server/handler.js");
assert!(
handler.contains("timeoutMs: 30000"),
"the run endpoint carries the WO-18 default deadline"
);
assert_eq!(probe(&log, "tags"), r#"["Started","Failed"]"#);
assert!(
probe(&log, "failure").contains("\"code\":\"AGENT_TIMEOUT\""),
"{log}"
);
let elapsed: u64 = probe(&log, "elapsedMs").parse().expect("elapsed ms");
assert!(
(29_000..60_000).contains(&elapsed),
"the run must end at its declared deadline, not before or long after: {elapsed} ms"
);
assert_eq!(probe(&log, "persistedState"), "\"Failed\"");
}
#[test]
fn cancellation_stops_the_model_call_the_run_and_the_session() {
let fixture = Fixture::new("cancel", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"cancel",
r#"
holdOpen = true;
const controller = new AbortController();
const response = await handler.fetch(new Request("http://noxid.test/_noxid/agents/Support/runs", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ input: { prompt: "hang" } }),
signal: controller.signal,
}), { sessionId: "session-1" }, {});
const reader = response.body.getReader();
const first = await reader.read();
report("firstFrame", new TextDecoder().decode(first.value).includes("\"tag\":\"Started\""));
setTimeout(() => controller.abort("client left"), 50);
const rest = [];
for (;;) {
const chunk = await reader.read();
if (chunk.done) break;
rest.push(new TextDecoder().decode(chunk.value));
}
report("afterCancel", tags(rest.join("")));
report("providerCalls", recorded.length);
const keys = await storage("agent_runs").list("");
report("persistedState", (await storage("agent_runs").get(keys[0])).state);
done();
"#,
r#"{"agents.Support.run": true}"#,
&[],
);
assert_success(&output, "run the cancellation probe");
let log = stdout(&output);
assert_eq!(probe(&log, "firstFrame"), "true");
assert_eq!(probe(&log, "afterCancel"), r#"["Cancelled"]"#);
assert_eq!(
probe(&log, "providerCalls"),
"1",
"the wedged provider call is abandoned rather than retried: {log}"
);
assert_eq!(probe(&log, "persistedState"), "\"Cancelled\"");
}
#[test]
fn every_event_in_the_declared_algebra_is_emitted() {
let fixture = Fixture::new("algebra", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"algebra",
r#"
const seen = new Set();
function observe(text) { for (const tag of tags(text)) seen.add(tag); }
// Started + Token, then a Completed answer parsed from plain text.
expect(textTurn(JSON.stringify({ answer: "hello" })));
observe((await post("/_noxid/agents/Support/runs", { input: { prompt: "greet" } })).text);
// ToolStarted + ToolCompleted.
expect(toolTurn("ReadTickets", { customer: "42" }));
expect(toolTurn("noxid_final_answer", { answer: "found" }, "toolu_2"));
observe((await post("/_noxid/agents/Support/runs", { input: { prompt: "look up" } })).text);
// Failed, through an output that violates the declared type.
expect(toolTurn("noxid_final_answer", { answer: 7 }, "toolu_3"));
const invalid = await post("/_noxid/agents/Support/runs", { input: { prompt: "bad output" } });
observe(invalid.text);
report("outputInvalid", frames(invalid.text).at(-1).data.value.code);
// PermissionRequired + Paused, through a deferred capability.
process.env.PROBE_AUTHORIZER = JSON.stringify({ "agents.Support.run": true, "tickets.read": "defer" });
expect(toolTurn("ReadTickets", { customer: "9" }, "toolu_4"));
const paused = await post("/_noxid/agents/Support/runs", { input: { prompt: "needs approval" } });
observe(paused.text);
report("pausedRunId", frames(paused.text).at(-1).data.value);
// Cancelled.
holdOpen = true;
const controller = new AbortController();
const response = await handler.fetch(new Request("http://noxid.test/_noxid/agents/Support/runs", {
method: "POST", headers: { "content-type": "application/json" },
body: JSON.stringify({ input: { prompt: "hang" } }), signal: controller.signal,
}), { sessionId: "session-1" }, {});
const reader = response.body.getReader();
await reader.read();
setTimeout(() => controller.abort("client left"), 50);
const rest = [];
for (;;) { const chunk = await reader.read(); if (chunk.done) break; rest.push(new TextDecoder().decode(chunk.value)); }
observe(rest.join(""));
report("algebra", [...seen].sort());
done();
"#,
r#"{"agents.Support.run": true, "tickets.read": true}"#,
&[],
);
assert_success(&output, "run the event-algebra probe");
let log = stdout(&output);
assert_eq!(probe(&log, "outputInvalid"), "\"AGENT_OUTPUT_INVALID\"");
assert_eq!(
probe(&log, "algebra"),
r#"["Cancelled","Completed","Failed","Paused","PermissionRequired","Started","Token","ToolCompleted","ToolStarted"]"#,
"every declared event, compiler-owned and developer-declared, must be reachable"
);
assert!(
probe(&log, "pausedRunId").len() >= 18,
"the Paused event carries the run id the resume endpoint takes: {log}"
);
}
#[test]
fn the_client_session_handles_paused_and_resumes_by_run_id() {
let fixture = Fixture::new("client", "fs");
assert_success(&fixture.build(), "build the agent fixture");
fixture.write(
"dist/assets/client-probe.mjs",
r#"import { Support } from "./Page.agents.js";
import { createOwner } from "./noxid-runtime.js";
const owner = createOwner();
let phase = 0;
const session = Support.create({ prompt: "hi" }, {
owner,
async invoke(input, api) {
phase += 1;
if (phase === 1) {
api.emit("Token", "thinking");
api.pause("run-01234567890123456789");
return null;
}
console.log(`PROBE resumedWith ${JSON.stringify(api.resume)}`);
return { answer: "done" };
},
});
console.log(`PROBE engine ${JSON.stringify(Support.engine)}`);
await session.start();
console.log(`PROBE pausedStatus ${JSON.stringify(session.status.get())}`);
const runId = session.events.get().find((entry) => entry.event.tag === "Paused").event.value;
console.log(`PROBE pausedRunId ${JSON.stringify(runId)}`);
await session.resume(runId);
console.log(`PROBE resumedStatus ${JSON.stringify(session.status.get())}`);
console.log(`PROBE output ${JSON.stringify(session.output.get())}`);
try { await session.resume(runId); } catch (error) { console.log(`PROBE refused ${JSON.stringify(error.code)}`); }
"#,
);
let output = Command::new("node")
.arg("client-probe.mjs")
.current_dir(fixture.root.join("dist/assets"))
.output()
.expect("execute the client probe");
assert_success(&output, "run the client session probe");
let log = stdout(&output);
let engine = probe(&log, "engine");
assert!(engine.contains("\"agent\":\"Support\""), "{log}");
assert!(
engine.contains("\"resumePath\":\"/_noxid/agents/Support/runs/\""),
"{log}"
);
assert!(
engine.contains("\"resumeCapability\":\"agents.Support.resume\""),
"{log}"
);
assert_eq!(probe(&log, "pausedStatus"), "\"Paused\"");
assert_eq!(probe(&log, "pausedRunId"), "\"run-01234567890123456789\"");
assert_eq!(probe(&log, "resumedWith"), "\"run-01234567890123456789\"");
assert_eq!(probe(&log, "resumedStatus"), "\"Completed\"");
assert_eq!(probe(&log, "output"), r#"{"answer":"done"}"#);
assert_eq!(probe(&log, "refused"), "\"AGENT_NOT_PAUSED\"");
}
#[test]
fn agent_spans_carry_identity_and_tokens_and_no_content() {
let fixture = Fixture::new("spans", "fs");
assert_success(&fixture.build(), "build the agent fixture");
let output = fixture.run(
"spans",
r#"
expect(toolTurn("ReadTickets", { customer: "secret-customer-4242" }));
expect(toolTurn("noxid_final_answer", { answer: "confidential-answer-text" }, "toolu_2"));
await post("/_noxid/agents/Support/runs", { input: { prompt: "confidential-prompt-text" } });
done();
"#,
r#"{"agents.Support.run": true, "tickets.read": true}"#,
&[],
);
assert_success(&output, "run the spans probe");
let log = stdout(&output);
let spans = log
.lines()
.filter(|line| line.contains("\"event\":\"agent."))
.collect::<Vec<_>>();
assert!(
spans
.iter()
.any(|line| line.contains("\"event\":\"agent.run\""))
&& spans
.iter()
.any(|line| line.contains("\"event\":\"agent.turn\""))
&& spans
.iter()
.any(|line| line.contains("\"event\":\"agent.tool\"")),
"all three spans must reach the kernel:\n{log}"
);
let run_span = spans
.iter()
.find(|line| line.contains("\"event\":\"agent.run\""))
.expect("agent.run span");
assert!(run_span.contains("\"agent\":\"Support\""), "{run_span}");
assert!(run_span.contains("\"agentRun\":\""), "{run_span}");
assert!(run_span.contains("\"tokensInput\":"), "{run_span}");
let tool_span = spans
.iter()
.find(|line| line.contains("\"event\":\"agent.tool\""))
.expect("agent.tool span");
assert!(
tool_span.contains("\"toolEndpoint\":\"endpoint:ReadTickets@1\""),
"{tool_span}"
);
for span in &spans {
for secret in [
"confidential-prompt-text",
"confidential-answer-text",
"secret-customer-4242",
"support agent",
] {
assert!(
!span.contains(secret),
"an agent span must never carry prompt, argument, or result content: {span}"
);
}
}
}
struct ComposePostgres {
file: PathBuf,
project: String,
}
impl ComposePostgres {
fn start() -> Option<Self> {
for probe in [vec!["info"], vec!["compose", "version"]] {
if !Command::new("docker")
.args(&probe)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.is_ok_and(|status| status.success())
{
eprintln!(
"WO-31 durable agent pause: SKIP (Docker unavailable; the restart-and-resume path was not exercised)"
);
return None;
}
}
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("clock after epoch")
.as_nanos();
let postgres = Self {
file: Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/wo19-postgres/compose.yaml"),
project: format!("noxidwo31{}{}", std::process::id(), nonce),
};
let output = postgres
.command()
.args(["up", "--detach"])
.output()
.expect("start agent-run Postgres");
assert!(
output.status.success(),
"Docker was available but Postgres failed to start: {}",
String::from_utf8_lossy(&output.stderr)
);
for _ in 0..480 {
if postgres
.command()
.args([
"exec",
"--no-TTY",
"postgres",
"pg_isready",
"-U",
"noxid_test",
"-d",
"noxid_test",
])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.expect("probe agent-run Postgres")
.success()
{
return Some(postgres);
}
thread::sleep(Duration::from_millis(250));
}
panic!("agent-run Postgres did not become ready within 120 seconds");
}
fn command(&self) -> Command {
let mut command = Command::new("docker");
command
.args(["compose", "-f"])
.arg(&self.file)
.args(["--project-name", &self.project]);
command
}
fn database_url(&self) -> String {
let output = self
.command()
.args(["port", "postgres", "5432"])
.output()
.expect("resolve agent-run Postgres port");
assert!(output.status.success());
let mapping = String::from_utf8(output.stdout).expect("UTF-8 port mapping");
let port = mapping
.trim()
.rsplit_once(':')
.map(|(_, port)| port)
.expect("mapped port");
format!("postgres://noxid_test:noxid_test@127.0.0.1:{port}/noxid_test")
}
}
impl Drop for ComposePostgres {
fn drop(&mut self) {
let _ = self
.command()
.args(["down", "--volumes", "--remove-orphans"])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
}
}
#[test]
fn a_permission_pause_survives_a_restart_and_resumes_through_the_endpoint() {
let Some(postgres) = ComposePostgres::start() else {
return;
};
let database_url = postgres.database_url();
let fixture = Fixture::new("pause", "postgres").with_queue();
if !fixture.link_postgres_driver() {
return;
}
assert_success(&fixture.build(), "build the postgres agent fixture");
let pause = fixture.run(
"pause",
r#"
expect(toolTurn("ReadTickets", { customer: "42" }));
const run = await post("/_noxid/agents/Support/runs", { input: { prompt: "needs approval" } });
report("tags", tags(run.text));
report("permission", frames(run.text).find((frame) => frame.data.tag === "PermissionRequired").data.value);
report("runId", frames(run.text).at(-1).data.value);
await exitProbe();
"#,
r#"{"agents.Support.run": true, "tickets.read": "defer"}"#,
&[("DATABASE_URL", &database_url)],
);
assert_success(&pause, "run the pause probe");
let pause_log = stdout(&pause);
assert_eq!(
probe(&pause_log, "tags"),
r#"["Started","PermissionRequired","Paused"]"#
);
assert!(
probe(&pause_log, "permission").contains("\"capability\":\"tickets.read\""),
"{pause_log}"
);
let run_id = probe(&pause_log, "runId");
let run_id = run_id.trim_matches('"').to_string();
let resume = fixture.run(
"resume",
&format!(
r#"
const runId = {run_id:?};
// The queue worker reconciles the store on startup: paused runs stay
// resumable, and any run left `Running` by the previous process is failed.
const worker = handler.startQueueWorker({{ queue: "AuditTrail", pollIntervalMs: 25 }});
await new Promise((resolve) => setTimeout(resolve, 300));
await worker.stop();
const beforeResume = await storage("agent_runs").get(`Support:${{runId}}`);
report("survivedState", beforeResume.state);
report("survivedPending", beforeResume.pending.endpoint);
report("survivedTurns", beforeResume.turns.length);
// Without the resume capability the endpoint refuses before the run is
// touched at all.
process.env.PROBE_AUTHORIZER = JSON.stringify({{ "agents.Support.resume": false, "tickets.read": true }});
const refused = await post(`/_noxid/agents/Support/runs/${{runId}}/resume`, {{}});
report("refusedStatus", refused.status);
report("refusedCode", JSON.parse(refused.text).error.code);
report("stateAfterRefusal", (await storage("agent_runs").get(`Support:${{runId}}`)).state);
// With the resume capability but the deferred capability still deferred, the
// run stays paused: approval is re-checked with the resuming principal.
process.env.PROBE_AUTHORIZER = JSON.stringify({{ "agents.Support.resume": true, "tickets.read": "defer" }});
const stillDeferred = await post(`/_noxid/agents/Support/runs/${{runId}}/resume`, {{}});
report("stillDeferredStatus", stillDeferred.status);
report("stillDeferredCode", JSON.parse(stillDeferred.text).error.code);
// With both, the approved call is the first work the resumed run does.
process.env.PROBE_AUTHORIZER = JSON.stringify({{ "agents.Support.resume": true, "tickets.read": true }});
expect(toolTurn("noxid_final_answer", {{ answer: "Ticket T-42 is a password reset" }}, "toolu_2"));
const resumed = await post(`/_noxid/agents/Support/runs/${{runId}}/resume`, {{}});
report("resumedTags", tags(resumed.text));
report("resumedAnswer", frames(resumed.text).at(-1).data.value);
report("providerCallsAfterResume", recorded.length);
report("finalState", (await storage("agent_runs").get(`Support:${{runId}}`)).state);
// A finished run does not resume twice.
const again = await post(`/_noxid/agents/Support/runs/${{runId}}/resume`, {{}});
report("againStatus", again.status);
report("againCode", JSON.parse(again.text).error.code);
await exitProbe();
"#
),
r#"{"agents.Support.resume": true, "tickets.read": true}"#,
&[("DATABASE_URL", &database_url)],
);
assert_success(&resume, "run the restart-and-resume probe");
let log = stdout(&resume);
assert_eq!(probe(&log, "survivedState"), "\"Paused\"");
assert_eq!(probe(&log, "survivedPending"), "\"ReadTickets\"");
assert_eq!(probe(&log, "survivedTurns"), "1");
assert_eq!(probe(&log, "refusedStatus"), "403");
assert_eq!(probe(&log, "refusedCode"), "\"ENDPOINT_CAPABILITY_DENIED\"");
assert_eq!(
probe(&log, "stateAfterRefusal"),
"\"Paused\"",
"a refused resume must leave the run exactly as it was"
);
assert_eq!(probe(&log, "stillDeferredStatus"), "403");
assert_eq!(
probe(&log, "stillDeferredCode"),
"\"AGENT_PERMISSION_DENIED\""
);
assert_eq!(
probe(&log, "resumedTags"),
r#"["Started","ToolStarted","ToolCompleted","Completed"]"#
);
assert_eq!(
probe(&log, "resumedAnswer"),
r#"{"answer":"Ticket T-42 is a password reset"}"#
);
assert_eq!(
probe(&log, "providerCallsAfterResume"),
"1",
"the resumed run replays the approved call, then asks the model once: {log}"
);
assert_eq!(probe(&log, "finalState"), "\"Completed\"");
assert_eq!(probe(&log, "againStatus"), "409");
assert_eq!(probe(&log, "againCode"), "\"AGENT_RUN_NOT_PAUSED\"");
}
const ADAPTED_RUN_DOOR_DRIVER: &str = r##"import http from "node:http";
import net from "node:net";
import { spawn } from "node:child_process";
const sse = (events) => events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join("");
const textTurn = (text) => sse([
{ type: "message_start", message: { usage: { input_tokens: 11 } } },
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
...[...text].map((character) => ({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: character } })),
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 3 } },
]);
const scripted = [textTurn(JSON.stringify({ answer: "adapted" }))];
let providerRequests = 0;
const sink = http.createServer((request, response) => {
request.resume();
request.on("end", () => {
providerRequests += 1;
const next = scripted.shift();
if (next === undefined) {
response.writeHead(500, { "content-type": "application/json" });
response.end(JSON.stringify({ error: { type: "sink_unscripted", code: "sink_unscripted" } }));
return;
}
response.writeHead(200, { "content-type": "text/event-stream" });
response.end(next);
});
});
await new Promise((resolve) => sink.listen(0, "127.0.0.1", resolve));
const gateway = `http://127.0.0.1:${sink.address().port}`;
const freePort = () => new Promise((resolve, reject) => {
const probe = net.createServer();
probe.on("error", reject);
probe.listen(0, "127.0.0.1", () => {
const chosen = probe.address().port;
probe.close(() => resolve(chosen));
});
});
const children = [];
// Only ever the children this driver spawned.
const stopAll = () => { for (const child of children) { try { child.kill("SIGKILL"); } catch {} } };
process.on("exit", stopAll);
const start = async (authorizer) => {
const port = await freePort();
const child = spawn(process.execPath, ["server.mjs"], {
cwd: process.cwd(),
env: {
...process.env,
PORT: String(port),
MODEL_GATEWAY_URL: gateway,
ANTHROPIC_API_KEY: "wo31-adapted-key",
PROBE_AUTHORIZER: authorizer,
},
stdio: "ignore",
});
children.push(child);
for (let attempt = 0; attempt < 300; attempt += 1) {
try {
const response = await fetch(`http://127.0.0.1:${port}/`);
await response.arrayBuffer();
if (response.status === 200) return port;
} catch {}
await new Promise((resolve) => setTimeout(resolve, 100));
}
throw new Error("the adapted server never served its shell");
};
const post = async (port, path, body) => {
const response = await fetch(`http://127.0.0.1:${port}${path}`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body ?? {}),
});
return { status: response.status, contentType: response.headers.get("content-type") ?? "", text: await response.text() };
};
function frames(text) {
const out = [];
for (const block of text.split("\n\n")) {
if (block.trim().length === 0) continue;
const lines = block.split("\n");
const event = lines.find((line) => line.startsWith("event: "))?.slice(7) ?? "message";
const data = lines.filter((line) => line.startsWith("data: ")).map((line) => line.slice(6)).join("\n");
out.push({ event, data: JSON.parse(data) });
}
return out;
}
const tags = (text) => frames(text).map((frame) => frame.event === "message" ? frame.data.tag : `!${frame.data.error.code}`);
const report = (name, value) => console.log(`PROBE ${name} ${JSON.stringify(value)}`);
const code = (text) => { try { return JSON.parse(text)?.error?.code ?? null; } catch { return null; } };
const allowed = await start('{"agents.Support.run": true, "agents.Support.resume": true, "tickets.read": true}');
const run = await post(allowed, "/_noxid/agents/Support/runs", { input: { prompt: "look up 42" } });
report("run-status", run.status);
report("run-content-type", run.contentType);
const runTags = run.contentType.includes("text/event-stream") ? tags(run.text) : [run.text.slice(0, 120)];
report("run-tags", [runTags[0] ?? null, runTags.at(-1) ?? null]);
report("run-token-events", runTags.filter((tag) => tag === "Token").length > 0);
report("run-completed", run.contentType.includes("text/event-stream") ? (frames(run.text).at(-1)?.data?.value ?? null) : null);
report("provider-requests", providerRequests);
// The door this build already forwarded, unchanged: it still reaches the
// engine and still answers structurally.
const resume = await post(allowed, "/_noxid/agents/Support/runs/aaaaaaaaaaaaaaaa/resume", {});
report("resume-status", resume.status);
report("resume-code", code(resume.text));
// A path that is not a run door must still fall through to the document.
const shell = await fetch(`http://127.0.0.1:${allowed}/_noxid/agents/Support/runs/aaaaaaaaaaaaaaaa`, {
method: "POST",
headers: { "content-type": "application/json" },
body: "{}",
});
await shell.arrayBuffer();
report("not-a-door-status", shell.status);
const denied = await start('{"agents.Support.run": false}');
const refused = await post(denied, "/_noxid/agents/Support/runs", { input: { prompt: "look up 42" } });
report("denied-status", refused.status);
report("denied-content-type", refused.contentType);
report("denied-code", code(refused.text));
report("provider-requests-after-refusal", providerRequests);
stopAll();
sink.close();
await new Promise((resolve) => process.stdout.write("\n", resolve));
process.exit(0);
"##;
#[test]
fn a_run_starts_through_the_adapted_front_door_and_the_door_enforces_the_run_capability() {
let fixture = Fixture::new("adapted-run-door", "fs");
let adapted = fixture.adapt("deploy");
assert_success(&adapted, "adapt the agent fixture for node");
fixture.write("deploy/driver.mjs", ADAPTED_RUN_DOOR_DRIVER);
let driven = Command::new("node")
.arg("driver.mjs")
.current_dir(fixture.root.join("deploy"))
.output()
.expect("drive the adapted front door");
let log = stdout(&driven);
assert!(
driven.status.success(),
"{log}\nstderr:\n{}",
String::from_utf8_lossy(&driven.stderr)
);
assert_eq!(probe(&log, "run-status"), "200", "{log}");
assert!(
probe(&log, "run-content-type").contains("text/event-stream"),
"the run start did not reach the engine; it fell through to the document:\n{log}"
);
assert_eq!(
probe(&log, "run-tags"),
r#"["Started","Completed"]"#,
"{log}"
);
assert_eq!(
probe(&log, "run-completed"),
r#"{"answer":"adapted"}"#,
"{log}"
);
assert_eq!(
probe(&log, "provider-requests"),
"1",
"the scripted provider was never reached, so the run did not execute:\n{log}"
);
assert_eq!(probe(&log, "resume-status"), "404", "{log}");
assert_eq!(
probe(&log, "resume-code"),
"\"AGENT_RUN_NOT_FOUND\"",
"{log}"
);
assert_eq!(probe(&log, "not-a-door-status"), "200", "{log}");
assert_eq!(probe(&log, "denied-status"), "403", "{log}");
assert!(
probe(&log, "denied-content-type").contains("application/json"),
"{log}"
);
assert_eq!(
probe(&log, "denied-code"),
"\"ENDPOINT_CAPABILITY_DENIED\"",
"{log}"
);
assert_eq!(
probe(&log, "provider-requests-after-refusal"),
"1",
"a refused run still called the model:\n{log}"
);
}