use std::ffi::OsString;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::sync::atomic::{AtomicU64, Ordering};
static NEXT_FIXTURE: AtomicU64 = AtomicU64::new(0);
struct Fixture {
root: PathBuf,
command_path: OsString,
}
impl Fixture {
fn new(label: &str, timeout: &str) -> Self {
let root = std::env::temp_dir().join(format!(
"noxid-wo46-round2-{label}-{}-{}",
std::process::id(),
NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir_all(root.join("bin")).expect("create round-2 fixture");
fs::write(
root.join("Qa.nox"),
format!(
r#"endpoint Qa {{
body {{ name: String }}
result: Boolean
timeout: {timeout}
property GetterSafety {{ runs: 1 expect: validates or refuses }}
}}
"#
),
)
.expect("write endpoint fixture");
fs::write(
root.join("weak.validators.js"),
r#"export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:endpoint.Qa.body": (value) => value.name,
});
"#,
)
.expect("write weakened validator");
let original_path = std::env::var_os("PATH").expect("PATH is available");
let real_node = find_program(&original_path, "node").expect("find Node.js");
let bin = root.join("bin");
write_executable(
&bin.join("node"),
r#"#!/bin/sh
case "$1" in
*property-case.mjs)
if [ -n "${NOXID_R2_VALIDATOR:-}" ]; then
cp "$NOXID_R2_VALIDATOR" "$PWD/Qa.validators.js"
fi
;;
esac
exec "$NOXID_R2_REAL_NODE" "$@"
"#,
);
write_executable(
&bin.join("noxid"),
"#!/bin/sh\nexec \"$NOXID_R2_REAL_CLI\" \"$@\"\n",
);
let command_path =
std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths(&original_path)))
.expect("construct fixture PATH");
let fixture = Self { root, command_path };
fixture.write_environment(real_node);
fixture
}
fn write_environment(&self, real_node: PathBuf) {
fs::write(
self.root.join("environment"),
real_node.to_string_lossy().as_bytes(),
)
.expect("record real Node path");
}
fn command(&self, program: &str) -> Command {
self.command_with_validator(program, Some(&self.root.join("weak.validators.js")))
}
fn command_with_validator(&self, program: &str, validator: Option<&Path>) -> Command {
let mut command = Command::new(program);
let real_node =
fs::read_to_string(self.root.join("environment")).expect("read real Node path");
command
.env("PATH", &self.command_path)
.env("NOXID_R2_REAL_NODE", real_node)
.env("NOXID_R2_REAL_CLI", env!("CARGO_BIN_EXE_noxid"));
if let Some(validator) = validator {
command.env("NOXID_R2_VALIDATOR", validator);
}
command
}
fn noxid_test(&self, validator: Option<&Path>) -> Output {
self.command_with_validator(env!("CARGO_BIN_EXE_noxid"), validator)
.arg("test")
.arg(self.root.join("Qa.nox"))
.arg("--json")
.output()
.expect("run property fixture")
}
fn initial_failure(&self) -> Output {
self.command(env!("CARGO_BIN_EXE_noxid"))
.arg("test")
.arg(self.root.join("Qa.nox"))
.arg("--json")
.output()
.expect("run initial weakened property")
}
fn paste_repro(&self, repro: &str) -> Output {
self.command("sh")
.arg("-c")
.arg(repro)
.output()
.expect("paste printed repro command")
}
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.root);
}
}
fn find_program(path: &OsString, name: &str) -> Option<PathBuf> {
std::env::split_paths(path)
.map(|directory| directory.join(name))
.find(|candidate| candidate.is_file())
.and_then(|candidate| fs::canonicalize(candidate).ok())
}
fn write_executable(path: &Path, contents: &str) {
fs::write(path, contents).expect("write executable fixture");
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut permissions = fs::metadata(path).expect("read fixture mode").permissions();
permissions.set_mode(0o755);
fs::set_permissions(path, permissions).expect("make fixture executable");
}
}
fn stdout(output: &Output) -> String {
String::from_utf8(output.stdout.clone()).expect("CLI stdout is UTF-8")
}
fn json_string_field(document: &str, name: &str) -> String {
let prefix = format!("\"{name}\":\"");
let start = document.find(&prefix).expect("JSON field exists") + prefix.len();
let mut value = String::new();
let mut characters = document[start..].chars();
while let Some(character) = characters.next() {
match character {
'"' => return value,
'\\' => match characters.next().expect("JSON escape has a value") {
'"' => value.push('"'),
'\\' => value.push('\\'),
'/' => value.push('/'),
'b' => value.push('\u{0008}'),
'f' => value.push('\u{000c}'),
'n' => value.push('\n'),
'r' => value.push('\r'),
't' => value.push('\t'),
escape => panic!("unsupported JSON escape in test output: {escape}"),
},
other => value.push(other),
}
}
panic!("unterminated JSON string field `{name}`")
}
fn assert_property_failure(output: &Output) -> String {
assert_failure(output, "PROPERTY_INVARIANT_VIOLATED")
}
fn assert_failure(output: &Output, code: &str) -> String {
let stdout = stdout(output);
assert!(
!output.status.success(),
"expected property failure containing {code}\nstdout:\n{stdout}\nstderr:\n{}",
String::from_utf8_lossy(&output.stderr)
);
assert!(stdout.contains(code), "{stdout}");
stdout
}
#[test]
fn printed_seed_repro_replays_the_weakened_validator_twice() {
let fixture = Fixture::new("seed-repro", "1s");
let initial = assert_property_failure(&fixture.initial_failure());
let repro = json_string_field(&initial, "repro");
let counterexample = json_string_field(&initial, "counterexample");
assert!(repro.starts_with("noxid test '"), "{repro}");
assert!(repro.contains(" --seed "), "{repro}");
assert!(
counterexample.contains("Object.defineProperty"),
"{counterexample}"
);
assert!(
!counterexample.contains("getterBomb: true"),
"{counterexample}"
);
let literal_probe = fixture.root.join("counterexample-probe.mjs");
fs::write(
&literal_probe,
format!(
"const value = {counterexample};\nconst descriptor = Object.getOwnPropertyDescriptor(value, \"name\");\nif (typeof descriptor?.get !== \"function\") throw new Error(\"counterexample did not reconstruct the getter value\");\n"
),
)
.expect("write counterexample literal probe");
let probe = fixture
.command("node")
.arg(literal_probe)
.output()
.expect("execute counterexample literal");
assert!(
probe.status.success(),
"counterexample literal must reconstruct the validator input: {}",
String::from_utf8_lossy(&probe.stderr)
);
let first_replay = assert_property_failure(&fixture.paste_repro(&repro));
let second_replay = assert_property_failure(&fixture.paste_repro(&repro));
assert_eq!(first_replay, second_replay);
for field in ["seed", "counterexample", "repro"] {
assert_eq!(
json_string_field(&initial, field),
json_string_field(&first_replay, field),
"printed repro must preserve `{field}`"
);
}
}
#[test]
fn tight_timeout_excludes_node_startup_under_parallel_load() {
let fixture = Fixture::new("parallel-tight-timeout", "50ms");
let outputs = std::thread::scope(|scope| {
let handles = (0..8)
.map(|_| scope.spawn(|| fixture.noxid_test(None)))
.collect::<Vec<_>>();
handles
.into_iter()
.map(|handle| handle.join().expect("parallel property worker"))
.collect::<Vec<_>>()
});
for output in outputs {
let stdout = stdout(&output);
assert!(
output.status.success(),
"correct 50ms property was charged for Node startup\nstdout:\n{stdout}\nstderr:\n{}",
String::from_utf8_lossy(&output.stderr)
);
assert!(stdout.contains("\"status\":\"pass\""), "{stdout}");
assert!(!stdout.contains("PROPERTY_TIMEOUT_EXCEEDED"), "{stdout}");
}
}
#[test]
fn startup_safety_budget_reports_runner_unavailable_without_a_case() {
let fixture = Fixture::new("startup-unavailable", "1s");
fs::write(
fixture.root.join("hook-entry.mjs"),
"import { register } from \"node:module\";\nregister(\"./hook.mjs\", import.meta.url);\n",
)
.expect("write startup-delay loader entry");
fs::write(
fixture.root.join("hook.mjs"),
r#"export async function load(url, context, next) {
const result = await next(url, context);
if (url.endsWith(".validators.js")) {
return { ...result, source: 'Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000);\n' + result.source };
}
return result;
}
"#,
)
.expect("write startup-delay loader hook");
let output = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.env(
"NODE_OPTIONS",
format!("--import {}", fixture.root.join("hook-entry.mjs").display()),
)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.arg("--json")
.output()
.expect("run startup-starved property fixture");
let stdout = stdout(&output);
assert!(!output.status.success(), "{stdout}");
assert!(stdout.contains("PROPERTY_RUNNER_UNAVAILABLE"), "{stdout}");
assert!(stdout.contains("\"seed\":null"), "{stdout}");
assert!(stdout.contains("\"counterexample\":null"), "{stdout}");
assert!(stdout.contains("\"repro\":null"), "{stdout}");
assert!(!stdout.contains("PROPERTY_TIMEOUT_EXCEEDED"), "{stdout}");
assert!(!stdout.contains("shrunk counterexample"), "{stdout}");
}
#[test]
fn seeded_replay_selects_one_property_in_a_multi_property_input() {
let fixture = Fixture::new("multi-property-replay", "1s");
fs::write(
fixture.root.join("Qa.nox"),
r#"endpoint Alpha {
body { name: String }
result: Boolean
property AlphaSafety { runs: 1 expect: validates or refuses }
}
endpoint Beta {
body { name: String }
result: Boolean
property BetaSafety { runs: 1 expect: validates or refuses }
}
"#,
)
.expect("write two-property fixture");
let unselected = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.args(["--json", "--seed", "123"])
.output()
.expect("run unselected multi-property replay");
let stderr = String::from_utf8_lossy(&unselected.stderr);
assert!(!unselected.status.success(), "{stderr}");
assert!(
stderr.contains("PROPERTY_REPLAY_SELECTOR_REQUIRED"),
"{stderr}"
);
assert!(stderr.contains("--property <semantic-id>"), "{stderr}");
assert!(
stderr.contains("property:endpoint.Alpha.AlphaSafety"),
"{stderr}"
);
assert!(
stderr.contains("property:endpoint.Beta.BetaSafety"),
"{stderr}"
);
fs::write(
fixture.root.join("hook-entry.mjs"),
"import { register } from \"node:module\";\nregister(\"./hook.mjs\", import.meta.url);\n",
)
.expect("write replay loader entry");
fs::write(
fixture.root.join("hook.mjs"),
r#"export async function load(url, context, next) {
if (url.endsWith(".validators.js")) {
return { format: "module", shortCircuit: true, source: `
export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:endpoint.Alpha.body": () => { throw new Error("BROKEN_ALPHA"); },
"validator:endpoint.Beta.body": () => { throw new Error("BROKEN_BETA"); },
});
` };
}
return next(url, context);
}
"#,
)
.expect("write replay loader hook");
let alpha_seed =
noxid_property_gen::property_seed("property:endpoint.Alpha.AlphaSafety", 123).to_string();
let selected = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.env(
"NODE_OPTIONS",
format!("--import {}", fixture.root.join("hook-entry.mjs").display()),
)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.args([
"--json",
"--seed",
&alpha_seed,
"--property",
"property:endpoint.Alpha.AlphaSafety",
])
.output()
.expect("run selected multi-property replay");
let selected_stdout = stdout(&selected);
assert!(!selected.status.success(), "{selected_stdout}");
assert!(
selected_stdout.contains("property:endpoint.Alpha.AlphaSafety"),
"{selected_stdout}"
);
assert!(!selected_stdout.contains("BetaSafety"), "{selected_stdout}");
assert!(
selected_stdout.contains(&format!("\"seed\":\"{alpha_seed}\"")),
"{selected_stdout}"
);
assert!(
json_string_field(&selected_stdout, "repro")
.ends_with("--property 'property:endpoint.Alpha.AlphaSafety'"),
"{selected_stdout}"
);
}
#[test]
fn unscoped_seed_must_belong_to_the_only_declared_property() {
let fixture = Fixture::new("single-property-seed-identity", "1s");
let foreign_seed =
noxid_property_gen::property_seed("property:endpoint.Foreign.ForeignSafety", 7).to_string();
let output = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.args(["--json", "--seed", &foreign_seed])
.output()
.expect("run unscoped foreign-seed replay");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!output.status.success(), "{stderr}");
assert!(output.stdout.is_empty(), "{}", stdout(&output));
assert!(stderr.contains("PROPERTY_REPLAY_SEED_MISMATCH"), "{stderr}");
assert!(stderr.contains(&foreign_seed), "{stderr}");
assert!(
stderr.contains("property:endpoint.Qa.GetterSafety"),
"{stderr}"
);
assert!(
stderr.contains("copy the seed from a failing report for this property"),
"{stderr}"
);
let own_seed =
noxid_property_gen::property_seed("property:endpoint.Qa.GetterSafety", 7).to_string();
let replay = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.args(["--json", "--seed", &own_seed])
.output()
.expect("run unscoped matching-seed replay");
assert!(
!String::from_utf8_lossy(&replay.stderr).contains("PROPERTY_REPLAY_SEED_MISMATCH"),
"{}",
String::from_utf8_lossy(&replay.stderr)
);
}
#[test]
fn gate_refuses_a_seed_instead_of_silently_dropping_its_run_floor() {
let fixture = Fixture::new("gate-seed-conflict", "1s");
let output = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.args(["--json", "--gate", "--seed", "123"])
.output()
.expect("run gate plus seed conflict");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!output.status.success(), "{stderr}");
assert!(stderr.contains("PROPERTY_REPLAY_GATE_CONFLICT"), "{stderr}");
assert!(stderr.contains("100-run property floor"), "{stderr}");
assert!(stderr.contains("run the gate without --seed"), "{stderr}");
assert!(
stderr.contains("--seed <n> with --property <semantic-id> separately"),
"{stderr}"
);
}
#[test]
fn validator_cpu_budget_has_a_one_second_wall_clock_kill_backstop() {
let fixture = Fixture::new("wall-backstop", "50ms");
fs::write(
fixture.root.join("hook-entry.mjs"),
"import { register } from \"node:module\";\nregister(\"./hook.mjs\", import.meta.url);\n",
)
.expect("write wall-backstop loader entry");
fs::write(
fixture.root.join("hook.mjs"),
r#"export async function load(url, context, next) {
if (url.endsWith(".validators.js")) {
return { format: "module", shortCircuit: true, source: `
export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:endpoint.Qa.body": () => {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1200);
},
});
` };
}
return next(url, context);
}
"#,
)
.expect("write wall-backstop loader hook");
let output = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.env(
"NODE_OPTIONS",
format!("--import {}", fixture.root.join("hook-entry.mjs").display()),
)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.arg("--json")
.output()
.expect("run blocking validator against the wall backstop");
let stdout = stdout(&output);
assert!(!output.status.success(), "{stdout}");
assert!(stdout.contains("PROPERTY_TIMEOUT_EXCEEDED"), "{stdout}");
assert!(
stdout.contains("50ms budget plus the 1000ms runner kill allowance"),
"{stdout}"
);
assert!(
stdout.contains("outer allowance preserves killability"),
"{stdout}"
);
}
#[test]
fn validator_stdout_cannot_spoof_the_structured_outcome() {
let fixture = Fixture::new("stdout-spoof", "1s");
fs::write(
fixture.root.join("hook-entry.mjs"),
"import { register } from \"node:module\";\nregister(\"./hook.mjs\", import.meta.url);\n",
)
.expect("write stdout-spoof loader entry");
fs::write(
fixture.root.join("hook.mjs"),
r#"export async function load(url, context, next) {
if (url.endsWith(".validators.js")) {
return { format: "module", shortCircuit: true, source: `
export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:endpoint.Qa.body": () => {
process.stdout.write('{"outcome":"timeout"}\\n');
throw new Error("SPOOF_MUST_REMAIN_A_VIOLATION");
},
});
` };
}
return next(url, context);
}
"#,
)
.expect("write stdout-spoof loader hook");
let output = fixture
.command_with_validator(env!("CARGO_BIN_EXE_noxid"), None)
.env(
"NODE_OPTIONS",
format!("--import {}", fixture.root.join("hook-entry.mjs").display()),
)
.arg("test")
.arg(fixture.root.join("Qa.nox"))
.arg("--json")
.output()
.expect("run stdout-spoofing validator");
let stdout = stdout(&output);
assert!(!output.status.success(), "{stdout}");
assert!(stdout.contains("PROPERTY_INVARIANT_VIOLATED"), "{stdout}");
assert!(!stdout.contains("PROPERTY_TIMEOUT_EXCEEDED"), "{stdout}");
assert!(stdout.contains("SPOOF_MUST_REMAIN_A_VIOLATION"), "{stdout}");
}
#[test]
fn tight_timeout_still_kills_a_synchronously_hanging_validator() {
let fixture = Fixture::new("tight-timeout-hang", "50ms");
let hanging = fixture.root.join("hanging.validators.js");
fs::write(
&hanging,
r#"export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:endpoint.Qa.body": () => { while (true) {} },
});
"#,
)
.expect("write hanging validator");
let started = std::time::Instant::now();
let output = fixture.noxid_test(Some(&hanging));
let stdout = assert_failure(&output, "PROPERTY_TIMEOUT_EXCEEDED");
assert!(stdout.contains("was killed"), "{stdout}");
assert!(stdout.contains("\"seed\":\""), "{stdout}");
assert!(started.elapsed() < std::time::Duration::from_secs(5));
}