use noxid_ai_eval::AffectedTestSelection;
use noxid_codegen_js::emit_scenario_expression;
use noxid_ir::{
AgentDefinition, AgentScenarioTurn, ComponentDefinition, EndpointDefinition,
PropertyDefinition, QueueDefinition, QueueHandler, ScenarioGivenKind, SemanticExpr,
SemanticExprKind, SemanticId, SemanticTemplatePart, TaskDefinition, TaskHandler,
};
use noxid_property_gen::{GeneratedCase, GeneratedValue, Schema};
use noxid_source::{SourceFile, js_escape, json_escape};
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use crate::modules;
/// A scenario runs a compiler-owned task or queue handler outside the server
/// dispatcher, so the handler context is supplied here. Tasks execute under
/// the system principal at runtime (`__NOXID_SYSTEM_PRINCIPAL`) and a queue
/// scenario enqueues without a request, so `System` is the honest principal
/// for both — and the shape is exactly the frozen runtime value ADR 0137
/// rule 5 keeps unchanged.
const SCENARIO_SYSTEM_CONTEXT: &str = "Object.freeze({ principal: Object.freeze({ kind: \"system\", canonical: \"system\", scope: null, agent: null }) })";
static NEXT_SCRATCH_DIRECTORY: AtomicU64 = AtomicU64::new(0);
const SCENARIO_HARNESS_TIMEOUT: Duration = Duration::from_secs(300);
const PROPERTY_NODE_STARTUP_TIMEOUT: Duration = Duration::from_secs(5);
const PROPERTY_VALIDATION_KILL_ALLOWANCE: Duration = Duration::from_secs(1);
const PROPERTY_VALIDATION_MARKER: &str = ".noxid-property-validation-started";
const PROPERTY_OUTCOME_SENTINEL: &str = "__NOXID_PROPERTY_OUTCOME__:";
pub(crate) struct Options {
pub(crate) gate: bool,
pub(crate) json_only: bool,
}
pub(crate) struct ScenarioRunReport {
pub(crate) json: String,
pub(crate) stderr: String,
pub(crate) success: bool,
}
pub(crate) fn affected_report_json(
selection: &AffectedTestSelection,
execution: &ScenarioRunReport,
) -> String {
let changed = selection
.changed
.iter()
.map(|id| format!("\"{}\"", json_escape(id.as_str())))
.collect::<Vec<_>>()
.join(",");
let uncovered = selection
.uncovered_changes
.iter()
.map(|id| format!("\"{}\"", json_escape(id.as_str())))
.collect::<Vec<_>>()
.join(",");
format!(
"{{\"schemaVersion\":1,\"mode\":\"emitted-artifact\",\"ok\":{},\"changed\":[{changed}],\"selectedCount\":{},\"uncoveredChanges\":[{uncovered}],\"execution\":{}}}",
execution.success,
selection.scenarios.len(),
execution.json,
)
}
#[derive(Clone)]
struct TestComponent {
definition: ComponentDefinition,
source: SourceFile,
}
#[derive(Clone)]
struct TestEndpoint {
definition: EndpointDefinition,
source: SourceFile,
}
#[derive(Clone)]
struct TestAgent {
definition: AgentDefinition,
source: SourceFile,
}
#[derive(Clone)]
struct TestTask {
definition: TaskDefinition,
source: SourceFile,
}
#[derive(Clone)]
struct TestQueue {
definition: QueueDefinition,
source: SourceFile,
}
#[derive(Clone)]
struct TestProperty {
definition: PropertyDefinition,
boundary_kind: &'static str,
boundary_name: String,
validator_module: String,
validators: Vec<(SemanticId, Schema)>,
timeout: Duration,
}
struct PropertyExecution {
javascript: String,
}
struct ScratchDirectory(PathBuf);
#[derive(Clone, Copy)]
struct ScenarioCheckpoint<'a> {
phase: &'static str,
index: usize,
target: Option<&'a str>,
}
impl ScenarioCheckpoint<'_> {
fn as_javascript(self) -> String {
let target = self
.target
.map(|target| format!("\"{}\"", json_escape(target)))
.unwrap_or_else(|| "null".into());
format!(
"{{ phase: \"{}\", index: {}, target: {target} }}",
self.phase, self.index
)
}
}
impl Drop for ScratchDirectory {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
pub(crate) fn run(
input: &Path,
options: Options,
seed: Option<u64>,
property_selector: Option<&str>,
) -> Result<(), String> {
let contract_update = options
.gate
.then(|| crate::project::prepare_api_contract_gate(input))
.transpose()?
.flatten();
let output = report_from_output(execute_with_selection_and_seed(
input,
&options,
None,
seed,
property_selector,
)?)?;
println!("{}", output.json);
// The split is the contract, and `--json` does not change it: stdout is
// the one report line and everything the run logged — every trace record
// the emitted server wrote at any `[server] tracing` level, and every
// refusal it printed — stays NDJSON on stderr. `--json` only suppresses
// the harness's own human summary line, which it does at the source.
eprint!("{}", output.stderr);
if output.success {
if let Some(update) = contract_update {
crate::project::apply_api_contract_update(update)?;
}
Ok(())
} else {
Err("one or more Noxid scenarios failed".into())
}
}
fn execute(input: &Path, options: &Options) -> Result<Output, String> {
execute_with_selection_and_seed(input, options, None, None, None)
}
pub(crate) fn execute_report(input: &Path, options: &Options) -> Result<ScenarioRunReport, String> {
report_from_output(execute(input, options)?)
}
pub(crate) fn execute_selected_report(
input: &Path,
options: &Options,
selected: &BTreeSet<SemanticId>,
) -> Result<ScenarioRunReport, String> {
report_from_output(execute_with_selection(input, options, Some(selected))?)
}
fn report_from_output(output: Output) -> Result<ScenarioRunReport, String> {
let json = String::from_utf8(output.stdout)
.map_err(|error| format!("scenario runner returned non-UTF-8 JSON: {error}"))?;
let stderr = String::from_utf8(output.stderr)
.map_err(|error| format!("scenario runner returned non-UTF-8 diagnostics: {error}"))?;
Ok(ScenarioRunReport {
json: json.trim().into(),
stderr,
success: output.status.success(),
})
}
fn execute_with_selection(
input: &Path,
options: &Options,
selected: Option<&BTreeSet<SemanticId>>,
) -> Result<Output, String> {
execute_with_selection_and_seed(input, options, selected, None, None)
}
fn execute_with_selection_and_seed(
input: &Path,
options: &Options,
selected: Option<&BTreeSet<SemanticId>>,
seed: Option<u64>,
property_selector: Option<&str>,
) -> Result<Output, String> {
let root = project_root(input)?;
let analysis_options = crate::project::project_analysis_options(input);
let entries = scenario_entries(input, &root)?;
let scratch = scratch_directory()?;
let project_components = root.join("src/components");
let sibling_components = root.join("components");
let auto_components = if project_components.exists() {
project_components
} else {
sibling_components
};
let auto_components = auto_components
.exists()
.then_some(auto_components.as_path());
let mut runtime_imports = BTreeSet::from(["flush".to_string()]);
let mut components = BTreeMap::<SemanticId, TestComponent>::new();
let mut component_origins = BTreeMap::<SemanticId, PathBuf>::new();
let mut endpoint_sources = BTreeMap::<SemanticId, SourceFile>::new();
let mut endpoint_origins = BTreeMap::<SemanticId, PathBuf>::new();
let mut endpoint_scenario_ids = BTreeSet::<SemanticId>::new();
let mut agent_sources = BTreeMap::<SemanticId, SourceFile>::new();
let mut agent_scenario_ids = BTreeSet::<SemanticId>::new();
let mut tasks = BTreeMap::<SemanticId, TestTask>::new();
let mut task_origins = BTreeMap::<SemanticId, PathBuf>::new();
let mut queues = BTreeMap::<SemanticId, TestQueue>::new();
let mut queue_origins = BTreeMap::<SemanticId, PathBuf>::new();
let mut properties = BTreeMap::<SemanticId, TestProperty>::new();
let mut emitted_modules = BTreeSet::new();
for entry in entries {
let graph = modules::compile_module_graph_with_options(
&entry,
&root,
auto_components,
&BTreeMap::new(),
&analysis_options,
)?;
refuse_transitive_boundaries(&graph, selected)?;
for (_, module) in graph.modules() {
for diagnostic in &module.compilation.diagnostics {
if !options.json_only {
eprintln!("{}", diagnostic.render(&module.source));
}
}
if selected.is_none() {
collect_module_properties(module, &mut properties)?;
}
runtime_imports.extend(module.compilation.runtime_imports());
for component in &module.compilation.program.components {
let origin = module.source.path().to_path_buf();
if let Some(existing) = component_origins.get(&component.id) {
if existing != &origin {
return Err(format!(
"DUPLICATE_COMPONENT_ID: `{}` is declared by both {} and {}; project tests require one stable owner per component semantic ID",
component.id,
existing.display(),
origin.display()
));
}
} else {
component_origins.insert(component.id.clone(), origin);
}
}
for endpoint in &module.compilation.program.endpoints {
endpoint_scenario_ids.extend(
endpoint
.scenarios
.iter()
.map(|scenario| scenario.id.clone()),
);
let origin = module.source.path().to_path_buf();
if let Some(existing) = endpoint_origins.get(&endpoint.id) {
if existing != &origin {
return Err(format!(
"DUPLICATE_ENDPOINT_ID: `{}` is declared by both {} and {}; project tests require one stable owner per endpoint semantic ID",
endpoint.id,
existing.display(),
origin.display()
));
}
} else {
endpoint_origins.insert(endpoint.id.clone(), origin);
endpoint_sources.insert(endpoint.id.clone(), module.source.clone());
}
}
for agent in &module.compilation.program.agents {
agent_scenario_ids
.extend(agent.scenarios.iter().map(|scenario| scenario.id.clone()));
agent_sources
.entry(agent.id.clone())
.or_insert_with(|| module.source.clone());
}
for task in &module.compilation.program.tasks {
let origin = module.source.path().to_path_buf();
if let Some(existing) = task_origins.get(&task.id) {
if existing != &origin {
return Err(format!(
"DUPLICATE_TASK_ID: `{}` is declared by both {} and {}; project tests require one stable owner per task semantic ID",
task.id,
existing.display(),
origin.display()
));
}
continue;
}
task_origins.insert(task.id.clone(), origin);
let mut definition = task.clone();
if let Some(selected) = selected {
definition
.scenarios
.retain(|scenario| selected.contains(&scenario.id));
}
if !definition.scenarios.is_empty() {
tasks.insert(
definition.id.clone(),
TestTask {
definition,
source: module.source.clone(),
},
);
}
}
for queue in &module.compilation.program.queues {
let origin = module.source.path().to_path_buf();
if let Some(existing) = queue_origins.get(&queue.id) {
if existing != &origin {
return Err(format!(
"DUPLICATE_QUEUE_ID: `{}` is declared by both {} and {}; project tests require one stable owner per queue semantic ID",
queue.id,
existing.display(),
origin.display()
));
}
continue;
}
queue_origins.insert(queue.id.clone(), origin);
let mut definition = queue.clone();
if let Some(selected) = selected {
definition
.scenarios
.retain(|scenario| selected.contains(&scenario.id));
}
if !definition.scenarios.is_empty() {
queues.insert(
definition.id.clone(),
TestQueue {
definition,
source: module.source.clone(),
},
);
}
}
emit_compilation(&scratch.0, module, &mut emitted_modules)?;
for component in &module.compilation.program.components {
let mut definition = component.clone();
if let Some(selected) = selected {
definition
.scenarios
.retain(|scenario| selected.contains(&scenario.id));
}
if definition.scenarios.is_empty()
&& (selected.is_some() || definition.requirements.is_empty())
{
continue;
}
components
.entry(definition.id.clone())
.or_insert_with(|| TestComponent {
definition,
source: module.source.clone(),
});
}
}
}
let mut endpoints = BTreeMap::<SemanticId, TestEndpoint>::new();
let mut agents = BTreeMap::<SemanticId, TestAgent>::new();
let has_selected_endpoint_scenario = endpoint_scenario_ids
.iter()
.any(|scenario| selected.is_none_or(|selected| selected.contains(scenario)));
// An agent scenario drives `POST /_noxid/agents/<Agent>/runs`, which the
// same emitted server handler serves, so it needs the same artifact an
// endpoint scenario needs — and it needs the project's derived tool
// registry, which a single-file compilation of the route that declares the
// agent cannot see.
let has_selected_agent_scenario = agent_scenario_ids
.iter()
.any(|scenario| selected.is_none_or(|selected| selected.contains(scenario)));
let endpoint_handler_module = if (!endpoint_sources.is_empty()
&& has_selected_endpoint_scenario)
|| (!agent_sources.is_empty() && has_selected_agent_scenario)
{
let (project_input, _) = crate::project::source_import_context(input)?.ok_or_else(|| {
"error[ENDPOINT_SCENARIO_PROJECT_REQUIRED]: endpoint and agent scenarios execute the shipped file-based Fetch handler and therefore require an enclosing Noxid.toml project; place this endpoint under server/api/ or server/routes/ in a project and run `noxid test <project> --gate`".to_string()
})?;
let endpoint_out = scratch.0.join("endpoint-dist");
let surface =
crate::project::build_endpoint_scenario_artifact(&project_input, &endpoint_out)?;
install_endpoint_scenario_boundary_stubs(&endpoint_out)?;
for mut definition in surface.endpoints {
let Some(source) = endpoint_sources.get(&definition.id).cloned() else {
continue;
};
if let Some(selected) = selected {
definition
.scenarios
.retain(|scenario| selected.contains(&scenario.id));
}
if definition.scenarios.is_empty() {
continue;
}
endpoints.insert(definition.id.clone(), TestEndpoint { definition, source });
}
for mut definition in surface.agents {
let Some(source) = agent_sources.get(&definition.id).cloned() else {
continue;
};
if let Some(selected) = selected {
definition
.scenarios
.retain(|scenario| selected.contains(&scenario.id));
}
if definition.scenarios.is_empty() {
continue;
}
agents.insert(definition.id.clone(), TestAgent { definition, source });
}
Some("./endpoint-dist/server/handler.js")
} else {
None
};
write_file(
&scratch.0.join("noxid-runtime.js"),
&noxid_compiler_core::scenario_runtime_javascript_for_imports(&runtime_imports),
)?;
write_file(
&scratch.0.join("package.json"),
"{\"private\":true,\"type\":\"module\"}\n",
)?;
link_project_packages(&root, &scratch.0)?;
let property_execution = execute_properties(
&scratch.0,
input,
&properties,
options,
seed,
property_selector,
)?;
let harness = generate_harness(
&components,
&endpoints,
&agents,
&tasks,
&queues,
endpoint_handler_module,
&property_execution,
options,
)?;
let harness_path = scratch.0.join("scenarios.mjs");
write_file(&harness_path, &harness)?;
run_node_with_timeout(&harness_path, &scratch.0, SCENARIO_HARNESS_TIMEOUT)
}
fn run_node_with_timeout(
harness_path: &Path,
current_dir: &Path,
timeout: Duration,
) -> Result<Output, String> {
let mut command = Command::new("node");
command
.arg(harness_path)
.current_dir(current_dir)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
configure_node_process_group(&mut command);
let mut child = command
.spawn()
.map_err(|error| format!("cannot execute Node.js scenario harness: {error}"))?;
let mut stdout = child
.stdout
.take()
.ok_or_else(|| "Node.js scenario harness stdout was not piped".to_string())?;
let mut stderr = child
.stderr
.take()
.ok_or_else(|| "Node.js scenario harness stderr was not piped".to_string())?;
let stdout_drain = thread::spawn(move || {
let mut output = Vec::new();
stdout.read_to_end(&mut output).map(|_| output)
});
let stderr_drain = thread::spawn(move || {
let mut output = Vec::new();
stderr.read_to_end(&mut output).map(|_| output)
});
let started = Instant::now();
loop {
match child.try_wait() {
Ok(Some(status)) => {
let stdout = join_scenario_pipe(stdout_drain, "stdout")?;
let stderr = join_scenario_pipe(stderr_drain, "stderr")?;
return Ok(Output {
status,
stdout,
stderr,
});
}
Ok(None) if started.elapsed() < timeout => {
thread::sleep(Duration::from_millis(10));
}
Ok(None) => {
terminate_node_process_group(&mut child);
let _ = stdout_drain.join();
let _ = stderr_drain.join();
return Err(format!(
"SCENARIO_TIMEOUT_EXCEEDED: Node.js scenario harness exceeded its {}ms wall-clock budget and was killed; add a finite boundary timeout or remove the non-terminating scenario behavior",
timeout.as_millis()
));
}
Err(error) => {
terminate_node_process_group(&mut child);
let _ = stdout_drain.join();
let _ = stderr_drain.join();
return Err(format!(
"cannot inspect Node.js scenario harness status: {error}"
));
}
}
}
}
fn configure_node_process_group(command: &mut Command) {
#[cfg(unix)]
{
use std::os::unix::process::CommandExt as _;
command.process_group(0);
}
}
fn terminate_node_process_group(child: &mut Child) {
#[cfg(unix)]
{
let process_group = format!("-{}", child.id());
let _ = Command::new("kill")
.args(["-s", "KILL", &process_group])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
}
let _ = child.kill();
let _ = child.wait();
}
fn join_scenario_pipe(
drain: thread::JoinHandle<std::io::Result<Vec<u8>>>,
name: &str,
) -> Result<Vec<u8>, String> {
drain
.join()
.map_err(|_| {
format!("cannot collect Node.js scenario harness {name}: drain thread panicked")
})?
.map_err(|error| format!("cannot collect Node.js scenario harness {name}: {error}"))
}
fn run_node_with_validation_timeout(
harness_path: &Path,
current_dir: &Path,
validation_timeout: Duration,
) -> Result<Output, PropertyRunnerError> {
let marker = current_dir.join(PROPERTY_VALIDATION_MARKER);
match fs::remove_file(&marker) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => {
return Err(PropertyRunnerError::Other(format!(
"cannot reset property validation marker {}: {error}",
marker.display()
)));
}
}
let mut command = Command::new("node");
command
.arg(harness_path)
.current_dir(current_dir)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
configure_node_process_group(&mut command);
let mut child = command.spawn().map_err(|error| {
PropertyRunnerError::Other(format!("cannot execute Node.js property harness: {error}"))
})?;
let mut stdout = child.stdout.take().ok_or_else(|| {
PropertyRunnerError::Other("Node.js property harness stdout was not piped".into())
})?;
let mut stderr = child.stderr.take().ok_or_else(|| {
PropertyRunnerError::Other("Node.js property harness stderr was not piped".into())
})?;
let stdout_drain = thread::spawn(move || {
let mut output = Vec::new();
stdout.read_to_end(&mut output).map(|_| output)
});
let stderr_drain = thread::spawn(move || {
let mut output = Vec::new();
stderr.read_to_end(&mut output).map(|_| output)
});
let startup_started = Instant::now();
let mut validation_started = None;
loop {
match child.try_wait() {
Ok(Some(status)) => {
let stdout = join_property_pipe(stdout_drain, "stdout")?;
let stderr = join_property_pipe(stderr_drain, "stderr")?;
return Ok(Output {
status,
stdout,
stderr,
});
}
Ok(None) => {
if validation_started.is_none() && marker.is_file() {
validation_started = Some(Instant::now());
}
let kill_timeout = validation_timeout + PROPERTY_VALIDATION_KILL_ALLOWANCE;
if validation_started.is_some_and(|started| started.elapsed() >= kill_timeout) {
terminate_node_process_group(&mut child);
let _ = stdout_drain.join();
let _ = stderr_drain.join();
return Err(PropertyRunnerError::ValidationTimeout(format!(
"SCENARIO_TIMEOUT_EXCEEDED: boundary validation did not return within its {}ms budget plus the {}ms runner kill allowance and was killed; Node measures finite validator work internally, while this outer allowance preserves killability for a synchronous hang",
validation_timeout.as_millis(),
PROPERTY_VALIDATION_KILL_ALLOWANCE.as_millis()
)));
}
if validation_started.is_none()
&& startup_started.elapsed() >= PROPERTY_NODE_STARTUP_TIMEOUT
{
terminate_node_process_group(&mut child);
let _ = stdout_drain.join();
let _ = stderr_drain.join();
return Err(PropertyRunnerError::Unavailable(format!(
"Node.js property harness did not reach validation within its {}ms startup safety budget and was killed; retry when the Node runner is available",
PROPERTY_NODE_STARTUP_TIMEOUT.as_millis()
)));
}
thread::sleep(Duration::from_millis(10));
}
Err(error) => {
terminate_node_process_group(&mut child);
let _ = stdout_drain.join();
let _ = stderr_drain.join();
return Err(PropertyRunnerError::Other(format!(
"cannot inspect Node.js property harness status: {error}"
)));
}
}
}
}
fn join_property_pipe(
drain: thread::JoinHandle<std::io::Result<Vec<u8>>>,
name: &str,
) -> Result<Vec<u8>, PropertyRunnerError> {
drain
.join()
.map_err(|_| {
PropertyRunnerError::Other(format!(
"cannot collect Node.js property harness {name}: drain thread panicked"
))
})?
.map_err(|error| {
PropertyRunnerError::Other(format!(
"cannot collect Node.js property harness {name}: {error}"
))
})
}
enum PropertyRunnerError {
Unavailable(String),
ValidationTimeout(String),
Other(String),
}
fn collect_module_properties(
module: &noxid_workspace::CompiledModule,
properties: &mut BTreeMap<SemanticId, TestProperty>,
) -> Result<(), String> {
let stem = module
.source
.path()
.file_stem()
.and_then(|value| value.to_str())
.ok_or_else(|| format!("{} has no valid file stem", module.source.path().display()))?;
let validator_module = format!("./{stem}.validators.js");
let validation = &module.compilation.validation;
for endpoint in &module.compilation.program.endpoints {
let validators = validation
.endpoint_boundaries
.iter()
.filter(|boundary| {
boundary.endpoint == endpoint.id
&& boundary.section != noxid_validation_ir::EndpointValidationSection::Result
})
.filter_map(|boundary| {
validation
.validators
.iter()
.find(|validator| validator.id == boundary.validator)
.map(|validator| {
(
boundary.validator.clone(),
Schema::from_validator(&validator.node, &validation.validators),
)
})
})
.collect::<Vec<_>>();
for property in &endpoint.properties {
insert_test_property(
properties,
TestProperty {
definition: property.clone(),
boundary_kind: "endpoint",
boundary_name: endpoint.name.clone(),
validator_module: validator_module.clone(),
validators: validators.clone(),
timeout: Duration::from_millis(if endpoint.timeout.defaulted {
1_000
} else {
endpoint.timeout.milliseconds.max(50)
}),
},
)?;
}
}
for queue in &module.compilation.program.queues {
let validator_id = SemanticId::queue_validator(&queue.name);
let validators = validation
.validators
.iter()
.find(|validator| validator.id == validator_id)
.map(|validator| {
vec![(
validator_id,
Schema::from_validator(&validator.node, &validation.validators),
)]
})
.unwrap_or_default();
for property in &queue.properties {
insert_test_property(
properties,
TestProperty {
definition: property.clone(),
boundary_kind: "queue",
boundary_name: queue.name.clone(),
validator_module: validator_module.clone(),
validators: validators.clone(),
timeout: Duration::from_secs(1),
},
)?;
}
}
Ok(())
}
fn insert_test_property(
properties: &mut BTreeMap<SemanticId, TestProperty>,
property: TestProperty,
) -> Result<(), String> {
if let Some(existing) = properties.get(&property.definition.id) {
if existing.validator_module == property.validator_module {
return Ok(());
}
return Err(format!(
"DUPLICATE_PROPERTY_ID: `{}` is declared by both {} and {}; property identities require one stable owner",
property.definition.id, existing.validator_module, property.validator_module
));
}
properties.insert(property.definition.id.clone(), property);
Ok(())
}
enum PropertyCaseOutcome {
Pass,
Violation(String),
Timeout(String),
RunnerUnavailable(String),
}
fn execute_properties(
scratch: &Path,
input: &Path,
properties: &BTreeMap<SemanticId, TestProperty>,
options: &Options,
seed: Option<u64>,
property_selector: Option<&str>,
) -> Result<PropertyExecution, String> {
let replay_property = replay_property(properties, seed, property_selector)?;
let include_property_selector = properties.len() > 1;
let mut javascript = String::new();
for property in properties.values() {
if replay_property.is_some_and(|id| id != &property.definition.id) {
continue;
}
if property.validators.is_empty() {
return Err(format!(
"PROPERTY_NO_FUZZABLE_BOUNDARY: property `{}` has no generated input validator",
property.definition.name
));
}
let declared_runs = if options.gate {
property.definition.runs.max(100)
} else {
property.definition.runs
};
let runs = if seed.is_some() { 1 } else { declared_runs };
let mut failure = None;
for scheduled_run in 0..runs {
let run_index = seed.map_or(scheduled_run, |seed| seed as u32);
let validator_index = run_index as usize % property.validators.len();
let (validator, schema) = &property.validators[validator_index];
let case = seed.map_or_else(
|| {
noxid_property_gen::generate_case(
schema,
property.definition.id.as_str(),
run_index,
)
},
|seed| noxid_property_gen::generate_case_from_seed(schema, seed),
);
match execute_property_case(scratch, property, validator, &case.value)? {
PropertyCaseOutcome::Pass => {}
PropertyCaseOutcome::Timeout(message) => {
failure = Some(PropertyFailure::case(
run_index,
&case,
&case.value,
"PROPERTY_TIMEOUT_EXCEEDED",
&message,
input,
include_property_selector.then_some(property.definition.id.as_str()),
));
break;
}
PropertyCaseOutcome::RunnerUnavailable(message) => {
failure = Some(PropertyFailure::RunnerUnavailable { message });
break;
}
PropertyCaseOutcome::Violation(message) => {
let mut shrunk = case.value.clone();
for candidate in noxid_property_gen::shrink_candidates_bounded(&case.value, 128)
{
if matches!(
execute_property_case(scratch, property, validator, &candidate)?,
PropertyCaseOutcome::Violation(_)
) {
shrunk = candidate;
break;
}
}
failure = Some(PropertyFailure::case(
run_index,
&case,
&shrunk,
"PROPERTY_INVARIANT_VIOLATED",
&message,
input,
include_property_selector.then_some(property.definition.id.as_str()),
));
break;
}
}
}
javascript.push_str(&property_result_javascript(
property,
runs,
failure.as_ref(),
));
}
Ok(PropertyExecution { javascript })
}
fn replay_property<'a>(
properties: &'a BTreeMap<SemanticId, TestProperty>,
seed: Option<u64>,
property_selector: Option<&str>,
) -> Result<Option<&'a SemanticId>, String> {
if seed.is_none() {
return property_selector.map_or(Ok(None), |_| {
Err("PROPERTY_REPLAY_SEED_REQUIRED: --property selects a seeded replay; add --seed <n>, or omit --property to run every declared property".into())
});
}
if let Some(selector) = property_selector {
let property = properties
.keys()
.find(|id| id.as_str() == selector)
.ok_or_else(|| {
let available = properties
.keys()
.map(SemanticId::as_str)
.collect::<Vec<_>>()
.join(", ");
format!(
"PROPERTY_REPLAY_SELECTOR_UNKNOWN: --property `{selector}` does not name a property in this input; choose one of: {available}"
)
})?;
let seed = seed.expect("seed presence checked above");
let identity_mask = 0xffff_ffff_0000_0000;
if seed & identity_mask
!= noxid_property_gen::property_seed(property.as_str(), 0) & identity_mask
{
return Err(format!(
"PROPERTY_REPLAY_SEED_MISMATCH: --seed {seed} does not belong to --property `{selector}`; copy the seed and --property pair from the same failing report, or omit both flags to generate a new run"
));
}
return Ok(Some(property));
}
if properties.len() > 1 {
let available = properties
.keys()
.map(SemanticId::as_str)
.collect::<Vec<_>>()
.join(", ");
return Err(format!(
"PROPERTY_REPLAY_SELECTOR_REQUIRED: --seed on an input with multiple properties requires --property <semantic-id>; choose one of: {available}"
));
}
let property = properties.keys().next();
if let Some(property) = property {
let seed = seed.expect("seed presence checked above");
let identity_mask = 0xffff_ffff_0000_0000;
if seed & identity_mask
!= noxid_property_gen::property_seed(property.as_str(), 0) & identity_mask
{
return Err(format!(
"PROPERTY_REPLAY_SEED_MISMATCH: --seed {seed} does not belong to the only property `{property}` in this input; copy the seed from a failing report for this property, or omit --seed to generate a new run"
));
}
}
Ok(property)
}
enum PropertyFailure {
Case {
seed: u64,
run_index: u32,
code: &'static str,
message: String,
counterexample: String,
repro: String,
},
RunnerUnavailable {
message: String,
},
}
impl PropertyFailure {
fn case(
run_index: u32,
case: &GeneratedCase,
shrunk: &GeneratedValue,
code: &'static str,
message: &str,
input: &Path,
property_selector: Option<&str>,
) -> Self {
let property_argument = property_selector.map_or_else(String::new, |property| {
format!(" --property {}", shell_quote(property))
});
Self::Case {
seed: case.seed,
run_index,
code,
message: message.to_string(),
counterexample: shrunk.to_repro_json(),
repro: format!(
"noxid test {} --seed {}{}",
shell_quote(&input.to_string_lossy()),
case.seed,
property_argument,
),
}
}
}
fn shell_quote(value: &str) -> String {
format!("'{}'", value.replace('\'', "'\"'\"'"))
}
fn property_result_javascript(
property: &TestProperty,
runs: u32,
failure: Option<&PropertyFailure>,
) -> String {
let identity = format!(
"id: \"{}\", name: \"{}\", property: true, {}: \"{}\", semanticUnit: \"{}\", runs: {runs}, assertions: [], invariantFailures: [], covers: []",
json_escape(property.definition.id.as_str()),
json_escape(&property.definition.name),
property.boundary_kind,
json_escape(&property.boundary_name),
json_escape(property.definition.boundary.as_str()),
);
failure.map_or_else(
|| {
format!(
"scenarioResults.push({{ {identity}, status: \"pass\", seed: null, counterexample: null, repro: null, failure: null }});\n"
)
},
|failure| match failure {
PropertyFailure::Case {
seed,
run_index,
code,
message,
counterexample,
repro,
} => {
let message = format!(
"{code}: property `{}` run {run_index} escaped `{message}`; seed {seed}; shrunk counterexample {counterexample}; repro {repro}",
property.definition.name,
);
format!(
"scenarioResults.push({{ {identity}, status: \"fail\", seed: \"{seed}\", counterexample: \"{}\", repro: \"{}\", failure: \"{}\" }});\n",
js_escape(counterexample),
js_escape(repro),
js_escape(&message),
)
}
PropertyFailure::RunnerUnavailable { message } => {
let message = format!(
"PROPERTY_RUNNER_UNAVAILABLE: property `{}` could not run because `{message}`",
property.definition.name,
);
format!(
"scenarioResults.push({{ {identity}, status: \"fail\", seed: null, counterexample: null, repro: null, failure: \"{}\" }});\n",
js_escape(&message),
)
}
},
)
}
fn execute_property_case(
scratch: &Path,
property: &TestProperty,
validator: &SemanticId,
value: &GeneratedValue,
) -> Result<PropertyCaseOutcome, String> {
let script = format!(
r#"import {{ writeFileSync }} from "node:fs";
import {{ typeValidators, ExternalValidationError }} from "{}";
const validator = typeValidators["{}"];
const value = {};
writeFileSync("{}", "started", {{ encoding: "utf8" }});
const validationStarted = process.cpuUsage();
let result;
let escaped = false;
try {{
if (typeof validator !== "function") throw new Error("PROPERTY_VALIDATOR_MISSING");
validator(value, true);
result = {{ outcome: "accepted" }};
}} catch (error) {{
const structured = error instanceof ExternalValidationError
&& error.code === "EXTERNAL_VALIDATION_FAILED"
&& Array.isArray(error.path)
&& typeof error.expected === "string"
&& typeof error.actual === "string";
if (structured) result = {{ outcome: "refused", code: error.code }};
else {{
result = {{ outcome: "escaped", code: error?.code ?? null, message: error?.message ?? String(error) }};
escaped = true;
}}
}}
const validationUsage = process.cpuUsage(validationStarted);
const validationCpuMs = (validationUsage.user + validationUsage.system) / 1000;
if (validationCpuMs > {}) {{
result = {{ outcome: "timeout", cpuMs: validationCpuMs }};
escaped = true;
}}
process.stdout.write("\n{}" + JSON.stringify(result) + "\n");
if (escaped) process.exitCode = 1;
"#,
json_escape(&property.validator_module),
json_escape(validator.as_str()),
value.to_javascript(),
PROPERTY_VALIDATION_MARKER,
property.timeout.as_millis(),
PROPERTY_OUTCOME_SENTINEL,
);
let path = scratch.join("property-case.mjs");
write_file(&path, &script)?;
match run_node_with_validation_timeout(&path, scratch, property.timeout) {
Ok(output) => {
let stdout = String::from_utf8(output.stdout)
.map_err(|error| format!("property runner returned non-UTF-8 JSON: {error}"))?;
let Some((outcome, record)) = property_outcome(&stdout) else {
return Ok(PropertyCaseOutcome::Violation(format!(
"PROPERTY_RUNNER_PROTOCOL_INVALID: property runner did not emit its sentinel-prefixed final outcome record; stdout was {}",
stdout.trim()
)));
};
match outcome {
"timeout" => Ok(PropertyCaseOutcome::Timeout(format!(
"SCENARIO_TIMEOUT_EXCEEDED: boundary validation consumed more than {}ms of CPU after Node startup and module import; {record}",
property.timeout.as_millis(),
))),
"accepted" | "refused" if output.status.success() => Ok(PropertyCaseOutcome::Pass),
"escaped" => Ok(PropertyCaseOutcome::Violation(record.to_string())),
other => Ok(PropertyCaseOutcome::Violation(format!(
"PROPERTY_RUNNER_PROTOCOL_INVALID: outcome `{other}` disagreed with process status {}; record {record}",
output.status
))),
}
}
Err(PropertyRunnerError::Unavailable(error)) => {
Ok(PropertyCaseOutcome::RunnerUnavailable(error))
}
Err(PropertyRunnerError::ValidationTimeout(error)) => {
Ok(PropertyCaseOutcome::Timeout(error))
}
Err(PropertyRunnerError::Other(error)) => Err(error),
}
}
fn property_outcome(stdout: &str) -> Option<(&str, &str)> {
let record = stdout
.lines()
.rev()
.find_map(|line| line.strip_prefix(PROPERTY_OUTCOME_SENTINEL))?;
let outcome = record.strip_prefix("{\"outcome\":\"")?;
let outcome = outcome.split_once('"')?.0;
Some((outcome, record))
}
fn refuse_transitive_boundaries(
graph: &noxid_workspace::ModuleGraph,
selected: Option<&BTreeSet<SemanticId>>,
) -> Result<(), String> {
let definitions = graph
.modules()
.flat_map(|(_, module)| module.compilation.program.components.iter())
.map(|component| (component.name.as_str(), component))
.collect::<BTreeMap<_, _>>();
for component in definitions.values() {
let executable_scenarios = component.scenarios.iter().filter(|scenario| {
selected.is_none_or(|selected| selected.contains(&scenario.id))
&& (!scenario.typed_given.is_empty()
|| !scenario.typed_when.is_empty()
|| !scenario.typed_expect.is_empty())
});
for scenario in executable_scenarios {
for reachable_name in graph.reachable_components(&component.name) {
let Some(reachable) = definitions.get(reachable_name.as_str()) else {
continue;
};
if reachable.id == component.id {
continue;
}
if let Some(resource) = reachable.resources.first() {
return Err(format!(
"SCENARIO_RESOURCE_BOUNDARY_UNSTUBBED: scenario `{}` mounts component `{}` with unstubbed resource `{}`; put the scenario beside `{}` and add its lifecycle given there, or remove that child boundary from this deterministic scenario",
scenario.name, reachable.name, resource.name, resource.name
));
}
if let Some(stream) = reachable.streams.first() {
return Err(format!(
"SCENARIO_STREAM_BOUNDARY_UNSTUBBED: scenario `{}` mounts component `{}` with unstubbed stream `{}`; put the scenario beside `{}` and add its finite event given there, or remove that child boundary from this deterministic scenario",
scenario.name, reachable.name, stream.name, stream.name
));
}
if let Some(agent) = reachable.agents.first() {
return Err(format!(
"SCENARIO_AGENT_BOUNDARY_UNSTUBBED: scenario `{}` mounts component `{}` with unstubbed agent session `{}`; extract plain state/actions into a deterministic component until typed agent-session givens are supported",
scenario.name, reachable.name, agent.name
));
}
}
}
}
Ok(())
}
fn project_root(input: &Path) -> Result<PathBuf, String> {
let canonical = fs::canonicalize(input)
.map_err(|error| format!("cannot resolve {}: {error}", input.display()))?;
if canonical.is_dir() {
return Ok(canonical);
}
if canonical.file_name().and_then(|value| value.to_str()) == Some("Noxid.toml") {
return canonical
.parent()
.map(Path::to_path_buf)
.ok_or_else(|| format!("{} has no project directory", canonical.display()));
}
canonical
.parent()
.map(Path::to_path_buf)
.ok_or_else(|| format!("{} has no source directory", canonical.display()))
}
fn scenario_entries(input: &Path, root: &Path) -> Result<Vec<PathBuf>, String> {
let canonical = fs::canonicalize(input)
.map_err(|error| format!("cannot resolve {}: {error}", input.display()))?;
if canonical.extension().and_then(|value| value.to_str()) == Some("nox") {
return Ok(vec![canonical]);
}
let mut entries = Vec::new();
// Scan the project root rather than assuming `src/`: Noxid.toml may point
// routes and components at custom directories. Generated/dependency
// directories are pruned by `collect_noxid_files` below.
collect_noxid_files(root, &mut entries)?;
entries.sort();
if entries.is_empty() {
return Err(format!("{} contains no .nox source files", root.display()));
}
Ok(entries)
}
fn collect_noxid_files(directory: &Path, output: &mut Vec<PathBuf>) -> Result<(), String> {
let mut entries = fs::read_dir(directory)
.map_err(|error| format!("cannot read {}: {error}", directory.display()))?
.collect::<Result<Vec<_>, _>>()
.map_err(|error| format!("cannot read {}: {error}", directory.display()))?;
entries.sort_by_key(|entry| entry.path());
for entry in entries {
let path = entry.path();
if path.is_dir() {
if matches!(
path.file_name().and_then(|value| value.to_str()),
Some("target" | "dist" | "node_modules" | ".git")
) {
continue;
}
collect_noxid_files(&path, output)?;
} else if path.extension().and_then(|value| value.to_str()) == Some("nox") {
output.push(path);
}
}
Ok(())
}
fn scratch_directory() -> Result<ScratchDirectory, String> {
let ordinal = NEXT_SCRATCH_DIRECTORY.fetch_add(1, Ordering::Relaxed);
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|error| format!("system clock cannot create scenario directory: {error}"))?
.as_nanos();
let path = std::env::temp_dir().join(format!(
"noxid-scenarios-{}-{nonce}-{ordinal}",
std::process::id()
));
fs::create_dir_all(&path)
.map_err(|error| format!("cannot create {}: {error}", path.display()))?;
Ok(ScratchDirectory(path))
}
fn emit_compilation(
directory: &Path,
module: &noxid_workspace::CompiledModule,
emitted_modules: &mut BTreeSet<String>,
) -> Result<(), String> {
let stem = module
.source
.path()
.file_stem()
.and_then(|value| value.to_str())
.ok_or_else(|| format!("{} has no valid file stem", module.source.path().display()))?;
if let Some(generated) = &module.compilation.generated {
if generated.modules.is_empty() {
for component in &module.compilation.program.components {
let name = format!("{}.js", component.name);
if emitted_modules.insert(name.clone()) {
write_file(&directory.join(name), &generated.javascript)?;
}
}
} else {
for generated_module in &generated.modules {
let name = format!("{}.js", generated_module.component);
if emitted_modules.insert(name.clone()) {
write_file(&directory.join(name), &generated_module.javascript)?;
}
}
}
}
for (suffix, contents) in [
(
"validators",
module.compilation.generated_validators.as_deref(),
),
(
"resources",
module.compilation.generated_resources.as_deref(),
),
("streams", module.compilation.generated_streams.as_deref()),
("agents", module.compilation.generated_agents.as_deref()),
] {
if let Some(contents) = contents {
let name = format!("{stem}.{suffix}.js");
if emitted_modules.insert(name.clone()) {
write_file(&directory.join(name), contents)?;
}
}
}
Ok(())
}
/// The scenario harness runs the shipped server handler out of a scratch
/// directory, so bare package specifiers in emitted server modules (the vetted
/// Drizzle adapter imports `drizzle-orm` and `postgres`) have no resolution
/// root. Point one at the project's installed packages; an absent
/// `node_modules` is not an error here, because a project whose server modules
/// import nothing needs none, and one that does gets Node's own
/// `ERR_MODULE_NOT_FOUND` naming the package to install.
fn link_project_packages(project_root: &Path, scratch: &Path) -> Result<(), String> {
let packages = project_root.join("node_modules");
if !packages.is_dir() {
return Ok(());
}
let link = scratch.join("node_modules");
#[cfg(unix)]
let linked = std::os::unix::fs::symlink(&packages, &link);
#[cfg(windows)]
let linked = std::os::windows::fs::symlink_dir(&packages, &link);
linked.map_err(|error| {
format!(
"cannot link {} into the scenario scratch directory: {error}",
packages.display()
)
})
}
fn install_endpoint_scenario_boundary_stubs(out_dir: &Path) -> Result<(), String> {
let server_dir = out_dir.join("server");
// WO-30: a scenario that declares model givens is exercising its host
// implementation with the model as its only external boundary, so the
// project's real host is kept beside the stub and reached only for the
// exact host keys that scenario authorized. Every other boundary stays
// stubbed and live `fetch` stays banned by the harness.
let real_host = server_dir.join("host.js");
let delegate = server_dir.join("host.scenario-real.js");
let delegation = if real_host.is_file() {
fs::rename(&real_host, &delegate).map_err(|error| {
format!("cannot set aside the project host for scenario delegation: {error}")
})?;
r#" const delegated = globalThis.__NOXID_MODEL_SCENARIO__?.hostDelegation;
if (delegated?.has(String(key))) {
const host = await import("./host.scenario-real.js");
const table = host.endpoints ?? host.actions ?? host.default ?? Object.create(null);
const implementation = table[String(key)];
if (typeof implementation !== "function") {
throw new Error(`SCENARIO_MODEL_HOST_MISSING: ${String(key)} declares model givens but the project host implements no ${String(key)}`);
}
return implementation(...args);
}
"#
} else {
""
};
write_file(
&server_dir.join("host.js"),
&format!(
r#"const unavailable = (key) => {{
throw new Error(`SCENARIO_ENDPOINT_BOUNDARY_UNSTUBBED: bodyless endpoint ${{key}} has no exact typed scenario given`);
}};
export const actions = new Proxy(Object.create(null), {{
has() {{ return true; }},
getOwnPropertyDescriptor() {{ return {{ configurable: true, enumerable: true }}; }},
get(_target, key) {{
return async (...args) => {{
{delegation} const stubs = globalThis.__NOXID_ENDPOINT_SCENARIO_STUBS__;
return stubs?.has(String(key)) ? structuredClone(stubs.get(String(key))) : unavailable(String(key));
}};
}}
}});
// Deny by default, exactly as before, with two declared exceptions. An agent
// scenario is the first: while one is running, the capabilities it declared
// deferred answer `"defer"` once and every other capability answers `true`,
// which is how `given: authorizer defers <capability>` produces a real
// `PermissionRequired` pause through the ordinary authorizer path.
//
// Otherwise capability authorization is the host's decision, and `authorize`
// is a host export, so it is stubbed with the rest of the host. A scenario
// that declares this endpoint's boundary given has stubbed the whole host, its
// authorizer included, and gets the capabilities the declaration names; so has
// a WO-30 scenario whose model givens delegate this key to the project host. A
// scenario that declares no boundary for the endpoint has no host at all, so
// its authority is undeclared and still fails closed with
// ENDPOINT_CAPABILITY_DENIED.
export async function authorize(request) {{
const decide = globalThis.__NOXID_MODEL_SCENARIO__?.authorizeAgentCapability;
// The agent authorizer answers `true` or `"defer"` only while an agent
// scenario is running; `false` is exactly "no agent scenario is active", so
// it is the fall-through into the endpoint-scenario stub check rather than a
// denial. Endpoint scenarios and agent scenarios never both own a run.
const agentDecision =
typeof decide === "function" ? decide(request?.capability) : false;
if (agentDecision === true || agentDecision === "defer") return agentDecision;
const key = String(request?.semanticId);
return (
globalThis.__NOXID_ENDPOINT_SCENARIO_STUBS__?.has(key) === true ||
globalThis.__NOXID_MODEL_SCENARIO__?.hostDelegation?.has(key) === true
);
}}
export default actions;
"#
),
)?;
write_file(
&server_dir.join("middleware.js"),
r#"const allow = async () => ({ allow: true });
export const middleware = new Proxy(Object.create(null), { get() { return allow; } });
export const globalMiddlewareHandlers = Object.freeze({});
export const globalMiddleware = Object.freeze([]);
"#,
)?;
install_scenario_io_boundary(&server_dir)
}
/// The delegated host runs real project code, so every persistent boundary it
/// could still reach has to be closed here or the scenario's "no I/O"
/// determinism is a claim rather than a fact.
///
/// `storage(...)` keeps working, because a host that cannot store anything is
/// not the host the project ships; it is redirected to an in-memory store
/// created for the scenario run and dropped with the process, so nothing
/// survives to the next scenario or to the developer's disk. Queue enqueue and
/// the database adapter have no such ephemeral equivalent — a fake queue or a
/// fake table would answer questions the real one would answer differently —
/// so they refuse with `MODEL_SCENARIO_IO_FORBIDDEN` naming the call. Live
/// `fetch` stays banned by the harness itself.
fn install_scenario_io_boundary(server_dir: &Path) -> Result<(), String> {
let bridge = server_dir.join("noxid-server.js");
if bridge.is_file() {
let source = fs::read_to_string(&bridge)
.map_err(|error| format!("cannot read the emitted server bridge: {error}"))?;
write_file(&bridge, &scenario_server_bridge(&source)?)?;
}
// The database adapter is compiler-owned and identified the same way the
// build identifies it: it is the module that installs the runtime principal
// authority. Replacing it keeps the real driver out of the process
// entirely, so a scenario cannot open a connection by accident.
for module in server_modules_under(server_dir)? {
let source = fs::read_to_string(&module)
.map_err(|error| format!("cannot read {}: {error}", module.display()))?;
if !source.contains("__installNoxidPrincipalAuthority") {
continue;
}
write_file(&module, &scenario_database_refusal(&source))?;
}
Ok(())
}
fn server_modules_under(directory: &Path) -> Result<Vec<PathBuf>, String> {
let mut modules = Vec::new();
let mut pending = vec![directory.join("modules")];
while let Some(current) = pending.pop() {
let Ok(entries) = fs::read_dir(¤t) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
pending.push(path);
} else if path.extension().and_then(|value| value.to_str()) == Some("js") {
modules.push(path);
}
}
}
modules.sort();
Ok(modules)
}
/// Rewrite the emitted `noxid:server` bridge so its storage is ephemeral and
/// its queue enqueue refuses. The two markers are compiler-generated text, so a
/// missing one means the generator changed shape and this transform would
/// silently leave a live boundary open — fail closed instead.
fn scenario_server_bridge(source: &str) -> Result<String, String> {
const STORAGE: &str = "export function storage(namespace) {";
const ENQUEUE: &str = "export async function enqueue(queue, payload, options) {";
let mut rewritten = source.to_string();
if rewritten.contains(STORAGE) {
rewritten = rewritten.replace(STORAGE, "function __noxidPersistentStorage(namespace) {");
} else {
return Err(
"error[SCENARIO_IO_BOUNDARY_UNRECOGNIZED]: the emitted `noxid:server` bridge no longer declares `storage`, so scenario runs cannot prove they reach no persistent store".into(),
);
}
if rewritten.contains(ENQUEUE) {
rewritten = rewritten.replace(
ENQUEUE,
"async function __noxidPersistentEnqueue(queue, payload, options) {",
);
} else {
return Err(
"error[SCENARIO_IO_BOUNDARY_UNRECOGNIZED]: the emitted `noxid:server` bridge no longer declares `enqueue`, so scenario runs cannot prove they reach no durable queue".into(),
);
}
rewritten.push_str(SCENARIO_IO_BRIDGE);
Ok(rewritten)
}
const SCENARIO_IO_BRIDGE: &str = r#"
// WO-30 scenario I/O boundary. `noxid test` replaces the persistent storage
// and queue surfaces with these, so a delegated host reaches no boundary that
// outlives the run. The persistent implementations above are kept only so the
// emitted module still parses as the compiler wrote it.
void __noxidPersistentStorage;
void __noxidPersistentEnqueue;
// The store is looked up per call, never captured, so the harness can drop it
// between scenarios: one scenario's writes must not be another's fixture, the
// same way a component scenario always remounts.
function __noxidScenarioStore() {
let store = globalThis.__NOXID_SCENARIO_STORAGE__;
if (store === undefined || store === null) {
store = new Map();
globalThis.__NOXID_SCENARIO_STORAGE__ = store;
}
return store;
}
function __noxidScenarioIoForbidden(call, instead) {
return Object.assign(
new Error(`MODEL_SCENARIO_IO_FORBIDDEN: a scenario called \`${call}\`, which reaches a boundary that outlives the run; ${instead}`),
{ code: "MODEL_SCENARIO_IO_FORBIDDEN", call },
);
}
export function storage(namespace) {
__assertName(namespace, "namespace");
const bucket = () => {
const store = __noxidScenarioStore();
let records = store.get(namespace);
if (!records) {
records = new Map();
store.set(namespace, records);
}
return records;
};
const read = (records, key) => {
const record = records.get(key);
if (!record) return null;
if (record.expiresAt !== null && record.expiresAt <= Date.now()) {
records.delete(key);
return null;
}
return __cloneJson(record.value);
};
return Object.freeze({
async get(key) {
__assertName(key, "key");
return read(bucket(), key);
},
async set(key, value, options) {
__assertName(key, "key");
bucket().set(key, { value: __cloneJson(value), expiresAt: __expiresAt(options) });
},
async compareAndSet(key, expected, value, options) {
__assertName(key, "key");
__assertExpected(expected);
const copied = __cloneJson(value);
const expiresAt = __expiresAt(options);
const records = bucket();
// No `await` between the read and the write, exactly as the ephemeral
// driver this stands in for.
if (!__expectedMatch(read(records, key), expected)) return false;
records.set(key, { value: copied, expiresAt });
return true;
},
async delete(key) {
__assertName(key, "key");
return bucket().delete(key);
},
async list(prefix = "") {
__assertName(prefix, "list prefix", true);
const records = bucket();
const keys = [];
for (const [key, record] of records) {
if (record.expiresAt !== null && record.expiresAt <= Date.now()) {
records.delete(key);
} else if (key.startsWith(prefix)) {
keys.push(key);
}
}
return Object.freeze(keys.sort());
},
});
}
export async function enqueue(queue, payload, options) {
void payload;
void options;
throw __noxidScenarioIoForbidden(
`enqueue(${typeof queue === "string" ? JSON.stringify(queue) : String(queue)})`,
"a scenario cannot hand work to a durable queue, because the work would outlive it; assert the handler through that queue's own scenario instead",
);
}
"#;
/// Replace the compiler-owned database adapter with one that refuses. Every
/// name the real module exports is re-exported as a refusal, so a host that
/// imports the adapter still links and fails with a structured code at the
/// call rather than with a module-resolution error.
fn scenario_database_refusal(source: &str) -> String {
let mut names = BTreeSet::new();
for line in source.lines() {
let rest = match line.trim_start().strip_prefix("export ") {
Some(rest) => rest,
None => continue,
};
let rest = rest.strip_prefix("async ").unwrap_or(rest);
let Some(rest) = ["function ", "class ", "const ", "let ", "var "]
.iter()
.find_map(|keyword| rest.strip_prefix(keyword))
else {
continue;
};
let name = rest
.trim_start()
.split(|character: char| !character.is_alphanumeric() && character != '_')
.find(|segment| !segment.is_empty())
.unwrap_or_default();
if !name.is_empty() {
names.insert(name.to_string());
}
}
let mut output = String::from(
r#"// WO-30 scenario I/O boundary. `noxid test` replaces the compiler-owned
// database adapter with this module, so the real driver is never imported and
// a scenario cannot open a connection. Every export the adapter declares is
// present, and every one of them refuses.
function __noxidScenarioDatabaseRefusal(call) {
const refuse = (...args) => {
void args;
throw Object.assign(
new Error(`MODEL_SCENARIO_IO_FORBIDDEN: a scenario called \`${call}\` on the database adapter, which reaches a boundary that outlives the run; give the endpoint that owns this data an exact typed scenario given instead`),
{ code: "MODEL_SCENARIO_IO_FORBIDDEN", call },
);
};
return refuse;
}
"#,
);
for name in &names {
// The generated handler installs the principal authority at module
// load, before any scenario runs, so that one export is a no-op rather
// than a refusal: refusing it would fail the whole harness at import
// instead of failing the call that actually reached for data.
if name == "__installNoxidPrincipalAuthority" {
output
.push_str("export function __installNoxidPrincipalAuthority() { return null; }\n");
continue;
}
output.push_str(&format!(
"export const {name} = __noxidScenarioDatabaseRefusal(\"{name}\");\n"
));
}
output.push_str("export default __noxidScenarioDatabaseRefusal(\"default\");\n");
output
}
#[allow(clippy::too_many_arguments)]
fn generate_harness(
components: &BTreeMap<SemanticId, TestComponent>,
endpoints: &BTreeMap<SemanticId, TestEndpoint>,
agents: &BTreeMap<SemanticId, TestAgent>,
tasks: &BTreeMap<SemanticId, TestTask>,
queues: &BTreeMap<SemanticId, TestQueue>,
endpoint_handler_module: Option<&str>,
property_execution: &PropertyExecution,
options: &Options,
) -> Result<String, String> {
let mut javascript = String::new();
javascript.push_str(noxid_runtime::NODE_TEST_DOM);
javascript.push_str(
r#"
const runtime = await import("./noxid-runtime.js");
const registrations = new Map();
globalThis.__NOXID_HMR__ = {
initialState(_semanticId, initial) { return initial; },
registerInstance(instance) { registrations.set(instance.component, instance); },
};
// The `noxid test` output contract: stdout carries exactly one line, the JSON
// report. Everything the code under test logs goes to stderr — including every
// trace record the emitted server writes at any `[server] tracing` level, which
// stays NDJSON, one record per line. Without this split a project with
// `tracing = "full"` interleaves trace records with the report and nothing can
// parse stdout as JSON. The report is written through the captured writer so
// the redirect cannot swallow it.
const __noxidWriteReport = (line) => process.stdout.write(line + "\n");
console.log = (...args) => { console.error(...args); };
const scenarioResults = [];
const proseOnlyScenarios = [];
const requirementEntries = [];
globalThis.__NOXID_ENDPOINT_SCENARIO_STUBS__ = new Map();
globalThis.fetch = async () => { throw new Error("SCENARIO_LIVE_IO_FORBIDDEN: endpoint scenarios cannot perform live fetch I/O; add an exact typed given for the boundary"); };
// The WO-30 no-I/O model controller. It is installed for the whole harness,
// not per scenario, so a model call from any scenario is served from a
// declared stub or fails closed with the stubbing syntax — it can never
// reach a provider.
const modelScenarioStubs = new Map();
const modelScenarioDelegation = new Set();
// The compiler-owned semantic id of the declaration whose scenario is running.
// It is what makes a MODEL_STUB_REQUIRED refusal say *which* call had no stub
// left, in the runtime error and in the structured scenario report alike.
let modelScenarioCallSite = "";
const modelScenarioRefusals = [];
// A refusal is a run log, so it is written where the run's logs go: one NDJSON
// record on stderr, beside the emitted server's trace stream. The report says
// *that* a stub was missing; this says which call asked for it, at the moment
// it asked, and it is the only external evidence that a scripted run failed
// closed instead of reaching a provider.
function recordModelStubRefusal(model) {
modelScenarioRefusals.push({ model, callSite: modelScenarioCallSite });
console.error(JSON.stringify({
schema: "noxid.scenario.refusal.v1",
code: "MODEL_STUB_REQUIRED",
model,
callSite: modelScenarioCallSite,
}));
}
// The WO-31 agent side of the same controller. An agent turn is scripted per
// agent rather than per model because the engine calls the provider itself,
// with the registry's tool schemas attached; one scripted entry is one whole
// provider turn, assembled from a run of `text` entries and at most one
// `tool`/`final` entry — exactly the shape a real turn has.
const agentScenarioState = {
active: false, agent: "", script: [], cursor: 0, turn: 0,
deferred: new Set(), deferredSeen: new Set(), toolCalls: [],
};
globalThis.__NOXID_MODEL_SCENARIO__ = {
hostDelegation: modelScenarioDelegation,
callSite: modelScenarioCallSite,
take(model) {
const queue = modelScenarioStubs.get(model);
if (queue === undefined || queue.length === 0) {
recordModelStubRefusal(model);
return null;
}
return queue.shift();
},
takeAgentTurn(agent) {
if (!agentScenarioState.active || agentScenarioState.agent !== agent) {
recordModelStubRefusal(`agent:${agent}`);
return null;
}
if (agentScenarioState.cursor >= agentScenarioState.script.length) {
recordModelStubRefusal(`agent:${agent}`);
return null;
}
const text = [];
let call = null;
while (agentScenarioState.cursor < agentScenarioState.script.length) {
const entry = agentScenarioState.script[agentScenarioState.cursor];
agentScenarioState.cursor += 1;
if (entry.kind === "text") { text.push(entry.text); continue; }
call = entry;
break;
}
const index = agentScenarioState.turn;
agentScenarioState.turn += 1;
return { index, text, call, inputTokens: 0, outputTokens: 0 };
},
// Three-valued, like the host authorizer it stands in for. A capability the
// scenario declared deferred defers the *first* time the run asks for it and
// is allowed afterwards, which is the human-in-the-loop shape: the pause
// asks a person, and the resume re-asks with that person's answer.
authorizeAgentCapability(capability) {
if (!agentScenarioState.active) return false;
if (agentScenarioState.deferred.has(capability) && !agentScenarioState.deferredSeen.has(capability)) {
agentScenarioState.deferredSeen.add(capability);
return "defer";
}
return true;
},
recordAgentToolCall(record) {
if (agentScenarioState.active) agentScenarioState.toolCalls.push(record);
},
};
function installAgentScenario(agent, script, deferred) {
agentScenarioState.active = true;
agentScenarioState.agent = agent;
agentScenarioState.script = script;
agentScenarioState.cursor = 0;
agentScenarioState.turn = 0;
agentScenarioState.deferred = new Set(deferred);
agentScenarioState.deferredSeen = new Set();
agentScenarioState.toolCalls = [];
}
function clearAgentScenario() { agentScenarioState.active = false; }
function agentScenarioToolCalls() { return agentScenarioState.toolCalls; }
// How many provider turns the run actually consumed from the script.
function agentScenarioTurnCount() { return agentScenarioState.turn; }
// One SSE body, decoded into the `{ tag, value }` events the engine emitted.
// A `noxid-error` frame is the run's terminal refusal.
async function readAgentEvents(response, sink) {
const text = await response.text();
for (const frame of text.split("\n\n")) {
if (frame.trim().length === 0) continue;
let name = "message";
const data = [];
for (const line of frame.split("\n")) {
if (line.startsWith("event: ")) name = line.slice(7);
else if (line.startsWith("data: ")) data.push(line.slice(6));
}
if (data.length === 0) continue;
let payload = null;
try { payload = JSON.parse(data.join("\n")); } catch { continue; }
if (name === "noxid-error") { sink.refusal = payload?.code ?? payload?.error?.code ?? "STREAM_ERROR"; continue; }
if (payload === null || typeof payload !== "object" || typeof payload.tag !== "string") continue;
sink.events.push(payload.tag);
if (payload.tag === "Completed") sink.output = payload.value ?? null;
if (payload.tag === "Failed") sink.refusal = payload.value?.code ?? "AGENT_RUN_FAILED";
if (payload.tag === "Paused") { sink.paused = true; sink.runId = payload.value ?? null; }
}
}
function agentValuesEqual(left, right) { return endpointValuesEqual(left, right); }
function installModelStubs(stubs, delegated, callSite = "") {
modelScenarioStubs.clear();
modelScenarioDelegation.clear();
modelScenarioRefusals.length = 0;
// The scenario storage bridge keeps its records here; dropping them is what
// makes the ephemeral store per-scenario rather than per-process.
globalThis.__NOXID_SCENARIO_STORAGE__?.clear();
modelScenarioCallSite = callSite;
globalThis.__NOXID_MODEL_SCENARIO__.callSite = callSite;
clearAgentScenario();
for (const stub of stubs) {
const queue = modelScenarioStubs.get(stub.model) ?? [];
queue.push(stub);
modelScenarioStubs.set(stub.model, queue);
}
for (const key of delegated) modelScenarioDelegation.add(key);
}
function modelStubRefusals() {
return modelScenarioRefusals.map((refusal) => ({ model: refusal.model, callSite: refusal.callSite }));
}
function actualValues(scope, references) {
const values = {};
for (const [semanticId, name] of references) {
try {
const value = scope[name]?.get?.();
values[semanticId] = semanticId.startsWith("stream-use:")
? value?.map?.(($noxEnvelope) => $noxEnvelope.event)
: value;
}
catch (error) { values[semanticId] = { unreadable: error?.message ?? String(error) }; }
}
return values;
}
function failureMessage(error) { return error?.message ?? String(error); }
function endpointQueryValue(value) {
return typeof value === "object" ? JSON.stringify(value) : String(value);
}
function endpointValuesEqual(left, right) {
if (Object.is(left, right)) return true;
if (Array.isArray(left) || Array.isArray(right)) return Array.isArray(left) && Array.isArray(right) && left.length === right.length && left.every((value, index) => endpointValuesEqual(value, right[index]));
if (left && right && typeof left === "object" && typeof right === "object") {
const leftKeys = Object.keys(left).sort();
const rightKeys = Object.keys(right).sort();
return leftKeys.length === rightKeys.length && leftKeys.every((key, index) => key === rightKeys[index] && endpointValuesEqual(left[key], right[key]));
}
return false;
}
"#,
);
javascript.push_str(&property_execution.javascript);
for (index, test_component) in components.values().enumerate() {
let component = &test_component.definition;
let module_var = format!("componentModule{index}");
javascript.push_str(&format!(
"const {module_var} = await import(\"./{}.js\");\n",
json_escape(&component.name)
));
for requirement in &component.requirements {
javascript.push_str(&format!(
"requirementEntries.push([\"{}\", \"{}\"]);\n",
json_escape(&component.name),
json_escape(requirement.id.as_str())
));
}
for scenario in &component.scenarios {
let has_typed_steps = !scenario.typed_given.is_empty()
|| !scenario.typed_when.is_empty()
|| !scenario.typed_expect.is_empty();
let has_prose = !scenario.given.is_empty()
|| !scenario.when.is_empty()
|| !scenario.expect.is_empty();
if !has_typed_steps {
let reason = if has_prose {
"prose steps are not executable"
} else {
"scenario has no executable typed steps"
};
javascript.push_str(&format!(
"proseOnlyScenarios.push(\"{}\");\nscenarioResults.push({{ id: \"{}\", name: \"{}\", component: \"{}\", status: \"unsupported\", assertions: [], failure: \"{reason}\", covers: {} }});\n",
json_escape(scenario.id.as_str()),
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&component.name),
ids_as_javascript(&scenario.covers),
));
continue;
}
javascript.push_str(&scenario_script(
&module_var,
component,
scenario,
&test_component.source,
)?);
}
}
if let Some(endpoint_handler_module) = endpoint_handler_module {
javascript.push_str(&format!(
"const endpointModule = await import(\"{}\");\n",
json_escape(endpoint_handler_module)
));
let mut scenario_ordinal = 1usize;
for test_endpoint in endpoints.values() {
let endpoint = &test_endpoint.definition;
for scenario in &endpoint.scenarios {
javascript.push_str(&endpoint_scenario_script(
endpoint,
scenario,
&test_endpoint.source,
scenario_ordinal,
)?);
scenario_ordinal += 1;
}
}
for test_agent in agents.values() {
let agent = &test_agent.definition;
for scenario in &agent.scenarios {
javascript.push_str(&agent_scenario_script(
agent,
scenario,
&test_agent.source,
scenario_ordinal,
)?);
scenario_ordinal += 1;
}
}
}
for test_task in tasks.values() {
for scenario in &test_task.definition.scenarios {
javascript.push_str(&task_scenario_script(
&test_task.definition,
scenario,
&test_task.source,
)?);
}
}
for test_queue in queues.values() {
for scenario in &test_queue.definition.scenarios {
javascript.push_str(&queue_scenario_script(
&test_queue.definition,
scenario,
&test_queue.source,
)?);
}
}
javascript.push_str(&format!(
r#"
const passedCoverage = new Set(scenarioResults.filter((scenario) => scenario.status === "pass").flatMap((scenario) => scenario.covers.map((id) => `${{scenario.component}}\u0000${{id}}`)));
const uncoveredRequirementEntries = {} ? requirementEntries.filter(([component, id]) => !passedCoverage.has(`${{component}}\u0000${{id}}`)) : [];
const uncoveredRequirements = uncoveredRequirementEntries.map(([, id]) => id);
const duplicateRequirementIds = new Set(requirementEntries.filter(([, id], index, entries) => entries.findIndex(([, candidate]) => candidate === id) !== index).map(([, id]) => id));
const uncoveredRequirementDeclarations = uncoveredRequirementEntries.filter(([, id]) => duplicateRequirementIds.has(id)).map(([component, id]) => ({{ component, id }}));
const failed = scenarioResults.filter((scenario) => scenario.status === "fail").length;
const passed = scenarioResults.filter((scenario) => scenario.status === "pass").length;
const unsupported = scenarioResults.filter((scenario) => scenario.status === "unsupported").length;
const gateFailed = {} && (uncoveredRequirements.length > 0 || proseOnlyScenarios.length > 0);
const report = {{
schemaVersion: 1,
ok: failed === 0 && !gateFailed,
summary: {{ total: scenarioResults.length, passed, failed, unsupported }},
scenarios: scenarioResults,
gate: {{ enabled: {}, uncoveredRequirements, ...(uncoveredRequirementDeclarations.length > 0 ? {{ uncoveredRequirementDeclarations }} : {{}}), proseOnlyScenarios }},
}};
__noxidWriteReport(JSON.stringify(report));
if (!{}) console.error(`noxid test: ${{passed}} passed, ${{failed}} failed, ${{unsupported}} unsupported${{gateFailed ? ", gate failed" : ""}}`);
if (!report.ok) process.exitCode = 1;
"#,
options.gate, options.gate, options.gate, options.json_only
));
Ok(javascript)
}
fn queue_scenario_script(
queue: &QueueDefinition,
scenario: &noxid_ir::QueueScenario,
source: &SourceFile,
) -> Result<String, String> {
let payload = scenario
.payload
.iter()
.map(|argument| {
Ok((
argument.name.as_str(),
emit_scenario_expression(&argument.value)?,
))
})
.collect::<Result<BTreeMap<_, _>, String>>()?;
let invocation = match &queue.handler {
QueueHandler::CompilerOwned { statements, .. } => {
let arguments = queue
.payload
.iter()
.map(|field| {
let value = payload
.get(field.name.as_str())
.cloned()
.unwrap_or_else(|| "null".into());
format!("\"{}\": structuredClone({value})", json_escape(&field.name))
})
.collect::<Vec<_>>()
.join(", ");
let statements =
noxid_codegen_server_js::compiler_statements_javascript(statements, 4)?;
format!(
"await (async () => {{\n const args = Object.freeze({{ {arguments} }});\n const context = {SCENARIO_SYSTEM_CONTEXT};\n{statements}\n}})()"
)
}
QueueHandler::Host { key } => match scenario
.typed_given
.iter()
.find(|given| &given.target == key)
{
Some(given) => format!(
"structuredClone({})",
emit_scenario_expression(&given.value)?
),
None => format!(
"(() => {{ throw new Error(\"SCENARIO_QUEUE_BOUNDARY_UNSTUBBED: bodyless queue {} has no exact typed scenario given\"); }})()",
json_escape(key.as_str())
),
},
};
let mut script = format!(
"{{\n{} const $noxQueueTest = {{ failure: null, assertions: [] }};\n try {{\n const $noxQueueValue = {invocation};\n const value = {{ get() {{ return $noxQueueValue; }} }};\n const refusal = {{ get() {{ return \"\"; }} }};\n const attempts = {{ get() {{ return 1; }} }};\n const runAt = {{ get() {{ return new Date(\"{}\"); }} }};\n const $noxScope = {{ value, refusal, attempts, runAt }};\n",
model_stub_script(&scenario.model_stubs, &[], queue.id.as_str())?,
json_escape(&scenario.clock),
);
for expectation in &scenario.expectations {
let mut references = BTreeSet::new();
collect_references(expectation, &mut references);
let reference_pairs = reference_pairs_as_javascript(&references);
let expression_source = source.slice(expectation.span).trim();
script.push_str(&format!(
" {{ const $noxActual = actualValues($noxScope, [{reference_pairs}]); const $noxPassed = !!({}); $noxQueueTest.assertions.push({{ expression: \"{}\", passed: $noxPassed, actual: $noxActual }}); if (!$noxPassed && $noxQueueTest.failure === null) $noxQueueTest.failure = \"expectation failed: {}\"; }}\n",
emit_scenario_expression(expectation)?,
json_escape(expression_source),
json_escape(expression_source),
));
}
script.push_str(&format!(
" }} catch ($noxError) {{ $noxQueueTest.failure = $noxQueueTest.failure ?? failureMessage($noxError); }}\n scenarioResults.push({{ id: \"{}\", name: \"{}\", queue: \"{}\", semanticUnit: \"{}\", status: $noxQueueTest.failure === null ? \"pass\" : \"fail\", assertions: $noxQueueTest.assertions, failure: $noxQueueTest.failure, modelStubRefusals: modelStubRefusals(), covers: {} }});\n}}\n",
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&queue.name),
json_escape(queue.id.as_str()),
strings_as_javascript(&scenario.covers),
));
Ok(script)
}
fn task_scenario_script(
task: &TaskDefinition,
scenario: &noxid_ir::TaskScenario,
source: &SourceFile,
) -> Result<String, String> {
if !scenario.given.is_empty() {
return Ok(format!(
"proseOnlyScenarios.push(\"{}\");\nscenarioResults.push({{ id: \"{}\", name: \"{}\", task: \"{}\", semanticUnit: \"{}\", status: \"unsupported\", assertions: [], failure: \"prose task givens are not executable; use an exact typed boundary given\", covers: {} }});\n",
json_escape(scenario.id.as_str()),
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&task.name),
json_escape(task.id.as_str()),
strings_as_javascript(&scenario.covers),
));
}
let invocation = match &task.handler {
TaskHandler::CompilerOwned { statements, .. } => {
let statements =
noxid_codegen_server_js::compiler_statements_javascript(statements, 4)?;
format!(
"await (async () => {{\n const context = {SCENARIO_SYSTEM_CONTEXT};\n{statements}\n}})()"
)
}
TaskHandler::Host { key } => {
let stub = scenario
.typed_given
.iter()
.find(|given| &given.target == key)
.map(|given| emit_scenario_expression(&given.value))
.transpose()?;
stub.map_or_else(
|| {
format!(
"(() => {{ throw new Error(\"SCENARIO_TASK_BOUNDARY_UNSTUBBED: bodyless task {} has no exact typed scenario given\"); }})()",
json_escape(key.as_str())
)
},
|value| format!("structuredClone({value})"),
)
}
};
let mut script = format!(
"{{\n{} const $noxTaskTest = {{ failure: null, assertions: [] }};\n try {{\n const $noxTaskValue = {invocation};\n const value = {{ get() {{ return $noxTaskValue; }} }};\n const refusal = {{ get() {{ return \"\"; }} }};\n const $noxScope = {{ value, refusal }};\n",
model_stub_script(&scenario.model_stubs, &[], task.id.as_str())?,
);
for expectation in &scenario.expectations {
let mut references = BTreeSet::new();
collect_references(expectation, &mut references);
let reference_pairs = reference_pairs_as_javascript(&references);
let expression_source = source.slice(expectation.span).trim();
script.push_str(&format!(
" {{ const $noxActual = actualValues($noxScope, [{reference_pairs}]); const $noxPassed = !!({}); $noxTaskTest.assertions.push({{ expression: \"{}\", passed: $noxPassed, actual: $noxActual }}); if (!$noxPassed && $noxTaskTest.failure === null) $noxTaskTest.failure = \"expectation failed: {}\"; }}\n",
emit_scenario_expression(expectation)?,
json_escape(expression_source),
json_escape(expression_source),
));
}
script.push_str(&format!(
" }} catch ($noxError) {{ $noxTaskTest.failure = $noxTaskTest.failure ?? failureMessage($noxError); }}\n scenarioResults.push({{ id: \"{}\", name: \"{}\", task: \"{}\", semanticUnit: \"{}\", status: $noxTaskTest.failure === null ? \"pass\" : \"fail\", assertions: $noxTaskTest.assertions, failure: $noxTaskTest.failure, modelStubRefusals: modelStubRefusals(), covers: {} }});\n}}\n",
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&task.name),
json_escape(task.id.as_str()),
strings_as_javascript(&scenario.covers),
));
Ok(script)
}
#[derive(Clone, Debug)]
struct EndpointFileFixture {
field: String,
bytes: Vec<u8>,
mime: String,
}
fn decode_scenario_file_base64(value: &str, field: &str) -> Result<Vec<u8>, String> {
if !value.len().is_multiple_of(4) {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_INVALID]: file fixture `{field}` bytes must be canonical padded base64"
));
}
let decode = |byte: u8| -> Option<u8> {
match byte {
b'A'..=b'Z' => Some(byte - b'A'),
b'a'..=b'z' => Some(byte - b'a' + 26),
b'0'..=b'9' => Some(byte - b'0' + 52),
b'+' => Some(62),
b'/' => Some(63),
_ => None,
}
};
let input = value.as_bytes();
let mut output = Vec::with_capacity(input.len() / 4 * 3);
for (index, quartet) in input.chunks_exact(4).enumerate() {
let last = index + 1 == input.len() / 4;
let padding = if quartet[2] == b'=' {
2
} else if quartet[3] == b'=' {
1
} else {
0
};
if (!last && padding != 0)
|| (padding == 2 && quartet[3] != b'=')
|| quartet[0] == b'='
|| quartet[1] == b'='
{
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_INVALID]: file fixture `{field}` bytes must be canonical padded base64"
));
}
let a = decode(quartet[0]);
let b = decode(quartet[1]);
let c = (padding < 2).then(|| decode(quartet[2])).flatten();
let d = (padding == 0).then(|| decode(quartet[3])).flatten();
let (Some(a), Some(b)) = (a, b) else {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_INVALID]: file fixture `{field}` bytes must be canonical padded base64"
));
};
if (padding < 2 && c.is_none()) || (padding == 0 && d.is_none()) {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_INVALID]: file fixture `{field}` bytes must be canonical padded base64"
));
}
if (padding == 2 && b & 0x0f != 0) || (padding == 1 && c.expect("checked") & 0x03 != 0) {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_INVALID]: file fixture `{field}` bytes must use canonical zero padding bits"
));
}
output.push((a << 2) | (b >> 4));
if let Some(c) = c {
output.push((b << 4) | (c >> 2));
if let Some(d) = d {
output.push((c << 6) | d);
}
}
}
Ok(output)
}
fn scenario_fixture_mime(value: &str) -> bool {
let Some((kind, subtype)) = value.split_once('/') else {
return false;
};
let token = |part: &str| {
!part.is_empty()
&& part.len() <= 64
&& part.bytes().all(|byte| {
byte.is_ascii_alphanumeric()
|| matches!(
byte,
b'!' | b'#' | b'$' | b'&' | b'^' | b'_' | b'.' | b'+' | b'-'
)
})
};
token(kind) && token(subtype)
}
fn endpoint_file_fixtures(
endpoint: &EndpointDefinition,
scenario: &noxid_ir::EndpointScenario,
) -> Result<Option<Vec<EndpointFileFixture>>, String> {
let mut fixtures = Vec::new();
for given in &scenario.given {
let Some(rest) = given.strip_prefix("file ") else {
return Ok(None);
};
let words = rest.split_ascii_whitespace().collect::<Vec<_>>();
if words.len() != 5 || words[1] != "bytes" || words[3] != "as" {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_INVALID]: endpoint scenario `{}` file givens use `file <body-field> bytes <canonical-base64> as <mime>`",
scenario.name,
));
}
let field = words[0];
let Some(contract) = endpoint
.body
.iter()
.find(|candidate| candidate.name == field)
.and_then(|candidate| candidate.file.as_ref())
else {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_UNKNOWN_FIELD]: endpoint scenario `{}` gives file `{field}`, but `{field}` is not a declared File body field",
scenario.name,
));
};
if !scenario_fixture_mime(words[4]) {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_INVALID]: endpoint scenario `{}` file `{field}` must declare one syntactically valid MIME type after `as`",
scenario.name,
));
}
let bytes = decode_scenario_file_base64(words[2], field)?;
if bytes.len() as u64 > contract.max_size_bytes.saturating_add(1) {
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_TOO_LARGE]: endpoint scenario `{}` file `{field}` has {} bytes; fixtures may reach the declared {} byte cap plus exactly one refusal byte",
scenario.name,
bytes.len(),
contract.max_size_bytes,
));
}
fixtures.push(EndpointFileFixture {
field: field.to_string(),
bytes,
mime: words[4].to_string(),
});
}
let upload_fields = endpoint
.body
.iter()
.filter_map(|field| field.file.as_ref().map(|contract| (field, contract)))
.collect::<Vec<_>>();
if upload_fields.is_empty() {
return if fixtures.is_empty() {
Ok(Some(fixtures))
} else {
Err(format!(
"error[SCENARIO_FILE_FIXTURE_UNKNOWN_FIELD]: endpoint scenario `{}` declares a file fixture for an endpoint with no File body field",
scenario.name,
))
};
}
for (field, contract) in upload_fields {
let count = fixtures
.iter()
.filter(|fixture| fixture.field == field.name)
.count();
if count == 0 || (!contract.multiple && count > 1) {
let rule = if contract.multiple {
"requires at least one fixture"
} else {
"requires exactly one fixture"
};
return Err(format!(
"error[SCENARIO_FILE_FIXTURE_CARDINALITY]: endpoint scenario `{}` {rule} for File body field `{}`; found {count}",
scenario.name, field.name,
));
}
}
Ok(Some(fixtures))
}
fn endpoint_scenario_script(
endpoint: &EndpointDefinition,
scenario: &noxid_ir::EndpointScenario,
source: &SourceFile,
scenario_ordinal: usize,
) -> Result<String, String> {
let Some(file_fixtures) = endpoint_file_fixtures(endpoint, scenario)? else {
return Ok(format!(
"proseOnlyScenarios.push(\"{}\");\nscenarioResults.push({{ id: \"{}\", name: \"{}\", endpoint: \"{}\", semanticUnit: \"{}\", status: \"unsupported\", assertions: [], failure: \"prose endpoint givens are not executable; use an exact typed boundary given\", covers: {} }});\n",
json_escape(scenario.id.as_str()),
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&endpoint.name),
json_escape(endpoint.id.as_str()),
strings_as_javascript(&scenario.covers),
));
};
let route = endpoint
.route
.as_ref()
.expect("scenario project endpoints are route-enriched by the canonical project builder");
let request_group = |name: &str| -> Result<String, String> {
match scenario
.request
.iter()
.find(|argument| argument.name == name)
{
Some(argument) => emit_endpoint_json_expression(&argument.value),
None => Ok("{}".into()),
}
};
let params = request_group("params")?;
let query = request_group("query")?;
let body = request_group("body")?;
let delegated = if scenario.model_stubs.is_empty()
&& (file_fixtures.is_empty() || !scenario.typed_given.is_empty())
{
Vec::new()
} else {
vec![endpoint.id.as_str().to_string()]
};
let mut script = format!(
"{{\n globalThis.__NOXID_ENDPOINT_SCENARIO_STUBS__.clear();\n{} const $noxEndpointTest = {{ failure: null, assertions: [] }};\n try {{\n const $noxParams = {params};\n const $noxQuery = {query};\n const $noxBody = {body};\n let $noxPath = \"{}\";\n",
model_stub_script(&scenario.model_stubs, &delegated, endpoint.id.as_str())?,
json_escape(&route.path),
);
for parameter in &route.dynamic_params {
script.push_str(&format!(
" $noxPath = $noxPath.replace(\"[{}]\", encodeURIComponent(String($noxParams[\"{}\"])));\n",
json_escape(parameter),
json_escape(parameter),
));
}
script.push_str(" const $noxUrl = new URL(`http://noxid.test${$noxPath}`);\n");
for field in &endpoint.query {
let encoded_value = if endpoint_query_is_array(&field.ty) {
"JSON.stringify($noxRawValue)"
} else {
"endpointQueryValue($noxRawValue)"
};
let present = if matches!(&field.ty, noxid_types::Type::Optional(_)) {
"$noxRawValue !== null"
} else {
"true"
};
script.push_str(&format!(
" {{ const $noxRawValue = $noxQuery[\"{}\"]; if ({present}) $noxUrl.searchParams.append(\"{}\", {encoded_value}); }}\n",
json_escape(&field.name),
json_escape(&field.name),
));
}
for given in &scenario.typed_given {
script.push_str(&format!(
" globalThis.__NOXID_ENDPOINT_SCENARIO_STUBS__.set(\"{}\", {});\n",
json_escape(given.target.as_str()),
emit_scenario_expression(&given.value)?,
));
}
script.push_str(" const $noxHeaders = new Headers();\n");
if !file_fixtures.is_empty() {
script.push_str(" const $noxMultipart = new FormData();\n");
for field in endpoint.body.iter().filter(|field| field.file.is_none()) {
let encode = match &field.ty {
noxid_types::Type::String
| noxid_types::Type::Date
| noxid_types::Type::Int
| noxid_types::Type::Number
| noxid_types::Type::Float
| noxid_types::Type::Boolean => "String($noxRawValue)",
noxid_types::Type::Optional(inner)
if matches!(
inner.as_ref(),
noxid_types::Type::String
| noxid_types::Type::Date
| noxid_types::Type::Int
| noxid_types::Type::Number
| noxid_types::Type::Float
| noxid_types::Type::Boolean
) =>
{
"String($noxRawValue)"
}
_ => "JSON.stringify($noxRawValue)",
};
script.push_str(&format!(
" {{ const $noxRawValue = $noxBody[\"{}\"]; if ($noxRawValue !== null && $noxRawValue !== undefined) $noxMultipart.append(\"{}\", {encode}); }}\n",
json_escape(&field.name),
json_escape(&field.name),
));
}
for fixture in &file_fixtures {
let bytes = fixture
.bytes
.iter()
.map(u8::to_string)
.collect::<Vec<_>>()
.join(",");
script.push_str(&format!(
" $noxMultipart.append(\"{}\", new Blob([Uint8Array.from([{bytes}])], {{ type: \"{}\" }}), \"{}\");\n",
json_escape(&fixture.field),
json_escape(&fixture.mime),
json_escape(&fixture.field),
));
}
} else if !endpoint.body.is_empty() {
script.push_str(" $noxHeaders.set(\"content-type\", \"application/json\");\n");
}
if endpoint.idempotent {
script.push_str(&format!(
" $noxHeaders.set(\"idempotency-key\", \"{}\");\n",
json_escape(scenario.id.as_str()),
));
}
let body_option = if !file_fixtures.is_empty() {
", body: $noxMultipart"
} else if endpoint.body.is_empty() {
""
} else {
", body: JSON.stringify($noxBody)"
};
script.push_str(&format!(
" const $noxRequest = new Request($noxUrl, {{ method: \"{}\", headers: $noxHeaders{body_option} }});\n const $noxEnvironment = Object.freeze({{ sessionId: \"noxid-scenario:{}\", requestIdentity: Object.freeze({{ ip: \"2001:db8::{scenario_ordinal:x}\" }}) }});\n const $noxResponse = await endpointModule.fetch($noxRequest, $noxEnvironment, {{ waitUntil() {{}} }});\n const $noxResponseText = await $noxResponse.text();\n let $noxPayload = null;\n try {{ $noxPayload = $noxResponseText === \"\" ? null : JSON.parse($noxResponseText); }} catch {{ $noxPayload = {{ raw: $noxResponseText }}; }}\n const status = {{ get() {{ return $noxResponse.status; }} }};\n const value = {{ get() {{ return $noxPayload?.value; }} }};\n const refusal = {{ get() {{ return $noxPayload?.refusal ?? $noxPayload?.error?.code ?? \"\"; }} }};\n const $noxScope = {{ status, value, refusal }};\n",
route.method.as_str().to_ascii_uppercase(),
json_escape(scenario.id.as_str()),
));
for expectation in &scenario.expectations {
let mut references = BTreeSet::new();
collect_references(expectation, &mut references);
let reference_pairs = reference_pairs_as_javascript(&references);
let expression_source = source.slice(expectation.span).trim();
script.push_str(&format!(
" {{ const $noxActual = actualValues($noxScope, [{reference_pairs}]); const $noxPassed = !!({}); $noxEndpointTest.assertions.push({{ expression: \"{}\", passed: $noxPassed, actual: $noxActual }}); if (!$noxPassed && $noxEndpointTest.failure === null) $noxEndpointTest.failure = \"expectation failed: {}\"; }}\n",
emit_endpoint_expectation_expression(expectation)?,
json_escape(expression_source),
json_escape(expression_source),
));
}
script.push_str(&format!(
" }} catch ($noxError) {{ $noxEndpointTest.failure = $noxEndpointTest.failure ?? failureMessage($noxError); }}\n scenarioResults.push({{ id: \"{}\", name: \"{}\", endpoint: \"{}\", semanticUnit: \"{}\", status: $noxEndpointTest.failure === null ? \"pass\" : \"fail\", assertions: $noxEndpointTest.assertions, failure: $noxEndpointTest.failure, modelStubRefusals: modelStubRefusals(), covers: {} }});\n}}\n",
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&endpoint.name),
json_escape(endpoint.id.as_str()),
strings_as_javascript(&scenario.covers),
));
Ok(script)
}
/// One agent scenario, as executable JavaScript.
///
/// The scenario drives the real generated door: `POST
/// /_noxid/agents/<Agent>/runs` with the typed input as its body, then reads
/// the SSE session the run streams. Nothing about the loop is simulated — the
/// only things the scenario replaces are the provider turn (scripted) and the
/// host authorizer's answer for a deferred capability. Every tool call
/// therefore crosses the endpoint's own validator, middleware, limits, and
/// audit, exactly as a production run's would.
fn agent_scenario_script(
agent: &AgentDefinition,
scenario: &noxid_ir::AgentScenario,
source: &SourceFile,
scenario_ordinal: usize,
) -> Result<String, String> {
let mut script_entries = Vec::new();
let mut scripted_tools = Vec::new();
for turn in &scenario.turns {
match turn {
AgentScenarioTurn::Text(text) => script_entries.push(format!(
"{{ kind: \"text\", text: \"{}\" }}",
json_escape(text)
)),
AgentScenarioTurn::Tool {
endpoint,
arguments,
} => {
let record = agent_scenario_record(arguments)?;
scripted_tools.push(format!(
"{{ tool: \"{}\", arguments: {record} }}",
json_escape(endpoint)
));
script_entries.push(format!(
"{{ kind: \"tool\", endpoint: \"{}\", arguments: {record} }}",
json_escape(endpoint)
));
}
AgentScenarioTurn::Final { arguments } => script_entries.push(format!(
"{{ kind: \"final\", arguments: {} }}",
agent_scenario_record(arguments)?
)),
}
}
let input = match &scenario.input {
Some(value) => emit_endpoint_json_expression(value)?,
None => "null".into(),
};
let deferred = strings_as_javascript(&scenario.deferred);
let mut script = format!(
"{{\n globalThis.__NOXID_ENDPOINT_SCENARIO_STUBS__.clear();\n installModelStubs([], [], \"{}\");\n installAgentScenario(\"{}\", [{}], {deferred});\n const $noxAgentTest = {{ failure: null, assertions: [] }};\n try {{\n const $noxSink = {{ events: [], output: null, refusal: \"\", paused: false, resumed: false, runId: null }};\n const $noxEnvironment = Object.freeze({{ sessionId: \"noxid-scenario:{}\", requestIdentity: Object.freeze({{ ip: \"2001:db8::{scenario_ordinal:x}\" }}) }});\n const $noxHeaders = new Headers({{ \"content-type\": \"application/json\" }});\n const $noxStart = new Request(\"http://noxid.test/_noxid/agents/{}/runs\", {{ method: \"POST\", headers: $noxHeaders, body: JSON.stringify({{ input: {input} }}) }});\n await readAgentEvents(await endpointModule.fetch($noxStart, $noxEnvironment, {{ waitUntil() {{}} }}), $noxSink);\n",
json_escape(scenario.id.as_str()),
json_escape(&agent.name),
script_entries.join(", "),
json_escape(scenario.id.as_str()),
json_escape(&agent.name),
);
if scenario.resume {
script.push_str(&format!(
" if ($noxSink.paused && typeof $noxSink.runId === \"string\") {{\n $noxSink.resumed = true;\n const $noxResume = new Request(`http://noxid.test/_noxid/agents/{}/runs/${{encodeURIComponent($noxSink.runId)}}/resume`, {{ method: \"POST\", headers: new Headers({{ \"content-type\": \"application/json\" }}), body: \"{{}}\" }});\n await readAgentEvents(await endpointModule.fetch($noxResume, $noxEnvironment, {{ waitUntil() {{}} }}), $noxSink);\n }}\n",
json_escape(&agent.name),
));
}
// Every scripted tool call is checked against what the endpoint actually
// validated. This is not an expectation the author writes: a scenario that
// scripts a call the endpoint would have rejected, or that reached the
// endpoint with different arguments, is a failed scenario by construction.
script.push_str(&format!(
" {{\n const $noxScripted = [{}];\n const $noxDispatched = agentScenarioToolCalls();\n for (let $noxIndex = 0; $noxIndex < $noxDispatched.length; $noxIndex += 1) {{\n const $noxActualCall = $noxDispatched[$noxIndex];\n const $noxExpectedCall = $noxScripted[$noxIndex] ?? null;\n const $noxCallOk = $noxExpectedCall !== null\n && $noxActualCall.tool === $noxExpectedCall.tool\n && agentValuesEqual($noxActualCall.arguments, $noxExpectedCall.arguments)\n && $noxActualCall.status < 400;\n $noxAgentTest.assertions.push({{ expression: `tool ${{$noxActualCall.tool}} validated the scripted request`, passed: $noxCallOk, actual: {{ tool: $noxActualCall.tool, arguments: $noxActualCall.arguments, status: $noxActualCall.status }} }});\n if (!$noxCallOk && $noxAgentTest.failure === null) $noxAgentTest.failure = `tool ${{$noxActualCall.tool}} did not validate the scripted request (status ${{$noxActualCall.status}})`;\n }}\n }}\n",
scripted_tools.join(", "),
));
script.push_str(" const emitted = { get() { return $noxSink.events; } };\n const tools = { get() { return agentScenarioToolCalls().map((call) => call.tool); } };\n const output = { get() { return $noxSink.output; } };\n const refusal = { get() { return $noxSink.refusal; } };\n const paused = { get() { return $noxSink.paused; } };\n const resumed = { get() { return $noxSink.resumed; } };\n const turns = { get() { return agentScenarioTurnCount(); } };\n const $noxScope = { emitted, tools, output, refusal, paused, resumed, turns };\n");
for expectation in &scenario.expectations {
let mut references = BTreeSet::new();
collect_references(expectation, &mut references);
let reference_pairs = reference_pairs_as_javascript(&references);
let expression_source = source.slice(expectation.span).trim();
script.push_str(&format!(
" {{ const $noxActual = actualValues($noxScope, [{reference_pairs}]); const $noxPassed = !!({}); $noxAgentTest.assertions.push({{ expression: \"{}\", passed: $noxPassed, actual: $noxActual }}); if (!$noxPassed && $noxAgentTest.failure === null) $noxAgentTest.failure = \"expectation failed: {}\"; }}\n",
emit_endpoint_expectation_expression(expectation)?,
json_escape(expression_source),
json_escape(expression_source),
));
}
script.push_str(&format!(
" }} catch ($noxError) {{ $noxAgentTest.failure = $noxAgentTest.failure ?? failureMessage($noxError); }}\n clearAgentScenario();\n scenarioResults.push({{ id: \"{}\", name: \"{}\", agent: \"{}\", semanticUnit: \"{}\", status: $noxAgentTest.failure === null ? \"pass\" : \"fail\", assertions: $noxAgentTest.assertions, failure: $noxAgentTest.failure, modelStubRefusals: modelStubRefusals(), covers: {} }});\n}}\n",
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&agent.name),
json_escape(agent.id.as_str()),
strings_as_javascript(&scenario.covers),
));
Ok(script)
}
/// A scripted turn's `{ field = value }` record, as a JSON object literal.
fn agent_scenario_record(arguments: &[noxid_ir::AgentScenarioArgument]) -> Result<String, String> {
let fields = arguments
.iter()
.map(|argument| {
Ok(format!(
"\"{}\": {}",
json_escape(&argument.name),
emit_endpoint_json_expression(&argument.value)?
))
})
.collect::<Result<Vec<_>, String>>()?;
Ok(format!("{{ {} }}", fields.join(", ")))
}
/// Emit the `installModelStubs(...)` call for one scenario.
///
/// `delegated` names the host keys this scenario's model stubs authorize the
/// boundary host stub to run for real. A scenario that declares model givens
/// is declaring that its host implementation is exercised and that the model
/// is its only external boundary; every other boundary stays stubbed and live
/// `fetch` stays banned.
fn model_stub_script(
stubs: &[noxid_ir::ScenarioModelStub],
delegated: &[String],
call_site: &str,
) -> Result<String, String> {
let entries = stubs
.iter()
.map(|stub| {
let detail = match &stub.kind {
noxid_ir::ScenarioModelStubKind::Text(text) => {
format!("kind: \"text\", text: \"{}\"", json_escape(text))
}
noxid_ir::ScenarioModelStubKind::Object { value, .. } => {
format!(
"kind: \"object\", value: {}",
emit_endpoint_json_expression(value)?
)
}
noxid_ir::ScenarioModelStubKind::Tokens(tokens) => format!(
"kind: \"tokens\", tokens: [{}]",
tokens
.iter()
.map(|token| format!("\"{}\"", json_escape(token)))
.collect::<Vec<_>>()
.join(", ")
),
noxid_ir::ScenarioModelStubKind::Fails(code) => {
format!("kind: \"fails\", code: \"{}\"", json_escape(code))
}
};
Ok(format!(
"{{ model: \"{}\", {detail}, inputTokens: 0, outputTokens: 0 }}",
json_escape(&stub.model)
))
})
.collect::<Result<Vec<_>, String>>()?
.join(", ");
let delegated = delegated
.iter()
.map(|key| format!("\"{}\"", json_escape(key)))
.collect::<Vec<_>>()
.join(", ");
Ok(format!(
" installModelStubs([{entries}], [{delegated}], \"{}\");\n",
json_escape(call_site),
))
}
fn emit_endpoint_json_expression(expression: &SemanticExpr) -> Result<String, String> {
Ok(match &expression.kind {
SemanticExprKind::Array(values) => format!(
"[{}]",
values
.iter()
.map(emit_endpoint_json_expression)
.collect::<Result<Vec<_>, String>>()?
.join(", ")
),
SemanticExprKind::Struct { fields, .. } => format!(
"{{ {} }}",
fields
.iter()
.map(|field| {
Ok(format!(
"\"{}\": {}",
json_escape(&field.name),
emit_endpoint_json_expression(&field.value)?
))
})
.collect::<Result<Vec<_>, String>>()?
.join(", ")
),
SemanticExprKind::Variant {
variant, payload, ..
} if matches!(&expression.ty, noxid_types::Type::Optional(_)) => {
if variant.as_str().ends_with(".None") {
"null".into()
} else if let Some(payload) = payload {
emit_endpoint_json_expression(payload)?
} else {
"null".into()
}
}
_ if matches!(&expression.ty, noxid_types::Type::Optional(_)) => format!(
"(($noxOptional) => $noxOptional?.tag === \"None\" ? null : $noxOptional?.tag === \"Some\" ? $noxOptional.value : $noxOptional)({})",
emit_scenario_expression(expression)?
),
_ => emit_scenario_expression(expression)?,
})
}
fn emit_endpoint_expectation_expression(expression: &SemanticExpr) -> Result<String, String> {
Ok(match &expression.kind {
SemanticExprKind::Binary { left, op, right } => {
let left = emit_endpoint_expectation_expression(left)?;
let right = emit_endpoint_expectation_expression(right)?;
match op {
noxid_ir::SemanticBinaryOp::Equal => {
format!("endpointValuesEqual({left}, {right})")
}
noxid_ir::SemanticBinaryOp::NotEqual => {
format!("(!endpointValuesEqual({left}, {right}))")
}
noxid_ir::SemanticBinaryOp::Coalesce => format!("({left} ?? {right})"),
other => format!("({left} {} {right})", other.as_str()),
}
}
SemanticExprKind::Unary { op, operand } => format!(
"({}{})",
op.as_str(),
emit_endpoint_expectation_expression(operand)?
),
SemanticExprKind::FieldAccess { base, name, .. } => format!(
"{}[\"{}\"]",
emit_endpoint_expectation_expression(base)?,
json_escape(name)
),
SemanticExprKind::Array(_) | SemanticExprKind::Struct { .. } => {
emit_endpoint_json_expression(expression)?
}
SemanticExprKind::Variant { .. }
if matches!(&expression.ty, noxid_types::Type::Optional(_)) =>
{
emit_endpoint_json_expression(expression)?
}
_ => emit_scenario_expression(expression)?,
})
}
fn scenario_script(
module_var: &str,
component: &ComponentDefinition,
scenario: &noxid_ir::ApplicationScenario,
source: &SourceFile,
) -> Result<String, String> {
let mut script = String::new();
let scope_names = component_scope_names(component).join(", ");
let factory = format!("__noxidCreate{}Actions", component.name);
let scenario_stubs = scenario
.typed_given
.iter()
.filter_map(|given| match given.kind {
ScenarioGivenKind::Resource | ScenarioGivenKind::Stream => {
Some(Ok(format!("\"{}\"", json_escape(given.target.as_str()))))
}
ScenarioGivenKind::RemoteAction => {
Some(emit_scenario_expression(&given.value).map(|result| {
format!(
"{{ kind: \"remote-action\", id: \"{}\", result: {result} }}",
json_escape(given.target.as_str()),
)
}))
}
ScenarioGivenKind::State => None,
})
.collect::<Result<BTreeSet<_>, String>>()?
.into_iter()
.collect::<Vec<_>>()
.join(", ");
let declared_capabilities = component
.capabilities
.iter()
.map(|capability| format!("\"{}\"", json_escape(capability.id.as_str())))
.collect::<Vec<_>>()
.join(", ");
let scenario_props = component
.props
.iter()
.filter(|prop| matches!(prop.ty, noxid_types::Type::Optional(_)))
.map(|prop| format!("\"{}\": null", json_escape(&prop.name)))
.collect::<Vec<_>>()
.join(", ");
script.push_str(&format!(
"{{\n registrations.delete(\"{}\");\n const $noxTest = {{ mounted: null, scope: null, expressionScope: null, actions: null, failure: null, assertions: [], invariantFailures: [], actualValues, runtime }};\n try {{\n const $noxHost = document.createElement(\"main\");\n const $noxDeclaredCapabilities = new Set([{declared_capabilities}]);\n const $noxScenarioOptions = runtime.createScenarioHarnessOptions([{scenario_stubs}]);\n const $noxRuntimeOptions = {{ ...$noxScenarioOptions, authorizeComponent($noxCapability, $noxSubject) {{ return $noxSubject === \"{}\" && $noxDeclaredCapabilities.has($noxCapability); }} }};\n $noxTest.mounted = {module_var}[\"mount{}\"]($noxHost, {{ {scenario_props} }}, {{}}, null, $noxRuntimeOptions);\n $noxTest.runtime.flush();\n const $noxRegistration = registrations.get(\"{}\");\n if (!$noxRegistration) throw new Error(\"generated mount did not register its semantic state scope\");\n $noxRegistration.patchActions({{ [\"{factory}\"](scope) {{ $noxTest.scope = scope; $noxTest.actions = {module_var}[\"{factory}\"](scope); return $noxTest.actions; }} }});\n if (!$noxTest.scope || !$noxTest.actions) throw new Error(\"generated action scope was not captured\");\n $noxTest.expressionScope = {{ ...$noxTest.scope }};\n{} {{\n const {{ {scope_names} }} = $noxTest.expressionScope;\n",
json_escape(&component.name),
json_escape(component.id.as_str()),
json_escape(&component.name),
json_escape(&component.name),
"",
));
emit_invariant_checker(&mut script, component, source)?;
emit_invariant_checkpoint(
&mut script,
ScenarioCheckpoint {
phase: "mount",
index: 0,
target: None,
},
);
for (index, given) in scenario.typed_given.iter().enumerate() {
emit_given_step(&mut script, given)?;
emit_invariant_checkpoint(
&mut script,
ScenarioCheckpoint {
phase: "given",
index,
target: Some(&given.target_name),
},
);
}
for (index, when) in scenario.typed_when.iter().enumerate() {
let arguments = when
.arguments
.iter()
.map(emit_scenario_expression)
.collect::<Result<Vec<_>, String>>()?
.join(", ");
script.push_str(&format!(
" await $noxTest.actions[\"{}\"]({arguments});\n $noxTest.runtime.flush();\n",
json_escape(&when.action_name)
));
emit_invariant_checkpoint(
&mut script,
ScenarioCheckpoint {
phase: "when",
index,
target: Some(&when.action_name),
},
);
}
for expression in &scenario.typed_expect {
let mut references = BTreeSet::new();
collect_references(expression, &mut references);
let reference_pairs = references
.iter()
.map(|id| {
format!(
"[\"{}\", \"{}\"]",
json_escape(id.as_str()),
json_escape(symbol_name(id))
)
})
.collect::<Vec<_>>()
.join(", ");
let expression_source = source.slice(expression.span).trim();
script.push_str(&format!(
" {{ const $noxPassed = !!({}); const $noxAssertion = {{ expression: \"{}\", expected: \"Boolean expression evaluates to true\", passed: $noxPassed, actual: $noxTest.actualValues($noxTest.expressionScope, [{reference_pairs}]) }}; $noxTest.assertions.push($noxAssertion); if (!$noxPassed && $noxTest.failure === null) $noxTest.failure = `assertion failed: ${{$noxAssertion.expression}}`; }}\n",
emit_scenario_expression(expression)?,
json_escape(expression_source),
));
}
script.push_str(&format!(
" }}\n }} catch ($noxError) {{ $noxTest.failure = $noxTest.failure ?? failureMessage($noxError); }} finally {{ try {{ $noxTest.mounted?.dispose(); }} catch ($noxError) {{ $noxTest.failure = $noxTest.failure ?? failureMessage($noxError); }} }}\n scenarioResults.push({{ id: \"{}\", name: \"{}\", component: \"{}\", status: $noxTest.failure === null ? \"pass\" : \"fail\", assertions: $noxTest.assertions, invariantFailures: $noxTest.invariantFailures, failure: $noxTest.failure, covers: {} }});\n}}\n",
json_escape(scenario.id.as_str()),
json_escape(&scenario.name),
json_escape(&component.name),
ids_as_javascript(&scenario.covers),
));
Ok(script)
}
fn emit_given_step(script: &mut String, given: &noxid_ir::ScenarioGivenStep) -> Result<(), String> {
let target_name = json_escape(&given.target_name);
let value = emit_scenario_expression(&given.value)?;
let statement = match given.kind {
ScenarioGivenKind::State => format!("$noxTest.scope[\"{target_name}\"].set({value});"),
ScenarioGivenKind::Resource => {
let age = given
.age_milliseconds
.map(|milliseconds| milliseconds.to_string())
.unwrap_or_else(|| "null".into());
format!("$noxTest.mounted.resources[\"{target_name}\"].seedScenario({value}, {age});")
}
ScenarioGivenKind::Stream => {
format!(
"$noxTest.mounted.streams[\"{target_name}\"].events.set(({value}).map(($noxEvent, $noxIndex) => ({{ sequence: $noxIndex + 1, event: $noxEvent }})));"
)
}
ScenarioGivenKind::RemoteAction => {
"/* remote-action given installed before mount by the scenario harness */".into()
}
};
script.push_str(&format!(
" {statement}\n $noxTest.runtime.flush();\n"
));
Ok(())
}
fn emit_invariant_checker(
script: &mut String,
component: &ComponentDefinition,
source: &SourceFile,
) -> Result<(), String> {
script.push_str(
" const $noxCheckInvariants = ($noxStep) => {\n const $noxFailureStart = $noxTest.invariantFailures.length;\n let $noxCheckpointFailed = false;\n",
);
for invariant in &component.invariants {
let Some(expression) = invariant.typed_assert.as_ref() else {
continue;
};
let mut references = BTreeSet::new();
collect_references(expression, &mut references);
let reference_pairs = reference_pairs_as_javascript(&references);
let expression_source = source.slice(expression.span).trim();
script.push_str(&format!(
" {{ const $noxActual = $noxTest.actualValues($noxTest.expressionScope, [{reference_pairs}]); const $noxPassed = !!({}); if (!$noxPassed) {{ $noxCheckpointFailed = true; $noxTest.invariantFailures.push({{ id: \"{}\", name: \"{}\", expression: \"{}\", step: $noxStep, actual: $noxActual }}); }} }}\n",
emit_scenario_expression(expression)?,
json_escape(invariant.id.as_str()),
json_escape(&invariant.name),
json_escape(expression_source),
));
}
script.push_str(
" if ($noxCheckpointFailed) { const $noxBreach = $noxTest.invariantFailures[$noxFailureStart]; $noxTest.failure = `invariant failed: ${$noxBreach.name} after ${$noxStep.phase}${$noxStep.target === null ? \"\" : ` ${$noxStep.target}`}`; throw new Error($noxTest.failure); }\n };\n",
);
Ok(())
}
fn emit_invariant_checkpoint(script: &mut String, checkpoint: ScenarioCheckpoint<'_>) {
script.push_str(&format!(
" $noxCheckInvariants({});\n",
checkpoint.as_javascript()
));
}
fn reference_pairs_as_javascript(references: &BTreeSet<SemanticId>) -> String {
references
.iter()
.map(|id| {
format!(
"[\"{}\", \"{}\"]",
json_escape(id.as_str()),
json_escape(symbol_name(id))
)
})
.collect::<Vec<_>>()
.join(", ")
}
fn component_scope_names(component: &ComponentDefinition) -> Vec<String> {
let mut names = BTreeSet::new();
names.extend(component.props.iter().map(|value| value.name.clone()));
names.extend(
component
.context_uses
.iter()
.flat_map(|context| context.fields.iter().map(|field| field.name.clone())),
);
names.extend(component.states.iter().map(|value| value.name.clone()));
names.extend(component.computed.iter().map(|value| value.name.clone()));
names.extend(component.resources.iter().map(|value| value.name.clone()));
names.extend(component.streams.iter().map(|value| value.name.clone()));
names.extend(component.agents.iter().map(|value| value.name.clone()));
names.into_iter().collect()
}
fn collect_references(expression: &SemanticExpr, references: &mut BTreeSet<SemanticId>) {
match &expression.kind {
SemanticExprKind::Int(_)
| SemanticExprKind::Float(_)
| SemanticExprKind::String(_)
| SemanticExprKind::Boolean(_) => {}
SemanticExprKind::Array(values) => {
for value in values {
collect_references(value, references);
}
}
SemanticExprKind::Struct { fields, .. } => {
for field in fields {
collect_references(&field.value, references);
}
}
SemanticExprKind::FieldAccess { base, .. }
| SemanticExprKind::Unary { operand: base, .. } => collect_references(base, references),
SemanticExprKind::CollectionQuery { base, value, .. } => {
collect_references(base, references);
if let Some(value) = value {
collect_references(value, references);
}
}
SemanticExprKind::Call { arguments, .. }
| SemanticExprKind::FunctionCall { arguments, .. } => {
for argument in arguments {
collect_references(argument, references);
}
}
SemanticExprKind::Reference(id) => {
references.insert(id.clone());
}
SemanticExprKind::Variant { payload, .. } => {
if let Some(payload) = payload {
collect_references(payload, references);
}
}
SemanticExprKind::Binary { left, right, .. } => {
collect_references(left, references);
collect_references(right, references);
}
SemanticExprKind::StringTemplate(parts) => {
for part in parts {
if let SemanticTemplatePart::Expression(expression) = part {
collect_references(expression, references);
}
}
}
}
}
fn ids_as_javascript(ids: &[SemanticId]) -> String {
format!(
"[{}]",
ids.iter()
.map(|id| format!("\"{}\"", json_escape(id.as_str())))
.collect::<Vec<_>>()
.join(", ")
)
}
fn strings_as_javascript(values: &[String]) -> String {
format!(
"[{}]",
values
.iter()
.map(|value| format!("\"{}\"", json_escape(value)))
.collect::<Vec<_>>()
.join(", ")
)
}
fn endpoint_query_is_array(ty: &noxid_types::Type) -> bool {
match ty {
noxid_types::Type::Array(_) => true,
noxid_types::Type::Optional(inner) => endpoint_query_is_array(inner),
_ => false,
}
}
fn symbol_name(id: &SemanticId) -> &str {
id.as_str().rsplit('.').next().unwrap_or(id.as_str())
}
fn write_file(path: &Path, contents: &str) -> Result<(), String> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.map_err(|error| format!("cannot create {}: {error}", parent.display()))?;
}
fs::write(path, contents).map_err(|error| format!("cannot write {}: {error}", path.display()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn upload_fixture_base64_is_canonical_and_exact() {
assert_eq!(
decode_scenario_file_base64("iVBORw0KGgo=", "avatar").unwrap(),
[0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
);
for invalid in ["abc", "ab=c", "a===", "AB==", "abc_"] {
let error = decode_scenario_file_base64(invalid, "avatar").unwrap_err();
assert!(error.contains("SCENARIO_FILE_FIXTURE_INVALID"), "{error}");
}
}
#[test]
fn node_harness_timeout_kills_a_hung_process_with_a_stable_code() {
let scratch = scratch_directory().unwrap();
let harness = scratch.0.join("hang.mjs");
write_file(&harness, "while (true) {}\n").unwrap();
let started = std::time::Instant::now();
let error =
run_node_with_timeout(&harness, &scratch.0, std::time::Duration::from_millis(50))
.unwrap_err();
assert!(error.contains("SCENARIO_TIMEOUT_EXCEEDED"), "{error}");
assert!(error.contains("50ms"), "{error}");
assert!(started.elapsed() < std::time::Duration::from_secs(2));
}
#[test]
fn node_harness_timeout_kills_a_grandchild_holding_stdout() {
let scratch = scratch_directory().unwrap();
let harness = scratch.0.join("grandchild.mjs");
write_file(
&harness,
r#"import { existsSync } from "node:fs";
import { spawn } from "node:child_process";
spawn(process.execPath, ["-e", "require('node:fs').writeFileSync('grandchild-started', '1'); setTimeout(() => {}, 10000);"], {
cwd: process.cwd(),
stdio: ["ignore", "inherit", "inherit"],
});
while (!existsSync("grandchild-started")) {}
while (true) {}
"#,
)
.unwrap();
let started = std::time::Instant::now();
let error =
run_node_with_timeout(&harness, &scratch.0, std::time::Duration::from_millis(500))
.unwrap_err();
assert!(scratch.0.join("grandchild-started").is_file());
assert!(error.contains("SCENARIO_TIMEOUT_EXCEEDED"), "{error}");
assert!(started.elapsed() < std::time::Duration::from_secs(2));
}
#[test]
fn property_validation_timeout_excludes_node_startup_and_module_import() {
let scratch = scratch_directory().unwrap();
write_file(
&scratch.0.join("package.json"),
"{\"private\":true,\"type\":\"module\"}\n",
)
.unwrap();
write_file(
&scratch.0.join("slow-import.validators.js"),
r#"await new Promise((resolve) => setTimeout(resolve, 150));
export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:test": () => true,
});
"#,
)
.unwrap();
let property = TestProperty {
definition: PropertyDefinition {
id: SemanticId::endpoint_property("Tight", "Terminates"),
name: "Terminates".into(),
runs: 1,
invariant: noxid_ir::PropertyInvariant::ValidatesOrRefuses,
boundary: SemanticId::endpoint("Tight"),
span: noxid_source::Span::new(0, 1),
},
boundary_kind: "endpoint",
boundary_name: "Tight".into(),
validator_module: "./slow-import.validators.js".into(),
validators: vec![(SemanticId::parse("validator:test").unwrap(), Schema::String)],
timeout: Duration::from_millis(50),
};
let started = Instant::now();
let outcome = execute_property_case(
&scratch.0,
&property,
&SemanticId::parse("validator:test").unwrap(),
&GeneratedValue::String("ready".into()),
)
.unwrap();
assert!(matches!(outcome, PropertyCaseOutcome::Pass));
assert!(started.elapsed() >= Duration::from_millis(100));
}
#[test]
fn property_failure_and_repro_use_the_shared_javascript_escape() {
let property = TestProperty {
definition: PropertyDefinition {
id: SemanticId::endpoint_property("Escapes", "Reported"),
name: "Reported".into(),
runs: 1,
invariant: noxid_ir::PropertyInvariant::ValidatesOrRefuses,
boundary: SemanticId::endpoint("Escapes"),
span: noxid_source::Span::new(0, 1),
},
boundary_kind: "endpoint",
boundary_name: "Escapes".into(),
validator_module: "./Escapes.validators.js".into(),
validators: vec![],
timeout: Duration::from_secs(1),
};
let failure = PropertyFailure::Case {
seed: 1,
run_index: 0,
code: "PROPERTY_INVARIANT_VIOLATED",
message: "failure\u{2028}line".into(),
counterexample: "counterexample".into(),
repro: "noxid test 'repro\u{2029}.nox' --seed 1".into(),
};
let javascript = property_result_javascript(&property, 1, Some(&failure));
assert!(
!javascript.contains('\u{2028}') && !javascript.contains('\u{2029}'),
"{javascript}"
);
assert!(javascript.contains("failure\\u2028line"), "{javascript}");
assert!(javascript.contains("repro\\u2029.nox"), "{javascript}");
}
fn fixture(name: &str, source: &str) -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let root = std::env::temp_dir().join(format!(
"noxid-scenario-cli-{name}-{}-{nonce}",
std::process::id()
));
fs::create_dir_all(&root).unwrap();
let path = root.join("Counter.nox");
fs::write(&path, source).unwrap();
path
}
fn endpoint_project_fixture(name: &str) -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let root = std::env::temp_dir().join(format!(
"noxid-endpoint-scenario-cli-{name}-{}-{nonce}",
std::process::id()
));
fs::create_dir_all(root.join("server/api/items")).unwrap();
fs::create_dir_all(root.join("src/components")).unwrap();
fs::write(root.join("Noxid.toml"), "[app]\ntitle = \"Scenarios\"\n").unwrap();
root
}
#[test]
fn emitted_endpoint_property_runs_hostile_cases_and_gate_raises_the_floor() {
let root = endpoint_project_fixture("property-pass");
fs::write(
root.join("server/api/items/validate.post.nox"),
r#"type Payload { name: String count: Int }
endpoint ValidatePayload {
body { payload: Payload }
result: Boolean
handler { return true }
property TotalValidation { runs: 1 expect: validates or refuses }
}"#,
)
.unwrap();
let output = execute(
&root,
&Options {
gate: true,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(output.status.success(), "{stdout}");
assert!(
stdout.contains("\"id\":\"property:endpoint.ValidatePayload.TotalValidation\""),
"{stdout}"
);
assert!(stdout.contains("\"property\":true"), "{stdout}");
assert!(stdout.contains("\"runs\":100"), "{stdout}");
assert!(stdout.contains("\"status\":\"pass\""), "{stdout}");
fs::remove_dir_all(root).unwrap();
}
#[test]
fn broken_validator_reports_seed_shrunk_counterexample_and_paste_ready_repro() {
let scratch = scratch_directory().unwrap();
write_file(
&scratch.0.join("package.json"),
"{\"private\":true,\"type\":\"module\"}\n",
)
.unwrap();
write_file(
&scratch.0.join("broken.validators.js"),
r#"export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:test": (value) => value.name,
});
"#,
)
.unwrap();
let definition = PropertyDefinition {
id: SemanticId::endpoint_property("Broken", "GetterSafety"),
name: "GetterSafety".into(),
runs: 1,
invariant: noxid_ir::PropertyInvariant::ValidatesOrRefuses,
boundary: SemanticId::endpoint("Broken"),
span: noxid_source::Span::new(0, 1),
};
let property = TestProperty {
definition: definition.clone(),
boundary_kind: "endpoint",
boundary_name: "Broken".into(),
validator_module: "./broken.validators.js".into(),
validators: vec![(
SemanticId::parse("validator:test").unwrap(),
Schema::Struct(vec![("name".into(), Schema::String)]),
)],
timeout: Duration::from_secs(1),
};
let execution = execute_properties(
&scratch.0,
Path::new("Broken.nox"),
&BTreeMap::from([(definition.id.clone(), property)]),
&Options {
gate: false,
json_only: true,
},
None,
None,
)
.unwrap();
assert!(
execution.javascript.contains("PROPERTY_INVARIANT_VIOLATED"),
"{}",
execution.javascript
);
assert!(execution.javascript.contains("seed"));
assert!(execution.javascript.contains("counterexample"));
assert!(
execution
.javascript
.contains("noxid test 'Broken.nox' --seed")
);
assert!(execution.javascript.contains("getterBomb"));
assert!(execution.javascript.contains("Object.defineProperty"));
}
#[test]
fn explicit_seed_replay_is_refused_after_a_property_rename() {
let scratch = scratch_directory().unwrap();
write_file(
&scratch.0.join("package.json"),
"{\"private\":true,\"type\":\"module\"}\n",
)
.unwrap();
write_file(
&scratch.0.join("broken.validators.js"),
r#"export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:test": (value) => value.name,
});
"#,
)
.unwrap();
let test_property = |name: &str| {
let definition = PropertyDefinition {
id: SemanticId::endpoint_property("Broken", name),
name: name.into(),
runs: 1,
invariant: noxid_ir::PropertyInvariant::ValidatesOrRefuses,
boundary: SemanticId::endpoint("Broken"),
span: noxid_source::Span::new(0, 1),
};
TestProperty {
definition,
boundary_kind: "endpoint",
boundary_name: "Broken".into(),
validator_module: "./broken.validators.js".into(),
validators: vec![(
SemanticId::parse("validator:test").unwrap(),
Schema::Struct(vec![("name".into(), Schema::String)]),
)],
timeout: Duration::from_secs(1),
}
};
let original = test_property("OriginalName");
let seed = noxid_property_gen::property_seed(original.definition.id.as_str(), 0);
let first = execute_properties(
&scratch.0,
Path::new("Broken.nox"),
&BTreeMap::from([(original.definition.id.clone(), original)]),
&Options {
gate: false,
json_only: true,
},
None,
None,
)
.unwrap();
let renamed = test_property("RenamedWithoutChangingTheSeed");
// A seed carries the identity of the property that produced it. After a
// rename the property is a different one, so the replay is refused with
// a teaching message rather than silently re-deriving the case under a
// name the seed never belonged to (Lane A follow-up sweep, round 1 QA).
let replay = execute_properties(
&scratch.0,
Path::new("Broken.nox"),
&BTreeMap::from([(renamed.definition.id.clone(), renamed)]),
&Options {
gate: false,
json_only: true,
},
Some(seed),
None,
);
let error = match replay {
Ok(_) => panic!("a renamed property must not accept the old seed"),
Err(error) => error.to_string(),
};
assert!(error.contains("PROPERTY_REPLAY_SEED_MISMATCH"), "{error}");
assert!(error.contains("RenamedWithoutChangingTheSeed"), "{error}");
assert!(first.javascript.contains(&format!("seed: \"{seed}\"")));
}
#[test]
fn queue_payload_property_executes_the_emitted_validator() {
let path = fixture(
"queue-property",
r#"type Payload { name: String }
queue ValidatePayload {
payload { value: Payload }
retry: 1
backoff: 1s
property Structured { runs: 6 expect: refusal is structured }
}"#,
);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(output.status.success(), "{stdout}");
assert!(
stdout.contains("\"id\":\"property:queue.ValidatePayload.Structured\""),
"{stdout}"
);
assert!(stdout.contains("\"runs\":6"), "{stdout}");
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn property_timeout_reports_the_offending_seed() {
let scratch = scratch_directory().unwrap();
write_file(
&scratch.0.join("package.json"),
"{\"private\":true,\"type\":\"module\"}\n",
)
.unwrap();
write_file(
&scratch.0.join("hung.validators.js"),
r#"export class ExternalValidationError extends Error {}
export const typeValidators = Object.freeze({
"validator:test": () => { while (true) {} },
});
"#,
)
.unwrap();
let definition = PropertyDefinition {
id: SemanticId::endpoint_property("Hung", "Terminates"),
name: "Terminates".into(),
runs: 1,
invariant: noxid_ir::PropertyInvariant::ValidatesOrRefuses,
boundary: SemanticId::endpoint("Hung"),
span: noxid_source::Span::new(0, 1),
};
let property = TestProperty {
definition: definition.clone(),
boundary_kind: "endpoint",
boundary_name: "Hung".into(),
validator_module: "./hung.validators.js".into(),
validators: vec![(SemanticId::parse("validator:test").unwrap(), Schema::String)],
timeout: Duration::from_millis(50),
};
let execution = execute_properties(
&scratch.0,
Path::new("Hung.nox"),
&BTreeMap::from([(definition.id.clone(), property)]),
&Options {
gate: false,
json_only: true,
},
None,
None,
)
.unwrap();
assert!(
execution.javascript.contains("PROPERTY_TIMEOUT_EXCEEDED"),
"{}",
execution.javascript
);
assert!(execution.javascript.contains("seed"));
}
#[test]
fn resource_given_emission_uses_the_scenario_seed_handshake_with_explicit_age() {
let span = noxid_source::Span::new(0, 1);
let resource_given = |age_milliseconds| noxid_ir::ScenarioGivenStep {
target: SemanticId::resource_acquisition("Catalog", "products"),
target_name: "products".into(),
kind: ScenarioGivenKind::Resource,
value: SemanticExpr {
kind: SemanticExprKind::Int(7),
ty: noxid_types::Type::Int,
span,
},
age_milliseconds,
span,
};
let mut aged = String::new();
emit_given_step(&mut aged, &resource_given(Some(45_000))).expect("given step emits");
assert!(aged.contains("$noxTest.mounted.resources[\"products\"].seedScenario(7, 45000);"));
let mut unaged = String::new();
emit_given_step(&mut unaged, &resource_given(None)).expect("given step emits");
assert!(unaged.contains("$noxTest.mounted.resources[\"products\"].seedScenario(7, null);"));
assert!(!aged.contains(".state.set("));
assert!(!unaged.contains(".state.set("));
}
#[test]
fn executes_generated_action_and_reports_failed_values() {
let path = fixture(
"failure",
r#"component Counter {
state { count: Int = 0 }
computed { doubled = count * 2 }
actions { increment() { count = count + 1 } }
scenario IncrementOnce {
description: "two increments"
given: count = 0
when: increment(), increment()
expect: count == 3, doubled == 4
}
view { <p>{count}</p><button +click={increment}>go</button> }
}"#,
);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(!output.status.success(), "{stdout}");
assert!(stdout.contains("\"expression\":\"count == 3\""));
assert!(stdout.contains("\"state:Counter.count\":2"));
assert!(stdout.contains("\"computed:Counter.doubled\":4"));
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn executes_typed_invariants_at_mount_and_after_each_scenario_step() {
let cases = [
(
"invariant-mount",
r#"component Counter {
state { count: Int = -1 }
invariant NonNegative { assert: count >= 0 }
scenario MountBreach {
description: "mount is checked"
expect: count == -1
}
view { <p>{count}</p> }
}"#,
"\"step\":{\"phase\":\"mount\",\"index\":0,\"target\":null}",
"\"state:Counter.count\":-1",
),
(
"invariant-given",
r#"component Counter {
state { count: Int = 0 delta: Int = 0 }
invariant NonNegative { assert: count >= 0 && delta >= 0 }
scenario GivenBreach {
description: "every given is checked"
given: count = 1, delta = -1
expect: delta == -1
}
view { <p>{count}</p> }
}"#,
"\"step\":{\"phase\":\"given\",\"index\":1,\"target\":\"delta\"}",
"\"state:Counter.delta\":-1",
),
(
"invariant-when",
r#"component Counter {
state { count: Int = 0 }
computed { doubled = count * 2 }
actions { increment() { count = count + 1 } }
invariant Bounded { assert: count <= 1 && doubled <= 2 }
scenario WhenBreach {
description: "every action is checked"
when: increment(), increment()
expect: count == 2
}
view { <p>{doubled}</p> }
}"#,
"\"step\":{\"phase\":\"when\",\"index\":1,\"target\":\"increment\"}",
"\"computed:Counter.doubled\":4",
),
];
for (name, source, step, actual) in cases {
let path = fixture(name, source);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(
!output.status.success(),
"{name} unexpectedly passed: {stdout}"
);
assert!(
stdout.contains("\"invariantFailures\":[{\"id\":\"invariant:Counter."),
"{name} did not report a structured invariant breach: {stdout}"
);
assert!(
stdout.contains(step),
"{name} lost its checkpoint: {stdout}"
);
assert!(
stdout.contains("\"expression\":"),
"{name} lost its invariant expression: {stdout}"
);
assert!(
stdout.contains(actual),
"{name} lost referenced actual values: {stdout}"
);
assert!(
stdout.contains("\"assertions\":[]"),
"{name} continued into expects after a breach: {stdout}"
);
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
}
#[test]
fn executes_resource_and_stream_givens_through_stubbed_runtime_handles() {
let path = fixture(
"boundary-givens",
r#"type Customer { id: Int name: String }
resource Customers(): Array<Customer> { get { GET "/must-not-fetch" } }
stream Feed() {
event Added(Customer)
event Settled
buffer 8
backpressure drop_oldest
}
component BoundaryHarness {
resources { customers = Customers() }
streams { feed = Feed() }
invariant ClosedLifecycle {
assert: customers == Idle || customers == Loading || customers == Ready([Customer(id = 7, name = "Ada")])
}
scenario ReadyAndEvents {
description: "ready payload and finite events reach emitted code"
given: customers = Ready([Customer(id = 7, name = "Ada")]), feed = [Added(Customer(id = 7, name = "Ada")), Settled]
expect: customers == Ready([Customer(id = 7, name = "Ada")])
}
scenario LoadingAndEmpty {
description: "non-ready lifecycle and empty stream remain deterministic"
given: customers = Loading, feed = []
expect: customers == Loading
}
view {
<section>
#match customers {
Idle { <p>idle</p> }
Loading { <p>loading</p> }
Ready(rows) { <p>{rows.count()}</p> }
Refreshing(rows) { <p>{rows.count()}</p> }
Failed(error) { <p>{error.code}</p> }
}
#stream feed {
Added(customer) { <p>{customer.name}</p> }
Settled { <p>settled</p> }
Completed { <p>completed</p> }
Failed(error) { <p>{error.code}</p> }
}
</section>
}
}"#,
);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(
output.status.success(),
"boundary scenarios failed:\nstdout={stdout}\nstderr={}",
String::from_utf8_lossy(&output.stderr)
);
assert!(stdout.contains("\"total\":2,\"passed\":2,\"failed\":0"));
assert_eq!(stdout.matches("\"invariantFailures\":[]").count(), 2);
assert!(stdout.contains("\"expression\":\"customers == Ready"));
assert!(stdout.contains("\"expression\":\"customers == Loading\""));
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn executes_resource_age_and_successful_mutation_invalidation_scenarios() {
let path = fixture(
"resource-cache-algebra",
r#"type Product { id: Int }
resource Products(): Array<Product> {
get { GET "/must-not-fetch" }
cache 30s
}
component CacheScenarios {
resources { products = Products() }
actions {
server saveProduct(product: Product): Product {}
server previewProduct(product: Product): Product invalidates none {}
save() {
let outcome = await saveProduct(product: Product(id = 2))
#match outcome {
Ok(saved) {}
Err(error) {}
}
}
preview() {
let outcome = await previewProduct(product: Product(id = 3))
#match outcome {
Ok(saved) {}
Err(error) {}
}
}
}
scenario StaleSeedRefreshes {
description: "a Ready seed older than its TTL refreshes"
given: products = Ready([Product(id = 1)]) aged 45s
expect: products == Refreshing([Product(id = 1)])
}
scenario SuccessfulMutationInvalidates {
description: "derived invalidation refreshes cached data"
given: products = Ready([Product(id = 1)]), saveProduct = Ok(Product(id = 2))
when: save()
expect: products == Refreshing([Product(id = 1)])
}
scenario ExplicitNonePreservesReady {
description: "invalidates none preserves cached data"
given: products = Ready([Product(id = 1)]), previewProduct = Ok(Product(id = 3))
when: preview()
expect: products == Ready([Product(id = 1)])
}
view {
#match products {
Idle { <p>idle</p> }
Loading { <p>loading</p> }
Ready(rows) { <p>{rows.count()}</p> }
Refreshing(rows) { <p>{rows.count()}</p> }
Failed(error) { <p>{error.code}</p> }
}
}
}"#,
);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(
output.status.success(),
"cache scenarios failed:\nstdout={stdout}\nstderr={}",
String::from_utf8_lossy(&output.stderr)
);
assert!(stdout.contains("\"total\":3,\"passed\":3,\"failed\":0"));
assert!(stdout.contains("\"name\":\"StaleSeedRefreshes\""));
assert!(stdout.contains("\"name\":\"SuccessfulMutationInvalidates\""));
assert!(stdout.contains("\"name\":\"ExplicitNonePreservesReady\""));
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn executes_remote_action_ok_and_err_givens_without_a_host_boundary() {
let path = fixture(
"remote-action-givens",
r#"type Incident { id: Int }
component RemoteForm {
state { savedId: Int = 0 message: String = "" }
actions {
server createIncident(request: Int): Incident {}
submit() {
savedId = -1
let outcome = await createIncident(request: 7)
#match outcome {
Ok(incident) { savedId = incident.id }
Err(error) { message = error.message }
}
}
}
scenario RemoteOk {
description: "typed success crosses the scenario-only boundary"
given: createIncident = Ok(Incident(id = 9))
when: submit()
expect: savedId == 9, message == ""
}
scenario RemoteErr {
description: "typed failure is consumed as RemoteError"
given: createIncident = Err(RemoteError(code = "CONFLICT", message = "already exists"))
when: submit()
expect: savedId == -1, message == "already exists"
}
view { <p>{savedId}</p> }
}"#,
);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
assert!(
output.status.success(),
"stdout={}\nstderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(stdout.contains("\"total\":2,\"passed\":2,\"failed\":0"));
assert!(stdout.contains("\"expression\":\"savedId == 9\""));
assert!(stdout.contains("\"expression\":\"message == \\\"already exists\\\"\""));
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn generated_harness_names_cannot_capture_component_scope() {
let path = fixture(
"hygiene",
r#"component HarnessHygiene {
state {
actualValues: Int = 0
assertions: Int = 0
failure: Int = 0
host: Int = 0
registration: Int = 0
runtime: Int = 0
scenarioScope: Int = 0
}
actions {
update() {
actualValues = 1
assertions = 2
failure = 3
host = 4
registration = 5
runtime = 6
scenarioScope = 7
}
}
scenario OrdinaryNames {
description: "harness implementation names stay private"
when: update()
expect: actualValues == 1, assertions == 2, failure == 3, host == 4, registration == 5, runtime == 6, scenarioScope == 7
}
view { <p>{actualValues}</p> }
}"#,
);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
assert!(
output.status.success(),
"stdout={}\nstderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn scenario_mount_grants_only_the_components_declared_capabilities() {
let path = fixture(
"declared-capability",
r#"component AuthorizedCounter {
requires [ counter.read ]
state { count: Int = 0 }
actions { increment() { count = count + 1 } }
scenario DeclaredAuthority {
description: "scenario mounts with compiler-declared authority"
when: increment()
expect: count == 1
}
view { <p>{count}</p> }
}"#,
);
let output = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap();
assert!(
output.status.success(),
"declared scenario authority was not granted:\nstdout={}\nstderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn gate_requires_passing_requirement_coverage() {
let path = fixture(
"gate",
r#"component Counter {
state { count: Int = 0 }
requirement COUNTER_001 {
description: "covered by a passing scenario"
verify: ["scenario"]
depends: [count]
}
view { <p>{count}</p> }
}"#,
);
let output = execute(
&path,
&Options {
gate: true,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(!output.status.success(), "{stdout}");
assert!(stdout.contains("requirement:COUNTER_001"));
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn refuses_unstubbed_boundaries_before_starting_node() {
let path = fixture(
"boundary",
r#"resource Customers(): String { get { GET "/customers" } }
component Counter {
resources { customers = Customers() }
state { count: Int = 0 }
scenario CannotReachNetwork {
description: "network is forbidden in phase one"
given: count = 0
expect: count == 0
}
view { <p>{count}</p> }
}"#,
);
let error = execute(
&path,
&Options {
gate: false,
json_only: true,
},
)
.unwrap_err();
assert!(
error.contains("SCENARIO_RESOURCE_BOUNDARY_UNSTUBBED"),
"{error}"
);
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
#[test]
fn executes_endpoint_scenarios_through_the_shipped_fetch_handler() {
let project = endpoint_project_fixture("fetch-handler");
fs::write(
project.join("server/api/items/[id].post.nox"),
r#"type Tagged { tag: String notes: Optional<Array<String>> }
type Echoed { id: String page: Int tag: String }
endpoint EchoItem {
params { id: String }
query { page: Int }
body { payload: Tagged }
result: Echoed
handler { return Echoed(id = id, page = page, tag = payload.tag) }
scenario RoundTrip {
description: "path, query, and body cross the emitted Fetch boundary"
when: request(params: Shape(id = "a/b"), query: Shape(page = 7), body: Shape(payload = Tagged(tag = "None", notes = Some(["</script><script>hostile()</script>"]))))
expect: status == 200, value == Echoed(id = "a/b", page = 7, tag = "None"), refusal == ""
}
scenario NestedOptionalNone {
description: "nested None crosses as JSON null without mangling an ordinary tag field"
when: request(params: Shape(id = "none"), query: Shape(page = 8), body: Shape(payload = Tagged(tag = "None", notes = None)))
expect: status == 200, value == Echoed(id = "none", page = 8, tag = "None")
}
scenario NestedOptionalSomeEmpty {
description: "nested Some empty array stays present as an empty JSON array"
when: request(params: Shape(id = "empty"), query: Shape(page = 9), body: Shape(payload = Tagged(tag = "Some", notes = Some([]))))
expect: status == 200, value == Echoed(id = "empty", page = 9, tag = "Some")
}
}"#,
)
.unwrap();
fs::write(
project.join("server/api/hosted.get.nox"),
r#"type HostedValue { message: String }
endpoint Hosted {
result: HostedValue
scenario ExactHostStub {
description: "the exact endpoint semantic key is stubbed"
given: Hosted = HostedValue(message = "safe")
when: request()
expect: status == 200, value == HostedValue(message = "safe"), refusal == ""
}
}"#,
)
.unwrap();
fs::write(
project.join("server/api/reject.get.nox"),
r#"type Accepted { value: Int }
type Refused { reason: String }
endpoint Reject {
result: Result<Accepted, Refused>
handler { return Err(Refused(reason = "no")) }
scenario TypedErr {
description: "typed Result Err crosses as 422"
when: request()
expect: status == 422, refusal == "ENDPOINT_RESULT_ERR"
}
}"#,
)
.unwrap();
fs::write(
project.join("server/api/denied.get.nox"),
r#"endpoint Denied {
result: Int
capabilities [secret.read]
handler { return 1 }
scenario Refusal {
description: "undeclared scenario authority fails closed"
when: request()
expect: status == 403, refusal == "ENDPOINT_CAPABILITY_DENIED"
}
}"#,
)
.unwrap();
fs::write(
project.join("server/api/arrays.get.nox"),
r#"endpoint Arrays {
query { tags: Array<String> ranks: Optional<Array<Int>> }
result: Int
handler { return tags.count() }
scenario JsonArray {
description: "array query values use one JSON-array URL value"
when: request(query: Shape(tags = ["a&b", "two"], ranks = Some([3, 4])))
expect: status == 200, value == 2
}
scenario EmptyAndAbsent {
description: "empty arrays and absent optional arrays remain distinct"
when: request(query: Shape(tags = [], ranks = None))
expect: status == 200, value == 0
}
scenario PresentEmptyOptional {
description: "Some empty array stays distinct from None on the wire"
when: request(query: Shape(tags = [""], ranks = Some([])))
expect: status == 200, value == 1
}
}"#,
)
.unwrap();
fs::write(
project.join("server/api/ip-limited.post.nox"),
r#"endpoint IpLimited {
body { value: Int }
result: Int
limit: 1 per minute per ip
handler { return value }
scenario FirstIpIdentity {
description: "the first scenario receives a trusted deterministic IP identity"
when: request(body: Shape(value = 11))
expect: status == 200, value == 11, refusal == ""
}
scenario SecondIpIdentity {
description: "a sibling scenario is isolated under its own deterministic IP identity"
when: request(body: Shape(value = 12))
expect: status == 200, value == 12, refusal == ""
}
}"#,
)
.unwrap();
fs::write(
project.join("server/api/session-limited.post.nox"),
r#"endpoint SessionLimited {
body { value: Int }
result: Int
limit: 1 per hour per session
handler { return value }
scenario FirstSessionIdentity {
description: "the first scenario receives a deterministic session identity"
when: request(body: Shape(value = 21))
expect: status == 200, value == 21, refusal == ""
}
scenario SecondSessionIdentity {
description: "a sibling scenario is isolated under its own deterministic session identity"
when: request(body: Shape(value = 22))
expect: status == 200, value == 22, refusal == ""
}
}"#,
)
.unwrap();
fs::write(
project.join("server/api/idempotent.post.nox"),
r#"endpoint Idempotent {
body { value: Int }
result: Int
idempotent
handler { return value }
scenario DeterministicReplayIdentity {
description: "scenario id supplies both replay key and trusted identity"
when: request(body: Shape(value = 31))
expect: status == 200, value == 31, refusal == ""
}
}"#,
)
.unwrap();
fs::write(
project.join("server/host.js"),
r#"throw new Error("REAL_HOST_EXECUTED");
export const actions = { "endpoint:Hosted@1": async () => fetch("https://must-not-run.invalid") };
"#,
)
.unwrap();
fs::write(
project.join("src/components/Counter.nox"),
r#"component Counter {
state { count: Int = 0 }
scenario ComponentAlongsideEndpoints {
description: "component and endpoint IDs share one deterministic report"
expect: count == 0
}
view { <p>{count}</p> }
}"#,
)
.unwrap();
let output = execute(
&project,
&Options {
gate: true,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(
output.status.success(),
"endpoint scenarios failed:\nstdout={stdout}\nstderr={}",
String::from_utf8_lossy(&output.stderr)
);
assert!(stdout.contains("\"total\":15,\"passed\":15,\"failed\":0"));
assert!(stdout.contains("\"id\":\"scenario:Counter.ComponentAlongsideEndpoints\""));
assert!(stdout.contains("\"semanticUnit\":\"endpoint:Arrays@1\""));
assert!(stdout.contains("\"semanticUnit\":\"endpoint:Denied@1\""));
assert!(stdout.contains("\"semanticUnit\":\"endpoint:EchoItem@1\""));
assert!(stdout.contains("\"semanticUnit\":\"endpoint:Hosted@1\""));
assert!(stdout.contains("\"semanticUnit\":\"endpoint:Reject@1\""));
assert!(!stdout.contains("REAL_HOST_EXECUTED"));
fs::remove_dir_all(project).unwrap();
}
#[test]
fn endpoint_expectation_failure_fails_the_gate_with_observed_values() {
let project = endpoint_project_fixture("failed-expectation");
fs::write(
project.join("server/api/failure.get.nox"),
r#"endpoint Failure { result: Int handler { return 7 }
scenario WrongStatus {
description: "a false emitted response expectation fails the command"
when: request()
expect: status == 201, value == 8
}
}"#,
)
.unwrap();
let output = execute(
&project,
&Options {
gate: true,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(!output.status.success(), "{stdout}");
assert!(stdout.contains("\"ok\":false"));
assert!(stdout.contains("\"endpoint-scenario-value:Failure.WrongStatus.status\":200"));
assert!(stdout.contains("\"endpoint-scenario-value:Failure.WrongStatus.value\":7"));
fs::remove_dir_all(project).unwrap();
}
#[test]
fn endpoint_file_input_selects_only_its_scenarios_from_the_project() {
let project = endpoint_project_fixture("single-file");
let selected_file = project.join("server/api/selected.get.nox");
fs::write(
&selected_file,
r#"endpoint Selected { result: Int handler { return 1 }
scenario Only { description: "selected source" when: request() expect: value == 1 }
}"#,
)
.unwrap();
fs::write(
project.join("server/api/unselected.get.nox"),
r#"endpoint Unselected { result: Int handler { return 2 }
scenario Other { description: "other source" when: request() expect: value == 2 }
}"#,
)
.unwrap();
let output = execute(
&selected_file,
&Options {
gate: true,
json_only: true,
},
)
.unwrap();
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(output.status.success(), "{stdout}");
assert!(stdout.contains("\"total\":1,\"passed\":1,\"failed\":0"));
assert!(stdout.contains("scenario:endpoint.Selected.Only"));
assert!(!stdout.contains("scenario:endpoint.Unselected.Other"));
fs::remove_dir_all(project).unwrap();
}
#[test]
fn affected_endpoint_selection_executes_only_the_selected_semantic_id() {
let project = endpoint_project_fixture("affected-selection");
fs::write(
project.join("server/api/selected.get.nox"),
r#"endpoint Selected { result: Int handler { return 1 }
scenario First { description: "not selected" when: request() expect: value == 1 }
scenario Second { description: "selected" when: request() expect: value == 1 }
}"#,
)
.unwrap();
let selected = BTreeSet::from([SemanticId::endpoint_scenario("Selected", "Second")]);
let report = execute_selected_report(
&project,
&Options {
gate: false,
json_only: true,
},
&selected,
)
.unwrap();
assert!(report.success, "{}\n{}", report.json, report.stderr);
assert!(
report
.json
.contains("\"total\":1,\"passed\":1,\"failed\":0")
);
assert!(report.json.contains("scenario:endpoint.Selected.Second"));
assert!(!report.json.contains("scenario:endpoint.Selected.First"));
fs::remove_dir_all(project).unwrap();
}
#[test]
fn standalone_endpoint_scenario_refuses_to_invent_file_routing() {
let path = fixture(
"endpoint-project-required",
r#"endpoint Standalone { result: Int handler { return 1 }
scenario One { description: "requires canonical routing" when: request() expect: value == 1 }
}"#,
);
let error = execute(
&path,
&Options {
gate: true,
json_only: true,
},
)
.unwrap_err();
assert!(
error.contains("ENDPOINT_SCENARIO_PROJECT_REQUIRED"),
"{error}"
);
fs::remove_dir_all(path.parent().unwrap()).unwrap();
}
}