use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
const GUIDE_BUDGET: usize = 6 * 1024;
const CHUNK_BUDGET: usize = 24 * 1024;
const CONTEXT_TYPE: &str = "text/plain; charset=utf-8";
const CONTEXT_PREAMBLE: &str = "Context first: call query_project { operation: \"context\" } with a byte budget and read only the surface it points you to; reach for this guide's detail after that, not before.";
const TOPICS: [&str; 14] = [
"core",
"scaffolding",
"interop",
"types",
"routing",
"storage",
"realtime",
"observability",
"reactivity",
"motion",
"remote-actions",
"ssr",
"testing",
"semantic-tools",
];
const CONTEXT_FILES: [&str; 3] = ["llms.txt", "llms-full.txt", "llms/quickstart.txt"];
static NEXT_TEMP: AtomicU64 = AtomicU64::new(0);
struct TempDir(PathBuf);
impl TempDir {
fn new(label: &str) -> Self {
let ordinal = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"noxid-wo53-qa2-{label}-{}-{ordinal}",
std::process::id()
));
let _ = fs::remove_dir_all(&path);
fs::create_dir_all(&path).expect("create QA temporary directory");
Self(path)
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
fn root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")
}
fn noxid(args: &[&str]) -> Output {
Command::new(env!("CARGO_BIN_EXE_noxid"))
.args(args)
.output()
.expect("run noxid")
}
fn output_text(output: &Output) -> String {
format!(
"status: {}\nstdout:\n{}\nstderr:\n{}",
output.status,
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
)
}
fn advertised_topics() -> Vec<String> {
let output = noxid(&[]);
assert!(
!output.status.success(),
"missing command unexpectedly succeeded"
);
let usage = format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let marker = "noxid agent-guide <";
let start = usage.find(marker).expect("usage advertises agent-guide") + marker.len();
let rest = &usage[start..];
let end = rest.find('>').expect("agent-guide topic list closes");
rest[..end].split('|').map(str::to_owned).collect()
}
fn mcp_describe(topic: &str, id: usize) -> Output {
let request = format!(
r#"{{"jsonrpc":"2.0","id":{id},"method":"tools/call","params":{{"name":"query_project","arguments":{{"operation":"describe","kind":"guide","name":"{topic}"}}}}}}"#
);
let mut child = Command::new(env!("CARGO_BIN_EXE_noxid"))
.args(["mcp", "../../examples/counter/Counter.nox"])
.current_dir(env!("CARGO_MANIFEST_DIR"))
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("start noxid MCP");
child
.stdin
.take()
.expect("MCP stdin")
.write_all(format!("{request}\n").as_bytes())
.expect("write MCP request");
child.wait_with_output().expect("finish MCP request")
}
#[test]
fn every_advertised_guide_fits_through_both_cli_doors_and_mcp() {
let topics = advertised_topics();
assert_eq!(
topics, TOPICS,
"usage must advertise exactly fourteen topics"
);
for (id, topic) in topics.iter().enumerate() {
let guide = noxid(&["agent-guide", topic]);
let described = noxid(&["describe", "guide", topic]);
assert!(
guide.status.success(),
"agent-guide {topic} failed: {}",
String::from_utf8_lossy(&guide.stderr)
);
assert!(
described.status.success(),
"describe guide {topic} failed: {}",
String::from_utf8_lossy(&described.stderr)
);
assert!(
guide.stdout.len() <= GUIDE_BUDGET,
"agent-guide {topic} is {} bytes, over {GUIDE_BUDGET}",
guide.stdout.len()
);
let guide_text = String::from_utf8(guide.stdout).expect("guide is UTF-8");
assert_eq!(
guide_text.lines().next(),
Some(CONTEXT_PREAMBLE),
"guide {topic} does not start context-first"
);
assert!(
described.stdout.len() <= GUIDE_BUDGET,
"describe guide {topic} is {} bytes, over {GUIDE_BUDGET}",
described.stdout.len()
);
let described_text = String::from_utf8(described.stdout).expect("described guide is UTF-8");
assert_eq!(
described_text.lines().next(),
Some(CONTEXT_PREAMBLE),
"described guide {topic} does not start context-first"
);
let heading = guide_text.lines().nth(1).expect("guide topic heading");
assert_eq!(
described_text.lines().nth(1),
Some(heading),
"describe guide {topic} returned the wrong topic"
);
let mcp = mcp_describe(topic, 5300 + id);
assert!(
mcp.status.success(),
"MCP describe guide {topic} failed: {}",
String::from_utf8_lossy(&mcp.stderr)
);
assert!(
mcp.stdout.len() <= GUIDE_BUDGET,
"complete MCP response for {topic} is {} bytes, over {GUIDE_BUDGET}",
mcp.stdout.len()
);
let response = String::from_utf8(mcp.stdout).expect("MCP response is UTF-8");
for expected in [
format!(r#""id":{}"#, 5300 + id),
r#""kind":"guide""#.to_string(),
format!(r#""name":"{topic}""#),
"Context first: call query_project".to_string(),
] {
assert!(
response.contains(&expected),
"MCP response for {topic} omitted {expected}: {response}"
);
}
}
}
fn copy_tree(from: &Path, to: &Path) {
fs::create_dir_all(to).expect("create export directory");
for entry in fs::read_dir(from).expect("read source directory") {
let entry = entry.expect("directory entry");
let target = to.join(entry.file_name());
if entry.file_type().expect("entry type").is_dir() {
copy_tree(&entry.path(), &target);
} else {
fs::copy(entry.path(), target).expect("copy export file");
}
}
}
fn export_generator() -> TempDir {
let temp = TempDir::new("cap-refusal");
let repository = root();
fs::create_dir_all(temp.0.join("website/src/routes")).expect("create website routes");
copy_tree(&repository.join("docs"), &temp.0.join("docs"));
copy_tree(&repository.join("tools"), &temp.0.join("tools"));
copy_tree(
&repository.join("grammar"),
&temp.0.join("grammar"),
);
fs::copy(repository.join("README.md"), temp.0.join("README.md")).expect("copy README");
#[cfg(unix)]
std::os::unix::fs::symlink(repository.join("node_modules"), temp.0.join("node_modules"))
.expect("link node_modules");
temp
}
fn chunk_snapshot(export: &Path) -> BTreeMap<String, Vec<u8>> {
fs::read_dir(export.join("llms"))
.expect("read generated chunks")
.map(|entry| {
let entry = entry.expect("chunk entry");
let name = entry
.file_name()
.into_string()
.expect("UTF-8 chunk filename");
let bytes = fs::read(entry.path()).expect("read chunk");
(name, bytes)
})
.collect()
}
fn run_generator(export: &Path, headroom: Option<&str>) -> Output {
let mut command = Command::new("node");
command
.arg(export.join("tools/docs-to-noxid.mjs"))
.current_dir(export);
if let Some(value) = headroom {
command.env("NOXID_DOCS_CONTINUATION_HEADROOM", value);
}
command.output().expect("run docs generator")
}
fn indexed_chunk_names(index: &str) -> Vec<String> {
index
.lines()
.filter_map(|line| line.strip_prefix("- llms/"))
.map(|row| {
row.split_once(" (")
.expect("indexed name and size")
.0
.to_string()
})
.collect()
}
fn chunk_payload(chunk: &[u8]) -> &[u8] {
let text = std::str::from_utf8(chunk).expect("chunk is valid UTF-8");
let first = text.lines().next().unwrap_or_default();
if first.starts_with("## ") && first.contains("(continued, part ") {
let heading_bytes = first.len() + 2;
assert_eq!(&text[first.len()..heading_bytes], "\n\n");
&chunk[heading_bytes..]
} else {
chunk
}
}
fn full_body(full: &[u8]) -> &[u8] {
let start = full
.windows(5)
.position(|window| window == b"\n\n## ")
.expect("full file has a header/body boundary")
+ 2;
&full[start..]
}
#[test]
fn a_cap_refusal_preserves_the_last_known_good_chunk_set() {
let export = export_generator();
let quickstart = export.0.join("docs/learn/quickstart.md");
let mut source = fs::read_to_string(&quickstart).expect("read quickstart source");
let probe = format!("{}😀{}", "x".repeat(24_319), "y".repeat(5_677));
assert_eq!(probe.len(), 30_000);
source.push_str("\n## Four-byte boundary probe\n\n");
source.push_str(&probe);
source.push('\n');
fs::write(&quickstart, source).expect("add overlong source line");
let first = run_generator(&export.0, None);
assert!(
first.status.success(),
"initial generation failed:\n{}\n{}",
String::from_utf8_lossy(&first.stdout),
String::from_utf8_lossy(&first.stderr)
);
let before = chunk_snapshot(&export.0);
assert!(!before.is_empty(), "initial generation emitted no chunks");
let index = fs::read_to_string(export.0.join("llms.txt")).expect("read generated index");
let full = fs::read(export.0.join("llms-full.txt")).expect("read generated full file");
let mut reconstructed = Vec::new();
let mut saw_scalar = false;
for name in indexed_chunk_names(&index) {
let chunk = before.get(&name).expect("indexed chunk resolves");
assert!(
chunk.len() <= CHUNK_BUDGET,
"{name} is {} bytes, over {CHUNK_BUDGET}",
chunk.len()
);
let payload = chunk_payload(chunk);
let payload_text = std::str::from_utf8(payload).expect("payload is independently UTF-8");
saw_scalar |= payload_text.contains('😀');
reconstructed.extend_from_slice(payload);
}
assert!(saw_scalar, "the four-byte scalar disappeared at the cut");
assert_eq!(
reconstructed,
full_body(&full),
"the split payloads do not reproduce the full body"
);
let refused = run_generator(&export.0, Some("0"));
assert!(
!refused.status.success(),
"over-cap generation unexpectedly succeeded"
);
let stderr = String::from_utf8_lossy(&refused.stderr);
assert!(
stderr.contains("docs-to-noxid: CHUNK_CAP_EXCEEDED"),
"cap refusal is not structured: {stderr}"
);
assert!(
stderr.contains("over the 24576-byte chunk cap"),
"cap refusal does not state the byte limit: {stderr}"
);
assert_eq!(
chunk_snapshot(&export.0),
before,
"a refused generation replaced the last known-good chunk set"
);
}
#[test]
fn committed_chunks_partition_the_body_and_reproduce_the_accounting_block() {
let repository = root();
let index = fs::read_to_string(repository.join("llms.txt")).expect("read llms index");
let full = fs::read(repository.join("llms-full.txt")).expect("read full context file");
let mut indexed_names = BTreeSet::new();
let mut reconstructed = Vec::new();
let mut payload_bytes = 0usize;
let mut continuation_bytes = 0usize;
let mut continued_parts = 0usize;
let mut indexed_bytes = 0usize;
for line in index.lines().filter(|line| line.starts_with("- llms/")) {
let row = line.strip_prefix("- llms/").expect("index prefix");
let (name, rest) = row.split_once(" (").expect("name and size");
let (declared, scope) = rest.split_once(" bytes) - ").expect("size and scope");
let declared = declared.parse::<usize>().expect("decimal size");
assert!(!scope.trim().is_empty(), "{name} has an empty scope");
assert!(
indexed_names.insert(name.to_string()),
"{name} is duplicated"
);
let chunk = fs::read(repository.join("llms").join(name)).expect("indexed chunk resolves");
assert_eq!(chunk.len(), declared, "the index mis-sizes {name}");
assert!(chunk.len() <= CHUNK_BUDGET, "{name} exceeds the cap");
let payload = chunk_payload(&chunk);
payload_bytes += payload.len();
let heading_bytes = chunk.len() - payload.len();
continuation_bytes += heading_bytes;
continued_parts += usize::from(heading_bytes > 0);
indexed_bytes += chunk.len();
reconstructed.extend_from_slice(payload);
}
let actual_names = fs::read_dir(repository.join("llms"))
.expect("read chunk directory")
.map(|entry| entry.expect("chunk entry").file_name())
.filter_map(|name| name.into_string().ok())
.filter(|name| name.ends_with(".txt"))
.collect::<BTreeSet<_>>();
assert_eq!(indexed_names, actual_names, "index is not a bijection");
assert_eq!(
reconstructed,
full_body(&full),
"chunk payload partition drifted"
);
assert_eq!(indexed_bytes, payload_bytes + continuation_bytes);
for claim in [
format!("{payload_bytes} payload bytes +"),
format!("{continuation_bytes} bytes of synthetic continuation headings across"),
format!("{continued_parts} continued part(s) = {indexed_bytes} bytes indexed above."),
] {
assert!(index.contains(&claim), "accounting omitted `{claim}`");
}
}
#[test]
fn published_surfaces_and_every_guide_route_agents_to_the_index() {
let repository = root();
for (relative, required) in [
("README.md", "The agent surface is budgeted"),
(
"docs/server/agent-surface.md",
"`llms.txt` is a chunk index",
),
(
"docs/language-reference.md",
"its own `llms/website-api.txt` chunk",
),
(
"website/src/routes/+page.nox",
"An index of budgeted context chunks",
),
] {
let text = fs::read_to_string(repository.join(relative)).expect("read published surface");
assert!(text.contains(required), "{relative} omitted `{required}`");
for stale in [
"whole language as one structured context file",
"whole documentation set into one file an agent can take as context",
"llms.txt API section",
] {
assert!(
!text.contains(stale),
"{relative} retained stale advice `{stale}`"
);
}
}
for topic in TOPICS {
let output = noxid(&["agent-guide", topic]);
assert!(output.status.success(), "agent-guide {topic} failed");
let guide = String::from_utf8(output.stdout).expect("guide is UTF-8");
for line in guide.lines().filter(|line| line.contains("llms-full.txt")) {
assert!(
line.contains("never"),
"agent-guide {topic} directs an agent to the full file: {line}"
);
}
}
}
fn write_context_project(project: &Path, ssr: bool) {
fs::create_dir_all(project.join("src/routes")).expect("create project routes");
fs::write(
project.join("Noxid.toml"),
"[app]\ntitle = \"WO-53 content types\"\n",
)
.expect("write project config");
fs::write(project.join("package.json"), "{\"type\":\"module\"}\n")
.expect("write package manifest");
let render = if ssr { " route { render: ssr }" } else { "" };
fs::write(
project.join("src/routes/+page.nox"),
format!("component Home {{{render} view {{ <main>Context</main> }} }}\n"),
)
.expect("write page");
if ssr {
fs::create_dir_all(project.join("server")).expect("create server directory");
fs::write(project.join("server/host.js"), "export default {};\n").expect("write SSR host");
}
}
fn publish_context_files(destination: &Path) {
let repository = root();
for relative in CONTEXT_FILES {
let target = destination.join(relative);
if let Some(parent) = target.parent() {
fs::create_dir_all(parent).expect("create context-file directory");
}
fs::copy(repository.join(relative), target).expect("publish context file");
}
}
fn free_port() -> u16 {
TcpListener::bind("127.0.0.1:0")
.expect("reserve port")
.local_addr()
.expect("read port")
.port()
}
struct ServerProcess {
child: Child,
port: u16,
}
impl ServerProcess {
#[allow(clippy::zombie_processes)]
fn start(mut command: Command, port: u16) -> Self {
command.stdout(Stdio::null()).stderr(Stdio::piped());
let mut child = command.spawn().expect("start server process");
let deadline = Instant::now() + Duration::from_secs(60);
while Instant::now() < deadline {
if TcpStream::connect(("127.0.0.1", port)).is_ok() {
return Self { child, port };
}
if let Some(status) = child.try_wait().expect("poll server") {
let mut stderr = String::new();
child
.stderr
.take()
.expect("server stderr")
.read_to_string(&mut stderr)
.expect("read server stderr");
panic!("server exited before listening: {status}: {stderr}");
}
std::thread::sleep(Duration::from_millis(50));
}
let _ = child.kill();
let _ = child.wait();
panic!("server did not listen on {port}");
}
fn get(&self, relative: &str) -> Vec<u8> {
let mut stream = TcpStream::connect(("127.0.0.1", self.port)).expect("connect to server");
stream
.set_read_timeout(Some(Duration::from_secs(10)))
.expect("set read timeout");
write!(
stream,
"GET /{relative} HTTP/1.1\r\nHost: 127.0.0.1\r\nAccept: */*\r\nConnection: close\r\n\r\n"
)
.expect("write HTTP request");
let mut response = Vec::new();
stream
.read_to_end(&mut response)
.expect("read HTTP response");
response
}
}
impl Drop for ServerProcess {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
fn assert_context_responses(server: &ServerProcess) {
for relative in CONTEXT_FILES {
let response = server.get(relative);
let boundary = response
.windows(4)
.position(|window| window == b"\r\n\r\n")
.expect("HTTP header boundary");
let headers = String::from_utf8_lossy(&response[..boundary]).to_ascii_lowercase();
assert!(headers.starts_with("http/1.1 200"), "{relative}: {headers}");
assert!(
headers.contains(&format!("content-type: {CONTEXT_TYPE}")),
"{relative} has the wrong content type: {headers}"
);
assert!(!response[boundary + 4..].is_empty(), "{relative} is empty");
}
}
#[test]
fn emitted_node_server_types_all_three_context_file_classes_as_plain_text() {
let fixture = TempDir::new("node-content-type");
write_context_project(&fixture.0, false);
let adapted = Command::new(env!("CARGO_BIN_EXE_noxid"))
.args(["adapt", ".", "--adapter", "node", "--out-dir", "deploy"])
.current_dir(&fixture.0)
.output()
.expect("adapt Node fixture");
assert!(adapted.status.success(), "{}", output_text(&adapted));
let deploy = fixture.0.join("deploy");
publish_context_files(&deploy);
let port = free_port();
let mut command = Command::new("node");
command
.arg("server.mjs")
.current_dir(&deploy)
.env("PORT", port.to_string());
let server = ServerProcess::start(command, port);
assert_context_responses(&server);
}
const DENO_DRIVER: &str = r#"import { readFile, realpath, stat } from "node:fs/promises";
globalThis.Deno = {
serve: (_options, handler) => { globalThis.__handler = handler; return { shutdown: async () => {} }; },
env: { get: () => undefined, toObject: () => ({}) },
readFile: (path) => readFile(path),
realPath: (path) => realpath(path),
stat: async (path) => { const value = await stat(path); return { isDirectory: value.isDirectory(), isFile: value.isFile(), size: value.size }; },
addSignalListener: () => {},
};
await import("./server.mjs");
for (const path of ["llms.txt", "llms-full.txt", "llms/quickstart.txt"]) {
const response = await globalThis.__handler(new Request(`http://noxid.test/${path}`, { headers: { accept: "*/*" } }));
console.log(`${path}|${response.status}|${response.headers.get("content-type")}|${(await response.arrayBuffer()).byteLength}`);
}
"#;
fn assert_deno_context_responses(deploy: &Path) {
fs::copy(deploy.join("server.ts"), deploy.join("server.mjs"))
.expect("stage Deno template for Node");
fs::write(deploy.join("deno-driver.mjs"), DENO_DRIVER).expect("write Deno driver");
let output = Command::new("node")
.arg("deno-driver.mjs")
.current_dir(deploy)
.output()
.expect("run Deno template under stub");
assert!(output.status.success(), "{}", output_text(&output));
let report = String::from_utf8(output.stdout).expect("Deno report is UTF-8");
for relative in CONTEXT_FILES {
let prefix = format!("{relative}|200|{CONTEXT_TYPE}|");
let line = report
.lines()
.find(|line| line.starts_with(&prefix))
.unwrap_or_else(|| panic!("missing `{prefix}` in {report}"));
assert!(
line.rsplit('|')
.next()
.and_then(|value| value.parse::<usize>().ok())
.is_some_and(|length| length > 0),
"{relative} has an empty Deno response: {line}"
);
}
}
#[test]
fn both_deno_templates_type_all_three_context_file_classes_as_plain_text() {
for (label, ssr) in [("deno-static", false), ("deno-provider", true)] {
let fixture = TempDir::new(label);
write_context_project(&fixture.0, ssr);
let adapted = Command::new(env!("CARGO_BIN_EXE_noxid"))
.args(["adapt", ".", "--adapter", "deno", "--out-dir", "deploy"])
.current_dir(&fixture.0)
.output()
.expect("adapt Deno fixture");
assert!(adapted.status.success(), "{}", output_text(&adapted));
let deploy = fixture.0.join("deploy");
publish_context_files(&deploy);
assert_deno_context_responses(&deploy);
}
}
#[test]
fn development_server_types_all_three_context_file_classes_as_plain_text() {
let fixture = TempDir::new("dev-content-type");
fs::write(
fixture.0.join("App.nox"),
"component App { view { <main>Context</main> } }\n",
)
.expect("write development fixture");
let output = fixture.0.join("dev-output");
let port = free_port();
let port_text = port.to_string();
let mut command = Command::new(env!("CARGO_BIN_EXE_noxid"));
command
.arg("dev")
.arg(fixture.0.join("App.nox"))
.args([
"--out-dir",
output.to_str().expect("UTF-8 output path"),
"--port",
&port_text,
])
.current_dir(&fixture.0);
let server = ServerProcess::start(command, port);
publish_context_files(&output);
assert_context_responses(&server);
}
#[test]
fn discarded_stage_d_failures_are_absent_and_the_comparison_stays_unproven() {
let repository = root();
let runs = repository.join("benchmarks/results/runs");
let mut bare_failures = Vec::new();
for entry in fs::read_dir(runs).expect("read benchmark ledger") {
let entry = entry.expect("ledger entry");
if entry.path().extension().and_then(|value| value.to_str()) != Some("json") {
continue;
}
let text = fs::read_to_string(entry.path()).expect("read ledger entry");
if text.contains("agent exited with 1") && !text.contains("\"model\":") {
bare_failures.push(entry.file_name());
}
}
assert!(
bare_failures.is_empty(),
"model-free agent failures remain in the ledger: {bare_failures:?}"
);
let order = fs::read_to_string(repository.join("docs/work-orders/wo-53-context-budgeting.md"))
.expect("read WO-53 status");
for claim in [
"Stage (d) is **measured-inconclusive**",
"neither establishes nor refutes the acceptance claim",
"**Three runs per arm**",
"**Guide consultation was uncontrolled.**",
"**Whole-session token totals only.**",
"belongs to\nthe **WO-52 ledger after the cut**",
] {
assert!(
order.contains(claim),
"the three-run comparison must not be presented as evidence; \
WO-53 status omitted `{claim}`"
);
}
assert!(
!order.contains(
"shows equal or better\ntask success at lower input tokens, recorded in the ledger"
),
"the acceptance text still asserts the unmeasured result"
);
}