noxid-cli 0.2.0

//! WO-53 context budgeting: the mechanical size accounting behind the chunk
//! index.
//!
//! `llms-full.txt` is the compatibility surface and `llms.txt` is an index
//! over chunks. Compatibility is **self-identity** (round 3, after round-2 QA
//! finding 1): the full file is the current generator's header block plus its
//! body — what the pre-split single-file algorithm would produce from today's
//! source documents — and the chunk payloads partition that same body
//! byte-for-byte. Comparing against a frozen artifact was round 1's oracle and
//! stopped being a contract once reconciliation added documentation pages; a
//! growing document set cannot be pinned to an old file's bytes.
//!
//! The order's acceptance requires the indexed sizes to sum to that body. They
//! can only do so if the accounting states what the synthetic continuation
//! headings are: they are chunk-local navigation metadata, never body content,
//! and the index's "## Accounting" block counts them separately.

use std::fs;
use std::path::{Path, PathBuf};

const CHUNK_CAP: usize = 24 * 1024;

fn root() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")
}

struct Row {
    name: String,
    declared: usize,
}

fn index_rows(index: &str) -> Vec<Row> {
    index
        .lines()
        .filter_map(|line| line.strip_prefix("- llms/"))
        .map(|row| {
            let (name, rest) = row.split_once(" (").expect("chunk name and size");
            let (size, _) = rest.split_once(" bytes) - ").expect("chunk size and scope");
            Row {
                name: name.to_string(),
                declared: size.parse().expect("decimal chunk size"),
            }
        })
        .collect()
}

/// A continuation heading is the only thing a chunk may carry that the body
/// does not. Strip it and the rest is verbatim body bytes.
fn split_continuation_heading(bytes: &[u8]) -> (&[u8], &[u8]) {
    let text = std::str::from_utf8(bytes).expect("chunk is UTF-8");
    let Some(first) = text.lines().next() else {
        return (&[], bytes);
    };
    if !(first.starts_with("## ") && first.contains("(continued, part ")) {
        return (&[], bytes);
    }
    let heading_len = first.len() + "\n\n".len();
    assert_eq!(
        &text[first.len()..heading_len],
        "\n\n",
        "continuation heading must be followed by one blank line"
    );
    bytes.split_at(heading_len)
}

#[test]
fn llms_full_keeps_the_pre_split_layout_and_the_chunks_partition_its_body() {
    let root = root();
    let full = fs::read(root.join("llms-full.txt")).expect("llms-full.txt");
    let index = fs::read_to_string(root.join("llms.txt")).expect("llms.txt");

    // Finding 1 (round 1): the split dropped the blank line between the common
    // header block and the first topic, so the compatibility file lost a byte
    // the single-file algorithm produced. The header ends with a blank line and
    // the body starts at the first `## ` heading; that layout is the standing
    // self-identity contract, checked here against the committed artifacts
    // rather than against any frozen historical file.
    let body_start = full
        .windows(5)
        .position(|window| window == b"\n\n## ")
        .expect("llms-full.txt separates the header block from the first topic")
        + 2;
    let body = &full[body_start..];
    assert!(
        body.starts_with(b"## "),
        "llms-full.txt body must start at a topic heading"
    );
    assert!(
        !full.windows(18).any(|w| w == b"(continued, part 2"),
        "llms-full.txt must not carry chunk-local continuation headings"
    );
    let header = &full[..body_start];
    let index_header_end = index
        .find("\nThis is an index.")
        .expect("index states that it is an index");
    assert_eq!(
        std::str::from_utf8(header)
            .expect("header is UTF-8")
            .trim_end(),
        index[..index_header_end].trim_end(),
        "the index and the full file must share the same header block"
    );

    // Finding 2: the chunk payloads partition the body byte-for-byte, in index
    // order. Anything else and "sizes sum to the former body" cannot be true.
    let rows = index_rows(&index);
    assert!(rows.len() >= 20, "unexpectedly small chunk topic set");
    let mut payload = Vec::new();
    let mut payload_bytes = 0usize;
    let mut continuation_bytes = 0usize;
    let mut continued_parts = 0usize;
    for row in &rows {
        let chunk = fs::read(root.join("llms").join(&row.name))
            .unwrap_or_else(|error| panic!("indexed chunk {} does not resolve: {error}", row.name));
        assert_eq!(
            chunk.len(),
            row.declared,
            "index mis-sized chunk {}",
            row.name
        );
        assert!(
            chunk.len() <= CHUNK_CAP,
            "chunk {} is {} bytes, over the {CHUNK_CAP}-byte cap",
            row.name,
            chunk.len()
        );
        let (heading, rest) = split_continuation_heading(&chunk);
        if !heading.is_empty() {
            continued_parts += 1;
        }
        continuation_bytes += heading.len();
        payload_bytes += rest.len();
        payload.extend_from_slice(rest);
    }
    assert_eq!(
        payload.len(),
        body.len(),
        "chunk payloads are {} bytes but llms-full.txt's body is {} bytes",
        payload.len(),
        body.len()
    );
    assert!(
        payload == body,
        "chunk payloads do not reproduce llms-full.txt's body byte-for-byte"
    );

    // The index must state the accounting, not leave a reader to infer it.
    let indexed: usize = rows.iter().map(|row| row.declared).sum();
    assert_eq!(indexed, payload_bytes + continuation_bytes);
    for claim in [
        format!("{payload_bytes} payload bytes +"),
        format!("{continuation_bytes} bytes of synthetic continuation headings across"),
        format!("{continued_parts} continued part(s) = {indexed} bytes indexed above."),
        format!("llms-full.txt is {} bytes:", full.len()),
        format!(
            "{} bytes of this header block plus the {payload_bytes}-byte body.",
            header.len()
        ),
    ] {
        assert!(
            index.contains(&claim),
            "llms.txt accounting block omitted `{claim}`"
        );
    }
}

// ---------------------------------------------------------------------------
// Finding 3: the chunk cap must not fail open.
// ---------------------------------------------------------------------------

fn copy_tree(from: &Path, to: &Path) {
    fs::create_dir_all(to).expect("create export directory");
    for entry in fs::read_dir(from).expect("read source directory") {
        let entry = entry.expect("directory entry");
        let target = to.join(entry.file_name());
        if entry.file_type().expect("entry type").is_dir() {
            copy_tree(&entry.path(), &target);
        } else {
            fs::copy(entry.path(), &target).expect("copy file");
        }
    }
}

/// The generator only needs the documents it transpiles, its own tools, the
/// shipping grammar, and a place to write the website routes.
fn docs_generator_export(label: &str) -> PathBuf {
    let root = root();
    let export =
        std::env::temp_dir().join(format!("noxid-wo53-budget-{label}-{}", std::process::id()));
    let _ = fs::remove_dir_all(&export);
    fs::create_dir_all(export.join("website/src/routes")).expect("create website route directory");
    copy_tree(&root.join("docs"), &export.join("docs"));
    copy_tree(&root.join("tools"), &export.join("tools"));
    copy_tree(
        &root.join("grammar"),
        &export.join("grammar"),
    );
    fs::copy(root.join("README.md"), export.join("README.md")).expect("copy README");
    #[cfg(unix)]
    std::os::unix::fs::symlink(root.join("node_modules"), export.join("node_modules"))
        .expect("link node_modules");
    export
}

fn run_generator(export: &Path, headroom: Option<&str>) -> std::process::Output {
    let mut command = std::process::Command::new("node");
    command
        .arg(export.join("tools/docs-to-noxid.mjs"))
        .current_dir(export);
    if let Some(headroom) = headroom {
        command.env("NOXID_DOCS_CONTINUATION_HEADROOM", headroom);
    }
    command.output().expect("run the docs generator")
}

#[test]
fn an_overlong_source_line_is_byte_split_and_an_over_cap_chunk_is_refused() {
    let export = docs_generator_export("overlong");

    // Round 1 broke the cap here: `splitToCap` only ever split at line
    // boundaries, so one 30 KB line escaped as a 30,036-byte chunk. The second
    // probe is multi-byte so a naive byte cut would corrupt it.
    let quickstart = export.join("docs/learn/quickstart.md");
    let mut source = fs::read_to_string(&quickstart).expect("read quickstart");
    source.push_str("\n## Overlong probe\n\n");
    source.push_str(&"x".repeat(30_000));
    source.push_str("\n\n## Overlong multibyte probe\n\n");
    source.push_str(&"\u{4e16}\u{754c}".repeat(9_000));
    source.push('\n');
    fs::write(&quickstart, &source).expect("write oversized quickstart");

    let output = run_generator(&export, None);
    assert!(
        output.status.success(),
        "generator refused a splittable document:\n{}\n{}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );

    let index = fs::read_to_string(export.join("llms.txt")).expect("generated index");
    let full = fs::read(export.join("llms-full.txt")).expect("generated full file");
    let body_start = full
        .windows(5)
        .position(|window| window == b"\n\n## ")
        .expect("generated full file has a body")
        + 2;
    let rows = index_rows(&index);
    let mut payload = Vec::new();
    let mut split_parts = 0usize;
    for row in &rows {
        let chunk = fs::read(export.join("llms").join(&row.name)).expect("generated chunk");
        assert_eq!(chunk.len(), row.declared, "index mis-sized {}", row.name);
        assert!(
            chunk.len() <= CHUNK_CAP,
            "chunk {} is {} bytes, over the {CHUNK_CAP}-byte cap: the cap failed open",
            row.name,
            chunk.len()
        );
        let (heading, rest) = split_continuation_heading(&chunk);
        if !heading.is_empty() {
            split_parts += 1;
        }
        // A byte-split must land on a code-point boundary, so every chunk is
        // still valid UTF-8 on its own.
        std::str::from_utf8(rest).expect("chunk payload is UTF-8");
        payload.extend_from_slice(rest);
    }
    assert!(
        split_parts > 0,
        "the oversized document did not split into parts"
    );
    assert!(
        payload == full[body_start..],
        "byte-splitting lost or invented body bytes"
    );

    // The cap enforcement itself must fail closed rather than publish a file it
    // mis-advertises. Documents alone cannot defeat the splitter's arithmetic,
    // so the guard is driven by removing the continuation headroom.
    let refusal = run_generator(&export, Some("0"));
    assert!(
        !refusal.status.success(),
        "generator published over-cap chunks instead of refusing"
    );
    let stderr = String::from_utf8_lossy(&refusal.stderr);
    assert!(
        stderr.contains("docs-to-noxid: CHUNK_CAP_EXCEEDED"),
        "refusal is not structured: {stderr}"
    );
    assert!(
        stderr.contains(&format!("over the {CHUNK_CAP}-byte chunk cap")),
        "refusal did not name the cap: {stderr}"
    );

    let _ = fs::remove_dir_all(&export);
}

// ---------------------------------------------------------------------------
// Finding 4: the served content type of the context files.
// ---------------------------------------------------------------------------

/// The three classes of context file WO-53 publishes. A model (or a browser)
/// that asks for any of them must get readable text, not a download.
const CONTEXT_FILE_CLASSES: [(&str, &str); 3] = [
    ("llms.txt", "the chunk index"),
    ("llms-full.txt", "the full concatenation"),
    ("llms/quickstart.txt", "one chunk"),
];

#[test]
fn the_emitted_node_server_serves_every_context_file_class_as_plain_text() {
    let project = std::env::temp_dir().join(format!("noxid-wo53-mime-{}", std::process::id()));
    let _ = fs::remove_dir_all(&project);
    fs::create_dir_all(project.join("src/routes")).expect("create project routes");
    fs::write(
        project.join("Noxid.toml"),
        "[app]\ntitle = \"Context files\"\n\n[deploy]\nadapter = \"node\"\n",
    )
    .expect("write project config");
    fs::write(
        project.join("src/routes/+page.nox"),
        "component Home {\n    view {\n        <p>{\"Home\"}</p>\n    }\n}\n",
    )
    .expect("write page");

    let adapted = std::process::Command::new(env!("CARGO_BIN_EXE_noxid"))
        .args(["adapt", ".", "--out-dir", "deploy", "--adapter", "node"])
        .current_dir(&project)
        .output()
        .expect("run noxid adapt");
    assert!(
        adapted.status.success(),
        "adapt the context-file fixture:\n{}\n{}",
        String::from_utf8_lossy(&adapted.stdout),
        String::from_utf8_lossy(&adapted.stderr)
    );

    // Publish the real repository artifacts, exactly as the SEO copy step does.
    let root = root();
    let deploy = project.join("deploy");
    fs::create_dir_all(deploy.join("llms")).expect("create chunk directory");
    for (name, _) in CONTEXT_FILE_CLASSES {
        fs::copy(root.join(name), deploy.join(name))
            .unwrap_or_else(|error| panic!("publish {name}: {error}"));
    }

    let port = std::net::TcpListener::bind("127.0.0.1:0")
        .expect("reserve a test port")
        .local_addr()
        .expect("read the test port")
        .port();
    let mut server = std::process::Command::new("node")
        .arg("server.mjs")
        .current_dir(&deploy)
        .env("PORT", port.to_string())
        .stdout(std::process::Stdio::piped())
        .stderr(std::process::Stdio::piped())
        .spawn()
        .expect("start the emitted Node server");
    let mut ready = String::new();
    {
        use std::io::BufRead;
        std::io::BufReader::new(server.stdout.take().expect("capture server stdout"))
            .read_line(&mut ready)
            .expect("read server readiness");
    }

    let names = format!(
        "[{}]",
        CONTEXT_FILE_CLASSES
            .iter()
            .map(|(name, _)| format!("\"{name}\""))
            .collect::<Vec<_>>()
            .join(", ")
    );
    let probe = format!(
        r#"const origin = "http://127.0.0.1:{port}";
const names = {names};
const lines = [];
for (const name of names) {{
  const response = await fetch(`${{origin}}/${{name}}`);
  lines.push(`${{name}} ${{response.status}} ${{response.headers.get("content-type")}} ${{(await response.text()).length}}`);
}}
console.log(lines.join("\n"));
"#
    );
    let client = std::process::Command::new("node")
        .args(["--input-type=module", "--eval", &probe])
        .output()
        .expect("probe the served context files");

    // Only ever kill the server this test started.
    let _ = server.kill();
    let _ = server.wait();
    let _ = fs::remove_dir_all(&project);

    assert!(
        ready.contains("Noxid Node adapter listening"),
        "server did not report readiness: {ready}"
    );
    assert!(
        client.status.success(),
        "probe failed:\n{}\n{}",
        String::from_utf8_lossy(&client.stdout),
        String::from_utf8_lossy(&client.stderr)
    );
    let observed = String::from_utf8_lossy(&client.stdout);
    for ((name, description), line) in CONTEXT_FILE_CLASSES.iter().zip(observed.lines()) {
        assert!(
            line.starts_with(&format!("{name} 200 text/plain; charset=utf-8 ")),
            "{description} ({name}) was not served as plain text: {line}"
        );
        let served: usize = line
            .rsplit(' ')
            .next()
            .and_then(|value| value.parse().ok())
            .expect("served length");
        assert!(served > 0, "{description} ({name}) served an empty body");
    }
}

/// The Node adapter is what the test above exercises end to end; the same
/// omission was visible in the emitted Deno maps and the local dev server, so
/// pin all three extension tables at the source.
#[test]
fn every_emitted_extension_map_and_the_dev_server_type_dot_txt_as_plain_text() {
    let sources = root().join("crates/cli/src");
    let deployment = fs::read_to_string(sources.join("deployment.rs")).expect("read deployment.rs");
    let maps = deployment
        .lines()
        .filter(|line| line.contains("\".html\": \"text/html; charset=utf-8\""))
        .count();
    assert!(
        maps >= 3,
        "expected the Node and Deno extension maps, found {maps}"
    );
    assert_eq!(
        deployment
            .lines()
            .filter(|line| line.contains("\".txt\": \"text/plain; charset=utf-8\""))
            .count(),
        maps,
        "an emitted extension map is missing its .txt entry"
    );
    let dev_server = fs::read_to_string(sources.join("app.rs")).expect("read app.rs");
    assert!(
        dev_server.contains("Some(\"txt\") => \"text/plain; charset=utf-8\""),
        "the dev-server MIME table is missing its .txt entry"
    );
}

// ---------------------------------------------------------------------------
// Round-2 QA finding 2: one canonical source for every guide topic.
// ---------------------------------------------------------------------------

/// Every topic `usage` advertises. Discovered from the binary rather than
/// hard-coded, so a new topic cannot skip this check by not being listed here.
fn advertised_topics() -> Vec<String> {
    let usage = std::process::Command::new(env!("CARGO_BIN_EXE_noxid"))
        .output()
        .expect("run noxid without a command");
    let usage = format!(
        "{}{}",
        String::from_utf8_lossy(&usage.stdout),
        String::from_utf8_lossy(&usage.stderr)
    );
    let marker = "noxid agent-guide <";
    let start = usage.find(marker).expect("usage advertises agent-guide") + marker.len();
    let rest = &usage[start..];
    let end = rest.find('>').expect("agent-guide topic list closes");
    rest[..end].split('|').map(str::to_owned).collect()
}

fn mcp_guide_response(topic: &str) -> String {
    use std::io::Write;
    let request = format!(
        r#"{{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{{"name":"query_project","arguments":{{"operation":"describe","kind":"guide","name":"{topic}"}}}}}}"#
    );
    let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_noxid"))
        .args(["mcp", "../../examples/counter/Counter.nox"])
        .current_dir(env!("CARGO_MANIFEST_DIR"))
        .stdin(std::process::Stdio::piped())
        .stdout(std::process::Stdio::piped())
        .stderr(std::process::Stdio::piped())
        .spawn()
        .expect("start noxid MCP");
    child
        .stdin
        .take()
        .expect("MCP stdin")
        .write_all(format!("{request}\n").as_bytes())
        .expect("write MCP request");
    let output = child.wait_with_output().expect("finish MCP request");
    assert!(
        output.status.success(),
        "MCP describe guide {topic} failed: {}",
        String::from_utf8_lossy(&output.stderr)
    );
    String::from_utf8(output.stdout).expect("MCP response is UTF-8")
}

#[test]
fn every_guide_topic_is_byte_identical_through_all_three_doors() {
    const GUIDE_BUDGET: usize = 6 * 1024;
    let topics = advertised_topics();
    assert!(
        topics.len() >= 12,
        "usage advertises {} topics",
        topics.len()
    );

    for topic in &topics {
        let guide = std::process::Command::new(env!("CARGO_BIN_EXE_noxid"))
            .args(["agent-guide", topic])
            .output()
            .expect("run agent-guide");
        let described = std::process::Command::new(env!("CARGO_BIN_EXE_noxid"))
            .args(["describe", "guide", topic])
            .output()
            .expect("run describe guide");
        assert!(guide.status.success(), "agent-guide {topic} failed");
        assert!(described.status.success(), "describe guide {topic} failed");

        // Round-2 QA found `agent-guide core` 196 bytes longer than
        // `describe guide core`: a sentence printed by an `if topic == "core"`
        // block in the command handler, outside `agent_guide()`, so the
        // on-demand doors silently dropped it. Presentation logic in the
        // command is how a guide grows a second source of truth.
        assert_eq!(
            guide.stdout, described.stdout,
            "`agent-guide {topic}` and `describe guide {topic}` returned different bytes"
        );

        let text = String::from_utf8(guide.stdout).expect("guide is UTF-8");
        let response = mcp_guide_response(topic);
        assert!(
            response.len() <= GUIDE_BUDGET,
            "complete MCP response for {topic} is {} bytes, over {GUIDE_BUDGET}",
            response.len()
        );
        // The MCP door carries the same bytes, JSON-escaped, and nothing else.
        let escaped = noxid_source::json_escape(text.trim_end_matches('\n'));
        assert!(
            response.contains(&format!("\"text\":\"{escaped}\"")),
            "the MCP response for {topic} is not the CLI guide verbatim"
        );
    }
}

/// The moved sentence must still reach a reader through the generated content.
/// Round 3 put it in `ssr` rather than `core`: `core` plus the sentence pushes
/// the complete MCP response to 6,171 bytes, over the 6,144-byte budget, and
/// the sentence is about SSR default-slot hydration.
#[test]
fn the_ssr_topic_carries_the_default_slot_hydration_sentence() {
    let guide = std::process::Command::new(env!("CARGO_BIN_EXE_noxid"))
        .args(["agent-guide", "ssr"])
        .output()
        .expect("run agent-guide ssr");
    assert!(guide.status.success(), "agent-guide ssr failed");
    let text = String::from_utf8(guide.stdout).expect("guide is UTF-8");
    assert!(
        text.contains(
            "On SSR routes, default-slot children keep the parent's scope; eager hydration \
             adopts their server marker range without recreating DOM. Deferred-hydration \
             islands with slot children fail closed."
        ),
        "the ssr guide lost the default-slot hydration sentence"
    );
}