const { spawnSync, spawn } = require("child_process");
const readline = require("readline");
const fs = require("fs");
const path = require("path");
const REPO_ROOT = path.resolve(__dirname, "..");
const ARGV = process.argv.slice(2); const UPDATE = ARGV.includes("--update");
const FIXTURE_NAME =
(ARGV.includes("--fixture") && ARGV[ARGV.indexOf("--fixture") + 1]) || "verify-tools";
const FIXTURE = path.join(REPO_ROOT, "tests/fixtures", FIXTURE_NAME);
const SNAP_DIR = path.join(FIXTURE, "snapshots");
const CASES = path.join(FIXTURE, "cases.jsonl");
const BIN_ARG =
(ARGV.includes("--bin") && ARGV[ARGV.indexOf("--bin") + 1]) ||
ARGV.find((a) => a.endsWith(".exe")) ||
path.join(REPO_ROOT, process.platform === "win32" ? "target/debug/symforge.exe" : "target/debug/symforge");
const BIN = path.resolve(REPO_ROOT, BIN_ARG);
const READ_TOOLS = new Set([
"search_symbols",
"search_text",
"find_references",
"get_symbol",
"get_file_context",
"get_symbol_context",
"batch_rename",
]);
function loadCases() {
return fs
.readFileSync(CASES, "utf8")
.split("\n")
.filter((l) => l.trim())
.map((l) => JSON.parse(l));
}
function grepOracle(pattern) {
const r = spawnSync("rg", ["-n", "--no-heading", pattern, "src"], {
cwd: FIXTURE,
encoding: "utf8",
});
if (r.error) return { hits: [], grepMissing: true };
const hits = (r.stdout || "")
.split("\n")
.filter((l) => l.trim())
.map((l) => l.trim());
return { hits, grepMissing: false };
}
function normalizeOracleText(text) {
return process.platform === "win32" ? text.replace(/\\/g, "/") : text;
}
function grepHitAnchor(hit) {
const match = normalizeOracleText(hit).match(/^(.*?):(\d+):/);
return match ? `${match[1]}:${match[2]}` : null;
}
function toolOutputAnchors(text) {
const anchors = new Set();
let groupedPath = null;
for (const rawLine of normalizeOracleText(text).split("\n")) {
const line = rawLine.trim();
const fileHeader = line.match(/^([A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)*\.[A-Za-z0-9]+)$/);
if (fileHeader) groupedPath = fileHeader[1];
for (const match of line.matchAll(/(?:^|[`(\s])([A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)*\.[A-Za-z0-9]+):(\d+)(?=$|[`),:\s])/g)) {
anchors.add(`${match[1]}:${match[2]}`);
}
const groupedLine = line.match(/^(?:>\s*)?(\d+):/);
if (groupedPath && groupedLine) anchors.add(`${groupedPath}:${groupedLine[1]}`);
const lineThenPath = line.match(/^(\d+):.*\(([A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)*\.[A-Za-z0-9]+)\)/);
if (lineThenPath) anchors.add(`${lineThenPath[2]}:${lineThenPath[1]}`);
}
return anchors;
}
function judgeOracleResult(text, c, hits, grepMissing) {
if (grepMissing) {
return { verdict: "REVIEW", reason: "rg not on PATH — grep oracle skipped", text };
}
const envelopeLines = text.split("\n").slice(0, 6);
const hasSuccessEnvelope = envelopeLines.some(
(line) => line.startsWith("Trust: ") || line.startsWith("Source authority: ")
);
const hasDeclaredEmptyResult =
typeof c.allow_empty === "string" && text.startsWith(c.allow_empty);
if (!hasSuccessEnvelope && !hasDeclaredEmptyResult) {
return {
verdict: "FAIL",
reason: "successful search terminal absent (expected envelope or declared empty result)",
text,
oracle: hits,
};
}
if (hasDeclaredEmptyResult) {
return {
verdict: "PASS",
reason: `declared empty result; grep saw ${hits.length} raw hit(s)`,
text,
oracle: hits,
};
}
const anchors = [...new Set(hits.map(grepHitAnchor).filter(Boolean))];
if (hits.length && !anchors.length) {
return {
verdict: "REVIEW",
reason: `could not derive path:line anchors from ${hits.length} grep hit(s)`,
text,
oracle: hits,
};
}
const observedAnchors = toolOutputAnchors(text);
const missingAnchors = anchors.filter((anchor) => !observedAnchors.has(anchor));
if (missingAnchors.length) {
const sample = missingAnchors.slice(0, 3).join(", ");
const more = missingAnchors.length > 3 ? ` (+${missingAnchors.length - 3} more)` : "";
return {
verdict: "REVIEW",
reason: `tool omitted ${missingAnchors.length}/${anchors.length} grep path:line anchor(s): ${sample}${more}`,
text,
oracle: hits,
};
}
return {
verdict: "PASS",
reason: `tool contains all ${anchors.length} grep path:line anchor(s)`,
text,
oracle: hits,
};
}
function runOracleComparatorSelfTest() {
const c = { allow_empty: "No references found" };
const hits = ["src/a.rs:2:fn alpha() {}", "src/b.rs:5:fn beta() {}"];
const complete = judgeOracleResult(
"Trust: exact | current index\nsrc/a.rs\n > 2: alpha\nsrc/b.rs\n 5: beta",
{},
hits,
false
);
const omitted = judgeOracleResult(
"Trust: exact | current index\nsrc/a.rs\n > 2: alpha",
{},
hits,
false
);
const declaredEmpty = judgeOracleResult("No references found for alpha", c, hits, false);
if (complete.verdict !== "PASS" || omitted.verdict !== "REVIEW" || declaredEmpty.verdict !== "PASS") {
throw new Error("grep path:line oracle comparator self-test failed");
}
}
function normalize(text) {
return text
.replace(/publication=\d+/g, "publication=<gen>")
.replace(/content=\d+/g, "content=<gen>")
.replace(/generation=\d+/g, "generation=<gen>")
.replace(/source=[0-9a-f]+/g, "source=<src>")
.replace(/content_hash=[0-9a-f]+/g, "content_hash=<hash>")
.replace(/link_id=[0-9a-f]+/g, "link_id=<id>")
.replace(/Original output is ~\d+ tokens/g, "Original output is ~<n> tokens")
.replace(/budget-limited \(was: full\)/g, "full");
}
function readSnapshot(name) {
const p = path.join(SNAP_DIR, name);
return fs.existsSync(p) ? fs.readFileSync(p, "utf8") : null;
}
function writeSnapshot(name, text) {
fs.writeFileSync(path.join(SNAP_DIR, name), text);
}
async function startSession(READINESS_PROBE) {
const proc = spawn(BIN, [], {
cwd: FIXTURE,
stdio: ["pipe", "pipe", "inherit"],
env: {
...process.env,
RUST_LOG: "warn",
SYMFORGE_SURFACE: "compact",
SYMFORGE_NO_DAEMON: "1",
SYMFORGE_WORKSPACE_ROOT: FIXTURE,
},
});
const pending = new Map();
let id = 1;
readline.createInterface({ input: proc.stdout }).on("line", (line) => {
if (!line.trim()) return;
let msg;
try {
msg = JSON.parse(line);
} catch {
return;
}
if (msg.id != null && pending.has(msg.id)) {
const { resolve, reject } = pending.get(msg.id);
pending.delete(msg.id);
if (msg.error) reject(new Error(JSON.stringify(msg.error)));
else resolve(msg.result);
}
});
function request(method, params) {
return new Promise((resolve, reject) => {
const myId = id++;
pending.set(myId, { resolve, reject });
proc.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: myId, method, params }) + "\n");
setTimeout(() => {
if (pending.has(myId)) {
pending.delete(myId);
reject(new Error(`timeout ${method}`));
}
}, 60000);
});
}
async function callTool(name, args) {
let toolName = name;
let toolArgs = args;
if (READ_TOOLS.has(name)) {
toolName = "symforge";
toolArgs = { _probe_legacy_tool: name, _probe_legacy_args: args };
}
const result = await request("tools/call", { name: toolName, arguments: toolArgs });
const text = (result.content || [])
.filter((c) => c.type === "text")
.map((c) => c.text)
.join("\n");
return { result, text, isError: !!result.isError };
}
await request("initialize", {
protocolVersion: "2024-11-05",
capabilities: {},
clientInfo: { name: "verify-tools", version: "1.0" },
});
proc.stdin.write(
JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized", params: {} }) + "\n"
);
const EVIDENCE_KEY = "symforge/project_evidence";
const EMPTY_BOOTSTRAP = "empty_bootstrap";
let ready = false;
let evidence = null;
for (let i = 0; i < 80 && !ready; i++) {
const status = await callTool("status", {});
evidence = (status.result && status.result._meta && status.result._meta[EVIDENCE_KEY]) || null;
const sourceBound =
!!evidence &&
evidence.index_state === "Ready" &&
evidence.load_source !== EMPTY_BOOTSTRAP &&
Number(evidence.index_files) > 0;
if (sourceBound) {
if (!READINESS_PROBE) {
ready = true; } else {
const probe = await callTool("search_symbols", { query: READINESS_PROBE });
if (!probe.isError && probe.text.includes(READINESS_PROBE)) ready = true;
}
}
if (!ready) await new Promise((r) => setTimeout(r, 250));
}
if (!ready) {
console.error(
"\n HARNESS ABORT — the index never reached a source-bound Ready generation " +
"(80 polls x 250ms, ~20s of sleep).\n" +
` last ${EVIDENCE_KEY}: ${JSON.stringify(evidence)}\n` +
` Wanted index_state="Ready" with load_source!="${EMPTY_BOOTSTRAP}" and index_files>0.\n` +
` load_source="${EMPTY_BOOTSTRAP}" => the real load never published (placeholder index).\n` +
' {"bound":false} => no workspace bound; check SYMFORGE_WORKSPACE_ROOT / the fixture path.\n' +
" null/absent evidence => the _meta key was NOT written at all. The server always writes\n" +
" it (unbound emits {\"bound\":false}, never omission), so this means the _meta contract\n" +
" changed; fix the gate, not the snapshots.\n"
);
proc.kill();
process.exit(2);
}
return { callTool, close: () => proc.kill() };
}
async function runCase(session, c) {
const isWrite = c.judge === "write_snapshot";
const writeAbs = isWrite ? path.join(FIXTURE, c.write_file) : null;
const original = isWrite ? fs.readFileSync(writeAbs, "utf8") : null;
let text, isError, writtenContent;
try {
({ text, isError } = await session.callTool(c.tool, c.args));
if (isWrite) writtenContent = fs.readFileSync(writeAbs, "utf8");
} finally {
if (isWrite) fs.writeFileSync(writeAbs, original);
}
if (isError || !text.trim()) {
return { verdict: "FAIL", reason: isError ? "tool returned error" : "empty response", text };
}
const missing = (c.must_contain || []).filter((s) => !text.includes(s));
if (missing.length) {
return { verdict: "FAIL", reason: `expected substrings absent: ${missing.join(", ")}`, text };
}
if (c.judge === "oracle") {
const { hits, grepMissing } = grepOracle(c.oracle_grep);
return judgeOracleResult(text, c, hits, grepMissing);
}
if (c.judge === "snapshot" || c.judge === "write_snapshot") {
const captured = normalize(isWrite ? writtenContent : text);
const snap = readSnapshot(c.snapshot);
if (snap === null) {
if (UPDATE) {
writeSnapshot(c.snapshot, captured);
return { verdict: "PASS", reason: `snapshot written (${c.snapshot})`, text: captured };
}
return { verdict: "REVIEW", reason: `no snapshot yet — run --update after eyeballing`, text: captured };
}
const snapN = normalize(snap);
if (snapN === captured) return { verdict: "PASS", reason: "byte-identical to snapshot", text: captured };
return { verdict: "FAIL", reason: `differs from snapshot ${c.snapshot}`, text: captured, snap: snapN };
}
return { verdict: "FAIL", reason: `unknown judge '${c.judge}'`, text };
}
const DIFF_LINES = 5;
function firstDiff(a, b) {
const al = (a || "").split("\n");
const bl = (b || "").split("\n");
const shown = [];
let differing = 0;
for (let i = 0; i < Math.max(al.length, bl.length); i++) {
if (al[i] === bl[i]) continue;
differing++;
if (shown.length < DIFF_LINES) {
shown.push(` L${i + 1}\n expected: ${JSON.stringify(bl[i])}\n actual: ${JSON.stringify(al[i])}`);
}
}
if (!differing) return "";
const more =
differing > shown.length ? `\n ... ${shown.length} of ${differing} differing line(s) shown` : "";
return shown.join("\n") + more;
}
(async () => {
if (ARGV.includes("--self-test-oracle")) {
runOracleComparatorSelfTest();
console.log("verify-tools oracle comparator self-test: PASS");
return;
}
if (!fs.existsSync(BIN)) {
console.error(`No binary at ${BIN}. Build it: cargo build`);
process.exit(2);
}
const cases = loadCases();
const probe = (cases.find((c) => c.tool === "search_symbols") || {}).args?.query || "";
const session = await startSession(probe);
const results = [];
for (const c of cases) {
try {
results.push({ c, r: await runCase(session, c) });
} catch (e) {
results.push({ c, r: { verdict: "FAIL", reason: `threw: ${e.message}`, text: "" } });
}
}
session.close();
const pad = (s, n) => (s + " ".repeat(n)).slice(0, n);
const mark = { PASS: "PASS ", REVIEW: "REVIEW", FAIL: "FAIL " };
console.log("\n TOOL-CORRECTNESS HARNESS — " + path.relative(REPO_ROOT, BIN) + "\n");
console.log(" " + pad("VERDICT", 8) + pad("CASE", 22) + pad("TOOL", 18) + "NOTE");
console.log(" " + "-".repeat(96));
const tally = { PASS: 0, REVIEW: 0, FAIL: 0 };
for (const { c, r } of results) {
tally[r.verdict]++;
console.log(" " + pad(mark[r.verdict], 8) + pad(c.id, 22) + pad(c.tool, 18) + r.reason);
if (r.verdict === "FAIL" && r.snap != null) {
const d = firstDiff(r.text, r.snap);
if (d) console.log(d);
}
}
console.log(" " + "-".repeat(96));
console.log(` ${tally.PASS} PASS ${tally.REVIEW} REVIEW ${tally.FAIL} FAIL\n`);
if (tally.REVIEW) console.log(" REVIEW = a human reads the diff (grep over-matches vs symbol-aware tools). Not a failure.");
if (tally.FAIL) console.log(" FAIL = a real regression: exact output/must-contain changed, a direct tool errored, or output went empty.\n");
process.exit(tally.FAIL > 0 ? 1 : 0);
})();