const { spawnSync, spawn } = require("child_process");
const readline = require("readline");
const fs = require("fs");
const path = require("path");
const REPO_ROOT = path.resolve(__dirname, "..");
const ARGV = process.argv.slice(2); const UPDATE = ARGV.includes("--update");
const FIXTURE_NAME =
(ARGV.includes("--fixture") && ARGV[ARGV.indexOf("--fixture") + 1]) || "verify-tools";
const FIXTURE = path.join(REPO_ROOT, "tests/fixtures", FIXTURE_NAME);
const SNAP_DIR = path.join(FIXTURE, "snapshots");
const CASES = path.join(FIXTURE, "cases.jsonl");
const BIN_ARG =
(ARGV.includes("--bin") && ARGV[ARGV.indexOf("--bin") + 1]) ||
ARGV.find((a) => a.endsWith(".exe")) ||
path.join(REPO_ROOT, process.platform === "win32" ? "target/debug/symforge.exe" : "target/debug/symforge");
const BIN = path.resolve(REPO_ROOT, BIN_ARG);
const READ_TOOLS = new Set([
"search_symbols",
"search_text",
"find_references",
"get_symbol",
"get_file_context",
"get_symbol_context",
"batch_rename",
]);
function loadCases() {
return fs
.readFileSync(CASES, "utf8")
.split("\n")
.filter((l) => l.trim())
.map((l) => JSON.parse(l));
}
function grepOracle(pattern) {
const r = spawnSync("rg", ["-n", "--no-heading", pattern, "src"], {
cwd: FIXTURE,
encoding: "utf8",
});
if (r.error) return { hits: [], grepMissing: true };
const hits = (r.stdout || "")
.split("\n")
.filter((l) => l.trim())
.map((l) => l.trim());
return { hits, grepMissing: false };
}
function normalize(text) {
return text
.replace(/publication=\d+/g, "publication=<gen>")
.replace(/content=\d+/g, "content=<gen>")
.replace(/generation=\d+/g, "generation=<gen>")
.replace(/source=[0-9a-f]+/g, "source=<src>")
.replace(/content_hash=[0-9a-f]+/g, "content_hash=<hash>")
.replace(/link_id=[0-9a-f]+/g, "link_id=<id>")
.replace(/Original output is ~\d+ tokens/g, "Original output is ~<n> tokens")
.replace(/budget-limited \(was: full\)/g, "full");
}
function readSnapshot(name) {
const p = path.join(SNAP_DIR, name);
return fs.existsSync(p) ? fs.readFileSync(p, "utf8") : null;
}
function writeSnapshot(name, text) {
fs.writeFileSync(path.join(SNAP_DIR, name), text);
}
async function startSession(READINESS_PROBE) {
const proc = spawn(BIN, [], {
cwd: FIXTURE,
stdio: ["pipe", "pipe", "ignore"],
env: {
...process.env,
RUST_LOG: "off",
SYMFORGE_SURFACE: "compact",
SYMFORGE_NO_DAEMON: "1",
SYMFORGE_WORKSPACE_ROOT: FIXTURE,
},
});
const pending = new Map();
let id = 1;
readline.createInterface({ input: proc.stdout }).on("line", (line) => {
if (!line.trim()) return;
let msg;
try {
msg = JSON.parse(line);
} catch {
return;
}
if (msg.id != null && pending.has(msg.id)) {
const { resolve, reject } = pending.get(msg.id);
pending.delete(msg.id);
if (msg.error) reject(new Error(JSON.stringify(msg.error)));
else resolve(msg.result);
}
});
function request(method, params) {
return new Promise((resolve, reject) => {
const myId = id++;
pending.set(myId, { resolve, reject });
proc.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: myId, method, params }) + "\n");
setTimeout(() => {
if (pending.has(myId)) {
pending.delete(myId);
reject(new Error(`timeout ${method}`));
}
}, 60000);
});
}
async function callTool(name, args) {
let toolName = name;
let toolArgs = args;
if (READ_TOOLS.has(name)) {
toolName = "symforge";
toolArgs = { _probe_legacy_tool: name, _probe_legacy_args: args };
}
const result = await request("tools/call", { name: toolName, arguments: toolArgs });
const text = (result.content || [])
.filter((c) => c.type === "text")
.map((c) => c.text)
.join("\n");
return { result, text, isError: !!result.isError };
}
await request("initialize", {
protocolVersion: "2024-11-05",
capabilities: {},
clientInfo: { name: "verify-tools", version: "1.0" },
});
proc.stdin.write(
JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized", params: {} }) + "\n"
);
let ready = false;
for (let i = 0; i < 30 && !ready; i++) {
const status = (await callTool("status", {})).text;
const files = (status.match(/index_files"?\s*:?\s*(\d+)/) || [])[1];
if (files && Number(files) > 0) {
if (!READINESS_PROBE) {
ready = true; } else {
const probe = await callTool("search_symbols", { query: READINESS_PROBE });
if (!probe.isError && probe.text.includes(READINESS_PROBE)) ready = true;
}
}
if (!ready) await new Promise((r) => setTimeout(r, 250));
}
if (!ready) throw new Error("index never became queryable (cold-start timeout)");
return { callTool, close: () => proc.kill() };
}
async function runCase(session, c) {
const isWrite = c.judge === "write_snapshot";
const writeAbs = isWrite ? path.join(FIXTURE, c.write_file) : null;
const original = isWrite ? fs.readFileSync(writeAbs, "utf8") : null;
let text, isError, writtenContent;
try {
({ text, isError } = await session.callTool(c.tool, c.args));
if (isWrite) writtenContent = fs.readFileSync(writeAbs, "utf8");
} finally {
if (isWrite) fs.writeFileSync(writeAbs, original);
}
if (isError || !text.trim()) {
return { verdict: "FAIL", reason: isError ? "tool returned error" : "empty response", text };
}
const missing = (c.must_contain || []).filter((s) => !text.includes(s));
if (c.judge === "oracle") {
const { hits, grepMissing } = grepOracle(c.oracle_grep);
if (grepMissing) return { verdict: "REVIEW", reason: "rg not on PATH — grep oracle skipped", text };
if (missing.length) {
return { verdict: "REVIEW", reason: `expected substrings absent: ${missing.join(", ")}`, text, oracle: hits };
}
return { verdict: "PASS", reason: `grep saw ${hits.length} raw hit(s); tool contains required anchors`, text, oracle: hits };
}
if (c.judge === "snapshot" || c.judge === "write_snapshot") {
const captured = normalize(isWrite ? writtenContent : text);
const snap = readSnapshot(c.snapshot);
if (snap === null) {
if (UPDATE) {
writeSnapshot(c.snapshot, captured);
return { verdict: "PASS", reason: `snapshot written (${c.snapshot})`, text: captured };
}
return { verdict: "REVIEW", reason: `no snapshot yet — run --update after eyeballing`, text: captured };
}
const snapN = normalize(snap);
if (snapN === captured) return { verdict: "PASS", reason: "byte-identical to snapshot", text: captured };
return { verdict: "FAIL", reason: `differs from snapshot ${c.snapshot}`, text: captured, snap: snapN };
}
return { verdict: "FAIL", reason: `unknown judge '${c.judge}'`, text };
}
function firstDiff(a, b) {
const al = (a || "").split("\n");
const bl = (b || "").split("\n");
for (let i = 0; i < Math.max(al.length, bl.length); i++) {
if (al[i] !== bl[i]) return ` L${i + 1}\n expected: ${JSON.stringify(bl[i])}\n actual: ${JSON.stringify(al[i])}`;
}
return "";
}
(async () => {
if (!fs.existsSync(BIN)) {
console.error(`No binary at ${BIN}. Build it: cargo build`);
process.exit(2);
}
const cases = loadCases();
const probe = (cases.find((c) => c.tool === "search_symbols") || {}).args?.query || "";
const session = await startSession(probe);
const results = [];
for (const c of cases) {
try {
results.push({ c, r: await runCase(session, c) });
} catch (e) {
results.push({ c, r: { verdict: "FAIL", reason: `threw: ${e.message}`, text: "" } });
}
}
session.close();
const pad = (s, n) => (s + " ".repeat(n)).slice(0, n);
const mark = { PASS: "PASS ", REVIEW: "REVIEW", FAIL: "FAIL " };
console.log("\n TOOL-CORRECTNESS HARNESS — " + path.relative(REPO_ROOT, BIN) + "\n");
console.log(" " + pad("VERDICT", 8) + pad("CASE", 22) + pad("TOOL", 18) + "NOTE");
console.log(" " + "-".repeat(96));
const tally = { PASS: 0, REVIEW: 0, FAIL: 0 };
for (const { c, r } of results) {
tally[r.verdict]++;
console.log(" " + pad(mark[r.verdict], 8) + pad(c.id, 22) + pad(c.tool, 18) + r.reason);
if (r.verdict === "FAIL" && r.snap != null) {
const d = firstDiff(r.text, r.snap);
if (d) console.log(d);
}
}
console.log(" " + "-".repeat(96));
console.log(` ${tally.PASS} PASS ${tally.REVIEW} REVIEW ${tally.FAIL} FAIL\n`);
if (tally.REVIEW) console.log(" REVIEW = a human reads the diff (grep over-matches vs symbol-aware tools). Not a failure.");
if (tally.FAIL) console.log(" FAIL = a real regression: exact-output tool changed, or tool errored/went empty.\n");
process.exit(tally.FAIL > 0 ? 1 : 0);
})();