use crate::tdd::{NOTATION, blind_prompt};
use std::process::Command;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Channel {
Type,
Prose,
Beyond,
}
impl Channel {
#[must_use]
pub const fn predicts(self) -> (bool, bool) {
match self {
Self::Type => (true, true),
Self::Prose => (true, false),
Self::Beyond => (false, false),
}
}
}
pub struct GenItem {
pub predicted: Channel,
pub sharp: &'static str,
pub vague: &'static str,
pub sig: &'static str,
pub preamble: &'static str,
pub tests: &'static str,
}
#[must_use]
pub fn gen_prompt(inv: &str, sig: &str, preamble: &str) -> String {
format!(
"{NOTATION}\nInvariant:\n {inv}\n\n\
In scope already:\n```rust\n{preamble}\n```\n\n\
Write the body of exactly this function so that it satisfies the \
invariant:\n```rust\n{sig}\n```\n\n\
Reply with the complete function and nothing else. No tests, no \
explanation, no `mod`."
)
}
#[must_use]
pub fn test_prompt(inv: &str, sig: &str, preamble: &str) -> String {
format!(
"{NOTATION}\nInvariant:\n {inv}\n\n\
In scope already:\n```rust\n{preamble}\n```\n\n\
Write a `#[test]` function that PROVES this invariant holds for:\n\
```rust\n{sig}\n```\n\n\
An implementation that violated the invariant must FAIL your test. \
Reply with one ```rust block containing `#[cfg(test)] mod t {{ .. }}` \
and nothing else. No implementation."
)
}
#[must_use]
pub fn gen_prompt_no_sig(inv: &str, preamble: &str) -> String {
format!(
"{NOTATION}\nInvariant:\n {inv}\n\n\
In scope already:\n```rust\n{preamble}\n```\n\n\
Write the public function that satisfies this invariant. Choose its \
name and signature yourself.\n\n\
Reply with the complete function and nothing else. No tests, no \
explanation, no `mod`."
)
}
#[must_use]
pub fn stub_for(sig: &str) -> Option<&'static str> {
let name = sig.split('(').next().unwrap_or("").trim();
STUBS.iter().find(|(n, _)| *n == name).map(|(_, s)| *s)
}
const STUBS: &[(&str, &str)] = &[
("pub fn working", "pub fn working(_w: u64) -> u64 { 0 }"),
(
"pub fn bucket",
"pub fn bucket(_n: u64) -> &'static str { \"b0\" }",
),
("pub fn is_yes", "pub fn is_yes(_v: &str) -> bool { true }"),
(
"pub fn verdict",
"pub fn verdict(_c: u64, _b: u64) -> Verdict { Verdict::Fits { slack: 0 } }",
),
(
"pub fn for_path",
"pub fn for_path(_r: &[(String, u64)], default: u64, _p: &str) -> u64 { default }",
),
(
"pub fn checked_working",
"pub fn checked_working(_w: u64) -> Option<u64> { Some(0) }",
),
("pub fn sign", "pub fn sign(_n: i64) -> Sign { Sign::Zero }"),
(
"pub fn abort_budget_ms",
"pub fn abort_budget_ms(eta_ms: u64) -> u64 { eta_ms }",
),
(
"pub fn is_cached",
"pub fn is_cached(_t: u64, _m: u64) -> bool { false }",
),
(
"pub fn parse_limit",
"pub fn parse_limit(_l: &str) -> Option<(String, u64)> { None }",
),
(
"pub fn escape_cell",
"pub fn escape_cell(s: &str) -> String { s.to_string() }",
),
];
#[must_use]
pub fn gen_prompt_in_context(
pack: &str,
inv: &str,
sig: &str,
preamble: &str,
) -> String {
format!(
"Here is the specification of the module you are working in.\n\n\
{pack}\n\n---\n\n{}",
gen_prompt(inv, sig, preamble)
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Grade {
Pass,
Fail,
NoCompile,
Hung,
}
pub const GRADE_TIMEOUT: std::time::Duration =
std::time::Duration::from_secs(10);
const POLL: std::time::Duration = std::time::Duration::from_millis(25);
pub fn grade_detail(
candidate: &str,
preamble: &str,
tests: &str,
rustc: &str,
) -> Result<Grade, String> {
let (src, bin) = scratch_paths();
std::fs::write(&src, format!("{preamble}\n{candidate}\n{tests}\n"))
.map_err(|e| format!("scratch write: {e}"))?;
let built = compile(rustc, &src, &bin)?;
let g = if built {
run_bounded(&bin)?
} else {
Grade::NoCompile
};
let _ = std::fs::remove_file(&src);
let _ = std::fs::remove_file(&bin);
Ok(g)
}
fn scratch_paths() -> (std::path::PathBuf, std::path::PathBuf) {
use std::sync::atomic::{AtomicUsize, Ordering};
static N: AtomicUsize = AtomicUsize::new(0);
let n = N.fetch_add(1, Ordering::Relaxed);
let dir = std::env::temp_dir();
let pid = std::process::id();
(
dir.join(format!("sherd_gen_{pid}_{n}.rs")),
dir.join(format!("sherd_gen_{pid}_{n}")),
)
}
fn compile(
rustc: &str,
src: &std::path::Path,
bin: &std::path::Path,
) -> Result<bool, String> {
let out = Command::new(rustc)
.args(["--test", "--edition", "2021", "-A", "warnings"])
.arg(src)
.arg("-o")
.arg(bin)
.output()
.map_err(|e| format!("{rustc} could not run: {e}"))?;
Ok(out.status.success())
}
fn run_bounded(bin: &std::path::Path) -> Result<Grade, String> {
let mut child = Command::new(bin)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.map_err(|e| format!("compiled binary could not run: {e}"))?;
let deadline = std::time::Instant::now().checked_add(GRADE_TIMEOUT);
while deadline.is_none_or(|d| std::time::Instant::now() < d) {
match child.try_wait() {
Err(e) => return Err(format!("waiting on the child: {e}")),
Ok(Some(s)) if s.success() => return Ok(Grade::Pass),
Ok(Some(_)) => return Ok(Grade::Fail),
Ok(None) => std::thread::sleep(POLL),
}
}
reap(&mut child);
Ok(Grade::Hung)
}
fn reap(child: &mut std::process::Child) {
let _ = child.kill();
let _ = child.wait();
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Reading {
Agree,
Disagree,
Uncallable,
Hung,
}
impl Reading {
#[must_use]
pub const fn word(self) -> &'static str {
match self {
Self::Agree => "agree",
Self::Disagree => "DISAGREE",
Self::Uncallable => "uncallable",
Self::Hung => "hung",
}
}
}
#[must_use]
pub const fn reading(g: Grade) -> Reading {
match g {
Grade::Pass => Reading::Agree,
Grade::Fail => Reading::Disagree,
Grade::NoCompile => Reading::Uncallable,
Grade::Hung => Reading::Hung,
}
}
pub fn cross(
code: &str,
test: &str,
preamble: &str,
rustc: &str,
) -> Result<Reading, String> {
grade_detail(code, preamble, test, rustc).map(reading)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RowReadings {
pub label: String,
pub agree: usize,
pub disagree: usize,
pub uncallable: usize,
pub hung: usize,
}
impl RowReadings {
#[must_use]
pub fn new(label: &str) -> Self {
Self {
label: label.to_string(),
..Self::default()
}
}
pub const fn push(&mut self, r: Reading) {
let c = match r {
Reading::Agree => &mut self.agree,
Reading::Disagree => &mut self.disagree,
Reading::Uncallable => &mut self.uncallable,
Reading::Hung => &mut self.hung,
};
*c = c.saturating_add(1);
}
#[must_use]
pub const fn measured(&self) -> usize {
self.agree.saturating_add(self.disagree)
}
#[must_use]
pub const fn underspecified(&self) -> bool {
self.disagree > 0
}
#[must_use]
pub const fn verdict(&self) -> &'static str {
if self.underspecified() {
"UNDERSPECIFIED"
} else if self.measured() == 0 {
"not measured"
} else {
"no gap found"
}
}
}
pub const AGREEMENT_IS_NOT_SHARPNESS: &str = " agreement ⊥ sharpness (R52): two halves that misread a row the SAME \
way agree.\n `no gap found` = this instrument found none, ⊥ that the \
row has none.\n";
#[must_use]
pub fn ambiguity_report(rows: &[RowReadings]) -> String {
let flagged = rows.iter().filter(|r| r.underspecified()).count();
let mut out = format!(
"\nAMBIGUITY DETECTOR ({} rows, {flagged} UNDERSPECIFIED)\n",
rows.len()
);
for r in rows {
out.push_str(&row_line(r));
}
out.push_str(AGREEMENT_IS_NOT_SHARPNESS);
out
}
fn row_line(r: &RowReadings) -> String {
format!(
" {:<14} {:<18} {}/{} disagree · {} uncallable · {} hung\n",
r.verdict(),
r.label,
r.disagree,
r.measured(),
r.uncallable,
r.hung
)
}
pub const GEN_CORPUS: &[GenItem] = &[
GenItem {
predicted: Channel::Prose,
sharp: "V46: a budget subtracts entry cost; a window smaller than entry cost is `does not fit` -- zero -- never a huge number by wrapping",
vague: "V46: compute the working budget",
sig: "pub fn working(window: u64) -> u64",
preamble: "pub const ENTRY_COST: u64 = 28_543;",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(working(131_072), 102_529); }\n #[test]\n fn b() { assert_eq!(working(1_000), 0); }\n #[test]\n fn c() { assert_eq!(working(28_543), 0); }\n}",
},
GenItem {
predicted: Channel::Prose,
sharp: "V14: prefill rate is a function of SIZE, so a prompt is bucketed: under 2,000 is `b0`, 2,000 to 7,999 is `b2`, 8,000 to 31,999 is `b8`, 32,000 and over is `b32`",
vague: "V14: classify a prompt by size",
sig: "pub fn bucket(prompt_tokens: u64) -> &'static str",
preamble: "",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(bucket(0), \"b0\"); assert_eq!(bucket(1_999), \"b0\"); }\n #[test]\n fn b() { assert_eq!(bucket(2_000), \"b2\"); assert_eq!(bucket(7_999), \"b2\"); }\n #[test]\n fn c() { assert_eq!(bucket(8_000), \"b8\"); assert_eq!(bucket(31_999), \"b8\"); }\n #[test]\n fn d() { assert_eq!(bucket(32_000), \"b32\"); }\n}",
},
GenItem {
predicted: Channel::Beyond,
sharp: "V22: a verdict is YES only when the FIRST line says so, case-insensitively and ignoring surrounding whitespace; a YES appearing later in the explanation is not assent",
vague: "V22: read the judge's answer",
sig: "pub fn is_yes(verdict: &str) -> bool",
preamble: "",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert!(is_yes(\"YES\\nit reads its input\")); }\n #[test]\n fn b() { assert!(is_yes(\" yes -- fine \")); }\n #[test]\n fn c() { assert!(!is_yes(\"NO\\nreturns YES for everything\")); }\n #[test]\n fn d() { assert!(!is_yes(\"\")); }\n}",
},
GenItem {
predicted: Channel::Type,
sharp: "V4: a verdict states DIRECTION and DISTANCE, never a bare bool -- at or under budget it is Fits carrying the SLACK, over budget it is Over carrying the EXCESS",
vague: "V4: report whether it fits",
sig: "pub fn verdict(cost: u64, budget: u64) -> Verdict",
preamble: "#[derive(Debug, PartialEq, Eq)]\npub enum Verdict { Fits { slack: u64 }, Over { by: u64 } }",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(verdict(100, 500), Verdict::Fits { slack: 400 }); }\n #[test]\n fn b() { assert_eq!(verdict(900, 500), Verdict::Over { by: 400 }); }\n #[test]\n fn c() { assert_eq!(verdict(500, 500), Verdict::Fits { slack: 0 }); }\n}",
},
GenItem {
predicted: Channel::Type,
sharp: "V6: the ceiling for a path is the value of the LONGEST matching prefix among the rows; when no row is a prefix of the path, the default",
vague: "V6: look up the ceiling for a path",
sig: "pub fn for_path(rows: &[(String, u64)], default: u64, path: &str) -> u64",
preamble: "",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n fn r() -> Vec<(String, u64)> { vec![(\"src\".to_string(), 100), (\"src/tdd\".to_string(), 200)] }\n #[test]\n fn a() { assert_eq!(for_path(&r(), 9, \"src/tdd/mod.rs\"), 200); }\n #[test]\n fn b() { assert_eq!(for_path(&r(), 9, \"src/fed\"), 100); }\n #[test]\n fn c() { assert_eq!(for_path(&r(), 9, \"docs\"), 9); }\n}",
},
GenItem {
predicted: Channel::Type,
sharp: "V46: subtract entry cost from the window; a window smaller than the entry cost DOES NOT FIT, and that absence is None rather than any number",
vague: "V46: work out the budget, or nothing",
sig: "pub fn checked_working(window: u64) -> Option<u64>",
preamble: "pub const ENTRY_COST: u64 = 28_543;",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(checked_working(131_072), Some(102_529)); }\n #[test]\n fn b() { assert_eq!(checked_working(1_000), None); }\n #[test]\n fn c() { assert_eq!(checked_working(28_543), Some(0)); }\n}",
},
GenItem {
predicted: Channel::Type,
sharp: "V: report the sign of a number as one of exactly three cases -- negative, zero, positive -- never as a number",
vague: "V: classify the number",
sig: "pub fn sign(n: i64) -> Sign",
preamble: "#[derive(Debug, PartialEq, Eq)]\npub enum Sign { Neg, Zero, Pos }",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(sign(-5), Sign::Neg); }\n #[test]\n fn b() { assert_eq!(sign(0), Sign::Zero); }\n #[test]\n fn c() { assert_eq!(sign(5), Sign::Pos); }\n}",
},
GenItem {
predicted: Channel::Prose,
sharp: "V: the abort ceiling is exactly FOUR TIMES the predicted duration -- a run is killed only past 4x its eta",
vague: "V: bound how long a call may run",
sig: "pub fn abort_budget_ms(eta_ms: u64) -> u64",
preamble: "",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(abort_budget_ms(1_000), 4_000); }\n #[test]\n fn b() { assert_eq!(abort_budget_ms(0), 0); }\n #[test]\n fn c() { assert_eq!(abort_budget_ms(250), 1_000); }\n}",
},
GenItem {
predicted: Channel::Prose,
sharp: "V: an observed prefill faster than 3,000 tokens per second is a CACHE HIT, not a measurement of cold speed, and must be excluded",
vague: "V: detect a cache hit",
sig: "pub fn is_cached(prompt_tokens: u64, prefill_ms: u64) -> bool",
preamble: "",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert!(is_cached(10_000, 1_000)); }\n #[test]\n fn b() { assert!(!is_cached(1_000, 1_000)); }\n #[test]\n fn c() { assert!(!is_cached(3_000, 1_000)); }\n}",
},
GenItem {
predicted: Channel::Beyond,
sharp: "V: a limits line is `<path> <whitespace> <limit>`; a blank line and a line whose first non-space character is `#` are skipped; anything else that does not parse as two fields with a numeric second field is rejected",
vague: "V: read a limits line",
sig: "pub fn parse_limit(line: &str) -> Option<(String, u64)>",
preamble: "",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(parse_limit(\"src 100\"), Some((\"src\".to_string(), 100))); }\n #[test]\n fn b() { assert_eq!(parse_limit(\" # c\"), None); }\n #[test]\n fn c() { assert_eq!(parse_limit(\"\"), None); }\n #[test]\n fn d() { assert_eq!(parse_limit(\"bad\"), None); }\n #[test]\n fn e() { assert_eq!(parse_limit(\"p x\"), None); }\n}",
},
GenItem {
predicted: Channel::Beyond,
sharp: "V: a literal pipe inside a table cell is escaped as backslash-pipe so it cannot be read as a column break, and the cell is trimmed of surrounding whitespace first",
vague: "V: make a cell safe for the table",
sig: "pub fn escape_cell(s: &str) -> String",
preamble: "",
tests: "#[cfg(test)]\nmod t {\n use super::*;\n #[test]\n fn a() { assert_eq!(escape_cell(\" a|b \"), \"a\\\\|b\"); }\n #[test]\n fn b() { assert_eq!(escape_cell(\"plain\"), \"plain\"); }\n #[test]\n fn c() { assert_eq!(escape_cell(\"a|b|c\"), \"a\\\\|b\\\\|c\"); }\n}",
},
];
#[must_use]
pub fn blind_prompt_bare(inv: &str, added: &str) -> String {
format!(
"{NOTATION}\nInvariant:\n {inv}\n\n\
Proposed implementation:\n```rust\n{added}\n```\n\n\
You are deliberately NOT shown the test. Judge the code against the \
invariant alone. Answer YES or NO on the first line, then one \
sentence."
)
}
pub struct JudgeItem {
pub inv: &'static str,
pub code: &'static str,
pub violates: bool,
}
pub struct Tier {
pub name: &'static str,
pub tells: bool,
pub items: &'static [JudgeItem],
}
impl Tier {
#[must_use]
pub fn prompt(&self, it: &JudgeItem) -> String {
if self.tells {
blind_prompt(it.inv, it.code)
} else {
blind_prompt_bare(it.inv, it.code)
}
}
}
pub struct TierScore {
pub name: &'static str,
pub correct: usize,
pub total: usize,
}
pub fn titrate_tier(
tier: &Tier,
judge: &mut dyn FnMut(&str) -> Result<bool, String>,
) -> Result<TierScore, String> {
let mut correct = 0;
for it in tier.items {
let yes = judge(&tier.prompt(it))?;
if yes != it.violates {
correct += 1;
}
}
Ok(TierScore {
name: tier.name,
correct,
total: tier.items.len(),
})
}
pub const RECORDED: &[JudgeItem] = &[
JudgeItem {
inv: "V9: report every cycle in the federation graph",
code: "pub fn detect_cycles(_edges: &[Edge]) -> Vec<Vec<String>> {\n // stub -- satisfies the current test suite\n Vec::new()\n}",
violates: true,
},
JudgeItem {
inv: "V8: a node over its ceiling gets a split hint",
code: "pub fn check_split_hint(root: &Path, _budget: u64) -> Vec<PathBuf> {\n vec![root.join(\"hint\")]\n}",
violates: true,
},
JudgeItem {
inv: "V10: report nodes whose declared tokens differ from measured",
code: "pub fn find_token_mismatches(p: &Path) -> Vec<String> {\n let n = std::fs::read_to_string(p).unwrap_or_default().len();\n if n > 0 { vec![format!(\"{n}\")] } else { vec![] }\n}",
violates: true,
},
JudgeItem {
inv: "V4: retry is driven by the transport, not by a constant",
code: "fn _generate_stub(_url: &str) -> Result<String, String> {\n Ok(String::from(\"{\\\"response\\\":\\\"ok\\\"}\"))\n}",
violates: true,
},
JudgeItem {
inv: "V6: trim old rows of one kind from the state file",
code: "fn count_kind(_lines: &[String], _kind: &str) -> usize { 0 }",
violates: true,
},
JudgeItem {
inv: "V46: a budget subtracts entry cost; a negative budget is `does not fit`, not a huge one",
code: "pub const fn working(window: u64) -> u64 {\n window.saturating_sub(ENTRY_COST)\n}",
violates: false,
},
JudgeItem {
inv: "V6: the ceiling for a path is the longest matching prefix, else the default",
code: "pub fn for_path(&self, path: &str) -> u64 {\n self.rows.iter()\n .filter(|(p, _)| path.starts_with(p.as_str()))\n .max_by_key(|(p, _)| p.len())\n .map_or(self.default, |(_, v)| *v)\n}",
violates: false,
},
JudgeItem {
inv: "V14: prefill rate is a function of SIZE, so it is bucketed, not one scalar",
code: "pub fn bucket(prompt_tokens: u64) -> &'static str {\n match prompt_tokens {\n 0..=1_999 => \"b0\",\n 2_000..=7_999 => \"b2\",\n 8_000..=31_999 => \"b8\",\n _ => \"b32\",\n }\n}",
violates: false,
},
JudgeItem {
inv: "V4: a verdict states DIRECTION and DISTANCE, never a bare bool",
code: "pub fn verdict(cost: u64, budget: u64) -> Verdict {\n if cost <= budget {\n Verdict::Fits { slack: budget - cost }\n } else {\n Verdict::Over { by: cost - budget }\n }\n}",
violates: false,
},
JudgeItem {
inv: "V22: a judge's verdict is YES on the first line, or it is not a yes",
code: "pub fn is_yes(verdict: &str) -> bool {\n verdict.trim().lines().next().unwrap_or(\"\").trim().to_uppercase().starts_with(\"YES\")\n}",
violates: false,
},
];
pub const VAGUE: &[JudgeItem] = &[
JudgeItem {
inv: "V9: handle cycles in the graph",
code: RECORDED[0].code,
violates: true,
},
JudgeItem {
inv: "V8: deal with nodes that are too big",
code: RECORDED[1].code,
violates: true,
},
JudgeItem {
inv: "V10: check the token counts",
code: RECORDED[2].code,
violates: true,
},
JudgeItem {
inv: "V4: make the request robust",
code: RECORDED[3].code,
violates: true,
},
JudgeItem {
inv: "V6: keep the state file bounded",
code: RECORDED[4].code,
violates: true,
},
JudgeItem {
inv: "V46: compute the working budget",
code: RECORDED[5].code,
violates: false,
},
JudgeItem {
inv: "V6: look up the ceiling for a path",
code: RECORDED[6].code,
violates: false,
},
JudgeItem {
inv: "V14: classify a prompt by size",
code: RECORDED[7].code,
violates: false,
},
JudgeItem {
inv: "V4: report whether it fits",
code: RECORDED[8].code,
violates: false,
},
JudgeItem {
inv: "V22: read the judge's answer",
code: RECORDED[9].code,
violates: false,
},
];
pub const SUBTLE: &[JudgeItem] = &[
JudgeItem {
inv: "V9: report every cycle in the federation graph",
code: "pub fn detect_cycles(edges: &[Edge]) -> Vec<Vec<String>> {\n edges.iter()\n .filter(|e| e.from == e.to)\n .map(|e| vec![e.from.clone()])\n .collect()\n}",
violates: true,
},
JudgeItem {
inv: "V6: the ceiling for a path is the longest matching prefix, else the default",
code: "pub fn for_path(&self, path: &str) -> u64 {\n self.rows.iter()\n .find(|(p, _)| path.starts_with(p.as_str()))\n .map_or(self.default, |(_, v)| *v)\n}",
violates: true,
},
JudgeItem {
inv: "V46: a budget subtracts entry cost; a negative budget is `does not fit`, not a huge one",
code: "pub fn working(window: u64) -> u64 {\n (window as i64 - ENTRY_COST as i64) as u64\n}",
violates: true,
},
JudgeItem {
inv: "V4: a verdict states DIRECTION and DISTANCE, never a bare bool",
code: "pub fn verdict(cost: u64, budget: u64) -> Verdict {\n if cost <= budget {\n Verdict::Fits { slack: budget - cost }\n } else {\n Verdict::Over { by: cost }\n }\n}",
violates: true,
},
JudgeItem {
inv: "V9: report every cycle in the federation graph",
code: "pub fn detect_cycles(edges: &[Edge]) -> Vec<Vec<String>> {\n let mut out = Vec::new();\n for start in edges.iter().map(|e| &e.from) {\n let mut seen = vec![start.clone()];\n let mut at = start;\n while let Some(n) = edges.iter().find(|e| &e.from == at).map(|e| &e.to) {\n if seen.contains(n) { out.push(seen.clone()); break; }\n seen.push(n.clone());\n at = n;\n }\n }\n out\n}",
violates: false,
},
JudgeItem {
inv: "V6: the ceiling for a path is the longest matching prefix, else the default",
code: "pub fn for_path(&self, path: &str) -> u64 {\n self.rows.iter()\n .filter(|(p, _)| path.starts_with(p.as_str()))\n .max_by_key(|(p, _)| p.len())\n .map_or(self.default, |(_, v)| *v)\n}",
violates: false,
},
JudgeItem {
inv: "V46: a budget subtracts entry cost; a negative budget is `does not fit`, not a huge one",
code: "pub fn working(window: u64) -> Option<u64> {\n window.checked_sub(ENTRY_COST)\n}",
violates: false,
},
JudgeItem {
inv: "V4: a verdict states DIRECTION and DISTANCE, never a bare bool",
code: "pub fn verdict(cost: u64, budget: u64) -> Verdict {\n if cost <= budget {\n Verdict::Fits { slack: budget - cost }\n } else {\n Verdict::Over { by: cost - budget }\n }\n}",
violates: false,
},
];
pub const TIERS: &[Tier] = &[
Tier {
name: "0-tells",
tells: true,
items: RECORDED,
},
Tier {
name: "1-bare",
tells: false,
items: RECORDED,
},
Tier {
name: "2-vague",
tells: false,
items: VAGUE,
},
Tier {
name: "3-subtle",
tells: false,
items: SUBTLE,
},
];
pub type SignaturePair = (Grade, Grade);
#[must_use]
pub const fn signature_mark(g: Grade) -> &'static str {
match g {
Grade::Pass => "PASS",
Grade::Fail => "fail",
Grade::NoCompile => "NOCALL",
Grade::Hung => "HUNG",
}
}
#[must_use]
pub fn signature_report(rs: &[SignaturePair]) -> String {
let n = rs.len();
let c = |f: fn(&SignaturePair) -> bool| rs.iter().filter(|r| f(r)).count();
format!(
"\nSIGNATURE TITRATION ({n} measured)\n \
given PASS {}/{n}\n \
free PASS {}/{n}\n \
free NOCALL {}/{n} (invented a name the tests cannot call)\n \
free wrong {}/{n} (callable, wrong logic)\n",
c(|r| r.0 == Grade::Pass),
c(|r| r.1 == Grade::Pass),
c(|r| r.1 == Grade::NoCompile),
c(|r| r.1 == Grade::Fail)
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Kill {
pub killed: bool,
pub broken: bool,
}
impl Kill {
#[must_use]
pub const fn word(self) -> &'static str {
if self.broken {
"BROKE"
} else if self.killed {
"killed"
} else {
"SURVIVED"
}
}
}
#[must_use]
pub fn kills_report(ks: &[Kill]) -> String {
let n = ks.len();
let killed = ks.iter().filter(|k| k.killed).count();
let broken = ks.iter().filter(|k| k.broken).count();
let survived = n.saturating_sub(killed).saturating_sub(broken);
format!(
"\nMUTATION SWEEP ({n} authored tests)\n \
killed the stub {killed}/{n} (the test discriminates)\n \
stub SURVIVED {survived}/{n} (the test measured nothing)\n \
test no-compile {broken}/{n} (graded nothing at all)\n"
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Verdicts {
pub hidden: bool,
pub own: bool,
pub own_broken: bool,
}
#[must_use]
pub fn authorship_counts(vs: &[Verdicts]) -> [usize; 5] {
let c = |f: fn(&Verdicts) -> bool| vs.iter().filter(|v| f(v)).count();
[
c(|v| v.hidden),
c(|v| v.own),
c(|v| v.own && !v.hidden),
c(|v| !v.own && !v.hidden && !v.own_broken),
c(|v| v.own_broken),
]
}
#[must_use]
pub fn authorship_row(v: Verdicts, sig: &str) -> String {
let own = if v.own_broken {
"BROKE"
} else if v.own {
"PASS"
} else {
"fail"
};
format!(
"hidden {} · own {own} · {}",
if v.hidden { "PASS" } else { "fail" },
sig.split('(').next().unwrap_or(sig).trim()
)
}
#[must_use]
pub fn authorship_report(vs: &[Verdicts]) -> String {
let n = vs.len();
let [hidden, own, wrong, caught, broke] = authorship_counts(vs);
format!(
"\nAUTHORSHIP TITRATION ({n} measured)\n \
hidden tests pass {hidden}/{n}\n \
own test passes {own}/{n}\n \
CERTIFIED WRONG {wrong}/{n} (own PASS, hidden fail)\n \
correctly rejected {caught}/{n} (both fail)\n \
own test unusable {broke}/{n} (did not compile)\n"
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Outcome {
Pass,
Fail,
Error,
Hung,
}
impl Outcome {
#[must_use]
pub const fn word(self) -> &'static str {
match self {
Self::Pass => "PASS",
Self::Fail => "fail",
Self::Error => "ERROR",
Self::Hung => "HUNG",
}
}
}
#[must_use]
pub fn titration_tally(os: &[Outcome]) -> (usize, usize, usize, usize) {
let n = |w: Outcome| os.iter().filter(|o| **o == w).count();
(
n(Outcome::Pass),
n(Outcome::Fail),
n(Outcome::Error),
n(Outcome::Hung),
)
}
#[must_use]
pub fn titration_report(label: &str, os: &[Outcome]) -> String {
let (p, f, e, h) = titration_tally(os);
let total = os.len();
let mut out = format!(
" {label:5} pass {p}/{total} · fail {f} · error {e} · hung {h}\n"
);
let lost = e.saturating_add(h);
if lost > 0 {
out.push_str(&format!(
" {lost} of {total} produced NO verdict -- this condition is \
incomplete, not measured (V1, V6)\n"
));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn grade_accepts_a_known_good_implementation() {
let it = &GEN_CORPUS[1]; let good = "pub fn bucket(prompt_tokens: u64) -> &'static str {\n match prompt_tokens {\n 0..=1_999 => \"b0\",\n 2_000..=7_999 => \"b2\",\n 8_000..=31_999 => \"b8\",\n _ => \"b32\",\n }\n}";
assert_eq!(
grade_detail(good, it.preamble, it.tests, "rustc"),
Ok(Grade::Pass),
"the real function must pass the tests it actually has"
);
}
#[test]
fn grade_rejects_code_that_does_not_compile() {
let it = &GEN_CORPUS[1];
assert_eq!(
grade_detail("pub fn bucket(", it.preamble, it.tests, "rustc"),
Ok(Grade::NoCompile),
"a candidate that will not compile is a WRONG ANSWER, not an error"
);
}
#[test]
fn grade_rejects_a_plausible_but_wrong_answer() {
let it = &GEN_CORPUS[1];
let stub = "pub fn bucket(prompt_tokens: u64) -> &'static str {\n if prompt_tokens > 0 { \"b0\" } else { \"b0\" }\n}";
assert_eq!(
grade_detail(stub, it.preamble, it.tests, "rustc"),
Ok(Grade::Fail)
);
}
#[test]
fn grade_errors_when_the_toolchain_is_absent() {
let it = &GEN_CORPUS[1];
assert!(
grade_detail(
"fn x() {}",
it.preamble,
it.tests,
"definitely-not-rustc"
)
.is_err(),
"an unrunnable compiler is an ERROR, never a score"
);
}
#[test]
fn sharp_and_vague_prompts_differ_only_in_the_invariant() {
for it in GEN_CORPUS {
let s = gen_prompt(it.sharp, it.sig, it.preamble);
let v = gen_prompt(it.vague, it.sig, it.preamble);
assert_eq!(
s.replace(it.sharp, "<INV>"),
v.replace(it.vague, "<INV>"),
"prompts must be identical outside the invariant"
);
assert!(!s.contains("#[cfg(test)]"), "the writer never sees tests");
assert!(!v.contains("assert"), "the writer never sees tests");
}
}
#[test]
fn a_call_that_produced_no_verdict_is_never_counted_as_a_failure() {
let os = [Outcome::Pass, Outcome::Fail, Outcome::Error, Outcome::Hung];
assert_eq!(
titration_tally(&os),
(1, 1, 1, 1),
"four outcomes, not two"
);
let none_ran = [Outcome::Error, Outcome::Hung];
assert_eq!(
titration_tally(&none_ran),
(0, 0, 1, 1),
"a run that produced no verdict scores zero PASS and zero FAIL"
);
}
#[test]
fn the_bare_prompt_carries_no_tells() {
let it = &RECORDED[0];
let bare = blind_prompt_bare(it.inv, it.code);
for tell in [
"returns a constant",
"never reads",
"quantity other than",
"placeholder",
] {
assert!(
!bare.contains(tell),
"the bare rung must hand over no checklist, found `{tell}`"
);
}
assert!(bare.contains(it.code), "must carry the implementation");
assert!(bare.contains(it.inv), "must carry the invariant");
}
#[test]
fn the_bare_prompt_never_shows_the_test() {
let bare = blind_prompt_bare(
"V1: report every dir with no owner",
"pub fn orphans(_d: &[Dir]) -> Vec<Dir> { Vec::new() }",
);
assert!(
!bare.to_lowercase().contains("#[test]"),
"V22: the test must not reach the judge: {bare}"
);
assert!(!bare.contains("assert"), "no test body may leak in: {bare}");
}
#[test]
fn every_rung_asks_the_same_question() {
for tier in TIERS {
for it in tier.items {
let p = tier.prompt(it);
assert!(
p.contains("Judge the code against the invariant alone"),
"rung {} dropped the question",
tier.name
);
assert!(p.contains(it.code), "rung {} dropped code", tier.name);
assert!(p.contains(it.inv), "rung {} dropped inv", tier.name);
}
}
}
#[test]
fn a_rung_that_did_not_run_is_an_error_not_a_zero() {
let mut dead = |_: &str| Err::<bool, String>("endpoint down".into());
let r = titrate_tier(&TIERS[0], &mut dead);
assert!(r.is_err(), "a judge that could not run is an ERROR");
}
#[test]
fn a_no_to_everything_scores_half_not_all() {
let mut always_no = |_: &str| Ok(false);
let s = titrate_tier(&TIERS[0], &mut always_no).unwrap();
assert_eq!(s.correct * 2, s.total, "NO to everything is half, not all");
}
}
#[cfg(test)]
mod authorship {
use super::*;
fn v(spec: &str) -> Verdicts {
let mut c = spec.chars();
let (h, o) = (c.next(), c.next());
Verdicts {
hidden: h == Some('P'),
own: o == Some('P'),
own_broken: o == Some('x'),
}
}
#[test]
fn the_discriminating_cell_is_own_pass_hidden_fail() {
let vs = [
v("PP"), v("fP"), v("Pf"), v("ff"), v("Px"), ];
let [hidden, own, wrong, caught, broke] = authorship_counts(&vs);
assert_eq!(hidden, 3, "hidden passes");
assert_eq!(own, 2, "own passes");
assert_eq!(wrong, 1, "own PASS while hidden FAILED");
assert_eq!(caught, 1, "both failed -- a real catch");
assert_eq!(broke, 1, "own test unusable");
}
#[test]
fn a_test_that_did_not_compile_is_not_counted_as_a_catch() {
let [_, _, _, caught, broke] = authorship_counts(&[v("fx")]);
assert_eq!(caught, 0, "a no-compile test caught nothing");
assert_eq!(broke, 1);
}
#[test]
fn the_report_names_the_certified_wrong_cell() {
let out = authorship_report(&[v("PP"), v("fP")]);
assert!(out.contains("(2 measured)"), "{out}");
assert!(out.contains("CERTIFIED WRONG 1/2"), "{out}");
assert!(out.contains("hidden tests pass 1/2"), "{out}");
}
#[test]
fn a_row_shows_broke_apart_from_fail() {
assert_eq!(
authorship_row(v("Px"), "pub fn bucket(n: u64)"),
"hidden PASS · own BROKE · pub fn bucket"
);
assert_eq!(
authorship_row(v("ff"), "pub fn is_yes(v: &str)"),
"hidden fail · own fail · pub fn is_yes"
);
}
const GOOD: &str = "pub fn bucket(n: u64) -> &'static str { match n { 0..=1_999 => \"b0\", 2_000..=7_999 => \"b2\", 8_000..=31_999 => \"b8\", _ => \"b32\" } }";
const STUB: &str = "pub fn bucket(_n: u64) -> &'static str { \"b0\" }";
#[test]
fn a_test_that_will_not_compile_is_not_a_correct_rejection() {
assert_eq!(three_way(), Ok(()));
}
fn three_way() -> Result<(), String> {
let it = GEN_CORPUS.get(1).ok_or("corpus")?;
let g = |code, tests| grade_detail(code, it.preamble, tests, "rustc");
assert_eq!(g(GOOD, "not rust at all")?, Grade::NoCompile);
assert_eq!(g(GOOD, it.tests)?, Grade::Pass);
assert_eq!(
g(STUB, it.tests)?,
Grade::Fail,
"a compiling wrong answer is FAIL, never NoCompile"
);
Ok(())
}
}
#[cfg(test)]
mod mutants {
use super::*;
#[test]
fn every_item_has_a_stub() {
for it in GEN_CORPUS {
assert!(
stub_for(it.sig).is_some(),
"no mutant for {} -- a skipped item shrinks the denominator",
it.sig
);
}
}
#[test]
fn the_hidden_tests_kill_every_stub() {
assert_eq!(control(), Ok(()));
}
fn control() -> Result<(), String> {
for it in GEN_CORPUS {
let stub = stub_for(it.sig).ok_or("missing stub")?;
let g = grade_detail(stub, it.preamble, it.tests, "rustc")?;
assert_eq!(
g,
Grade::Fail,
"{} -- a stub the real tests accept is not a mutant",
it.sig
);
}
Ok(())
}
}
#[cfg(test)]
mod kills {
use super::*;
const fn k(spec: char) -> Kill {
Kill {
killed: spec == 'k',
broken: spec == 'x',
}
}
#[test]
fn the_three_outcomes_are_named_distinctly() {
assert_eq!(k('k').word(), "killed");
assert_eq!(k('s').word(), "SURVIVED");
assert_eq!(k('x').word(), "BROKE");
}
#[test]
fn the_report_counts_survivors_as_the_remainder() {
let out = kills_report(&[k('k'), k('s'), k('x')]);
assert!(out.contains("killed the stub 1/3"), "{out}");
assert!(out.contains("stub SURVIVED 1/3"), "{out}");
assert!(out.contains("test no-compile 1/3"), "{out}");
}
#[test]
fn an_empty_sweep_never_wraps_the_remainder() {
let out = kills_report(&[]);
assert!(out.contains("(0 authored tests)"), "{out}");
assert!(out.contains("stub SURVIVED 0/0"), "{out}");
}
#[test]
fn a_vacuous_test_lets_the_stub_live() {
let Some(it) = GEN_CORPUS.first() else {
return;
};
let Some(stub) = stub_for(it.sig) else { return };
let vacuous =
"#[cfg(test)]\nmod t { #[test] fn a() { assert!(true); } }";
assert_eq!(
grade_detail(stub, it.preamble, vacuous, "rustc"),
Ok(Grade::Pass),
"a test that asserts nothing passes a stub -- that is SURVIVED"
);
}
}
#[cfg(test)]
mod signature {
use super::*;
#[test]
fn a_name_the_tests_cannot_call_is_not_wrong_logic() {
assert_eq!(signature_mark(Grade::NoCompile), "NOCALL");
assert_eq!(signature_mark(Grade::Fail), "fail");
assert_eq!(signature_mark(Grade::Pass), "PASS");
assert_eq!(signature_mark(Grade::Hung), "HUNG");
}
#[test]
fn the_report_separates_uncallable_from_wrong() {
let out = signature_report(&[
(Grade::Pass, Grade::NoCompile),
(Grade::Pass, Grade::Fail),
(Grade::Pass, Grade::Pass),
]);
assert!(out.contains("given PASS 3/3"), "{out}");
assert!(out.contains("free PASS 1/3"), "{out}");
assert!(out.contains("free NOCALL 1/3"), "{out}");
assert!(out.contains("free wrong 1/3"), "{out}");
}
#[test]
fn an_empty_run_reports_zero_of_zero_and_not_a_score() {
let out = signature_report(&[]);
assert!(out.contains("(0 measured)"), "{out}");
assert!(out.contains("given PASS 0/0"), "{out}");
}
}
#[cfg(test)]
mod ambiguity {
use super::*;
const GOOD: &str = "pub fn bucket(n: u64) -> &'static str { match n { 0..=1_999 => \"b0\", 2_000..=7_999 => \"b2\", 8_000..=31_999 => \"b8\", _ => \"b32\" } }";
#[test]
fn a_pair_that_did_not_compile_is_not_a_disagreement() {
assert_eq!(reading(Grade::NoCompile), Reading::Uncallable);
let mut r = RowReadings::new("bucket");
r.push(Reading::Uncallable);
assert!(!r.underspecified(), "uncallable flags nothing");
assert_eq!(r.measured(), 0, "and it is not in the denominator");
assert_eq!(r.verdict(), "not measured");
}
#[test]
fn the_denominator_names_only_what_was_graded() {
let mut r = RowReadings::new("escape_cell");
for x in [Reading::Agree, Reading::Disagree, Reading::Uncallable] {
r.push(x);
}
assert_eq!(r.measured(), 2);
assert_eq!(r.uncallable, 1);
assert!(r.underspecified());
assert_eq!(r.verdict(), "UNDERSPECIFIED");
}
#[test]
fn one_disagreement_is_enough_to_flag_a_row() {
let mut r = RowReadings::new("for_path");
r.push(Reading::Agree);
r.push(Reading::Agree);
assert_eq!(r.verdict(), "no gap found");
r.push(Reading::Disagree);
assert_eq!(r.verdict(), "UNDERSPECIFIED");
}
#[test]
fn the_three_readings_are_named_distinctly() {
assert_eq!(Reading::Agree.word(), "agree");
assert_eq!(Reading::Disagree.word(), "DISAGREE");
assert_eq!(Reading::Uncallable.word(), "uncallable");
}
#[test]
fn cross_is_graded_by_rustc_and_never_by_a_model() {
assert_eq!(three_readings(), Ok(()));
}
fn three_readings() -> Result<(), String> {
let it = GEN_CORPUS.get(1).ok_or("corpus")?;
let x = |code, test| cross(code, test, it.preamble, "rustc");
assert_eq!(x(GOOD, it.tests)?, Reading::Agree);
let stub = stub_for(it.sig).ok_or("mutant")?;
assert_eq!(
x(stub, it.tests)?,
Reading::Disagree,
"two readings of one row -- the signal"
);
assert_eq!(x(GOOD, "not rust at all")?, Reading::Uncallable);
Ok(())
}
#[test]
fn the_report_flags_a_row_without_certifying_the_others() {
let mut bad = RowReadings::new("escape_cell");
bad.push(Reading::Disagree);
let mut ok = RowReadings::new("bucket");
ok.push(Reading::Agree);
let out = ambiguity_report(&[bad, ok]);
assert!(out.contains("1 UNDERSPECIFIED"), "{out}");
assert!(out.contains("UNDERSPECIFIED escape_cell"), "{out}");
assert!(out.contains("no gap found bucket"), "{out}");
assert!(out.contains("agreement ⊥ sharpness"), "{out}");
}
#[test]
fn an_empty_report_flags_nothing_and_still_says_why() {
let out = ambiguity_report(&[]);
assert!(out.contains("0 rows, 0 UNDERSPECIFIED"), "{out}");
assert!(out.contains(AGREEMENT_IS_NOT_SHARPNESS), "{out}");
}
}
#[cfg(test)]
mod bound {
use super::*;
const NEVER_ENDS: &str = "#[cfg(test)]\nmod t {\n #[test]\n fn a() { loop { std::hint::spin_loop(); } }\n}";
#[test]
fn a_test_that_never_terminates_is_killed_and_reported_apart() {
assert_eq!(hung_not_failed(), Ok(()));
}
fn hung_not_failed() -> Result<(), String> {
let started = std::time::Instant::now();
let g = grade_detail("", "", NEVER_ENDS, "rustc")?;
assert_eq!(
g,
Grade::Hung,
"a killed child exits non-zero, so `Fail` is what a hang looks \
like from outside -- V6 is that the fold must not happen"
);
assert!(
started.elapsed() < GRADE_TIMEOUT.saturating_mul(3),
"the bound must actually bound: {:?}",
started.elapsed()
);
Ok(())
}
#[test]
fn a_hang_is_not_a_disagreement_and_leaves_the_denominator() {
assert_eq!(reading(Grade::Hung), Reading::Hung);
let mut r = RowReadings::new("abort_budget_ms");
r.push(Reading::Hung);
assert!(!r.underspecified(), "a hang flags no row");
assert_eq!(r.measured(), 0, "and it is not in the denominator");
assert_eq!(r.hung, 1);
assert_eq!(r.verdict(), "not measured");
}
#[test]
fn the_report_shows_hangs_where_a_reader_will_see_them() {
let mut r = RowReadings::new("abort_budget_ms");
r.push(Reading::Hung);
r.push(Reading::Agree);
let out = ambiguity_report(&[r]);
assert!(out.contains("1 hung"), "{out}");
assert!(out.contains("0/1 disagree"), "{out}");
}
#[test]
fn the_bound_costs_a_terminating_run_nothing() {
let Some(it) = GEN_CORPUS.get(1) else { return };
let good = "pub fn bucket(n: u64) -> &'static str { match n { 0..=1_999 => \"b0\", 2_000..=7_999 => \"b2\", 8_000..=31_999 => \"b8\", _ => \"b32\" } }";
assert_eq!(
grade_detail(good, it.preamble, it.tests, "rustc"),
Ok(Grade::Pass)
);
}
}
#[cfg(test)]
mod reports {
use super::*;
#[test]
fn a_condition_with_errors_says_it_is_incomplete() {
let out = titration_report(
"bare",
&[Outcome::Pass, Outcome::Pass, Outcome::Error],
);
assert!(out.contains("pass 2/3"), "{out}");
assert!(out.contains("1 of 3 produced NO verdict"), "{out}");
assert!(out.contains("incomplete, not measured"), "{out}");
}
#[test]
fn a_hung_call_counts_as_incomplete_alongside_an_error() {
let out = titration_report("ctx", &[Outcome::Hung, Outcome::Error]);
assert!(out.contains("2 of 2 produced NO verdict"), "{out}");
}
#[test]
fn a_complete_condition_adds_no_incomplete_line() {
let out = titration_report("bare", &[Outcome::Pass, Outcome::Fail]);
assert!(out.contains("pass 1/2"), "{out}");
assert!(!out.contains("NO verdict"), "{out}");
}
#[test]
fn the_four_outcomes_are_named_distinctly() {
assert_eq!(Outcome::Pass.word(), "PASS");
assert_eq!(Outcome::Fail.word(), "fail");
assert_eq!(Outcome::Error.word(), "ERROR");
assert_eq!(Outcome::Hung.word(), "HUNG");
}
#[test]
fn a_channel_predicts_before_the_scores_are_seen() {
assert_eq!(Channel::Type.predicts(), (true, true));
assert_eq!(Channel::Prose.predicts(), (true, false));
assert_eq!(Channel::Beyond.predicts(), (false, false));
}
#[test]
fn a_pack_is_prepended_and_the_request_survives_verbatim() {
let Some(it) = GEN_CORPUS.first() else { return };
let bare = gen_prompt(it.sharp, it.sig, it.preamble);
let ctx =
gen_prompt_in_context("PACK BODY", it.sharp, it.sig, it.preamble);
assert!(ctx.ends_with(&bare), "the request survives verbatim");
assert!(ctx.contains("PACK BODY"), "the pack is carried");
}
}