1use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8
9use regex::Regex;
10use sha2::{Digest, Sha256};
11
12use crate::tools::paths::{Class, Protected};
13use crate::tools::policy::Policy;
14use crate::worktree::{Change, ChangeKind};
15
16const SKIPPED_DIRS: &[&str] = &[
19 ".git",
20 "target",
21 "node_modules",
22 ".venv",
23 "venv",
24 "__pycache__",
25 ".tox",
26 ".mypy_cache",
27 ".pytest_cache",
28];
29
30#[derive(Debug, Clone, Default, PartialEq, Eq)]
34pub struct JudgeSnapshot(BTreeMap<String, String>);
35
36#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
38pub struct Drift {
39 pub path: String,
41 pub class: &'static str,
43 pub change: &'static str,
45}
46
47impl JudgeSnapshot {
48 #[must_use]
51 pub fn take(root: &Path, policy: &Policy) -> Self {
52 let mut files = BTreeMap::new();
53 let walk = ignore::WalkBuilder::new(root)
54 .hidden(false)
55 .ignore(false)
56 .git_ignore(false)
57 .git_global(false)
58 .git_exclude(false)
59 .parents(false)
60 .follow_links(false)
61 .filter_entry(|e| {
62 !(e.depth() > 0
63 && e.file_type().is_some_and(|t| t.is_dir())
64 && SKIPPED_DIRS.contains(&e.file_name().to_string_lossy().as_ref()))
65 })
66 .build();
67 for entry in walk.flatten() {
68 if !entry.file_type().is_some_and(|t| t.is_file()) {
69 continue;
70 }
71 let Ok(rel) = entry.path().strip_prefix(root) else {
72 continue;
73 };
74 let rel = rel.to_string_lossy().replace('\\', "/");
75 if class_of(policy, &rel).is_none() {
76 continue;
77 }
78 if let Ok(bytes) = std::fs::read(entry.path()) {
79 files.insert(rel, sha256_hex(&bytes));
80 }
81 }
82 Self(files)
83 }
84
85 #[must_use]
87 pub fn entries(&self) -> &BTreeMap<String, String> {
88 &self.0
89 }
90
91 #[must_use]
93 pub fn from_entries(entries: BTreeMap<String, String>) -> Self {
94 Self(entries)
95 }
96
97 pub fn set(&mut self, path: &str, hash: String) {
99 self.0.insert(path.to_owned(), hash);
100 }
101
102 #[must_use]
105 pub fn drift(&self, later: &Self, policy: &Policy, approved: &BTreeSet<String>) -> Vec<Drift> {
106 let mut out = Vec::new();
107 let paths: BTreeSet<&String> = self.0.keys().chain(later.0.keys()).collect();
108 for path in paths {
109 let change = match (self.0.get(path), later.0.get(path)) {
110 (None, Some(_)) => "added",
111 (Some(_), None) => "deleted",
112 (Some(a), Some(b)) if a != b => "modified",
113 _ => continue,
114 };
115 if approved.contains(path.as_str()) {
116 continue;
117 }
118 if let Some(class) = class_of(policy, path) {
119 out.push(Drift {
120 path: path.clone(),
121 class,
122 change,
123 });
124 }
125 }
126 out
127 }
128}
129
130fn class_of(policy: &Policy, rel: &str) -> Option<&'static str> {
131 match policy.write_protected(rel) {
132 Some(Class::Judge) => Some("judge"),
133 Some(Class::Intent) => Some("intent"),
134 Some(_) => None,
135 None if policy.protected().is_context(rel) => Some("context"),
136 None => None,
137 }
138}
139
140const TEST_MARKERS: &str = r"#\[(tokio::)?test\b|^\s*def test_|^\s*(async\s+)?fn test_|\b(it|test)\s*\(\s*['\x22`]|@Test\b|^\s*func Test";
142const SKIP_MARKERS: &str = r"#\[ignore\b|@pytest\.mark\.(skip|xfail)|\b(?:it|describe|test|suite|pytest)\.skip\s*\(|\bx(it|describe|test)\s*\(|@Disabled\b|@Ignore\b|\bt\.Skip(Now)?\s*\(|@unittest\.skip|\b(?:it|test)\.todo\s*\(";
144
145const TEST_NAMES: &str = r#"(?m)#\[(?:tokio::)?test\b[^\]]*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)|^\s*(?:async\s+)?def\s+(test_\w+)|^\s*func\s+(Test\w+)|\b(?:it|test)\s*\(\s*['"`]([^'"`]+)['"`]"#;
148
149#[must_use]
151pub fn removed_tests(before: &str, after: &str) -> Vec<String> {
152 let Ok(re) = Regex::new(TEST_NAMES) else {
153 return Vec::new();
154 };
155 let names = |text: &str| -> Vec<String> {
156 re.captures_iter(text)
157 .filter_map(|c| {
158 c.iter()
159 .skip(1)
160 .flatten()
161 .next()
162 .map(|m| m.as_str().to_owned())
163 })
164 .collect()
165 };
166 let kept: BTreeSet<String> = names(after).into_iter().collect();
167 let mut gone = names(before);
168 gone.retain(|n| !kept.contains(n));
169 gone.dedup();
170 gone
171}
172
173const PLACEHOLDER_MARKERS: &str = r#"\braise\s+NotImplementedError\b|\btodo!\s*\(|\bunimplemented!\s*\(|(?i:throw\s+new\s+Error\s*\(\s*['"`](?:not implemented|todo|placeholder))|(?i:\b(?:t\.Fatal|pytest\.fail|fail)\s*\(\s*['"`](?:not implemented|todo|placeholder))"#;
176
177#[must_use]
181pub fn has_placeholder(text: &str) -> bool {
182 Regex::new(PLACEHOLDER_MARKERS).is_ok_and(|r| r.is_match(text))
183}
184
185#[must_use]
189pub fn asserts_crlf_as_text(path: &str, text: &str) -> bool {
190 path.ends_with(".py")
191 && (text.contains("text=True") || text.contains("universal_newlines=True"))
192 && text
193 .lines()
194 .any(|l| l.contains("assert") && l.contains("\\r\\n"))
195}
196
197#[must_use]
200pub fn never_passes(path: &str, text: &str) -> Option<&'static str> {
201 if asserts_crlf_as_text(path, text) {
202 Some("it asserts \\r\\n in output read with text=True, which turns it into \\n")
203 } else if has_placeholder(text) {
204 Some("it raises \"not implemented\" instead of checking the behaviour")
205 } else {
206 None
207 }
208}
209
210#[must_use]
212pub fn has_skip_marker(text: &str) -> bool {
213 Regex::new(SKIP_MARKERS).is_ok_and(|r| r.is_match(text))
214}
215
216#[must_use]
219pub fn only_additions(old: &str, new: &str) -> bool {
220 let mut new_lines = new.lines();
221 old.lines().all(|line| new_lines.any(|n| n == line))
222}
223
224#[must_use]
230pub fn audit(
231 changes: &[Change],
232 patch: &str,
233 frozen: &BTreeSet<String>,
234 additions_ok: bool,
235 approved: &BTreeSet<String>,
236) -> Vec<String> {
237 audit_in(
238 &Protected::default(),
239 changes,
240 patch,
241 frozen,
242 additions_ok,
243 approved,
244 )
245}
246
247#[must_use]
249pub fn audit_in(
250 protected: &Protected,
251 changes: &[Change],
252 patch: &str,
253 frozen: &BTreeSet<String>,
254 additions_ok: bool,
255 approved: &BTreeSet<String>,
256) -> Vec<String> {
257 let mut findings = Vec::new();
258 let removed_lines = removed_lines(patch);
259
260 for c in changes {
261 if frozen
262 .iter()
263 .any(|f| f.to_lowercase() == c.path.to_lowercase())
264 {
265 match c.kind {
266 ChangeKind::Deleted => findings.push(format!("frozen test deleted: {}", c.path)),
267 ChangeKind::Modified
268 if additions_ok && removed_lines.get(&c.path).copied().unwrap_or(0) == 0 => {}
269 ChangeKind::Modified => findings.push(format!("frozen test modified: {}", c.path)),
270 ChangeKind::Added => {}
271 }
272 }
273 match protected.write_class(&c.path) {
274 Some(Class::Judge) => findings.push(format!("verification input changed: {}", c.path)),
275 Some(Class::Intent) => findings.push(format!("spec changed: {}", c.path)),
276 None if protected.is_context(&c.path) && !approved.contains(&c.path) => {
277 findings.push(format!("agent context file changed: {}", c.path));
278 }
279 _ => {}
280 }
281 }
282
283 let (Ok(test), Ok(skip)) = (Regex::new(TEST_MARKERS), Regex::new(SKIP_MARKERS)) else {
284 return findings;
285 };
286 let mut tests: BTreeMap<String, (usize, usize)> = BTreeMap::new(); let mut text: BTreeMap<String, (String, String)> = BTreeMap::new(); let mut skips: BTreeSet<String> = BTreeSet::new();
289 let mut file = String::new();
290 for line in patch.lines() {
291 if let Some(header) = line.strip_prefix("diff --git a/") {
292 file = header
295 .split_once(" b/")
296 .map_or(header, |(a, _)| a)
297 .to_owned();
298 } else if line.starts_with("+++ ") || line.starts_with("--- ") {
299 } else if let Some(added) = line.strip_prefix('+') {
301 let t = text.entry(file.clone()).or_default();
302 t.0.push_str(added);
303 t.0.push('\n');
304 if test.is_match(added) {
305 tests.entry(file.clone()).or_default().0 += 1;
306 }
307 if skip.is_match(added) {
308 skips.insert(file.clone());
309 }
310 } else if let Some(removed) = line.strip_prefix('-') {
311 let t = text.entry(file.clone()).or_default();
312 t.1.push_str(removed);
313 t.1.push('\n');
314 if test.is_match(removed) {
315 tests.entry(file.clone()).or_default().1 += 1;
316 }
317 }
318 }
319 let deleted: BTreeSet<&str> = changes
320 .iter()
321 .filter(|c| c.kind == ChangeKind::Deleted)
322 .map(|c| c.path.as_str())
323 .collect();
324 for (path, (added, removed)) in tests {
325 if removed > added && !deleted.contains(path.as_str()) {
327 let names = text
328 .get(&path)
329 .map(|(a, r)| removed_tests(r, a))
330 .unwrap_or_default();
331 let named = if names.is_empty() {
332 String::new()
333 } else {
334 format!(": {}", names.join(", "))
335 };
336 findings.push(format!(
337 "test removed in {path} ({removed} removed, {added} added){named}"
338 ));
339 }
340 }
341 for path in skips {
342 findings.push(format!("skip marker added in {path}"));
343 }
344 findings
345}
346
347fn removed_lines(patch: &str) -> BTreeMap<String, usize> {
349 let mut out: BTreeMap<String, usize> = BTreeMap::new();
350 let mut file = String::new();
351 for line in patch.lines() {
352 if let Some(header) = line.strip_prefix("diff --git a/") {
353 file = header
354 .split_once(" b/")
355 .map_or(header, |(a, _)| a)
356 .to_owned();
357 } else if line.starts_with('-') && !line.starts_with("--- ") {
358 *out.entry(file.clone()).or_default() += 1;
359 }
360 }
361 out
362}
363
364#[must_use]
367pub fn sha256_hex(bytes: &[u8]) -> String {
368 Sha256::digest(bytes)
369 .iter()
370 .map(|b| format!("{b:02x}"))
371 .collect()
372}
373
374pub(crate) fn untick(bytes: &[u8]) -> Vec<u8> {
376 let mut out = bytes.to_vec();
377 for i in 1..out.len().saturating_sub(1) {
378 if out[i - 1] == b'[' && out[i + 1] == b']' && matches!(out[i], b'x' | b'X') {
379 out[i] = b' ';
380 }
381 }
382 out
383}
384
385#[cfg(test)]
386mod tests {
387
388 use super::*;
389
390 fn change(path: &str, kind: ChangeKind) -> Change {
391 Change {
392 path: path.to_owned(),
393 kind,
394 }
395 }
396
397 #[test]
398 fn removed_tests_are_named_across_languages() {
399 let before = "#[cfg(test)]\nmod tests {\n #[test]\n fn it_works() {}\n #[tokio::test]\n async fn later() {}\n}\ndef test_py(): pass\nfunc TestGo(t *testing.T) {}\nit('greets', () => {})\n";
400 assert_eq!(
401 removed_tests(before, "#[test]\nfn later() {}\n"),
402 ["it_works", "test_py", "TestGo", "greets"]
403 );
404 assert!(removed_tests(before, before).is_empty());
405 }
406
407 #[test]
408 fn flags_tampering_and_passes_clean_diffs() {
409 let frozen: BTreeSet<String> = ["tests/api.rs".to_owned()].into();
410 let patch = "diff --git a/src/lib.rs b/src/lib.rs\n--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -1 +1 @@\n- #[test]\n+ #[ignore]\n+ #[test]\n";
411 let f = audit(
412 &[
413 change("tests/api.rs", ChangeKind::Deleted),
414 change("speccoder.toml", ChangeKind::Modified),
415 ],
416 patch,
417 &frozen,
418 false,
419 &BTreeSet::new(),
420 );
421 assert_eq!(
422 f,
423 [
424 "frozen test deleted: tests/api.rs",
425 "verification input changed: speccoder.toml",
426 "skip marker added in src/lib.rs",
427 ]
428 );
429
430 let removed = "diff --git a/src/lib.rs b/src/lib.rs\n--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -5,4 +4,0 @@\n- #[test]\n- fn a() {}\n";
431 assert_eq!(
432 audit(&[], removed, &BTreeSet::new(), false, &BTreeSet::new()),
433 ["test removed in src/lib.rs (1 removed, 0 added): a"]
434 );
435
436 let clean = "diff --git a/src/lib.rs b/src/lib.rs\n--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -1 +1,4 @@\n+pub fn add() {}\n+ #[test]\n+ fn adds() {}\n";
437 assert!(
438 audit(
439 &[
440 change("src/lib.rs", ChangeKind::Modified),
441 change("tests/new.rs", ChangeKind::Added)
442 ],
443 clean,
444 &frozen,
445 false,
446 &BTreeSet::new()
447 )
448 .is_empty()
449 );
450 }
451
452 #[test]
453 fn additions_to_a_frozen_test_pass_only_in_additions_mode() {
454 let frozen: BTreeSet<String> = ["tests/api.rs".to_owned()].into();
455 let added = "diff --git a/tests/api.rs b/tests/api.rs\n--- a/tests/api.rs\n+++ b/tests/api.rs\n@@ -1,2 +1,4 @@\n #[test]\n fn a() {}\n+#[test]\n+fn b() {}\n";
456 let changed = "diff --git a/tests/api.rs b/tests/api.rs\n--- a/tests/api.rs\n+++ b/tests/api.rs\n@@ -1,2 +1,2 @@\n #[test]\n-fn a() { assert!(x()) }\n+fn a() {}\n";
457 let modified = [change("tests/api.rs", ChangeKind::Modified)];
458 assert!(audit(&modified, added, &frozen, true, &BTreeSet::new()).is_empty());
459 assert_eq!(
460 audit(&modified, added, &frozen, false, &BTreeSet::new()),
461 ["frozen test modified: tests/api.rs"]
462 );
463 assert_eq!(
464 audit(&modified, changed, &frozen, true, &BTreeSet::new()),
465 ["frozen test modified: tests/api.rs"]
466 );
467
468 assert!(only_additions("a\nb\n", "a\nnew\nb\nmore\n"));
469 assert!(!only_additions("a\nb\n", "a\nB\n"));
470 assert!(!only_additions("a\nb\n", "b\na\n"));
471 }
472
473 #[test]
474 fn a_snapshot_sees_ignored_judge_inputs_and_reports_drift() {
475 use crate::tools::policy::Policy;
476 let dir = tempfile::tempdir().unwrap();
477 let root = dir.path();
478 std::fs::create_dir_all(root.join(".claude")).unwrap();
479 std::fs::create_dir_all(root.join("target/debug")).unwrap();
480 std::fs::write(root.join(".claude/settings.json"), "{}").unwrap();
481 std::fs::write(root.join("AGENTS.md"), "a").unwrap();
482 std::fs::write(root.join("speccoder.toml"), "x").unwrap();
483 std::fs::write(root.join("src.rs"), "free").unwrap();
484 std::fs::write(root.join("target/debug/speccoder.toml"), "skipped").unwrap();
485 let policy = Policy::new(&[], &[]);
486 let before = JudgeSnapshot::take(root, &policy);
487 assert_eq!(
488 before.0.len(),
489 3,
490 "{:?}",
491 before.0.keys().collect::<Vec<_>>()
492 );
493
494 std::fs::write(root.join(".claude/settings.local.json"), "{}").unwrap();
495 std::fs::write(root.join("AGENTS.md"), "b").unwrap();
496 std::fs::remove_file(root.join("speccoder.toml")).unwrap();
497 std::fs::write(root.join("src.rs"), "changed").unwrap();
498 let after = JudgeSnapshot::take(root, &policy);
499 let drift = before.drift(&after, &policy, &BTreeSet::new());
500 let seen: Vec<(&str, &str, &str)> = drift
501 .iter()
502 .map(|d| (d.path.as_str(), d.class, d.change))
503 .collect();
504 assert_eq!(
505 seen,
506 [
507 (".claude/settings.local.json", "judge", "added"),
508 ("AGENTS.md", "context", "modified"),
509 ("speccoder.toml", "judge", "deleted"),
510 ]
511 );
512 let approved: BTreeSet<String> = ["AGENTS.md".to_owned()].into();
514 assert_eq!(before.drift(&after, &policy, &approved).len(), 2);
515 std::fs::write(root.join("AGENTS.md"), "a").unwrap();
517 std::fs::write(root.join("speccoder.toml"), "x").unwrap();
518 std::fs::remove_file(root.join(".claude/settings.local.json")).unwrap();
519 assert!(
520 before
521 .drift(
522 &JudgeSnapshot::take(root, &policy),
523 &policy,
524 &BTreeSet::new()
525 )
526 .is_empty()
527 );
528 }
529
530 #[test]
534 fn a_judge_input_in_another_letter_case_is_drift() {
535 use crate::tools::policy::Policy;
536 let dir = tempfile::tempdir().unwrap();
537 let root = dir.path();
538 let policy = Policy::new(&[], &[]);
539 let before = JudgeSnapshot::take(root, &policy);
540 std::fs::create_dir_all(root.join(".CLAUDE")).unwrap();
541 std::fs::write(root.join(".CLAUDE/settings.json"), "{}").unwrap();
542 std::fs::create_dir_all(root.join(".Cursor")).unwrap();
543 std::fs::write(root.join(".Cursor/rules.json"), "{}").unwrap();
544 std::fs::write(root.join("Makefile.x"), "free").unwrap();
545 std::fs::write(root.join("MAKEFILE"), "all:").unwrap();
546 let after = JudgeSnapshot::take(root, &policy);
547 let drift: Vec<&str> = before
548 .drift(&after, &policy, &BTreeSet::new())
549 .iter()
550 .map(|d| d.class)
551 .collect();
552 assert_eq!(drift, ["judge", "judge", "judge"], "{:?}", after.0.keys());
553 let frozen: BTreeSet<String> = ["tests/api.rs".to_owned()].into();
555 assert_eq!(
556 audit(
557 &[change("Tests/API.rs", ChangeKind::Deleted)],
558 "",
559 &frozen,
560 false,
561 &BTreeSet::new()
562 ),
563 ["frozen test deleted: Tests/API.rs"]
564 );
565 }
566
567 #[test]
568 fn a_placeholder_test_is_known_in_each_language() {
569 for text in [
570 "def test_a():\n raise NotImplementedError(\"later\")\n",
571 "#[test] fn a() { todo!() }",
572 "#[test] fn a() { unimplemented!(\"x\") }",
573 "it('a', () => { throw new Error('Not implemented') })",
574 "func TestA(t *testing.T) { t.Fatal(\"not implemented\") }",
575 ] {
576 assert!(has_placeholder(text), "{text}");
577 }
578 for text in [
579 "def test_a():\n with pytest.raises(NotImplementedError):\n f()\n",
580 "#[test] fn a() { assert_eq!(f(), 2) }",
581 "it('a', () => expect(() => f()).toThrow('not implemented'))",
582 ] {
583 assert!(!has_placeholder(text), "{text}");
584 }
585 }
586
587 #[test]
590 fn crlf_asserted_on_text_output_never_passes() {
591 let run = "r = subprocess.run(cmd, input=t, text=True, capture_output=True)\n";
592 assert!(asserts_crlf_as_text(
593 "tests/test_us1.py",
594 &format!("{run}assert r.stdout == \"a,b\\r\\n\"\n")
595 ));
596 assert!(!asserts_crlf_as_text(
598 "tests/test_us1.py",
599 &format!("{run}assert r.stdout.splitlines() == [\"a,b\"]\n")
600 ));
601 assert!(!asserts_crlf_as_text(
602 "tests/test_us1.py",
603 "r = subprocess.run(cmd, input=b\"x\", capture_output=True)\nassert r.stdout == b\"a,b\\r\\n\"\n"
604 ));
605 assert!(!asserts_crlf_as_text(
606 "tests/test_us1.py",
607 &format!("t = \"| a |\\r\\n\"\n{run}assert r.stdout == \"a\\n\"\n")
608 ));
609 assert_eq!(
611 never_passes(
612 "tests/test_us1.py",
613 &format!("{run}assert r.stdout == \"a,b\\r\\n\"\n")
614 ),
615 Some("it asserts \\r\\n in output read with text=True, which turns it into \\n")
616 );
617 assert_eq!(
618 never_passes("tests/test_us1.py", &format!("{run}assert r.stdout\n")),
619 None
620 );
621 }
622}